diff --git a/.cargo/config.toml b/.cargo/config.toml index ac33e0c8..7c9f7c37 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,2 @@ [env] ROOT_DIR = { value = "", relative = true } -TEST_ORIGIN_KEY = "4527387f92a381cbe804593f33991d327d456a97" diff --git a/.dockerignore b/.dockerignore index 4cec9f56..73b3870e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,5 @@ # Only include the skyd binary !target/release/skyd !target/release/skysh -!examples/config-files/docker.toml \ No newline at end of file +!examples/config-files/dpkg/config.yaml +!pkg/docker/start-server.sh \ No newline at end of file diff --git a/.github/workflows/benches-release.yml b/.github/workflows/benches-release.yml deleted file mode 100644 index ce269f2b..00000000 --- a/.github/workflows/benches-release.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Update benches (release) - -on: - push: - tags: - - "v*" - -jobs: - request-bench: - name: Add request for updating bench - runs-on: ubuntu-latest - steps: - - name: Clone perf repo - run: git clone https://github.com/skytable/perf.git - - name: Configure git - run: | - git config --global user.name "Glydr" - git config --global user.email "${{ secrets.GLYDR_MAIL }}" - - name: Set env - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Add request and push - env: - REL: ${{ env.RELEASE_VERSION }} - run: | - cd perf - printf "\ntarget/release/skyreport update release $REL" >> requests.txt - git add . - git commit -m "Add update request for release ``$REL``" - git push https://glydr:${{ secrets.GLYDR }}@github.com/skytable/perf.git --all diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml deleted file mode 100644 index c2316934..00000000 --- a/.github/workflows/deploy-docs.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Rebuild docs - -on: - push: - branches: - - next -env: - IS_ACTIONS_DOC: "false" - -jobs: - rebuild-docs: - name: Build new actiondoc - runs-on: ubuntu-latest - steps: - - name: Checkout source code - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - - name: Setup environment - run: | - chmod +x ci/setvars.sh - ci/setvars.sh - - - name: Publish docs - env: - BOT_USER: ${{ secrets.BOT_INIT_USER }} - BOT_MAIL: ${{ secrets.BOT_INIT_MAIL }} - BOT_API: ${{ secrets.BOT_API_CALL }} - GIT_SHA: ${{ env.GITHUB_SHA }} - run: | - chmod +x ci/doc.sh - ci/doc.sh - if: env.IS_ACTIONS_DOC == 'true' diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index f74c86d3..a972a9d9 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -2,16 +2,13 @@ name: Docker image on: push: - # Publish `next` as Docker `latest` image. branches: - next - - # Publish `v1.2.3` tags as releases. tags: - v* env: - IMAGE_NAME: sdb + IMAGE_NAME: skytable BUILD: "false" jobs: @@ -34,14 +31,16 @@ jobs: command: build args: --release - name: Build image - run: docker build . --file Dockerfile --tag $IMAGE_NAME - if: env.BUILD == 'true' || github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v') + run: docker build . --file Dockerfile --tag $IMAGE_NAME:${{ github.ref == 'refs/heads/next' && 'next' || github.ref_name }} + if: github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/tags/v') - name: Push to Docker Hub uses: docker/build-push-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - repository: skytable/sdb - tags: latest - tag_with_ref: true - if: env.BUILD == 'true' || github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v') + repository: skytable/skytable + tags: | + ${{ github.ref == 'refs/heads/next' && 'next' || '' }} + ${{ startsWith(github.ref, 'refs/tags/v') && github.ref_name || '' }} + ${{ startsWith(github.ref, 'refs/tags/v') && 'latest' || '' }} + if: github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/tags/v') diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index b4390181..1dc798c7 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -36,12 +36,9 @@ jobs: - name: Install Rust run: | rustup update ${{ matrix.rust }} --no-self-update + rustup update rustup default ${{ matrix.rust }} if: env.BUILD == 'true' - - name: Install perl modules - uses: perl-actions/install-with-cpanm@v1 - with: - install: "HTML::Entities" - name: Run Tests run: make test env: diff --git a/.github/workflows/test-push.yml b/.github/workflows/test-push.yml index 45e6ef51..2684d037 100644 --- a/.github/workflows/test-push.yml +++ b/.github/workflows/test-push.yml @@ -43,10 +43,6 @@ jobs: brew install gnu-tar echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH if: runner.os == 'macOS' - - name: Install perl modules - uses: perl-actions/install-with-cpanm@v1 - with: - install: "HTML::Entities" - name: Setup environment run: | chmod +x ci/setvars.sh @@ -72,29 +68,6 @@ jobs: RUST_BACKTRACE: 1 TARGET: ${{ matrix.rust }} - - name: Archive artifacts - run: | - mkdir dist - cargo build --target ${{ matrix.rust }} - mv target/${{ matrix.rust }}/debug/skyd target/${{ matrix.rust }}/debug/skysh target/${{ matrix.rust }}/debug/sky-bench dist - if: runner.os != 'Windows' - - - name: Archive artifacts - shell: cmd - run: | - cargo build --target ${{ matrix.rust }} - rm -rf dist - mkdir dist - move target\${{ matrix.rust }}\debug\*.exe dist - env: - RUSTFLAGS: -Ctarget-feature=+crt-static - if: runner.os == 'Windows' - - - name: Save artifacts - uses: actions/upload-artifact@v2 - with: - name: ${{ github.sha }}-${{ matrix.rust }}-builds.zip - path: dist test_32bit: name: Test (32-bit) runs-on: ${{ matrix.os }} @@ -116,10 +89,6 @@ jobs: run: | chmod +x ci/setvars.sh ci/setvars.sh - - name: Install perl modules - uses: perl-actions/install-with-cpanm@v1 - with: - install: "HTML::Entities" - name: Restore cache uses: actions/cache@v3 @@ -132,6 +101,7 @@ jobs: - name: Install Rust run: | + rustup self update rustup default stable rustup target add ${{ matrix.rust }} @@ -141,29 +111,6 @@ jobs: RUST_BACKTRACE: 1 TARGET: ${{ matrix.rust }} - - name: Archive artifacts - run: | - mkdir dist - cargo build --target ${{ matrix.rust }} - mv target/${{ matrix.rust }}/debug/skyd target/${{ matrix.rust }}/debug/skysh target/${{ matrix.rust }}/debug/sky-bench dist - if: runner.os == 'Linux' - - - name: Archive artifacts - shell: cmd - run: | - cargo build --target ${{ matrix.rust }} - rm -rf dist - mkdir dist - move target\${{ matrix.rust }}\debug\*.exe dist - env: - RUSTFLAGS: -Ctarget-feature=+crt-static - if: runner.os == 'Windows' - - - name: Save artifacts - uses: actions/upload-artifact@v2 - with: - name: ${{ github.sha }}-${{ matrix.rust }}-builds.zip - path: dist test_musl64: name: Test MUSL x86_64 (Tier 2) runs-on: ${{ matrix.os }} @@ -178,10 +125,6 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 2 - - name: Install perl modules - uses: perl-actions/install-with-cpanm@v1 - with: - install: "HTML::Entities" - name: Setup environment run: | @@ -199,6 +142,7 @@ jobs: - name: Install Rust run: | + rustup self update rustup default stable rustup target add ${{ matrix.rust }} @@ -207,15 +151,3 @@ jobs: env: RUST_BACKTRACE: 1 TARGET: ${{ matrix.rust }} - - - name: Archive artifacts - run: | - mkdir dist - cargo build --target ${{ matrix.rust }} - mv target/${{ matrix.rust }}/debug/skyd target/${{ matrix.rust }}/debug/skysh target/${{ matrix.rust }}/debug/sky-bench dist - - - name: Save artifacts - uses: actions/upload-artifact@v2 - with: - name: ${{ github.sha }}-${{ matrix.rust }}-builds.zip - path: dist diff --git a/.gitignore b/.gitignore index 1fc42b43..9a175c06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ /target /.vscode *.bin -data +/data/ +/server/data /server/snapshots snapstore.bin snapstore.partmap @@ -13,4 +14,7 @@ snapstore.partmap .devcontainer *.deb .skytest_* -*.pem \ No newline at end of file +*.pem +passphrase.txt +*.db-tlog +*.db \ No newline at end of file diff --git a/ACKNOWLEDGEMENTS.txt b/ACKNOWLEDGEMENTS.txt new file mode 100644 index 00000000..200517e9 --- /dev/null +++ b/ACKNOWLEDGEMENTS.txt @@ -0,0 +1,18 @@ +---- +Notes for v0.8.0 +---- + +I'd like to thank a lot of the people that have very indirectly had some influence on Skytable's design. Here's a little list (in no +particular order): + +- First of all I'd like to thank Raymond F. Boyce and Donald D. Chamberlin for their work on SQL. While I'm not fortunate enough to have +any connection to them, a lot of their work have laid out guiding principles for my work. +- I'd also like to thank several people from the Rust community (listed in no particular order): + - Aaron Turon: Aaron's work on concurrency libraries have greatly helped in parts of "design thinking" + - Carl Lerche (@carllerche): For the immense amount of work Carl done on Tokio and related systems. + - Michael Vaner (@vorner): For their work on designing concurrency primitives. I'm a great admirer of Michael's work but + unfortunately haven't had the opportunity to directly talk. + - Amanieu d'Antras (@Amanieu): Amanieu's work on parking_lot and hashbrown have been eye openers for several things that I've designed + and implemented, both in and out of Skytable + +-- Sayan N. (Dec, 2023 ) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4582ce6b..93d1ac18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,23 +2,109 @@ All changes in this project will be noted in this file. -## Unreleased +## Version 0.8.0 + +> This is the first release of Skytable Octave, and it changes the query API entirely making all previous versions incompatible +> excluding the data files which are automatically upgraded per our backwards compatibility guarantees ### Additions +#### BlueQL query language +- DDL: + - `space`s are the equivalent of the `keyspace` from previous versions + - `model`s are the equivalent of `table`s from previous version + - The following queries were added: + - `CREATE SPACE [IF NOT EXISTS] ...` + - `CREATE MODEL [IF NOT EXISTS] ...` + - Nested lists are now supported + - Type definitions are now supported + - Multiple fields are now supported + - `ALTER SPACE ...` + - `ALTER MODEL ...` + - `DROP SPACE [IF EXISTS] ...` + - `DROP MODEL [IF EXISTS] ...` + - `USE `: + - works just like SQL + - **does not work with DDL queries**: the reason it works in this way is to prevent accidental deletes + - `INSPECT ...`: + - `INSPECT global`: can be used to inspect the global state, seeing all spaces currently present on the system, users and other information. Some information is limited to the root account only (as JSON) + - `INSPECT space `: can be used to inspect a single space, returning a list of models and relevant information (as JSON) + - `INSPECT model `: can be used to inspect a single model, returning declaration and relevant information (as JSON) +- DML: + - **All actions removed**: All the prior `SET`, `GET` and other actions have been removed in favor of the new query language + - The following queries were added: + - `INSERT INTO .(col, col2, col3, ...)` + - Insert queries can use several ways of insertion including: + - a standard order of declared columns like this: + ```sql + INSERT INTO myspace.mymodel(col1, col2, col3, ...) + ``` + - using a map: + ```sql + INSERT INTO myspace.mymodel { col1: val1, col2: val2, col4: val4, col3: val3 } + ``` + - Inserts can also make use of function calls during inserts: + - It can be called like this: `INSERT INTO myspace.mymodel(@uuidstr, ...)` + - The following functions are available: + - `@uuidstr`: returns a string with a randomly generated v4 UUID + - `@uuidbin`: same as `@uuidstr` but returns it as a blob + - `@timesec`: returns a 64-bit integer with the current time in seconds + - `SELECT [ALL] field1, field2, ... FROM . WHERE = [LIMIT n]` + - New data manipulation via `UPDATE` allows arithmetic operations, string manipulation and more! Examples: + - `UPDATE . SET col_num += 1 WHERE = ` + - `UPDATE . SET mystring += " last part of string" WHERE ...` + - `DELETE FROM . WHERE = ` +- DCL: + - `SYSCTL CREATE USER WITH { password: }` + - `SYSCTL ALTER USER WITH { password: }` + - `SYSCTL DROP USER ` + +#### Fractal engine + +- The fractal engine is the start of the development of advanced internal state management in Skytable +- Effectively balances performance and reliability tasks + +#### Skyhash 2 protocol +- The `Skyhash-2` protocol now uses a multi-stage connection sequence for improved security and performance +- Seamless auth +- More types +- Fewer retransmissions + +#### Storage engines +- **New `deltax` storage engine for data**: + - The `deltax` storage engine monitors the database for changes and only records the changes in an append only file + - The interval can be adjusted per requirements of reliability + - Faster and hugely more reliable than the previous engine +- **New DDL ACID transactions with with the `logx` engine**: + - DDL queries are now fully transactional which means that if they are executed, you can be sure that they were complete and synced to disk + - This largely improves reliability + +#### New shell + +- The new client shell easily authenticates based on the Skyhash-2 protocol +- More reliable + +#### Benchmark tool + +- New benchmark engines to enable more efficient load testing +- New benchmark engines use far lesser memory +- New engines can handle midway benchmark crashes + +### Breaking changes + - `skyd`: - - New protocol: Skyhash 2.0 - - Reduced bandwidth usage (as much as 50%) - - Even simpler client implementations - - Backward compatibility with Skyhash 1.0: - - Simply set the protocol version you want to use in the config file, env vars or pass it as a CLI - argument - - Even faster implementation, even for Skyhash 1.0 - - New query language: BlueQL - - `create keyspace` is now `create space` - - `create table` is now `create model` - - Similary, all `inspect` queries have been changed - - Entities are now of the form `space.model` instead of `ks:tbl` + - **The entire query API has changed as actions have been removed** + - **Spaces and models**: replace keyspaces and models respectively + - **Configuration**: + - The configuration system now uses YAML instead of TOML for better readability + - The configuration options have changed across CLI, ENV and the config file + - Authentication **must be enabled** irrespective of `dev`/`prod` mode +- `sky-bench`: + - New benchmark engines are completely different from the previous engines (see above) + - Configuration options have changed because of how the new Skytable engine works +- `skysh`: + - Configuration options have changed because of how the new Skytable engine works +- `sky-migrate`: **This tool is deprecated and has been removed**. The Skytable engine will now automatically manage data upgrades. (please see [this issue](https://github.com/skytable/skytable/issues/320) for discussion on the same) ## Version 0.7.6 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d5e985e..82c3dda5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,13 +50,6 @@ The main parts (ignorning CI scripts, stress test suite, test harness and custom - `cli`: REPL shell - `server`: database server - `sky-bench`: benchmark tool -- `sky-migrate`: migration tool - -### Jargon - -Each project has its own jargon — and so do we! - -- _actiondoc_ and _actions docs_ : This refers to the `actiondoc.yml` file, which is used by the Skytable documentation website for automatically building documentation for the actions ### Branches @@ -116,4 +109,4 @@ Testing is simple: just run this: make test ``` -> **NOTE**: Make sure ports 2003 and 2004 are not used by any applications. Also, make sure your _own instance_ isn't running on any of these ports; if that is the case, you might end up losing data due to conflicting entity names! The test suite creates a `testsuite` keyspace and some tables within it to run all the tests. +> **NOTE**: Make sure port 2003 and 2004 are not used by any applications. Also, make sure your _own instance_ isn't running on any of these ports; if that is the case, you might end up losing data due to conflicting entity names! The test suite creates multiple spaces and some models within it to run all the tests. diff --git a/Cargo.lock b/Cargo.lock index 9c439303..b35e3b21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -10,75 +19,33 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aes" -version = "0.7.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if", - "cipher 0.3.0", + "cipher", "cpufeatures", - "opaque-debug", -] - -[[package]] -name = "ahash" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "version_check", ] [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "async-trait" -version = "0.1.61" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", + "syn 2.0.39", ] [[package]] @@ -87,6 +54,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -95,15 +77,15 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64ct" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bb8" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1627eccf3aa91405435ba240be23513eeca466b5dc33866422672264de061582" +checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2" dependencies = [ "async-trait", "futures-channel", @@ -113,25 +95,22 @@ dependencies = [ ] [[package]] -name = "bincode" -version = "1.3.3" +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "1.3.2" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] @@ -143,26 +122,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7" dependencies = [ "byteorder", - "cipher 0.4.3", + "cipher", ] -[[package]] -name = "bumpalo" -version = "3.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" - [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.3.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bzip2" @@ -187,11 +160,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -200,134 +174,70 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chrono" -version = "0.4.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-integer", - "num-traits", - "time 0.1.45", - "wasm-bindgen", - "winapi", -] - -[[package]] -name = "cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" -dependencies = [ - "generic-array", -] - [[package]] name = "cipher" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", ] [[package]] -name = "clap" -version = "2.34.0" +name = "clipboard-win" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "c57002a5d9be777c1ef967e33674dac9ebd310d8893e4e3437b14d5f0f6372cc" dependencies = [ - "ansi_term", - "atty", - "bitflags", - "strsim 0.8.0", - "textwrap", - "unicode-width", - "vec_map", - "yaml-rust", + "error-code", ] [[package]] -name = "clap" -version = "4.0.32" +name = "constant_time_eq" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" -dependencies = [ - "bitflags", - "clap_derive", - "clap_lex", - "is-terminal", - "once_cell", - "strsim 0.10.0", - "termcolor", -] +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] -name = "clap_derive" -version = "4.0.21" +name = "core-foundation" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "core-foundation-sys", + "libc", ] [[package]] -name = "clap_lex" -version = "0.3.0" +name = "core-foundation-sys" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" -dependencies = [ - "os_str_bytes", -] +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] -name = "clipboard-win" -version = "4.5.0" +name = "cpufeatures" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ - "error-code", - "str-buf", - "winapi", + "libc", ] [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "crc" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" dependencies = [ - "termcolor", - "unicode-width", + "crc-catalog", ] [[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - -[[package]] -name = "core-foundation-sys" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" - -[[package]] -name = "cpufeatures" -version = "0.2.5" +name = "crc-catalog" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" -dependencies = [ - "libc", -] +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" @@ -340,30 +250,19 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.2" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", - "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", @@ -374,20 +273,20 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] [[package]] name = "crossterm" -version = "0.25.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags", + "bitflags 2.4.1", "crossterm_winapi", "libc", "mio", @@ -399,9 +298,9 @@ dependencies = [ [[package]] name = "crossterm_winapi" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" dependencies = [ "winapi", ] @@ -417,93 +316,25 @@ dependencies = [ ] [[package]] -name = "cxx" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.86" +name = "deranged" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" dependencies = [ - "proc-macro2", - "quote", - "syn", + "powerfmt", ] -[[package]] -name = "devtimer" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907339959a92f6b98846570500c0a567c9aecbb3871cef00561eb5d20d47b7c1" - [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", "subtle", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "either" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" - [[package]] name = "endian-type" version = "0.1.2" @@ -512,9 +343,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" dependencies = [ "humantime", "is-terminal", @@ -524,52 +355,49 @@ dependencies = [ ] [[package]] -name = "errno" -version = "0.2.8" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "errno" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] name = "error-code" -version = "2.3.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" -dependencies = [ - "libc", - "str-buf", -] +checksum = "281e452d3bad4005426416cdba5ccfd4f5c1280e10099e21db27f7c1c28347fc" + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fd-lock" -version = "3.0.8" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb21c69b9fea5e15dbc1049e4b77145dd0ba1c84019c488102de0dc4ea4b0a27" +checksum = "b93f7a0db71c99f68398f80653ed05afb0b00e062e1a20c7ff849c4edfabbbcc" dependencies = [ "cfg-if", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -590,38 +418,32 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" -[[package]] -name = "fs_extra" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" - [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -633,9 +455,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -643,15 +465,21 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + [[package]] name = "harness" version = "0.1.0" @@ -661,50 +489,37 @@ dependencies = [ "log", "openssl", "powershell_script", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)", "zip", ] [[package]] name = "hashbrown" -version = "0.13.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" -dependencies = [ - "ahash", -] - -[[package]] -name = "heck" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] -name = "hermit-abi" -version = "0.2.6" +name = "hmac" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "libc", + "digest", ] [[package]] -name = "hmac" -version = "0.12.1" +name = "home" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "digest", + "windows-sys 0.48.0", ] [[package]] @@ -714,27 +529,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] -name = "iana-time-zone" -version = "0.1.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "winapi", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.1" +name = "indexmap" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ - "cxx", - "cxx-build", + "equivalent", + "hashbrown", ] [[package]] @@ -746,50 +547,38 @@ dependencies = [ "generic-array", ] -[[package]] -name = "io-lifetimes" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" -dependencies = [ - "libc", - "windows-sys", -] - [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.2.6", - "io-lifetimes", + "hermit-abi", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jemalloc-sys" -version = "0.5.2+5.3.0-patched" +version = "0.5.4+5.3.0-patched" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134163979b6eed9564c98637b710b40979939ba351f59952708234ea11b5f3f8" +checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2" dependencies = [ "cc", - "fs_extra", "libc", ] [[package]] name = "jemallocator" -version = "0.5.0" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16c2514137880c52b0b4822b563fadd38257c1f380858addb74a400889696ea6" +checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc" dependencies = [ "jemalloc-sys", "libc", @@ -797,22 +586,13 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] -[[package]] -name = "js-sys" -version = "0.3.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" -dependencies = [ - "wasm-bindgen", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -821,44 +601,25 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libsky" version = "0.8.0" -[[package]] -name = "libstress" -version = "0.8.0" -dependencies = [ - "crossbeam-channel", - "log", - "rand", - "rayon", -] - -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -866,47 +627,62 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.5" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", ] [[package]] @@ -920,72 +696,47 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if", "libc", ] [[package]] -name = "ntapi" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc" -dependencies = [ - "winapi", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.15" +name = "num_cpus" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "autocfg", + "hermit-abi", + "libc", ] [[package]] -name = "num_cpus" -version = "1.15.0" +name = "object" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ - "hermit-abi 0.2.6", - "libc", + "memchr", ] [[package]] name = "once_cell" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -996,31 +747,36 @@ dependencies = [ [[package]] name = "openssl-macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.39", ] +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "openssl-src" -version = "111.24.0+1.1.1s" +version = "300.1.6+3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" dependencies = [ - "autocfg", "cc", "libc", "openssl-src", @@ -1028,12 +784,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - [[package]] name = "parking_lot" version = "0.12.1" @@ -1046,15 +796,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.5" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-targets 0.48.5", ] [[package]] @@ -1082,9 +832,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1094,15 +844,21 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "powershell_script" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54bde2e1a395c0aee9423072d781610da37b7b120edf17d4da99f83d04f2cd54" +checksum = "bef8336090917f3d3a044256bc0e5c51d5420e5d09dfa1df4868083c5231a454" [[package]] name = "ppv-lite86" @@ -1110,44 +866,20 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -1203,28 +935,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rayon" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "num_cpus", -] - [[package]] name = "rcrypt" version = "0.4.0" @@ -1238,29 +948,30 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] -name = "redox_users" -version = "0.4.3" +name = "regex" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ - "getrandom", - "redox_syscall", - "thiserror", + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", ] [[package]] -name = "regex" -version = "1.7.1" +name = "regex-automata" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -1269,41 +980,45 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rustc-demangle" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.36.6" +version = "0.38.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" dependencies = [ - "bitflags", + "bitflags 2.4.1", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "rustyline" -version = "10.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1cd5ae51d3f7bf65d7969d579d502168ef578f289452bd8ccc91de28fda20e" +checksum = "02a2d683a4ac90aeef5b1013933f6d977bd37d51ff3f4dad829d4931a7e6be86" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if", "clipboard-win", - "dirs-next", "fd-lock", + "home", "libc", "log", "memchr", "nix", "radix_trie", - "scopeguard", "unicode-segmentation", "unicode-width", "utf8parse", @@ -1312,67 +1027,95 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] [[package]] name = "scheduled-thread-pool" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" dependencies = [ "parking_lot", ] [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "scratch" -version = "1.0.3" +name = "security-framework" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "serde" -version = "1.0.152" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.39", ] [[package]] -name = "serde_json" -version = "1.0.91" +name = "serde_yaml" +version = "0.9.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" dependencies = [ + "indexmap", "itoa", "ryu", "serde", + "unsafe-libyaml", ] [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -1381,9 +1124,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -1392,9 +1135,9 @@ dependencies = [ [[package]] name = "signal-hook" -version = "0.3.14" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" dependencies = [ "libc", "signal-hook-registry", @@ -1413,9 +1156,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] @@ -1424,67 +1167,48 @@ dependencies = [ name = "sky-bench" version = "0.8.0" dependencies = [ - "clap 4.0.32", - "devtimer", - "env_logger", - "libstress", - "log", - "rand", - "serde", - "serde_json", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)", -] - -[[package]] -name = "sky-migrate" -version = "0.8.0" -dependencies = [ - "bincode", - "clap 4.0.32", + "crossbeam-channel", "env_logger", + "libsky", "log", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)", + "num_cpus", + "skytable", + "tokio", ] [[package]] name = "sky_macros" version = "0.8.0" dependencies = [ + "libsky", "proc-macro2", "quote", - "rand", - "syn", + "syn 1.0.109", ] [[package]] name = "skyd" version = "0.8.0" dependencies = [ - "ahash", - "base64", - "bincode", "bytes", - "cc", - "chrono", - "clap 2.34.0", + "crc", + "crossbeam-epoch", "env_logger", - "hashbrown", "jemallocator", "libc", "libsky", - "libstress", "log", "openssl", "parking_lot", "rand", "rcrypt", - "regex", "serde", + "serde_yaml", "sky_macros", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)", + "skytable", "tokio", "tokio-openssl", - "toml", + "uuid", "winapi", ] @@ -1492,105 +1216,74 @@ dependencies = [ name = "skysh" version = "0.8.0" dependencies = [ - "clap 4.0.32", "crossterm", - "lazy_static", "libsky", "rustyline", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)", - "tokio", + "skytable", ] [[package]] name = "skytable" version = "0.8.0" -source = "git+https://github.com/skytable/client-rust?branch=next#c57e46e6ddce57c221e41cd038e7eb8296d2d732" +source = "git+https://github.com/skytable/client-rust.git?branch=octave#091aa94a696e22abd2ce10845e3040c48ea30720" dependencies = [ "async-trait", "bb8", - "bytes", - "openssl", + "itoa", + "native-tls", "r2d2", + "rand", "tokio", - "tokio-openssl", -] - -[[package]] -name = "skytable" -version = "0.8.0" -source = "git+https://github.com/skytable/client-rust.git#c57e46e6ddce57c221e41cd038e7eb8296d2d732" -dependencies = [ - "r2d2", + "tokio-native-tls", ] [[package]] name = "slab" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "socket2" -version = "0.4.7" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] -name = "str-buf" -version = "1.0.6" +name = "subtle" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] -name = "stress-test" -version = "0.1.0" +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "crossbeam-channel", - "devtimer", - "env_logger", - "libstress", - "log", - "rand", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)", - "sysinfo", + "proc-macro2", + "quote", + "unicode-ident", ] -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "subtle" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" - [[package]] name = "syn" -version = "1.0.107" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -1598,106 +1291,54 @@ dependencies = [ ] [[package]] -name = "sysinfo" -version = "0.27.4" +name = "tempfile" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444c7f371193a054273b0a1bbfe1e617b804b7fd62fd3e2794b4be6b0a872b0c" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", - "core-foundation-sys", - "libc", - "ntapi", - "once_cell", - "rayon", - "winapi", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.48.0", ] [[package]] name = "termcolor" -version = "1.1.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" dependencies = [ "winapi-util", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "thiserror" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "time" -version = "0.1.45" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" -dependencies = [ - "itoa", + "deranged", + "powerfmt", "serde", "time-core", - "time-macros", ] [[package]] name = "time-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" - -[[package]] -name = "time-macros" -version = "0.2.6" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" -dependencies = [ - "time-core", -] +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "tokio" -version = "1.24.1" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", @@ -1705,18 +1346,28 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.39", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", ] [[package]] @@ -1731,128 +1382,81 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" -dependencies = [ - "serde", -] - [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - -[[package]] -name = "utf8parse" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "vec_map" -version = "0.8.2" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +name = "unsafe-libyaml" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +name = "utf8parse" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] -name = "wasm-bindgen" -version = "0.2.83" +name = "uuid" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ - "cfg-if", - "wasm-bindgen-macro", + "getrandom", + "rand", + "uuid-macro-internal", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.83" +name = "uuid-macro-internal" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "f49e7f3f3db8040a100710a11932239fd30697115e2ba4107080d8252939845e" dependencies = [ - "bumpalo", - "log", - "once_cell", "proc-macro2", "quote", - "syn", - "wasm-bindgen-shared", + "syn 2.0.39", ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.83" +name = "vcpkg" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.83" +name = "version_check" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "wasm-bindgen-shared" -version = "0.2.83" +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "winapi" @@ -1872,9 +1476,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -1887,72 +1491,141 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.42.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] -name = "yaml-rust" -version = "0.3.5" +name = "windows_x86_64_msvc" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "zip" -version = "0.6.3" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537ce7411d25e54e8ae21a7ce0b15840e7bfcff15b51d697ec3266cc76bdf080" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ "aes", "byteorder", @@ -1964,7 +1637,7 @@ dependencies = [ "hmac", "pbkdf2", "sha1", - "time 0.3.17", + "time", "zstd", ] @@ -1989,10 +1662,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.4+zstd.1.5.2" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa202f2ef00074143e219d15b62ffc317d17cc33909feac471c044087cad7b0" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index d5b6c93c..66bc3478 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,15 +1,6 @@ [workspace] -members = [ - "cli", - "server", - "libsky", - "sky-bench", - "sky-macros", - "libstress", - "stress-test", - "sky-migrate", - "harness", -] +resolver = "1" +members = ["cli", "server", "libsky", "sky-bench", "sky-macros", "harness"] [profile.release] opt-level = 3 diff --git a/Dockerfile b/Dockerfile index 03820a3b..c58a8ce8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,16 @@ -# # The Dockerfile for the Skytable server sdb -# FROM debian:stable - +# Copy the necessary binaries COPY target/release/skyd /usr/local/bin COPY target/release/skysh /usr/local/bin -RUN mkdir /etc/skytable +# Create necessary directories RUN mkdir /var/lib/skytable -COPY examples/config-files/docker.toml /etc/skytable/skyd.toml +COPY examples/config-files/dpkg/config.yaml /var/lib/skytable/config.yaml +COPY pkg/docker/start-server.sh /usr/local/bin/start-server.sh WORKDIR /var/lib/skytable -CMD ["skyd", "-c", "/etc/skytable/skyd.toml"] +# Install uuidgen for generating a random password +RUN apt-get update && apt-get install -y uuid-runtime +RUN chmod +x /usr/local/bin/start-server.sh +ENTRYPOINT ["/usr/local/bin/start-server.sh"] EXPOSE 2003/tcp diff --git a/README.md b/README.md index 20c0b792..095b6755 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,125 @@ - -
- -

Skytable

Your next NoSQL database

+

+ +

+

+ Skytable — A modern database for building powerful experiences +

+

+ Performance, scalability and flexibility. Choose three. +

+

+

+GitHub release (with filter) GitHub Workflow Status (with event) Discord Docs Static Badge +

-![GitHub Workflow Status]() ![Development](https://img.shields.io/badge/development-regular-32CD32?style=flat-square) ![GitHub release (latest SemVer including pre-releases)](https://img.shields.io/github/v/release/skytable/skytable?include_prereleases&sort=semver&style=flat-square) -[![Docs](https://img.shields.io/badge/readthedocs-here-blueviolet?style=flat-square)](https://docs.skytable.io) [![Contribute Now](https://img.shields.io/badge/%F0%9F%8C%9Fcontribute-now-a94064)](https://ohsayan.github.io/skythanks) [![Discord](https://img.shields.io/badge/talk-on%20discord-7289DA?logo=discord&style=flat-square")](https://discord.gg/QptWFdx) +## 💡 What is Skytable? -
- +Skytable is a **modern NoSQL database** that focuses on **performance, flexibility and scalability**. Our goal is to deliver **a rock-solid database** that you can use as the foundation for your next application — **minus the gimmicks**. -## What is Skytable? +Skytable makes every attempt to **remove the gotchas from SQL-systems**. For example, nonempty `model`s and `space`s cannot be dropped and **BlueQL is designed to greatly deter SQL injection vulnerabilities** with a new **mandatory parameterization** design and several other **secure query language design principles**. -Skytable is a free and open-source NoSQL database that aims to provide flexible data modeling at -scale. For us simplicity, performance and flexibility are our guiding design principles. -We were previously known as TerrabaseDB or Skybase and are nicknamed Sky, SDB or STable by the community. +Every component in Skytable has been **engineered from the ground up** to meet our design goals. **Skytable uses BlueQLTM** which is our own **new in-house query language** designed from the ground up for a **clean, powerful, modern and secure querying experience** and is **generally more secure than SQL**. -Features like keyspaces, tables, data types, authn+authz, snapshots and more are ready for you to use while we're working on [several new data models and features](https://github.com/skytable/skytable/issues/203). Skytable's key/value store is performant, secure and ready for you to deploy. +Skytable works with **structured and semi-structured data**. We're currently working on supporting unstructured data. -## Getting started 🚀 +> **You can read more about Skytable's architecture, including information on the clustering and HA implementation that we're currently working on, and limitations [on this page](https://docs.skytable.io/architecture).** -1. Download a bundle for your platform from [here ⬇️ ](https://github.com/skytable/skytable/releases) -2. Unzip the bundle -3. Make the files executable (run `chmod +x skyd skysh` on \*nix systems) -4. First run `skyd` to start the database server and then run `skysh` to start the interactive shell -5. Run commands like: `SET foo bar` , `GET bar` , `UPDATE cat mitten` or `DEL proprietary` on `skysh`! +## 🎨 Features -You can learn more about installation [here](https://docs.skytable.io/getting-started) +- **Spaces, models and more**: For flexible data definition +- **Powerful querying with BlueQL**: A modern query language, designed for the 21st century +- **Rich data modeling**: Use `model`s to define data with complex types, collections and more +- **Performant, in and out of the box**: Heavily multithreaded and optimized +- **Secure, query in and response out**: BlueQL is designed to strongly deter query injection pathways +- **SQL minus the gotchas**: Ever done a `DROP TABLE users` and lost all data? **That won't happen in Skytable**. +- **Designed to scale by enforcing best practices**: If you're building with Skytable today, the practices you'll learn here will let you easily take on the job of building large scale systems -## Features +> Learn more about [Skytable's features here](https://docs.skytable.io). -- **Insanely fast**: Scale to millions of queries per second per node. See [benchmarks here](https://github.com/ohsayan/sky-benches). -- **Multiple keyspaces/tables**: Seamlessly integrates with actions to provide a SQL-like experience -- **Key/value store**: `GET` , `SET` , `UPDATE` and [all that stuff](https://docs.skytable.io/all-actions). With the `str` and `binstr` types. -- **Authn/Authz**: Simple and secure authentication/authorization -- **Volatile tables**: For all the caching you need -- **Snapshots**: Automated (and tunable) snapshots for stress-free backups -- **Secure**: Secure connections are built into Skytable with SSL/TLS -- **Multithreaded**: Designed to exploit all CPU cores -- **Resource friendly**: The database server doesn't need more than 1MB to run -- **Convenient**: Without the setup hassle and system-specific dependencies +## 🚀 Getting started -**🛣️ There's a lot more coming! View our [roadmap](https://github.com/skytable/skytable/issues/203)** +1. **Set up Skytable on your machine**: You'll need to download a bundled release file [from the releases page](https://github.com/skytable/skytable/releases). Unzip the files and you're ready to go. +2. Start the database server: `./skyd --auth-root-password ` with your choice of a password for the `root` account. The `root` account is just like a `root` account on Unix based systems that has control over everything. +3. Start the interactive client REPL: `./skysh` and then enter your password. +4. You're ready to run queries! -## Clients 🔌 +> **For a more detailed guide on installation and deployment, [follow the guide here.](https://docs.skytable.io/installation)** -The project currently maintains an official [Rust driver](https://github.com/skytable/client-rust) and we have plans -to support more languages along the way! -We also maintain a list of [community supported drivers here](https://github.com/skytable/skytable/wiki/Drivers). -If you want to use a different language, for now you'll just need to implement the simple and performant [Skyhash Protocol](https://docs.skytable.io/protocol/skyhash). +## ⚡ Using Skytable -## Community 👐 +Skytable has `SPACE`s instead of `DATABASE`s due to signficant operational differences (and because `SPACE`s store a lot more than tabular data). -A project which is powered by the community believes in the power of community! If you get stuck anywhere - here are your options! +**With the REPL started, follow this guide**: - - - - +1. Create a `space` and switch to it: + ```sql + CREATE SPACE myspace + USE myspace + ``` +2. Create a `model`: + ```sql + CREATE MODEL myspace.mymodel(username: string, password: string, notes: list { type: string }) + ``` + The rough representation for this in Rust would be: + ```rust + pub struct MyModel { + username: String, + password: Strin, + notes: Vec, + } + ``` +3. `INSERT` some data: + ```sql + INSERT INTO mymodel('sayan', 'pass123', []) + ``` +4. `UPDATE` some data: + ```sql + UPDATE mymodel SET notes += "my first note" WHERE username = 'sayan' + ``` +5. `SELECT` some data + ```sql + SELECT * FROM mymodel WHERE username = 'sayan' + ``` +6. Poke around! **And then make sure you [read the documentation learn BlueQL](https://docs.skytable.io/blueql/overview).** -## Platforms 💻 +> **For a complete guide on Skytable, it's architecture, BlueQL, queries and more we strongly recommend you to [read the documentation here.](https://docs.skytable.io)** +> +> While you're seeing strings and other values being used here, this is so because the REPL client smartly parameterizes queries behind the scenes. **BlueQL has mandatory parameterization**. (See below to see how the Rust client handles this) -![Linux supported](https://img.shields.io/badge/Linux%2032--bit%2F64--bit-Supported%20✓-%23228B22?logo=linux) ![macOS supported](https://img.shields.io/badge/macOS%20x86__64%2Farm64-supported%20✓-228B22?style=flat-square&logo=apple) ![Windows supported](https://img.shields.io/badge/Windows%2032--bit%2F64--bit-supported%20✓-228B22?style=flat-square&logo=windows) +## 🧩 Find a client driver -## Versioning +You need a client driver to use Skytable in your programs. Officially, we maintain a regularly updated [Rust client driver](https://github.com/skytable/client-rust) which is liberally license under the Apache-2.0 license so that you can use it anywhere. -This project strictly follows semver, however, since this project is currently in the development phase (0.x.y), the API may change unpredictably +Using the Rust client driver, it's very straightforward to run queries thanks to Rust's powerful type system and macros: -## Contributing +```rust +use skytable::{Config, query}; + +fn main() { + let mut db = Config::new_default("username", "password").connect().unwrap(); + let query = query!("select username, password from myspace.mymodel where username = ?", "sayan"); + let (username, password): (String, Vec) = db.query_parse(&query).unwrap(); + // do something with it! +} +``` + +> **You can find more information on client drivers on [this page](https://docs.skytable.io/libraries). If you want to help write a client driver for your language of choice, *we're here to support your work*. Please reach out to: hey@skytable.io or leave a message on our Discord server!** + +## 🙋 Getting help -[![Contribute Now](https://img.shields.io/badge/%F0%9F%8C%9Fcontribute-now-a94064?style=for-the-badge)](https://ohsayan.github.io/skythanks) +We exclusively use [Discord](https://discord.gg/QptWFdx) for most real-time communications — you can chat with developers, maintainers, and our amazing users! Outside that, we recommend that you use our [GitHub Discussions page](https://github.com/skytable/skytable/discussions) for any questions or open a new issue if you think you've found a bug. + +*We're here to help!* + +## Contributing -You are welcome to contribute to Skytable! Beginner friendly issues are marked with the [](https://github.com/skytable/skytable/labels/L-easy) label. Read the guide [here](./CONTRIBUTING.md). +Please read the [contributing guide here](./CONTRIBUTING.md). -## Contributors +## Acknowledgements -You can see a full list of contributors [here](https://ohsayan.github.io/skythanks) +Please read the [acknowledgements](./ACKNOWLEDGEMENTS.txt) document. ## License -This project is licensed under the [AGPL-3.0 License](./LICENSE). +Skytable is distributed under the [AGPL-3.0 License](./LICENSE). **You may not use Skytable's logo for other projects.** diff --git a/actiondoc.yml b/actiondoc.yml deleted file mode 100644 index f12a48be..00000000 --- a/actiondoc.yml +++ /dev/null @@ -1,380 +0,0 @@ -# -# Created on Thu Aug 27 2020 -# -# This file is a part of Skytable -# Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source -# NoSQL database written by Sayan Nandan ("the Author") with the -# vision to provide flexibility in data modelling without compromising -# on performance, queryability or scalability. -# -# Copyright (c) 2020, Sayan Nandan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# -# -# This file is used by Skytable's documentation website for automatically -# generating documentation for the actions. It will also be used by the Skytable -# server in the future - -# the docbuilder expects: -# 'name': str, 'complexity': str, 'accept': [str] -# 'return': [str], 'syntax': [str], 'desc': str - -global: - - name: HEYA - complexity: O(1) - accept: [AnyArray] - syntax: [HEYA, HEYA ] - desc: | - Either returns a "HEY!" or returns the provided argument as an `str` - return: [String] - - name: DBSIZE - complexity: O(1) - accept: [AnyArray] - syntax: [DBSIZE, DBSIZE ] - desc: Check the number of entries stored in the current table or in the provided entity - return: [Integer] - - name: MKSNAP - complexity: O(n) - accept: [AnyArray] - syntax: [MKSNAP, MKSNAP ] - desc: | - This action can be used to create a snapshot. Do note that this action **requires - snapshotting to be enabled on the server side**, before it can create snapshots. - If you want to create snapshots **without** snapshots being enabled on the server-side, - pass a second argument `` to specify a snapshot name and a snapshot will - be create in a folder called `rsnap` under your data directory. For more - information on snapshots, read [this document](/snapshots) - return: [Rcode 0, err-snapshot-disabled, err-snapshot-busy] - - name: FLUSHDB - complexity: O(n) - accept: [AnyArray] - syntax: [FLUSHDB, FLUSHDB ] - desc: Removes all entries stored in the current table or in the provided entity - return: [Rcode 0, Rcode 5] - - name: WHEREAMI - complexity: O(1) - accept: [AnyArray] - syntax: [WHEREAMI] - desc: | - Returns an array with either the name of the current keyspace as the first element or if a default table - is set, then it returns the keyspace name as the first element and the table name as the second element - return: [Non-null array] - - name: AUTH - desc: Change global authn/authz settings - subactions: - - name: LOGIN - complexity: O(1) - accept: [AnyArray] - syntax: [AUTH LOGIN ] - desc: Attempts to log in using the provided credentials - return: [Rcode 0, Rcode 10] - - name: CLAIM - complexity: O(1) - accept: [AnyArray] - syntax: [AUTH CLAIM ] - desc: Attempts to claim the root account using the origin key - return: [String, Rcode 10] - - name: LOGOUT - complexity: O(1) - accept: [AnyArray] - syntax: [AUTH LOGOUT] - desc: Attempts to log out the currently logged in user - return: [Rcode 0, Rcode 10] - - name: ADDUSER - complexity: O(1) - accept: [AnyArray] - syntax: [AUTH ADDUSER ] - desc: Attempts to create a new user with the provided username, returning the token - return: [String, Rcode 11] - - name: DELUSER - complexity: O(1) - accept: [AnyArray] - syntax: [AUTH DELUSER ] - desc: Attempts to delete the user with the provided username - return: [Rcode 0, Rcode 10, Rcode 11] - - name: RESTORE - complexity: O(1) - accept: [AnyArray] - syntax: [AUTH RESTORE , AUTH RESTORE ] - desc: | - Attempts to restore the password for the provided user. This will regenerate the token - and return the newly issued token. However, if you aren't a root account, that is, you - lost your root password, then you'll need to run `AUTH RESTORE root`. - return: [String, Rcode 10, Rcode 11] - - name: LISTUSER - complexity: O(1) - accept: [AnyArray] - syntax: [AUTH LISTUSER] - desc: | - Attempts to return a list of users for the current database instance - return: [Non-null array] - - name: WHOAMI - complexity: O(1) - accept: [AnyArray] - syntax: [AUTH WHOAMI] - desc: | - Returns a string with the AuthID of the currently logged in user or errors if the user - is not logged in - return: [String] - - name: SYS - desc: | - Get system information and metrics - subactions: - - name: INFO - complexity: O(1) - accept: [AnyArray] - syntax: [sys info ] - return: [String, Float] - desc: | - Returns static properties of the system, i.e properties that do not change during runtime. - The following properties are available: - - `version`: Returns the server version (String) - - `protocol`: Returns the protocol version string (String) - - `protover`: Returns the protocol version (float) - - name: METRIC - complexity: O(1) - accept: [AnyArray] - syntax: [sys metric ] - return: [String, Float] - desc: | - Returns dynamic properties of the system, i.e metrics are properties that can change during - runtime. The following metrics are available: - - `health`: Returns "good" or "critical" depending on the system state (String) - - `storage`: Returns bytes used for on-disk storage (uint64) - -keyvalue: - generic: - - name: DEL - complexity: O(n) - accept: [AnyArray] - syntax: [DEL ...] - desc: | - Delete 'n' keys from the current table. This will return the number of keys that were deleted - as an unsigned integer - return: [Integer, Rcode 5] - - name: EXISTS - complexity: O(n) - accept: [AnyArray] - syntax: [EXISTS ...] - desc: | - Check if 'n' keys exist in the current table. This will return the number of keys that exist - as an unsigned integer. - return: [Integer] - - name: LSKEYS - complexity: O(n) - accept: [AnyArray] - syntax: [LSKEYS , LSKEYS , LSKEYS ] - desc: | - Returns a flat string array of keys present in the current table or in the provided entity. - If no `` is given, then a maximum of 10 keys are returned. If a limit is specified, - then a maximum of `` keys are returned. The order of keys is meaningless. - return: [Typed Array] - string: - - name: GET - complexity: O(1) - accept: [AnyArray] - syntax: [GET ] - desc: Get the value of a key from the current table, if it exists - return: [Rcode 1, String, Binstr] - - name: MGET - complexity: O(n) - accept: [AnyArray] - syntax: [MGET ...] - desc: Get the value of 'n' keys from the current table, if they exist - return: [Typed Array] - - name: SET - complexity: O(1) - accept: [AnyArray] - syntax: [SET ] - desc: Set the value of a key in the current table, if it doesn't already exist - return: [Rcode 0, Rcode 2, Rcode 5] - - name: MSET - complexity: O(n) - accept: [AnyArray] - syntax: [MSET ...] - desc: | - Set the value of 'n' keys in the current table, if they don't already exist. This will - return the number of keys that were set as an unsigned integer. - return: [Integer, Rcode 5] - - name: UPDATE - complexity: O(1) - accept: [AnyArray] - syntax: [UPDATE ] - desc: Update the value of an existing key in the current table - return: [Rcode 0, Rcode 1, Rcode 5] - - name: MUPDATE - complexity: O(n) - accept: [AnyArray] - syntax: [MUPDATE ...] - desc: | - Update the value of 'n' keys in the current table, if they already exist. This will return - the number of keys that were updated as an unsigned integer. - return: [Integer, Rcode 5] - - name: SSET - complexity: O(n) - accept: [AnyArray] - syntax: [SSET ...] - desc: Set all keys to the given values only if all of them don't exist in the current table - return: [Rcode 0, Rcode 2, Rcode 5] - - name: SDEL - complexity: O(n) - accept: [AnyArray] - syntax: [SDEL ...] - desc: | - Delete all keys if all of the keys exist in the current table. Do note that if a single key doesn't - exist, then a `Nil` code is returned. - return: [Rcode 0, Rcode 1, Rcode 5] - - name: SUPDATE - complexity: O(n) - accept: [AnyArray] - syntax: [SUPDATE ...] - desc: | - Update all keys if all of the keys exist in the current table. Do note that if a single key doesn't - exist, then a `Nil` code is returned. - return: [Rcode 0, Rcode 1, Rcode 5] - - name: USET - complexity: O(n) - accept: [AnyArray] - syntax: [USET ...] - desc: SET all keys if they don't exist, or UPDATE them if they do exist. This operation performs `USET`s in the current table - return: [Integer, Rcode 5] - - name: KEYLEN - complexity: O(1) - accept: [AnyArray] - syntax: [KEYLEN ] - desc: Returns the length of the UTF-8 string, if it exists in the current table - return: [Integer, Rcode 1] - - name: POP - complexity: O(1) - accept: [AnyArray] - syntax: [POP ] - desc: | - Deletes and return the value of the provided key from the current table. - If the database is poisoned, this will return a server error. - return: [String, Binstr, Rcode 5] - - name: MPOP - complexity: O(n) - accept: [AnyArray] - syntax: [MPOP ...] - desc: | - Deletes and returns the values of the provided 'n' keys from the current table. - If the database is poisoned, this will return a server error - return: [Typed Array, Rcode 5] - lists: - - name: LGET - desc: | - `LGET` can be used to access the items in a list. Through the sub-actions provided by `lget`, - you can access multiple or individual elements in lists. - subactions: - - name: LGET - complexity: O(n) - accept: [AnyArray] - syntax: [LGET ] - desc: | - Returns all the values contained in a the provided list, if it exists in the current - table. - return: [Typed Array, Rcode 1] - - name: limit - complexity: O(n) - accept: [AnyArray] - syntax: [LGET limit ] - desc: Returns a maximum of `limit` values from the provided list, if it exists in the current table - return: [Typed Array, Rcode 1] - - name: len - complexity: O(1) - accept: [AnyArray] - syntax: [LGET len] - desc: Returns the length of the list - return: [Integer, Rcode 1] - - name: valueat - complexity: O(1) - accept: [AnyArray] - syntax: [LGET valueat ] - desc: Returns the element present at the provided `index`, if it exists in the given list. - return: [String, binstr, Rcode 1, bad-list-index] - - name: first - complexity: O(1) - accept: [AnyArray] - syntax: [LGET first] - desc: Returns the first element present in the list, if it exists. - return: [String, binstr, Rcode 1, list-is-empty] - - name: last - complexity: O(1) - accept: [AnyArray] - syntax: [LGET last] - desc: Returns the last element present in the list, if it exists. - return: [String, binstr, Rcode 1, list-is-empty] - - name: range - complexity: O(n) - accept: [AnyArray] - syntax: [LGET range , LGET range ] - desc: | - Returns items in the given range. If no value for `stop` is provided, all the elements from that - index are returned. If a value for `stop` is provided, then a subarray is returned - return: [Typed Array, Rcode 1, bad-list-index] - - name: LMOD - desc: | - `LMOD` can be used to mutate the elements in a list - subactions: - - name: push - complexity: O(1) - accept: [AnyArray] - syntax: [LMOD push ...] - desc: Appends the elements to the end of the provided list, if it exists. - return: [Rcode 0, Rcode 1, Rcode 5] - - name: insert - complexity: O(1) - accept: [AnyArray] - syntax: [LMOD insert ] - desc: | - Inserts the element to the provided index, if it is valid while shifting elements - to the right if required - return: [Rcode 0, Rcode 1, Rcode 5, bad-list-index] - - name: pop - complexity: O(1) - accept: [AnyArray] - syntax: [LMOD pop, LMOD pop ] - desc: | - Removes the element from the end of the list if no index is provided or from the provided - index while shifting elements to the right if required. - return: [String, Binstr, Rcode 1, Rcode 5, bad-list-index] - - name: remove - complexity: O(1) - accept: [AnyArray] - syntax: [LMOD remove ] - desc: | - Removes the element at the provided index from the list, shifting elements to the right - if required. - return: [Rcode 0, Rcode 1, Rcode 5, bad-list-index] - - name: clear - complexity: O(n) - accept: [AnyArray] - syntax: [LMOD clear] - desc: | - Removes all the elements present in the list - return: [Rcode 0, Rcode 1, Rcode 5] - - name: LSET - desc: | - `LSET` can be used to create empty lists or lists with the provided values. - subactions: - - name: LSET - complexity: O(n) - accept: [AnyArray] - syntax: [LSET , LSET ...] - desc: | - Creates a list with the provided values, or simply creates an empty list if it doesn't - already exist in the table. - return: [Rcode 0, Rcode 2, Rcode 5] diff --git a/assets/logo.jpg b/assets/logo.jpg old mode 100755 new mode 100644 index e0886a72..3bccc249 Binary files a/assets/logo.jpg and b/assets/logo.jpg differ diff --git a/ci/server1.toml b/ci/server1.toml deleted file mode 100644 index dad19e6c..00000000 --- a/ci/server1.toml +++ /dev/null @@ -1,9 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 -noart = true - -[ssl] -key="../key.pem" -chain="../cert.pem" -port = 2004 diff --git a/ci/server1.yaml b/ci/server1.yaml new file mode 100644 index 00000000..ccf52e5e --- /dev/null +++ b/ci/server1.yaml @@ -0,0 +1,17 @@ +system: + mode: prod + +auth: + plugin: pwd + root_pass: mypassword12345678 + +endpoints: + insecure: + host: 127.0.0.1 + port: 2003 + secure: + host: 127.0.0.1 + port: 2004 + cert: ../cert.pem + private_key: ../key.pem + pkey_passphrase: ../passphrase.txt diff --git a/ci/server2.toml b/ci/server2.toml deleted file mode 100644 index 05083faa..00000000 --- a/ci/server2.toml +++ /dev/null @@ -1,12 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2005 -noart = true - -[auth] -origin_key = "4527387f92a381cbe804593f33991d327d456a97" - -[ssl] -key = "../key.pem" -chain = "../cert.pem" -port = 2006 diff --git a/ci/server3.toml b/ci/server3.toml deleted file mode 100644 index 69d6b24b..00000000 --- a/ci/server3.toml +++ /dev/null @@ -1,14 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2007 -noart = true - -[snapshot] -every = 3600 -atmost = 4 -failsafe = true - -[ssl] -key = "../key.pem" -chain = "../cert.pem" -port = 2008 diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 158d560c..aa0baecf 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -10,13 +10,7 @@ description = "The Skytable Shell (skysh)" [dependencies] # internal deps libsky = { path = "../libsky" } -skytable = { git = "https://github.com/skytable/client-rust", branch = "next", features = [ - "aio", - "aio-sslv", -], default-features = false } +skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" } # external deps -tokio = { version = "1.24.1", features = ["full"] } -clap = { version = "4.0.32", features = ["derive"] } -rustyline = "10.0.0" -crossterm = "0.25.0" -lazy_static = "1.4.0" +crossterm = "0.27.0" +rustyline = "13.0.0" diff --git a/cli/help_text/help b/cli/help_text/help new file mode 100644 index 00000000..8a4d6ab6 --- /dev/null +++ b/cli/help_text/help @@ -0,0 +1,30 @@ +skysh 0.8.0 +Sayan N. +The Skytable interactive shell (skysh) + +USAGE: + skysh [OPTIONS] + +FLAGS: + --help Diplays this help message + --version Displays the shell version + +OPTIONS: + --endpoint Set the endpoint for the connection + --user Set the user for this client session + --password Set the password for this client session + --tls-cert Set the TLS certificate to use (for TLS endpoints) + +NOTES: + - When no endpoint is specified, skysh will attempt to connect to the default + TCP endpoint `tcp@127.0.0.1:2003` + - When no user is specified, skysh will attempt to authenticate as root + - All connections need an username and password. If this is not provided + via arguments, it will be asked for interactively + - Endpoints are specified using the Skytable endpoint syntax. For example, + the default TCP endpoint is `tcp@127.0.0.1:2003` while the default TLS + endpoint is `tls@127.0.0.1:2004` + - If you choose to use a TLS endpoint, you must provide a certificate. + Failing to do so will throw an error, as expected + - All history is stored in the `.sky_history` file. If you wish to delete + it, simply remove the file \ No newline at end of file diff --git a/cli/help_text/welcome b/cli/help_text/welcome new file mode 100644 index 00000000..27c14bbd --- /dev/null +++ b/cli/help_text/welcome @@ -0,0 +1,16 @@ +Welcome to the Skytable Interactive shell (REPL environment). + +Here are a few tips to help you get started: +- Skytable uses its own query language called BlueQL. It is mostly like SQL +but with some important changes for security +- Since BlueQL doesn't need a query terminator ';' you do not need to use it +here for running queries +- You might be surprised to see that you can use literals in this REPL while +Skytable does not allow the use of literals for security concerns. This is +because whenever you run a query, the REPL turns it into a parameterized query. +- You can also run some `skysh` specific commands: + - `!help` displays this help message + - `clear` clears the terminal screen + - `exit` exits the REPL session + +Now, it's time to get querying! diff --git a/cli/src/argparse.rs b/cli/src/argparse.rs deleted file mode 100644 index d9dc26cf..00000000 --- a/cli/src/argparse.rs +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Created on Wed Jul 01 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{cli::Cli, runner::Runner, tokenizer}, - clap::Parser, - crossterm::{ - cursor, execute, - terminal::{Clear, ClearType}, - }, - libsky::{URL, VERSION}, - rustyline::{config::Configurer, error::ReadlineError, Editor}, - skytable::{Pipeline, Query}, - std::{io::stdout, process}, -}; - -const SKYSH_HISTORY_FILE: &str = ".sky_history"; - -const HELP_TEXT: &str = r#" -███████ ██  ██ ██  ██ ████████  █████  ██████  ██  ███████ -██      ██  ██   ██  ██     ██    ██   ██ ██   ██ ██  ██ -███████ █████    ████   ██  ███████ ██████  ██  █████ -     ██ ██  ██   ██   ██  ██   ██ ██   ██ ██  ██ -███████ ██  ██  ██  ██  ██  ██ ██████  ███████ ███████ - -Welcome to Skytable's interactive shell (REPL) environment. Using the Skytable -shell, you can create, read, update or delete data on your remote Skytable -instance. When you connect to your database instance, you'll be connected to -the `default` table in the `default` keyspace. This table has binary keys and -binary values as the default data type. Here's a brief guide on doing some -everyday tasks: - -(1) Running actions -================================================================================ -An action is like a shell command: it starts with a name and contains arguments! -To run actions, simply type them out, like "set x 100" or "inspect table mytbl" -and hit enter. - -(2) Running shell commands -================================================================================ -Shell commands are those which are provided by `skysh` and are not supported by -the server. These enable you to do convenient things like: -- "exit": exits the shell -- "clear": clears the terminal screen - -Apart from these, you can use the following shell commands: -- "!pipe": Lets you create a pipeline. Terminate with a semicolon (`;`) -- "!help": Brings up this help menu -- "?": Describes what the built-in shell command is for - -With Skytable in your hands, the sky is the only limit on what you can create!"#; - -const SKY_WELCOME: &str = " -Welcome to Skytable's interactive shell (REPL) environment. For usage and help -within the shell, you can run `!help` anytime. Now that you have Skytable in -your hands, the sky is the only limit on what you can create! -"; - -/// This creates a REPL on the command line and also parses command-line arguments -/// -/// Anything that is entered following a return, is parsed into a query and is -/// written to the socket (which is either `localhost:2003` or it is determined by -/// command line parameters) -pub async fn start_repl() { - let mut skysh_blank: String = " > ".to_owned(); - let mut skysh_prompt: String = "skysh@default:default> ".to_owned(); - let mut did_swap = false; - - macro_rules! readln { - ($editor:expr) => { - match $editor.readline(&skysh_blank) { - Ok(l) => l, - Err(ReadlineError::Interrupted | ReadlineError::Eof) => return, - Err(err) => fatal!("ERROR: Failed to read line with error: {}", err), - } - }; - } - - let cli = Cli::parse(); - let mut editor = match Editor::<()>::new() { - Ok(e) => e, - Err(e) => fatal!("Editor init error: {}", e), - }; - editor.set_auto_add_history(true); - editor.set_history_ignore_dups(true); - editor.bind_sequence( - rustyline::KeyEvent( - rustyline::KeyCode::BracketedPasteStart, - rustyline::Modifiers::NONE, - ), - rustyline::Cmd::Noop, - ); - let con = match cli.ssl_cert { - Some(cert) => Runner::new_secure(&cli.host, cli.port, &cert).await, - None => Runner::new_insecure(&cli.host, cli.port).await, - }; - let mut runner = match con { - Ok(c) => c, - Err(e) => fatal!("Failed to connect to server with error: {}", e), - }; - - macro_rules! checkswap { - () => { - if did_swap { - // noice, we need to poll for the location of the new entity - runner - .check_entity(&mut skysh_blank, &mut skysh_prompt) - .await; - } - }; - } - - if let Some(expressions) = cli.expressions { - for eval_expr in expressions { - if !eval_expr.is_empty() { - runner.run_query(&eval_expr).await; - } - } - process::exit(0x00); - } - println!("Skytable v{} | {}", VERSION, URL); - match editor.load_history(SKYSH_HISTORY_FILE) { - Ok(_) => {} - Err(e) => match e { - ReadlineError::Io(e) if e.kind() == std::io::ErrorKind::NotFound => { - println!("{}", SKY_WELCOME) - } - _ => fatal!("Failed to read history file with error: {}", e), - }, - } - loop { - match editor.readline(&skysh_prompt) { - Ok(mut line) => { - macro_rules! tokenize { - ($inp:expr) => { - match tokenizer::get_query($inp) { - Ok(q) => q, - Err(e) => { - eskysh!(e); - continue; - } - } - }; - () => { - tokenize!(line.as_bytes()) - }; - } - match line.to_lowercase().as_str() { - "exit" => break, - "clear" => { - clear_screen(); - continue; - } - "help" => { - println!("To get help, run `!help`"); - continue; - } - _ => { - if line.is_empty() { - continue; - } - match line.as_bytes()[0] { - b'#' => continue, - b'!' => { - match &line.as_bytes()[1..] { - b"" => eskysh!("Bad shell command"), - b"help" => println!("{}", HELP_TEXT), - b"pipe" => { - // so we need to handle a pipeline - let mut pipeline = Pipeline::new(); - line = readln!(editor); - loop { - did_swap = line - .get(..3) - .map(|v| v.eq_ignore_ascii_case("use")) - .unwrap_or(did_swap); - if !line.is_empty() { - if *(line.as_bytes().last().unwrap()) == b';' { - break; - } else { - let q: Query = tokenize!(); - pipeline.push(q); - } - } - line = readln!(editor); - } - if line.len() > 1 { - line.drain(line.len() - 1..); - let q: Query = tokenize!(); - pipeline.push(q); - } - runner.run_pipeline(pipeline).await; - checkswap!(); - } - _ => eskysh!("Unknown shell command"), - } - continue; - } - b'?' => { - // handle explanation for a shell command - print_help(&line); - continue; - } - _ => {} - } - while line.len() >= 2 && line[line.len() - 2..].as_bytes().eq(br#" \"#) { - // continuation on next line - let cl = readln!(editor); - line.drain(line.len() - 2..); - line.push_str(&cl); - } - did_swap = line - .get(..3) - .map(|v| v.eq_ignore_ascii_case("use")) - .unwrap_or(did_swap); - runner.run_query(&line).await; - checkswap!(); - } - } - } - Err(ReadlineError::Interrupted | ReadlineError::Eof) => break, - Err(err) => fatal!("ERROR: Failed to read line with error: {}", err), - } - } - editor - .save_history(SKYSH_HISTORY_FILE) - .map_err(|e| { - fatal!("ERROR: Failed to save history with error: '{}'", e); - }) - .unwrap(); -} - -fn print_help(line: &str) { - match &line.as_bytes()[1..] { - b"" => eskysh!("Bad shell command"), - b"help" => println!("`!help` shows the help menu"), - b"exit" => println!("`exit` ends the shell session"), - b"clear" => println!("`clear` clears the terminal screen"), - b"pipe" | b"!pipe" => println!("`!pipe` lets you run pipelines using the shell"), - _ => eskysh!("Unknown shell command"), - } -} - -fn clear_screen() { - let mut stdout = stdout(); - execute!(stdout, Clear(ClearType::All)).expect("Failed to clear screen"); - execute!(stdout, cursor::MoveTo(0, 0)).expect("Failed to move cursor to origin"); -} diff --git a/cli/src/args.rs b/cli/src/args.rs new file mode 100644 index 00000000..804afc16 --- /dev/null +++ b/cli/src/args.rs @@ -0,0 +1,192 @@ +/* + * Created on Wed Nov 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::error::{CliError, CliResult}, + crossterm::{ + event::{self, Event, KeyCode, KeyEvent}, + terminal, + }, + libsky::CliAction, + std::{ + collections::HashMap, + fs, + io::{self, Write}, + process::exit, + }, +}; + +const TXT_HELP: &str = include_str!("../help_text/help"); + +#[derive(Debug)] +pub struct ClientConfig { + pub kind: ClientConfigKind, + pub username: String, + pub password: String, +} + +impl ClientConfig { + pub fn new(kind: ClientConfigKind, username: String, password: String) -> Self { + Self { + kind, + username, + password, + } + } +} + +#[derive(Debug)] +pub enum ClientConfigKind { + Tcp(String, u16), + Tls(String, u16, String), +} + +#[derive(Debug)] +pub enum Task { + HelpMessage(String), + OpenShell(ClientConfig), +} + +enum TaskInner { + HelpMsg(String), + OpenShell(HashMap), +} + +fn load_env() -> CliResult { + let action = libsky::parse_cli_args_disallow_duplicate()?; + match action { + CliAction::Help => Ok(TaskInner::HelpMsg(TXT_HELP.into())), + CliAction::Version => Ok(TaskInner::HelpMsg(libsky::version_msg("skysh"))), + CliAction::Action(a) => Ok(TaskInner::OpenShell(a)), + } +} + +pub fn parse() -> CliResult { + let mut args = match load_env()? { + TaskInner::HelpMsg(msg) => return Ok(Task::HelpMessage(msg)), + TaskInner::OpenShell(args) => args, + }; + let endpoint = match args.remove("--endpoint") { + None => ClientConfigKind::Tcp("127.0.0.1".into(), 2003), + Some(ep) => { + // should be in the format protocol@host:port + let proto_host_port: Vec<&str> = ep.split("@").collect(); + if proto_host_port.len() != 2 { + return Err(CliError::ArgsErr("invalid value for --endpoint".into())); + } + let (protocol, host_port) = (proto_host_port[0], proto_host_port[1]); + let host_port: Vec<&str> = host_port.split(":").collect(); + if host_port.len() != 2 { + return Err(CliError::ArgsErr("invalid value for --endpoint".into())); + } + let (host, port) = (host_port[0], host_port[1]); + let port = match port.parse::() { + Ok(port) => port, + Err(e) => { + return Err(CliError::ArgsErr(format!( + "invalid value for endpoint port. {e}" + ))) + } + }; + let tls_cert = args.remove("--tls-cert"); + match protocol { + "tcp" => { + // TODO(@ohsayan): warn! + ClientConfigKind::Tcp(host.into(), port) + } + "tls" => { + // we need a TLS cert + match tls_cert { + Some(path) => { + let cert = fs::read_to_string(path)?; + ClientConfigKind::Tls(host.into(), port, cert) + } + None => { + return Err(CliError::ArgsErr(format!( + "must provide TLS cert when using TLS endpoint" + ))) + } + } + } + _ => { + return Err(CliError::ArgsErr(format!( + "unknown protocol scheme `{protocol}`" + ))) + } + } + } + }; + let username = match args.remove("--user") { + Some(u) => u, + None => { + // default + "root".into() + } + }; + let password = match args.remove("--password") { + Some(p) => p, + None => read_password("Enter password: ")?, + }; + if args.is_empty() { + Ok(Task::OpenShell(ClientConfig::new( + endpoint, username, password, + ))) + } else { + Err(CliError::ArgsErr(format!("found unknown arguments"))) + } +} + +fn read_password(prompt: &str) -> Result { + terminal::enable_raw_mode()?; + print!("{prompt}"); + io::stdout().flush()?; + let mut password = String::new(); + loop { + match event::read()? { + Event::Key(KeyEvent { + code: KeyCode::Char('c'), + modifiers: event::KeyModifiers::CONTROL, + .. + }) => { + terminal::disable_raw_mode()?; + println!(); + exit(0x00) + } + Event::Key(KeyEvent { code, .. }) => match code { + KeyCode::Backspace => { + let _ = password.pop(); + } + KeyCode::Char(c) => password.push(c), + KeyCode::Enter => break, + _ => {} + }, + _ => {} + } + } + terminal::disable_raw_mode()?; + println!(); + Ok(password) +} diff --git a/cli/src/error.rs b/cli/src/error.rs new file mode 100644 index 00000000..913c2e9f --- /dev/null +++ b/cli/src/error.rs @@ -0,0 +1,73 @@ +/* + * Created on Wed Nov 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use core::fmt; + +pub type CliResult = Result; + +#[derive(Debug)] +pub enum CliError { + QueryError(String), + ArgsErr(String), + ClientError(skytable::error::Error), + IoError(std::io::Error), +} + +impl From for CliError { + fn from(e: libsky::ArgParseError) -> Self { + match e { + libsky::ArgParseError::Duplicate(d) => { + Self::ArgsErr(format!("duplicate value for `{d}`")) + } + libsky::ArgParseError::MissingValue(m) => { + Self::ArgsErr(format!("missing value for `{m}`")) + } + } + } +} + +impl From for CliError { + fn from(cle: skytable::error::Error) -> Self { + Self::ClientError(cle) + } +} + +impl From for CliError { + fn from(ioe: std::io::Error) -> Self { + Self::IoError(ioe) + } +} + +impl fmt::Display for CliError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ArgsErr(e) => write!(f, "incorrect arguments. {e}"), + Self::ClientError(e) => write!(f, "client error. {e}"), + Self::IoError(e) => write!(f, "i/o error. {e}"), + Self::QueryError(e) => write!(f, "invalid query. {e}"), + } + } +} diff --git a/cli/src/macros.rs b/cli/src/macros.rs deleted file mode 100644 index cdc809d6..00000000 --- a/cli/src/macros.rs +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Created on Wed Nov 03 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -macro_rules! write_str { - ($st:ident) => { - println!("\"{}\"", $st) - }; - ($idx:ident, $st:ident) => { - println!("({}) \"{}\"", $idx, $st) - }; -} - -macro_rules! write_binstr { - ($st:ident) => { - println!("{}", BinaryData($st)) - }; - ($idx:ident, $st:ident) => { - println!("({}) {}", $idx, BinaryData($st)) - }; -} - -macro_rules! write_int { - ($int:ident) => { - println!("{}", $int) - }; - ($idx:ident, $st:ident) => { - println!("({}) \"{}\"", $idx, $st) - }; -} - -macro_rules! write_err { - ($idx:expr, $err:ident) => { - err!(if let Some(idx) = $idx { - format!("({}) ({})\n", idx, $err) - } else { - format!("({})\n", $err) - }) - }; - ($idx:ident, $err:literal) => { - err!( - (if let Some(idx) = $idx { - format!("({}) ({})\n", idx, $err) - } else { - format!("({})\n", $err) - }) - ) - }; -} - -macro_rules! write_okay { - () => { - crossterm::execute!( - std::io::stdout(), - SetForegroundColor(Color::Cyan), - Print("(Okay)\n".to_string()), - ResetColor - ) - .expect("Failed to write to stdout") - }; - ($idx:ident) => { - crossterm::execute!( - std::io::stdout(), - SetForegroundColor(Color::Cyan), - Print(format!("({}) (Okay)\n", $idx)), - ResetColor - ) - .expect("Failed to write to stdout") - }; -} - -macro_rules! err { - ($input:expr) => { - crossterm::execute!( - std::io::stdout(), - ::crossterm::style::SetForegroundColor(::crossterm::style::Color::Red), - ::crossterm::style::Print($input), - ::crossterm::style::ResetColor - ) - .expect("Failed to write to stdout") - }; -} - -macro_rules! eskysh { - ($e:expr) => { - err!(format!("[SKYSH ERROR] {}\n", $e)) - }; -} - -macro_rules! fatal { - ($e:expr) => {{ - err!($e); - ::std::process::exit(0x01); - }}; - ($e:expr, $desc:expr) => {{ - err!(format!($e, $desc)); - println!(); - ::std::process::exit(0x01) - }}; -} diff --git a/cli/src/main.rs b/cli/src/main.rs index afa7c311..c23a09b6 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,5 +1,5 @@ /* - * Created on Wed Jul 01 2020 + * Created on Wed Nov 15 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2020, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,22 +24,32 @@ * */ -#![deny(unused_crate_dependencies)] -#![deny(unused_imports)] +macro_rules! fatal { + ($($arg:tt)*) => {{ + eprintln!($($arg)*); + std::process::exit(0x01); + }} +} + +mod args; +mod error; +mod query; +mod repl; +mod resp; -#[macro_use] -mod macros; -mod argparse; -mod cli; -mod runner; -mod tokenizer; +use args::Task; -// tests -#[cfg(test)] -mod tests; +fn main() { + match run() { + Ok(()) => {} + Err(e) => fatal!("cli error: {e}"), + } +} -#[tokio::main] -async fn main() { - argparse::start_repl().await; - println!("Goodbye!"); +fn run() -> error::CliResult<()> { + match args::parse()? { + Task::HelpMessage(msg) => println!("{msg}"), + Task::OpenShell(cfg) => repl::start(cfg)?, + } + Ok(()) } diff --git a/cli/src/query.rs b/cli/src/query.rs new file mode 100644 index 00000000..fa6ec7da --- /dev/null +++ b/cli/src/query.rs @@ -0,0 +1,295 @@ +/* + * Created on Thu Nov 16 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::error::{CliError, CliResult}, + skytable::{ + error::ClientResult, query::SQParam, response::Response, Connection, ConnectionTls, Query, + }, +}; + +pub trait IsConnection { + fn execute_query(&mut self, q: Query) -> ClientResult; +} + +impl IsConnection for Connection { + fn execute_query(&mut self, q: Query) -> ClientResult { + self.query(&q) + } +} + +impl IsConnection for ConnectionTls { + fn execute_query(&mut self, q: Query) -> ClientResult { + self.query(&q) + } +} + +#[derive(Debug, PartialEq)] +enum Item { + UInt(u64), + SInt(i64), + Float(f64), + String(String), + Bin(Vec), +} + +impl SQParam for Item { + fn append_param(self, buf: &mut Vec) { + match self { + Item::UInt(u) => u.append_param(buf), + Item::SInt(s) => s.append_param(buf), + Item::Float(f) => f.append_param(buf), + Item::String(s) => s.append_param(buf), + Item::Bin(b) => SQParam::append_param(&*b, buf), + } + } +} + +pub struct Parameterizer { + buf: Vec, + i: usize, + params: Vec, + query: Vec, +} + +#[derive(Debug, PartialEq)] +pub enum ExecKind { + Standard(Query), + UseSpace(Query, String), + UseNull(Query), + PrintSpecial(Query), +} + +impl Parameterizer { + pub fn new(q: String) -> Self { + Self { + buf: q.into_bytes(), + i: 0, + params: vec![], + query: vec![], + } + } + pub fn parameterize(mut self) -> CliResult { + while self.not_eof() { + match self.buf[self.i] { + b if b.is_ascii_alphabetic() || b == b'_' => self.read_ident(), + b if b.is_ascii_digit() => self.read_unsigned_integer(), + b'-' => self.read_signed_integer(), + quote_style @ (b'"' | b'\'') => { + self.i += 1; + self.read_string(quote_style) + } + b'`' => { + self.i += 1; + self.read_binary() + } + sym => { + self.i += 1; + Vec::push(&mut self.query, sym); + Ok(()) + } + }? + } + match String::from_utf8(self.query) { + Ok(qstr) => { + let mut q = Query::new(&qstr); + self.params.into_iter().for_each(|p| { + q.push_param(p); + }); + Ok(if qstr.eq_ignore_ascii_case("use null") { + ExecKind::UseNull(q) + } else { + if qstr.len() > 8 { + let qstr = &qstr[..8]; + if qstr.eq_ignore_ascii_case("inspect ") { + return Ok(ExecKind::PrintSpecial(q)); + } + } + let mut splits = qstr.split_ascii_whitespace(); + let tok_use = splits.next(); + let tok_name = splits.next(); + match (tok_use, tok_name) { + (Some(tok_use), Some(tok_name)) + if tok_use.eq_ignore_ascii_case("use") + && !tok_name.eq_ignore_ascii_case("$current") => + { + ExecKind::UseSpace(q, tok_name.into()) + } + _ => ExecKind::Standard(q), + } + }) + } + Err(_) => Err(CliError::QueryError("query is not valid UTF-8".into())), + } + } + fn read_string(&mut self, quote_style: u8) -> CliResult<()> { + self.query.push(b'?'); + let mut string = Vec::new(); + let mut terminated = false; + while self.not_eof() && !terminated { + let b = self.buf[self.i]; + if b == b'\\' { + self.i += 1; + // escape sequence + if self.i == self.buf.len() { + // string was not terminated + return Err(CliError::QueryError("string not terminated".into())); + } + match self.buf[self.i] { + b'\\' => { + // escaped \ + string.push(b'\\'); + } + b if b == quote_style => { + // escape quote + string.push(quote_style); + } + _ => return Err(CliError::QueryError("unknown escape sequence".into())), + } + } + if b == quote_style { + terminated = true; + } else { + string.push(b); + } + self.i += 1; + } + if terminated { + match String::from_utf8(string) { + Ok(s) => self.params.push(Item::String(s)), + Err(_) => return Err(CliError::QueryError("invalid UTF-8 string".into())), + } + Ok(()) + } else { + return Err(CliError::QueryError("string not terminated".into())); + } + } + fn read_ident(&mut self) -> CliResult<()> { + // we're looking at an ident + let start = self.i; + self.i += 1; + while self.not_eof() { + if self.buf[self.i].is_ascii_alphanumeric() || self.buf[self.i] == b'_' { + self.i += 1; + } else { + break; + } + } + let stop = self.i; + self.query.extend(&self.buf[start..stop]); + Ok(()) + } + fn read_float(&mut self, start: usize) -> CliResult<()> { + self.read_until_number_escape(); + let stop = self.i; + match core::str::from_utf8(&self.buf[start..stop]).map(|v| v.parse()) { + Ok(Ok(num)) => self.params.push(Item::Float(num)), + _ => { + return Err(CliError::QueryError( + "invalid floating point literal".into(), + )) + } + } + Ok(()) + } + fn read_signed_integer(&mut self) -> CliResult<()> { + self.query.push(b'?'); + // we must have encountered a `-` + let start = self.i; + self.read_until_number_escape(); + let stop = self.i; + match core::str::from_utf8(&self.buf[start..stop]).map(|v| v.parse()) { + Ok(Ok(s)) => self.params.push(Item::SInt(s)), + _ => { + return Err(CliError::QueryError( + "invalid signed integer literal".into(), + )) + } + } + Ok(()) + } + fn read_unsigned_integer(&mut self) -> CliResult<()> { + self.query.push(b'?'); + let start = self.i; + let mut ret = 0u64; + while self.not_eof() { + match self.buf[self.i] { + b if b.is_ascii_digit() => { + self.i += 1; + ret = match ret + .checked_mul(10) + .map(|v| v.checked_add((b & 0x0f) as u64)) + { + Some(Some(r)) => r, + _ => return Err(CliError::QueryError("bad value for integer".into())), + }; + } + b'.' => { + self.i += 1; + // uh oh, that's a float + return self.read_float(start); + } + b if b == b' ' || b == b'\t' || b.is_ascii_punctuation() => { + break; + } + _ => { + // nothing else is valid here + return Err(CliError::QueryError( + "invalid unsigned integer literal".into(), + )); + } + } + } + self.params.push(Item::UInt(ret)); + Ok(()) + } + fn read_until_number_escape(&mut self) { + while self.not_eof() { + let b = self.buf[self.i]; + if b == b'\n' || b == b'\t' || b.is_ascii_punctuation() { + break; + } + self.i += 1; + } + } + fn read_binary(&mut self) -> CliResult<()> { + self.query.push(b'?'); + let start = self.i; + while self.not_eof() { + let b = self.buf[self.i]; + self.i += 1; + if b == b'`' { + self.params + .push(Item::Bin(self.buf[start..self.i].to_vec())); + return Ok(()); + } + } + Err(CliError::QueryError("binary literal not terminated".into())) + } + fn not_eof(&self) -> bool { + self.i < self.buf.len() + } +} diff --git a/cli/src/repl.rs b/cli/src/repl.rs new file mode 100644 index 00000000..80d31500 --- /dev/null +++ b/cli/src/repl.rs @@ -0,0 +1,163 @@ +/* + * Created on Thu Nov 16 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::query::ExecKind; + +use { + crate::{ + args::{ClientConfig, ClientConfigKind}, + error::{CliError, CliResult}, + query::{self, IsConnection}, + resp, + }, + crossterm::{cursor, execute, terminal}, + rustyline::{config::Configurer, error::ReadlineError, DefaultEditor}, + skytable::Config, + std::io::{stdout, ErrorKind}, +}; + +const SKYSH_HISTORY_FILE: &str = ".sky_history"; +const TXT_WELCOME: &str = include_str!("../help_text/welcome"); + +pub fn start(cfg: ClientConfig) -> CliResult<()> { + match cfg.kind { + ClientConfigKind::Tcp(host, port) => { + let c = Config::new(&host, port, &cfg.username, &cfg.password).connect()?; + println!( + "Authenticated as '{}' on {}:{} over Skyhash/TCP\n---", + &cfg.username, &host, &port + ); + repl(c) + } + ClientConfigKind::Tls(host, port, cert) => { + let c = Config::new(&host, port, &cfg.username, &cfg.password).connect_tls(&cert)?; + println!( + "Authenticated as '{}' on {}:{} over Skyhash/TLS\n---", + &cfg.username, &host, &port + ); + repl(c) + } + } +} + +fn repl(mut con: C) -> CliResult<()> { + let init_editor = || { + let mut editor = DefaultEditor::new()?; + editor.set_auto_add_history(true); + editor.set_history_ignore_dups(true)?; + editor.bind_sequence( + rustyline::KeyEvent( + rustyline::KeyCode::BracketedPasteStart, + rustyline::Modifiers::NONE, + ), + rustyline::Cmd::Noop, + ); + match editor.load_history(SKYSH_HISTORY_FILE) { + Ok(()) => {} + Err(e) => match e { + ReadlineError::Io(ref ioe) => match ioe.kind() { + ErrorKind::NotFound => { + println!("{TXT_WELCOME}"); + } + _ => return Err(e), + }, + e => return Err(e), + }, + } + rustyline::Result::Ok(editor) + }; + let mut editor = match init_editor() { + Ok(e) => e, + Err(e) => fatal!("error: failed to init REPL. {e}"), + }; + let mut prompt = "> ".to_owned(); + loop { + match editor.readline(&prompt) { + Ok(line) => match line.as_str() { + "!help" => println!("{TXT_WELCOME}"), + "exit" => break, + "clear" => clear_screen()?, + _ => { + if line.is_empty() { + continue; + } + match query::Parameterizer::new(line).parameterize() { + Ok(q) => { + let mut new_prompt = None; + let mut special = false; + let q = match q { + ExecKind::Standard(q) => q, + ExecKind::UseNull(q) => { + new_prompt = Some("> ".into()); + q + } + ExecKind::UseSpace(q, space) => { + new_prompt = Some(format!("{space}> ")); + q + } + ExecKind::PrintSpecial(q) => { + special = true; + q + } + }; + if resp::format_response(con.execute_query(q)?, special) { + if let Some(pr) = new_prompt { + prompt = pr; + } + } + } + Err(e) => match e { + CliError::QueryError(e) => { + eprintln!("[skysh error]: bad query. {e}"); + continue; + } + _ => return Err(e), + }, + }; + } + }, + Err(e) => match e { + ReadlineError::Interrupted | ReadlineError::Eof => { + // done + break; + } + ReadlineError::WindowResized => {} + e => fatal!("error: failed to read line REPL. {e}"), + }, + } + } + editor + .save_history(SKYSH_HISTORY_FILE) + .expect("failed to save history"); + println!("Goodbye!"); + Ok(()) +} + +fn clear_screen() -> std::io::Result<()> { + let mut stdout = stdout(); + execute!(stdout, terminal::Clear(terminal::ClearType::All))?; + execute!(stdout, cursor::MoveTo(0, 0)) +} diff --git a/cli/src/resp.rs b/cli/src/resp.rs new file mode 100644 index 00000000..276a9fd0 --- /dev/null +++ b/cli/src/resp.rs @@ -0,0 +1,134 @@ +/* + * Created on Thu Nov 16 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crossterm::style::Stylize, + skytable::response::{Response, Row, Value}, +}; + +pub fn format_response(resp: Response, print_special: bool) -> bool { + match resp { + Response::Empty => println!("{}", "(Okay)".cyan()), + Response::Error(e) => { + println!("{}", format!("(server error code: {e})").red()); + return false; + } + Response::Value(v) => { + print_value(v, print_special); + println!(); + } + Response::Row(r) => { + print_row(r); + println!(); + } + Response::Rows(rows) => { + if rows.is_empty() { + println!("{}", "[0 rows returned]".grey().italic()); + } else { + for (i, row) in rows.into_iter().enumerate().map(|(i, r)| (i + 1, r)) { + print!("{} ", format!("({i})").grey().bold()); + print_row(row); + println!(); + } + } + } + }; + true +} + +fn print_row(r: Row) { + print!("("); + let mut columns = r.into_values().into_iter().peekable(); + while let Some(cell) = columns.next() { + print_value(cell, false); + if columns.peek().is_some() { + print!(", "); + } + } + print!(")"); +} + +fn print_value(v: Value, print_special: bool) { + match v { + Value::Null => print!("{}", "null".grey().italic()), + Value::String(s) => print_string(&s, print_special), + Value::Binary(b) => print_binary(&b), + Value::Bool(b) => print!("{b}"), + Value::UInt8(i) => print!("{i}"), + Value::UInt16(i) => print!("{i}"), + Value::UInt32(i) => print!("{i}"), + Value::UInt64(i) => print!("{i}"), + Value::SInt8(i) => print!("{i}"), + Value::SInt16(i) => print!("{i}"), + Value::SInt32(i) => print!("{i}"), + Value::SInt64(i) => print!("{i}"), + Value::Float32(f) => print!("{f}"), + Value::Float64(f) => print!("{f}"), + Value::List(items) => { + print!("["); + let mut items = items.into_iter().peekable(); + while let Some(item) = items.next() { + print_value(item, print_special); + if items.peek().is_some() { + print!(", "); + } + } + print!("]"); + } + } +} + +fn print_binary(b: &[u8]) { + let mut it = b.into_iter().peekable(); + print!("["); + while let Some(byte) = it.next() { + print!("{byte}"); + if it.peek().is_some() { + print!(", "); + } + } + print!("]"); +} + +fn print_string(s: &str, print_special: bool) { + if print_special { + print!("{}", s.italic().grey()); + } else { + print!("\""); + for ch in s.chars() { + if ch == '"' { + print!("\\{ch}"); + } else if ch == '\t' { + print!("\\t"); + } else if ch == '\n' { + print!("\\n"); + } else { + print!("{ch}"); + } + } + print!("\""); + } +} diff --git a/cli/src/runner.rs b/cli/src/runner.rs deleted file mode 100644 index f9ce45ac..00000000 --- a/cli/src/runner.rs +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Created on Wed May 12 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::tokenizer, - core::fmt, - crossterm::style::{Color, Print, ResetColor, SetForegroundColor}, - skytable::{ - aio, error::Error, types::Array, types::FlatElement, Element, Pipeline, Query, RespCode, - }, -}; - -type SkyResult = Result; - -pub enum Runner { - Insecure(aio::Connection), - Secure(aio::TlsConnection), -} - -impl Runner { - pub async fn new_insecure(host: &str, port: u16) -> SkyResult { - let con = aio::Connection::new(host, port).await?; - Ok(Self::Insecure(con)) - } - pub async fn new_secure(host: &str, port: u16, cert: &str) -> SkyResult { - let con = aio::TlsConnection::new(host, port, cert).await?; - Ok(Self::Secure(con)) - } - pub async fn run_pipeline(&mut self, pipeline: Pipeline) { - let ret = match self { - Self::Insecure(con) => con.run_pipeline(pipeline).await, - Self::Secure(con) => con.run_pipeline(pipeline).await, - }; - let retok = match ret { - Ok(r) => r, - Err(e) => fatal!("An I/O error occurred while querying: {}", e), - }; - for (idx, resp) in retok - .into_iter() - .enumerate() - .map(|(idx, resp)| (idx + 1, resp)) - { - println!("[Response {}]", idx); - print_element(resp); - } - } - pub async fn run_query(&mut self, unescaped: &str) { - let query: Query = match tokenizer::get_query(unescaped.as_bytes()) { - Ok(q) => q, - Err(e) => { - err!(format!("[Syntax Error: {}]\n", e)); - return; - } - }; - let ret = match self { - Self::Insecure(con) => con.run_query_raw(&query).await, - Self::Secure(con) => con.run_query_raw(&query).await, - }; - match ret { - Ok(resp) => print_element(resp), - Err(e) => fatal!("An I/O error occurred while querying: {}", e), - } - } - pub async fn check_entity(&mut self, blank: &mut String, prompt: &mut String) { - let query: Query = tokenizer::get_query(b"whereami").unwrap(); - let ret = match self { - Self::Insecure(con) => con.run_query_raw(&query).await, - Self::Secure(con) => con.run_query_raw(&query).await, - }; - let ret = match ret { - Ok(resp) => resp, - Err(e) => fatal!("An I/O error occurred while querying: {}", e), - }; - match ret { - Element::Array(Array::NonNullStr(srr)) => match srr.len() { - 1 => { - *blank = format!(" {blank}> ", blank = " ".repeat(srr[0].len())); - *prompt = format!("skysh@{ks}> ", ks = srr[0]); - } - 2 => { - let ks = &srr[0]; - let tbl = &srr[1]; - *blank = format!( - " {blank}> ", - blank = " ".repeat(ks.len() + tbl.len() + 1) - ); - *prompt = format!("skysh@{ks}:{tbl}> ", ks = ks, tbl = tbl); - } - count => fatal!( - "The server returned {} IDs while checking entity state", - count - ), - }, - _ => fatal!("The server returned the wrong data type for entity state check"), - } - } -} - -fn print_float(float: f32, idx: Option) { - if let Some(idx) = idx { - println!("({idx}) {float}") - } else { - println!("{float}"); - } -} - -fn print_element(el: Element) { - match el { - Element::String(st) => write_str!(st), - Element::Binstr(st) => write_binstr!(st), - Element::Array(Array::Bin(brr)) => print_bin_array(brr), - Element::Array(Array::Str(srr)) => print_str_array(srr), - Element::RespCode(r) => print_rcode(r, None), - Element::UnsignedInt(int) => write_int!(int), - Element::Array(Array::Flat(frr)) => write_flat_array(frr), - Element::Array(Array::Recursive(a)) => print_array(a), - Element::Array(Array::NonNullBin(nbrr)) => print_array_nonnull_bin(nbrr), - Element::Array(Array::NonNullStr(nsrr)) => print_array_nonnull_str(nsrr), - Element::Float(float) => print_float(float, None), - _ => eskysh!("The server possibly sent a newer data type that we can't parse"), - } -} - -fn print_rcode(rcode: RespCode, idx: Option) { - match rcode { - RespCode::Okay => write_okay!(), - RespCode::ActionError => write_err!(idx, "Action Error"), - RespCode::ErrorString(st) => write_err!(idx, st), - RespCode::OtherError => write_err!(idx, "Other Error"), - RespCode::NotFound => write_err!(idx, "Not Found"), - RespCode::OverwriteError => write_err!(idx, "Overwrite Error"), - RespCode::PacketError => write_err!(idx, "Packet Error"), - RespCode::ServerError => write_err!(idx, "Server Error"), - RespCode::UnknownDataType => write_err!(idx, "Unknown data type"), - RespCode::EncodingError => write_err!(idx, "Encoding error"), - RespCode::AuthBadCredentials => write_err!(idx, "auth bad credentials"), - RespCode::AuthPermissionError => write_err!(idx, "auth permission error"), - _ => write_err!(idx, "Unknown error"), - } -} - -fn print_bin_array(bin_array: Vec>>) { - bin_array.into_iter().enumerate().for_each(|(idx, elem)| { - let idx = idx + 1; - match elem { - Some(ele) => { - write_binstr!(idx, ele); - } - None => print_rcode(RespCode::NotFound, Some(idx)), - } - }) -} - -fn print_str_array(str_array: Vec>) { - str_array.into_iter().enumerate().for_each(|(idx, elem)| { - let idx = idx + 1; - match elem { - Some(ele) => { - write_str!(idx, ele); - } - None => print_rcode(RespCode::NotFound, Some(idx)), - } - }) -} - -fn print_array_nonnull_str(str_array: Vec) { - str_array.into_iter().enumerate().for_each(|(idx, elem)| { - let idx = idx + 1; - write_str!(idx, elem) - }) -} - -fn print_array_nonnull_bin(str_array: Vec>) { - str_array.into_iter().enumerate().for_each(|(idx, elem)| { - let idx = idx + 1; - write_binstr!(idx, elem) - }) -} - -fn write_flat_array(flat_array: Vec) { - for (idx, item) in flat_array.into_iter().enumerate() { - let idx = idx + 1; - match item { - FlatElement::String(st) => write_str!(idx, st), - FlatElement::Binstr(st) => { - write_binstr!(idx, st) - } - FlatElement::RespCode(rc) => print_rcode(rc, Some(idx)), - FlatElement::UnsignedInt(int) => write_int!(int, idx), - _ => eskysh!("Element typed cannot yet be parsed"), - } - } -} - -fn print_array(array: Vec) { - for (idx, item) in array.into_iter().enumerate() { - let idx = idx + 1; - match item { - Element::String(st) => write_str!(idx, st), - Element::RespCode(rc) => print_rcode(rc, Some(idx)), - Element::UnsignedInt(int) => write_int!(idx, int), - Element::Array(Array::Bin(brr)) => print_bin_array(brr), - Element::Array(Array::Str(srr)) => print_str_array(srr), - _ => eskysh!("Nested arrays cannot be printed just yet"), - } - } -} - -pub struct BinaryData(Vec); - -impl fmt::Display for BinaryData { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!(f, "b\"")?; - for b in self.0.iter() { - let b = *b; - // See this: https://doc.rust-lang.org/reference/tokens.html#byte-escapes - // this idea was borrowed from the Bytes crate - #[allow(clippy::manual_range_contains)] - if b == b'\n' { - write!(f, "\\n")?; - } else if b == b'\r' { - write!(f, "\\r")?; - } else if b == b'\t' { - write!(f, "\\t")?; - } else if b == b'\\' || b == b'"' { - write!(f, "\\{}", b as char)?; - } else if b == b'\0' { - write!(f, "\\0")?; - // ASCII printable - } else if b >= 0x20 && b < 0x7f { - write!(f, "{}", b as char)?; - } else { - write!(f, "\\x{:02x}", b)?; - } - } - write!(f, "\"")?; - Ok(()) - } -} diff --git a/cli/src/tests.rs b/cli/src/tests.rs deleted file mode 100644 index d527b1f8..00000000 --- a/cli/src/tests.rs +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Created on Sun Oct 10 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::tokenizer::{get_query, TokenizerError}; - -fn query_from(input: &[u8]) -> Result, TokenizerError> { - get_query(input) -} - -#[test] -fn test_basic_tokenization() { - let input = "set x 100".as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec!["set".to_owned(), "x".to_owned(), "100".to_owned()] - ); -} - -#[test] -fn test_single_quote_tokens() { - let input = "set 'x with a whitespace' 100".as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "100".to_owned() - ] - ); -} - -#[test] -fn test_double_quote_tokens() { - let input = r#"set "x with a whitespace" 100"#.as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "100".to_owned() - ] - ); -} - -#[test] -fn test_single_and_double_quote_tokens() { - let input = r#"set "x with a whitespace" 'y with a whitespace'"#.as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "y with a whitespace".to_owned() - ] - ); -} - -#[test] -fn test_multiple_single_quote_tokens() { - let input = r#"'set' 'x with a whitespace' 'y with a whitespace'"#.as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "y with a whitespace".to_owned() - ] - ); -} - -#[test] -fn test_multiple_double_quote_tokens() { - let input = r#""set" "x with a whitespace" "y with a whitespace""#.as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "y with a whitespace".to_owned() - ] - ); -} - -#[test] -fn test_missing_single_quote() { - let input = r#"'get' 'x with a whitespace"#.as_bytes(); - let ret = format!("{}", query_from(input).unwrap_err()); - assert_eq!(ret, "mismatched quotes near end of: `x with a whitespace`"); -} - -#[test] -fn test_missing_double_quote() { - let input = r#"'get' "x with a whitespace"#.as_bytes(); - let ret = format!("{}", query_from(input).unwrap_err()); - assert_eq!(ret, "mismatched quotes near end of: `x with a whitespace`"); -} - -#[test] -fn test_extra_whitespace() { - let input = "set x '100'".as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec!["set".to_owned(), "x".to_owned(), "100".to_owned()] - ); -} - -#[test] -fn test_singly_quoted() { - let input = "set tables' wth".as_bytes(); - let ret = query_from(input).unwrap_err(); - assert_eq!(ret, TokenizerError::ExpectedWhitespace("tables".to_owned())); -} - -#[test] -fn test_text_after_quote_nospace() { - let input = "get 'rust'ferris".as_bytes(); - let ret = query_from(input).unwrap_err(); - assert_eq!(ret, TokenizerError::ExpectedWhitespace("rust'".to_owned())); -} - -#[test] -fn test_text_after_double_quote_nospace() { - let input = r#"get "rust"ferris"#.as_bytes(); - let ret = query_from(input).unwrap_err(); - assert_eq!(ret, TokenizerError::ExpectedWhitespace("rust\"".to_owned())); -} - -#[test] -fn test_inline_comment() { - let input = "set x 100 # sets x to 100".as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec!["set".to_owned(), "x".to_owned(), "100".to_owned()] - ) -} - -#[test] -fn test_full_comment() { - let input = "# what is going on?".as_bytes(); - let ret = query_from(input).unwrap(); - assert!(ret.is_empty()); -} - -#[test] -fn test_ignore_comment() { - let input = "set x \"# ooh la la\"".as_bytes(); - assert_eq!( - query_from(input).unwrap(), - vec!["set".to_owned(), "x".to_owned(), "# ooh la la".to_owned()] - ); - let input = "set x \"#\"".as_bytes(); - assert_eq!( - query_from(input).unwrap(), - vec!["set".to_owned(), "x".to_owned(), "#".to_owned()] - ); -} - -#[test] -fn test_blueql_query() { - let input = b"create model mymodel(string, binary)"; - assert_eq!( - query_from(input).unwrap(), - vec!["create model mymodel(string, binary)"] - ); -} diff --git a/cli/src/tokenizer.rs b/cli/src/tokenizer.rs deleted file mode 100644 index 715bec1b..00000000 --- a/cli/src/tokenizer.rs +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Created on Sat Oct 09 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! This module provides a simple way to avoid "the funk" with "funky input queries". It simply -//! tokenizes char-by-char analyzing quotes et al as required -//! - -use { - core::fmt, - skytable::{types::RawString, Query}, - std::collections::HashSet, -}; - -lazy_static::lazy_static! { - static ref BLUEQL_KW: HashSet<&'static [u8]> = { - let mut hs = HashSet::new(); - hs.insert("create".as_bytes()); - hs.insert(b"inspect"); - hs.insert(b"drop"); - hs.insert(b"use"); - hs - }; -} - -#[derive(Debug, PartialEq, Eq)] -pub enum TokenizerError { - QuoteMismatch(String), - BacktickMismatch(String), - ExpectedWhitespace(String), -} - -impl fmt::Display for TokenizerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::QuoteMismatch(expr) => write!(f, "mismatched quotes near end of: `{}`", expr), - Self::ExpectedWhitespace(expr) => { - write!(f, "expected whitespace near end of: `{}`", expr) - } - Self::BacktickMismatch(expr) => { - write!(f, "mismatched backticks near end of: `{}`", expr) - } - } - } -} - -pub trait SequentialQuery { - fn push(&mut self, input: &[u8]); - fn new() -> Self; - fn is_empty(&self) -> bool; -} - -// #[cfg(test)] -impl SequentialQuery for Vec { - fn push(&mut self, input: &[u8]) { - Vec::push(self, String::from_utf8_lossy(input).to_string()) - } - fn is_empty(&self) -> bool { - Vec::len(self) == 0 - } - fn new() -> Self { - Vec::new() - } -} - -impl SequentialQuery for Query { - fn push(&mut self, input: &[u8]) { - Query::push(self, RawString::from(input.to_owned())) - } - fn is_empty(&self) -> bool { - Query::len(self) == 0 - } - fn new() -> Self { - Query::new() - } -} - -// FIXME(@ohsayan): Fix this entire impl. At this point, it's almost like legacy code -pub fn get_query(inp: &[u8]) -> Result { - assert!(!inp.is_empty(), "Input is empty"); - let mut query = T::new(); - let mut it = inp.iter().peekable(); - macro_rules! pos { - () => { - inp.len() - it.len() - }; - } - macro_rules! expect_whitespace { - ($start:expr) => { - match it.peek() { - Some(b) => match **b { - b' ' => {} - _ => { - return Err(TokenizerError::ExpectedWhitespace( - String::from_utf8_lossy(&inp[$start..pos!()]).to_string(), - )) - } - }, - None => {} - } - }; - } - // skip useless starting whitespace - while let Some(b' ') = it.next() {} - let end_of_first = match it.position(|x| *x == b' ') { - Some(e) => e + 1, - None if it.len() == 0 => inp.len(), - None => { - return Err(TokenizerError::ExpectedWhitespace( - String::from_utf8_lossy(inp).to_string(), - )) - } - }; - if BLUEQL_KW.contains(inp[..end_of_first].to_ascii_lowercase().as_slice()) { - query.push(inp); - return Ok(query); - } else { - it = inp.iter().peekable(); - } - 'outer: while let Some(tok) = it.next() { - match tok { - b'\'' => { - // hmm, quotes; let's see where it ends - let pos = pos!(); - let qidx = it.position(|x| *x == b'\''); - match qidx { - Some(idx) => query.push(&inp[pos..idx + pos]), - None => { - let end = pos!(); - return Err(TokenizerError::QuoteMismatch( - String::from_utf8_lossy(&inp[pos..end]).to_string(), - )); - } - } - expect_whitespace!(pos); - } - b'"' => { - // hmm, quotes; let's see where it ends - let pos = pos!(); - let qidx = it.position(|x| *x == b'"'); - match qidx { - Some(idx) => query.push(&inp[pos..idx + pos]), - None => { - let end = pos!(); - return Err(TokenizerError::QuoteMismatch( - String::from_utf8_lossy(&inp[pos..end]).to_string(), - )); - } - } - expect_whitespace!(pos); - } - b'`' => { - // hmm, backtick? let's look for the end - let pos = pos!(); - let qidx = it.position(|x| *x == b'`'); - match qidx { - Some(idx) => query.push(&inp[pos..idx + pos]), - None => { - let end = pos!(); - return Err(TokenizerError::BacktickMismatch( - String::from_utf8_lossy(&inp[pos..end]).to_string(), - )); - } - } - expect_whitespace!(pos); - } - b' ' => { - // this just prevents control from being handed to the wildcard - continue; - } - b'#' => { - // so this is an inline comment; skip until newline - let _ = it.position(|x| *x == b'\n'); - } - _ => { - let start = pos!() - 1; - let mut end = start; - // alpha? cool, go on - 'inner: while let Some(tok) = it.peek() { - match **tok { - b' ' => { - it.next(); - break 'inner; - } - b'\'' | b'"' => { - return Err(TokenizerError::ExpectedWhitespace( - String::from_utf8_lossy(&inp[start..pos!()]).to_string(), - )) - } - b'#' => continue 'outer, - _ => { - end += 1; - it.next(); - continue 'inner; - } - } - } - end += 1; - query.push(&inp[start..end]); - } - } - } - Ok(query) -} diff --git a/examples/config-files/badcfg.toml b/examples/config-files/badcfg.toml deleted file mode 100644 index 4e7c7e47..00000000 --- a/examples/config-files/badcfg.toml +++ /dev/null @@ -1,3 +0,0 @@ -# This is a 'bad' configuration file since it contains an invalid port -[server] -port = 20033002 \ No newline at end of file diff --git a/examples/config-files/badcfg2.toml b/examples/config-files/badcfg2.toml deleted file mode 100644 index 066661b2..00000000 --- a/examples/config-files/badcfg2.toml +++ /dev/null @@ -1,8 +0,0 @@ -# This is a bad toml file since it contains an invalid value of `every` in BGSAVE -[server] -host = "127.0.0.1" -port = 2003 - -[bgsave] -every = 0.5 # Not possible! -enabled = true \ No newline at end of file diff --git a/examples/config-files/bgsave-justenabled.toml b/examples/config-files/bgsave-justenabled.toml deleted file mode 100644 index b1dfbeb9..00000000 --- a/examples/config-files/bgsave-justenabled.toml +++ /dev/null @@ -1,6 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 - -[bgsave] -enabled = true \ No newline at end of file diff --git a/examples/config-files/bgsave-justevery.toml b/examples/config-files/bgsave-justevery.toml deleted file mode 100644 index eb9e69a7..00000000 --- a/examples/config-files/bgsave-justevery.toml +++ /dev/null @@ -1,6 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 - -[bgsave] -every = 600 \ No newline at end of file diff --git a/examples/config-files/docker.toml b/examples/config-files/docker.toml deleted file mode 100644 index 3b5058bd..00000000 --- a/examples/config-files/docker.toml +++ /dev/null @@ -1,5 +0,0 @@ -# this is the default configuration file for Docker images. Modify it as required -[server] -host = "0.0.0.0" -port = 2003 -noart = true diff --git a/examples/config-files/dpkg/config.yaml b/examples/config-files/dpkg/config.yaml new file mode 100644 index 00000000..44408e3c --- /dev/null +++ b/examples/config-files/dpkg/config.yaml @@ -0,0 +1,14 @@ +# this is the default configuration for the Debian package. Modify to use your own settings. +system: + mode: prod + rs_window: 300 + +auth: + plugin: pwd + # replace with your root password of choice + root_pass: rootpass + +endpoints: + insecure: + host: 127.0.0.1 + port: 2003 diff --git a/examples/config-files/ipv6.toml b/examples/config-files/ipv6.toml deleted file mode 100644 index c98fe3bc..00000000 --- a/examples/config-files/ipv6.toml +++ /dev/null @@ -1,4 +0,0 @@ -# This makes use of an IPv6 address -[server] -host = "::1" -port = 2003 diff --git a/examples/config-files/secure-noart.toml b/examples/config-files/secure-noart.toml deleted file mode 100644 index 14adffca..00000000 --- a/examples/config-files/secure-noart.toml +++ /dev/null @@ -1,4 +0,0 @@ -[server] -host = '127.0.0.1' # i.e localhost -port = 2003 # The port to bind to -noart = true # No terminal artwork \ No newline at end of file diff --git a/examples/config-files/skyd.toml b/examples/config-files/skyd.toml deleted file mode 100644 index 7bf537d7..00000000 --- a/examples/config-files/skyd.toml +++ /dev/null @@ -1,5 +0,0 @@ -# This is a 'good' configuration file since it contains a valid port -# and appropriate keys -[server] -host = '127.0.0.1' -port = 2003 # Set the server's port to 2003 \ No newline at end of file diff --git a/examples/config-files/snapshot.toml b/examples/config-files/snapshot.toml deleted file mode 100644 index fd202c9f..00000000 --- a/examples/config-files/snapshot.toml +++ /dev/null @@ -1,17 +0,0 @@ -[server] -host = "127.0.0.1" # The IP address to which you want sdb to bind to -port = 2003 # The port to which you want sdb to bind to -# Set `noart` to true if you want to disable terminal artwork -noart = false - -[bgsave] -# Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run -# after every 2 minutes -enabled = true -every = 120 - -[snapshot] -# Create a snapshot every hour (1 hour = 60 minutes = 60 * 60 seconds = 3600 seconds) -every = 3600 -# How many of the snapshots to keep -atmost = 4 # keep the four most recent snapshots diff --git a/examples/config-files/ssl.toml b/examples/config-files/ssl.toml deleted file mode 100644 index c4e951e3..00000000 --- a/examples/config-files/ssl.toml +++ /dev/null @@ -1,17 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 -noart = false - -[ssl] -key = "/path/to/keyfile.pem" -chain = "/path/to/chain.pem" -port = 2004 - -[bgsave] -enabled = true -every = 120 - -[snapshot] -every = 3600 -atmost = 4 \ No newline at end of file diff --git a/examples/config-files/template.toml b/examples/config-files/template.toml deleted file mode 100644 index bc21d894..00000000 --- a/examples/config-files/template.toml +++ /dev/null @@ -1,40 +0,0 @@ -# This is a complete sdb configuration template which is always kept updated -# to include all the configuration options. I encourage you to always use this -# when you use a configuration file -# Instead of deleting entire sections from this file, comment them out, so that you -# now what you've kept enabled and what you've kept disabled. This helps avoid -# configuration problems during production - -# This is a *REQUIRED* key -[server] -host = "127.0.0.1" # The IP address to which you want sdb to bind to -port = 2003 # The port to which you want sdb to bind to -noart = false # Set `noart` to true if you want to disable terminal artwork -maxcon = 50000 # set the maximum number of clients that the server can accept -mode = "dev" # Set this to `prod` when you're running in production and `dev` when in development - -# This is an optional key -[auth] -# the origin key to be used to claim the root account -origin_key = "4527387f92a381cbe804593f33991d327d456a97" - -# This key is *OPTIONAL* -[bgsave] -# Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run -# after every 2 minutes -enabled = true -every = 120 - -# This key is *OPTIONAL* -[snapshot] -every = 3600 # Make a snapshot after every 1 hour (60min * 60sec= 3600secs) -atmost = 4 # Keep the 4 most recent snapshots -failsafe = true # stops accepting writes if snapshotting fails - -# This key is *OPTIONAL*, used for TLS/SSL config -[ssl] -key = "/path/to/keyfile.pem" -chain = "/path/to/chain.pem" -port = 2004 -only = true # optional to enable SSL-only requests -passin = "/path/to/cert/passphrase.txt" # optional to programmatically verify the TLS cert diff --git a/examples/config-files/template.yaml b/examples/config-files/template.yaml new file mode 100644 index 00000000..1e7f286c --- /dev/null +++ b/examples/config-files/template.yaml @@ -0,0 +1,22 @@ +system: + mode: prod + rs_window: 600 + +auth: + plugin: pwd + # replace with your root password of choice + root_pass: password + +endpoints: + secure: + host: 127.0.0.1 + port: 2004 + # replace `cert` with the path to your self-signed certificate + cert: cert.pem + # replace `private_key` with the path to your private key + private_key: private.key + # replace `passphrase.txt` with the path to your private key passphrase + pkey_passphrase: passphrase.txt + insecure: + host: 127.0.0.1 + port: 2003 diff --git a/examples/config-files/withcustombgsave.toml b/examples/config-files/withcustombgsave.toml deleted file mode 100644 index 34ee730b..00000000 --- a/examples/config-files/withcustombgsave.toml +++ /dev/null @@ -1,7 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 - -[bgsave] -enabled = true -every = 600 # Every 10 minutes diff --git a/harness/Cargo.toml b/harness/Cargo.toml index 5329474d..247cb9a4 100644 --- a/harness/Cargo.toml +++ b/harness/Cargo.toml @@ -7,13 +7,10 @@ edition = "2021" [dependencies] # internal deps -skytable = { git = "https://github.com/skytable/client-rust.git", features = [ - "sync", -], default-features = false } libsky = { path = "../libsky" } # external deps -env_logger = "0.10.0" -log = "0.4.17" -zip = { version = "0.6.3", features = ["deflate"] } -powershell_script = "1.0.4" -openssl = { version = "0.10.45", features = ["vendored"] } +env_logger = "0.10.1" +log = "0.4.20" +zip = { version = "0.6.6", features = ["deflate"] } +powershell_script = "1.1.0" +openssl = { version = "0.10.61", features = ["vendored"] } diff --git a/harness/src/build.rs b/harness/src/build.rs index 48ce8780..f640d915 100644 --- a/harness/src/build.rs +++ b/harness/src/build.rs @@ -34,7 +34,7 @@ use { }; /// The binaries that will be present in a bundle -pub const BINARIES: [&str; 4] = ["skyd", "sky-bench", "skysh", "sky-migrate"]; +pub const BINARIES: [&str; 3] = ["skyd", "sky-bench", "skysh"]; /// The build mode #[derive(Copy, Clone, PartialEq, Eq)] diff --git a/harness/src/linuxpkg.rs b/harness/src/linuxpkg.rs index 755afb41..1f8dbd44 100644 --- a/harness/src/linuxpkg.rs +++ b/harness/src/linuxpkg.rs @@ -76,6 +76,18 @@ pub fn create_linuxpkg(package_type: LinuxPackageType) -> HarnessResult<()> { LinuxPackageType::Deb => { // install cargo-deb util::handle_child("install cargo-deb", cmd!("cargo", "install", "cargo-deb"))?; + // make files executable + util::handle_child( + "make maintainer scripts executable", + cmd!( + "chmod", + "+x", + "pkg/debian/postinst", + "pkg/debian/preinst", + "pkg/debian/postrm", + "pkg/debian/prerm" + ), + )?; // assemble the command let mut build_args = vec!["cargo".into(), "deb".to_owned()]; if let Some(t) = util::get_var(util::VAR_TARGET) { diff --git a/harness/src/main.rs b/harness/src/main.rs index 70011fce..77e1087e 100644 --- a/harness/src/main.rs +++ b/harness/src/main.rs @@ -52,10 +52,13 @@ fn main() { Builder::new() .parse_filters(&env::var("SKYHARNESS_LOG").unwrap_or_else(|_| "info".to_owned())) .init(); - // avoid verbose logging - env::set_var("SKY_LOG", "error"); + env::set_var("SKY_LOG", "trace"); if let Err(e) = runner() { error!("harness failed with: {}", e); + error!("fetching logs from server processes"); + for ret in test::get_children() { + ret.print_logs(); + } process::exit(0x01); } } diff --git a/harness/src/test/mod.rs b/harness/src/test/mod.rs index ae2afda1..de97c421 100644 --- a/harness/src/test/mod.rs +++ b/harness/src/test/mod.rs @@ -35,8 +35,9 @@ use { bn::{BigNum, MsbOption}, error::ErrorStack, hash::MessageDigest, - pkey::{PKey, Private}, + pkey::PKey, rsa::Rsa, + symm::Cipher, x509::{ extension::{BasicConstraints, KeyUsage, SubjectKeyIdentifier}, X509NameBuilder, X509, @@ -45,6 +46,7 @@ use { std::{fs, io::Write}, }; mod svc; +pub use svc::get_children; /// Run the test suite pub fn run_test() -> HarnessResult<()> { @@ -89,16 +91,16 @@ fn append_target(args: &mut Vec) { /// - The standard test suite /// - The persistence test suite fn run_test_inner() -> HarnessResult<()> { + const TEST_PASSWORD: &str = "xCqe4yuVM7l2MnHZOFZDDieqjqmmL3qvO5LOEOhpXPE="; // first create the TLS keys info!("Creating TLS key+cert"); - let (cert, pkey) = mk_ca_cert().expect("Failed to create cert"); + let (cert, pkey) = mk_ca_cert(TEST_PASSWORD.as_bytes()).expect("Failed to create cert"); + let mut passfile = fs::File::create("passphrase.txt").unwrap(); + passfile.write_all(TEST_PASSWORD.as_bytes()).unwrap(); let mut certfile = fs::File::create("cert.pem").expect("failed to create cert.pem"); - certfile.write_all(&cert.to_pem().unwrap()).unwrap(); + certfile.write_all(&cert).unwrap(); let mut pkeyfile = fs::File::create("key.pem").expect("failed to create key.pem"); - pkeyfile - .write_all(&pkey.private_key_to_pem_pkcs8().unwrap()) - .unwrap(); - + pkeyfile.write_all(&pkey).unwrap(); // assemble commands let target_folder = util::get_target_folder(BuildMode::Debug); let mut standard_test_suite_args = vec!["cargo".to_owned(), "test".into()]; @@ -110,15 +112,9 @@ fn run_test_inner() -> HarnessResult<()> { ]; append_target(&mut build_cmd_args); append_target(&mut standard_test_suite_args); - let persist_test_suite_args = [ - standard_test_suite_args.as_slice(), - &["--features".to_owned(), "persist-suite".into()], - ] - .concat(); // get cmd let build_cmd = util::assemble_command_from_slice(build_cmd_args); let standard_test_suite = util::assemble_command_from_slice(standard_test_suite_args); - let persist_test_suite = util::assemble_command_from_slice(persist_test_suite_args); // build skyd info!("Building server binary ..."); @@ -130,20 +126,11 @@ fn run_test_inner() -> HarnessResult<()> { util::handle_child("standard test suite", standard_test_suite)?; Ok(()) })?; - - // run persistence tests; don't kill the servers because if either of the tests fail - // then we can ensure that they will be killed by `run_test` - svc::run_with_servers(&target_folder, false, move || { - info!("Running persistence test suite ..."); - util::handle_child("standard test suite", persist_test_suite)?; - Ok(()) - })?; - Ok(()) } /// Generate certificates -fn mk_ca_cert() -> Result<(X509, PKey), ErrorStack> { +fn mk_ca_cert(password: &[u8]) -> Result<(Vec, Vec), ErrorStack> { let rsa = Rsa::generate(2048)?; let key_pair = PKey::from_rsa(rsa)?; @@ -182,9 +169,8 @@ fn mk_ca_cert() -> Result<(X509, PKey), ErrorStack> { let subject_key_identifier = SubjectKeyIdentifier::new().build(&cert_builder.x509v3_context(None, None))?; cert_builder.append_extension(subject_key_identifier)?; - cert_builder.sign(&key_pair, MessageDigest::sha256())?; - let cert = cert_builder.build(); - + let cert = cert_builder.build().to_pem().unwrap(); + let key_pair = key_pair.private_key_to_pem_pkcs8_passphrase(Cipher::aes_256_cbc(), password)?; Ok((cert, key_pair)) } diff --git a/harness/src/test/svc.rs b/harness/src/test/svc.rs index 0c583969..55d5f57b 100644 --- a/harness/src/test/svc.rs +++ b/harness/src/test/svc.rs @@ -26,35 +26,97 @@ #[cfg(windows)] use std::os::windows::process::CommandExt; + use { crate::{ util::{self}, HarnessError, HarnessResult, ROOT_DIR, }, - skytable::{error::Error, Connection, SkyResult}, std::{ + cell::RefCell, io::ErrorKind, path::Path, - process::{Child, Command}, + process::{Child, Command, Output, Stdio}, }, }; +thread_local! { + static CHILDREN: RefCell> = RefCell::default(); +} + +pub struct ChildStatus { + id: &'static str, + stdout: String, + stderr: String, + exit_code: i32, +} + +impl ChildStatus { + pub fn new(id: &'static str, stdout: String, stderr: String, exit_code: i32) -> Self { + Self { + id, + stdout, + stderr, + exit_code, + } + } + pub fn print_logs(&self) { + println!( + "######################### LOGS FROM {} #########################", + self.id + ); + println!("-> exit code: `{}`", self.exit_code); + if !self.stdout.is_empty() { + println!("+++++++++++++++++++++ STDOUT +++++++++++++++++++++"); + println!("{}", self.stdout); + println!("++++++++++++++++++++++++++++++++++++++++++++++++++"); + } + if !self.stderr.is_empty() { + println!("+++++++++++++++++++++ STDERR +++++++++++++++++++++"); + println!("{}", self.stderr); + println!("++++++++++++++++++++++++++++++++++++++++++++++++++"); + } + println!("######################### ############ #########################"); + } +} + +pub fn get_children() -> Vec { + CHILDREN.with(|c| { + let mut ret = vec![]; + for (name, child) in c.borrow_mut().drain(..) { + let Output { + status, + stdout, + stderr, + } = child.wait_with_output().unwrap(); + ret.push(ChildStatus::new( + name, + String::from_utf8(stdout).unwrap(), + String::from_utf8(stderr).unwrap(), + status.code().unwrap(), + )) + } + ret + }) +} + #[cfg(windows)] /// The powershell script hack to send CTRL+C using kernel32 const POWERSHELL_SCRIPT: &str = include_str!("../../../ci/windows/stop.ps1"); #[cfg(windows)] /// Flag for new console Window const CREATE_NEW_CONSOLE: u32 = 0x00000010; -pub(super) const SERVERS: [(&str, [u16; 2]); 3] = [ - ("server1", [2003, 2004]), - ("server2", [2005, 2006]), - ("server3", [2007, 2008]), -]; +pub(super) const SERVERS: [(&str, [u16; 2]); 1] = [("server1", [2003, 2004])]; /// The test suite server host const TESTSUITE_SERVER_HOST: &str = "127.0.0.1"; /// The workspace root const WORKSPACE_ROOT: &str = env!("ROOT_DIR"); +fn connect_db(host: &str, port: u16) -> std::io::Result { + let tcp_stream = std::net::TcpStream::connect((host, port))?; + Ok(tcp_stream) +} + /// Get the command to start the provided server1 pub fn get_run_server_cmd(server_id: &'static str, target_folder: impl AsRef) -> Command { let args = vec![ @@ -63,11 +125,13 @@ pub fn get_run_server_cmd(server_id: &'static str, target_folder: impl AsRef HarnessResult<()> { Ok(()) } -fn connection_refused(input: SkyResult) -> HarnessResult { +fn connection_refused(input: std::io::Result) -> HarnessResult { match input { Ok(_) => Ok(false), - Err(Error::IoError(e)) + Err(e) if matches!( e.kind(), ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset @@ -118,7 +182,7 @@ fn wait_for_startup() -> HarnessResult<()> { for port in ports { let connection_string = format!("{TESTSUITE_SERVER_HOST}:{port}"); let mut backoff = 1; - let mut con = Connection::new(TESTSUITE_SERVER_HOST, port); + let mut con = connect_db(TESTSUITE_SERVER_HOST, port); while connection_refused(con)? { if backoff > 64 { // enough sleeping, return an error @@ -131,7 +195,7 @@ fn wait_for_startup() -> HarnessResult<()> { "Server at {connection_string} not started. Sleeping for {backoff} second(s) ..." ); util::sleep_sec(backoff); - con = Connection::new(TESTSUITE_SERVER_HOST, port); + con = connect_db(TESTSUITE_SERVER_HOST, port); backoff *= 2; } info!("Server at {connection_string} has started"); @@ -148,7 +212,7 @@ fn wait_for_shutdown() -> HarnessResult<()> { for port in ports { let connection_string = format!("{TESTSUITE_SERVER_HOST}:{port}"); let mut backoff = 1; - let mut con = Connection::new(TESTSUITE_SERVER_HOST, port); + let mut con = connect_db(TESTSUITE_SERVER_HOST, port); while !connection_refused(con)? { if backoff > 64 { // enough sleeping, return an error @@ -161,7 +225,7 @@ fn wait_for_shutdown() -> HarnessResult<()> { "Server at {connection_string} still active. Sleeping for {backoff} second(s) ..." ); util::sleep_sec(backoff); - con = Connection::new(TESTSUITE_SERVER_HOST, port); + con = connect_db(TESTSUITE_SERVER_HOST, port); backoff *= 2; } info!("Server at {connection_string} has stopped accepting connections"); @@ -173,15 +237,15 @@ fn wait_for_shutdown() -> HarnessResult<()> { } /// Start the servers returning handles to the child processes -fn start_servers(target_folder: impl AsRef) -> HarnessResult> { - let mut ret = Vec::with_capacity(SERVERS.len()); +fn start_servers(target_folder: impl AsRef) -> HarnessResult<()> { for (server_id, _ports) in SERVERS { let cmd = get_run_server_cmd(server_id, target_folder.as_ref()); info!("Starting {server_id} ..."); - ret.push(util::get_child(format!("start {server_id}"), cmd)?); + let child = util::get_child(format!("start {server_id}"), cmd)?; + CHILDREN.with(|c| c.borrow_mut().push((server_id, child))); } wait_for_startup()?; - Ok(ret) + Ok(()) } pub(super) fn run_with_servers( @@ -190,14 +254,12 @@ pub(super) fn run_with_servers( run_what: impl FnOnce() -> HarnessResult<()>, ) -> HarnessResult<()> { info!("Starting servers ..."); - let children = start_servers(target_folder.as_ref())?; + start_servers(target_folder.as_ref())?; run_what()?; if kill_servers_when_done { kill_servers()?; wait_for_shutdown()?; } - // just use this to avoid ignoring the children vector - assert_eq!(children.len(), SERVERS.len()); Ok(()) } diff --git a/libsky/src/lib.rs b/libsky/src/lib.rs index 15ee9247..0532bc3a 100644 --- a/libsky/src/lib.rs +++ b/libsky/src/lib.rs @@ -31,25 +31,142 @@ //! //! This contains modules which are shared by both the `cli` and the `server` modules -use std::error::Error; -/// A generic result -pub type TResult = Result>; -/// The size of the read buffer in bytes -pub const BUF_CAP: usize = 8 * 1024; // 8 KB per-connection /// The current version pub const VERSION: &str = env!("CARGO_PKG_VERSION"); /// The URL pub const URL: &str = "https://github.com/skytable/skytable"; -#[macro_export] -/// Don't use unwrap_or but use this macro as the optimizer fails to optimize away usages -/// of unwrap_or and creates a lot of LLVM IR bloat. use -// FIXME(@ohsayan): Fix this when https://github.com/rust-lang/rust/issues/68667 is addressed -macro_rules! option_unwrap_or { - ($try:expr, $fallback:expr) => { - match $try { - Some(t) => t, - None => $fallback, +pub mod test_utils { + pub const DEFAULT_USER_NAME: &str = "root"; + pub const DEFAULT_USER_PASS: &str = "mypassword12345678"; + pub const DEFAULT_HOST: &str = "127.0.0.1"; + pub const DEFAULT_PORT: u16 = 2003; +} + +use std::{ + collections::{hash_map::Entry, HashMap}, + env, +}; + +/// Returns a formatted version message `{binary} vx.y.z` +pub fn version_msg(binary: &str) -> String { + format!("{binary} v{VERSION}") +} + +#[derive(Debug, PartialEq)] +/// The CLI action that is expected to be performed +pub enum CliAction { + /// Display the `--help` message + Help, + /// Dipslay the `--version` + Version, + /// Perform an action using the given args + Action(A), +} + +pub type CliActionMulti = CliAction>>; +pub type CliActionSingle = CliAction>; + +/* + generic cli arg parser +*/ + +#[derive(Debug, PartialEq)] +/// Argument parse error +pub enum AnyArgsParseError { + /// The value for the given argument was either incorrectly formatted or missing + MissingValue(String), +} +/// Parse CLI args, allowing duplicates (bucketing them) +pub fn parse_cli_args_allow_duplicate() -> Result { + parse_args(env::args()) +} +/// Parse args allowing and bucketing any duplicates +pub fn parse_args( + args: impl IntoIterator, +) -> Result { + let mut ret: HashMap> = HashMap::new(); + let mut args = args.into_iter().skip(1).peekable(); + while let Some(arg) = args.next() { + if arg == "--help" { + return Ok(CliAction::Help); + } + if arg == "--version" { + return Ok(CliAction::Version); + } + let (arg, value) = extract_arg(arg, &mut args).map_err(AnyArgsParseError::MissingValue)?; + match ret.get_mut(&arg) { + Some(values) => { + values.push(value); + } + None => { + ret.insert(arg, vec![value]); + } + } + } + Ok(CliAction::Action(ret)) +} + +/* + no duplicate arg parser +*/ + +#[derive(Debug, PartialEq)] +/// Argument parse error +pub enum ArgParseError { + /// The given argument had a duplicate value + Duplicate(String), + /// The given argument did not have an appropriate value + MissingValue(String), +} +/// Parse all non-repeating CLI arguments +pub fn parse_cli_args_disallow_duplicate() -> Result { + parse_args_deny_duplicate(env::args()) +} +/// Parse all arguments but deny any duplicates +pub fn parse_args_deny_duplicate( + args: impl IntoIterator, +) -> Result { + let mut ret: HashMap = HashMap::new(); + let mut args = args.into_iter().skip(1).peekable(); + while let Some(arg) = args.next() { + if arg == "--help" { + return Ok(CliAction::Help); + } + if arg == "--version" { + return Ok(CliAction::Version); + } + let (arg, value) = extract_arg(arg, &mut args).map_err(ArgParseError::MissingValue)?; + match ret.entry(arg) { + Entry::Vacant(v) => { + v.insert(value); + } + Entry::Occupied(oe) => return Err(ArgParseError::Duplicate(oe.key().into())), + } + } + Ok(CliAction::Action(ret)) +} + +/// Extract an argument: +/// - `--arg=value` +/// - `--arg value` +fn extract_arg( + arg: String, + args: &mut impl Iterator, +) -> Result<(String, String), String> { + let this_args: Vec<&str> = arg.split("=").collect(); + let (arg, value) = if this_args.len() == 2 { + // self contained arg + (this_args[0].to_owned(), this_args[1].to_owned()) + } else { + if this_args.len() == 1 { + match args.next() { + None => return Err(arg), + Some(val) => (arg, val), + } + } else { + return Err(arg); } }; + Ok((arg, value)) } diff --git a/libstress/Cargo.toml b/libstress/Cargo.toml deleted file mode 100644 index 1caccf17..00000000 --- a/libstress/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "libstress" -version = "0.8.0" -authors = ["Sayan Nandan "] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -# external deps -crossbeam-channel = "0.5.6" -rayon = "1.6.1" -log = "0.4.17" -rand = "0.8.5" diff --git a/libstress/src/lib.rs b/libstress/src/lib.rs deleted file mode 100644 index 4a922773..00000000 --- a/libstress/src/lib.rs +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Created on Wed Jun 16 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # libstress -//! -//! Tools for emulating concurrent query behavior to _stress test_ the database server. -//! As of now, this crate provides a [`Workpool`] which is a generic synchronous threadpool -//! for doing multiple operations. But Workpool is a little different from standard threadpool -//! implementations in it categorizing a job to be made up of three parts, namely: -//! -//! - The init_pre_loop_var (the pre-loop stage) -//! - The on_loop (the in-loop stage) -//! - The on_exit (the post-loop stage) -//! -//! These stages form a part of the event loop. -//! -//! ## The event loop -//! -//! A task runs in a loop with the `on_loop` routine to which the a reference of the result of -//! the `init_pre_loop_var` is sent that is initialized. The loop proceeds whenever a worker -//! receives a task or else it blocks the current thread, waiting for a task. Hence the loop -//! cannot be terminated by an execute call. Instead, the _event loop_ is terminated when the -//! Workpool is dropped, either by scoping out, or by using the provided finish-like methods -//! (that call the destructor). -//! -//! ## Worker lifetime -//! -//! If a runtime panic occurs in the pre-loop stage, then the entire worker just terminates. Hence -//! this worker is no longer able to perform any tasks. Similarly, if a runtime panic occurs in -//! the in-loop stage, the worker terminates and is no longer available to do any work. This will -//! be reflected when the workpool attempts to terminate in entirety, i.e when the threads are joined -//! to the parent thread -//! - -#![deny(unused_crate_dependencies)] -#![deny(unused_imports)] - -pub mod traits; -pub use rayon; - -use { - core::marker::PhantomData, - crossbeam_channel::{bounded, unbounded, Receiver as CReceiver, Sender as CSender}, - rayon::prelude::{IntoParallelIterator, ParallelIterator}, - std::{fmt::Display, thread}, -}; - -#[derive(Debug)] -pub enum WorkpoolError { - ThreadStartFailure(usize, usize), -} - -impl Display for WorkpoolError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - WorkpoolError::ThreadStartFailure(expected, started) => { - write!( - f, - "couldn't start all threads. expected {expected} but started {started}" - ) - } - } - } -} - -pub type WorkpoolResult = Result; - -/// A Job. The UIn type parameter is the type that will be used to execute the action -/// Nothing is a variant used by the drop implementation to terminate all the workers -/// and call the exit_loop function -enum JobType { - Task(UIn), - Nothing, -} - -/// A worker -/// -/// The only reason we use option is to reduce the effort needed to implement [`Drop`] for the -/// [`Workpool`] -struct Worker { - thread: Option>, -} - -impl Worker { - /// Initialize a new worker - fn new( - id: usize, - job_receiver: CReceiver>, - init_pre_loop_var: Lv, - on_exit: Ex, - on_loop: Lp, - wgtx: CSender<()>, - ) -> Self - where - UIn: Send + Sync + 'static, - Lv: Fn() -> Inp + 'static + Send, - Lp: Fn(&mut Inp, UIn) + Send + Sync + 'static, - Ex: Fn(&mut Inp) + Send + 'static, - { - let thread = thread::Builder::new() - .name(format!("worker-{id}")) - .spawn(move || { - let on_loop = on_loop; - let mut pre_loop_var = init_pre_loop_var(); - wgtx.send(()).unwrap(); - drop(wgtx); - loop { - let action = job_receiver.recv().unwrap(); - match action { - JobType::Task(tsk) => on_loop(&mut pre_loop_var, tsk), - JobType::Nothing => { - on_exit(&mut pre_loop_var); - break; - } - } - } - }) - .unwrap(); - Self { - thread: Some(thread), - } - } -} - -/// A pool configuration setting to easily generate [`Workpool`]s without -/// having to clone an entire pool and its threads upfront -pub struct PoolConfig { - /// the pool size - count: usize, - /// the function that sets the pre-loop variable - init_pre_loop_var: Lv, - /// the function to be executed on worker termination - on_exit: Ex, - /// the function to be executed on loop - on_loop: Lp, - /// a marker for `Inp` since no parameters use it directly - _marker: PhantomData<(Inp, UIn)>, - /// check if self needs a pool for parallel iterators - needs_iterator_pool: bool, - /// expected maximum number of jobs - expected_max_sends: Option, -} - -impl PoolConfig -where - UIn: Send + Sync + 'static, - Inp: Sync, - Ex: Fn(&mut Inp) + Send + Sync + 'static + Clone, - Lv: Fn() -> Inp + Send + Sync + 'static + Clone, - Lp: Fn(&mut Inp, UIn) + Clone + Send + Sync + 'static, -{ - /// Create a new pool config - pub fn new( - count: usize, - init_pre_loop_var: Lv, - on_loop: Lp, - on_exit: Ex, - needs_iterator_pool: bool, - expected_max_sends: Option, - ) -> Self { - Self { - count, - init_pre_loop_var, - on_loop, - on_exit, - needs_iterator_pool, - _marker: PhantomData, - expected_max_sends, - } - } - /// Get a new [`Workpool`] from the current config - pub fn get_pool(&self) -> WorkpoolResult> { - self.get_pool_with_workers(self.count) - } - /// Get a [`Workpool`] with the base config but with a different number of workers - pub fn get_pool_with_workers( - &self, - count: usize, - ) -> WorkpoolResult> { - Workpool::new( - count, - self.init_pre_loop_var.clone(), - self.on_loop.clone(), - self.on_exit.clone(), - self.needs_iterator_pool, - self.expected_max_sends, - ) - } - /// Get a [`Workpool`] with the base config but with a custom loop-stage closure - pub fn with_loop_closure(&self, lp: Dlp) -> WorkpoolResult> - where - Dlp: Fn(&mut Inp, UIn) + Clone + Send + Sync + 'static, - { - Workpool::new( - self.count, - self.init_pre_loop_var.clone(), - lp, - self.on_exit.clone(), - self.needs_iterator_pool, - self.expected_max_sends, - ) - } -} - -/// # Workpool -/// -/// A Workpool is a generic synchronous thread pool that can be used to perform, well, anything. -/// A workpool has to be initialized with the number of workers, the pre_loop_variable (set this -/// to None if there isn't any). what to do on loop and what to do on exit of each worker. The -/// closures are kept as `Clone`able types just to reduce complexity with copy (we were lazy). -/// -/// ## Clones -/// -/// Workpool clones simply create a new workpool with the same on_exit, on_loop and init_pre_loop_var -/// configurations. This provides a very convenient interface if one desires to use multiple workpools -/// to do the _same kind of thing_ -/// -/// ## Actual thread count -/// -/// The actual thread count will depend on whether the caller requests the initialization of an -/// iterator pool or not. If the caller does request for an iterator pool, then the number of threads -/// spawned will be twice the number of the set workers. Else, the number of spawned threads is equal -/// to the number of workers. -pub struct Workpool { - /// the workers - workers: Vec, - /// the sender that sends jobs - job_distributor: CSender>, - /// the function that sets the pre-loop variable - init_pre_loop_var: Lv, - /// the function to be executed on worker termination - on_exit: Ex, - /// the function to be executed on loop - on_loop: Lp, - /// a marker for `Inp` since no parameters use it directly - _marker: PhantomData, - /// check if self needs a pool for parallel iterators - needs_iterator_pool: bool, - /// expected maximum number of sends - expected_max_sends: Option, -} - -impl Workpool -where - UIn: Send + Sync + 'static, - Ex: Fn(&mut Inp) + Send + Sync + 'static + Clone, - Lv: Fn() -> Inp + Send + Sync + 'static + Clone, - Lp: Fn(&mut Inp, UIn) + Send + Sync + 'static + Clone, - Inp: Sync, -{ - /// Create a new workpool - pub fn new( - count: usize, - init_pre_loop_var: Lv, - on_loop: Lp, - on_exit: Ex, - needs_iterator_pool: bool, - expected_max_sends: Option, - ) -> WorkpoolResult { - // init threadpool for iterator - if needs_iterator_pool { - // initialize a global threadpool for parallel iterators - let _ = rayon::ThreadPoolBuilder::new() - .num_threads(count) - .build_global(); - } - assert!(count != 0, "Runtime panic: Bad value `0` for thread count"); - let (sender, receiver) = match expected_max_sends { - Some(limit) => bounded(limit), - None => unbounded(), - }; - let (wgtx, wgrx) = bounded::<()>(count); - let mut workers = Vec::with_capacity(count); - for i in 0..count { - workers.push(Worker::new( - i, - receiver.clone(), - init_pre_loop_var.clone(), - on_exit.clone(), - on_loop.clone(), - wgtx.clone(), - )); - } - drop(wgtx); - let sum: usize = wgrx.iter().map(|_| 1usize).sum(); - if sum == count { - Ok(Self { - workers, - job_distributor: sender, - init_pre_loop_var, - on_exit, - on_loop, - _marker: PhantomData, - needs_iterator_pool, - expected_max_sends, - }) - } else { - Err(WorkpoolError::ThreadStartFailure(count, sum)) - } - } - pub fn clone_pool(&self) -> WorkpoolResult { - Self::new( - self.workers.len(), - self.init_pre_loop_var.clone(), - self.on_loop.clone(), - self.on_exit.clone(), - self.needs_iterator_pool, - self.expected_max_sends, - ) - } - /// Execute something - pub fn execute(&self, inp: UIn) { - self.job_distributor - .send(JobType::Task(inp)) - .expect("Worker thread crashed") - } - /// Execute something that can be executed as a parallel iterator - /// For the best performance, it is recommended that you pass true for `needs_iterator_pool` - /// on initialization of the [`Workpool`] - pub fn execute_iter(&self, iter: impl IntoParallelIterator) { - iter.into_par_iter().for_each(|inp| self.execute(inp)) - } - /// Does the same thing as [`execute_iter`] but drops self ensuring that all the - /// workers actually finish their tasks - pub fn execute_and_finish_iter(self, iter: impl IntoParallelIterator) { - self.execute_iter(iter); - drop(self); - } - /// Initialize a new [`Workpool`] with the default count of threads. This is equal - /// to 2 * the number of logical cores. - pub fn new_default_threads( - init_pre_loop_var: Lv, - on_loop: Lp, - on_exit: Ex, - needs_iterator_pool: bool, - expected_max_sends: Option, - ) -> WorkpoolResult { - // we'll naively use the number of CPUs present on the system times 2 to determine - // the number of workers (sure the scheduler does tricks all the time) - let worker_count = thread::available_parallelism().map_or(1, usize::from) * 2; - Self::new( - worker_count, - init_pre_loop_var, - on_loop, - on_exit, - needs_iterator_pool, - expected_max_sends, - ) - } -} - -impl Drop for Workpool { - fn drop(&mut self) { - for _ in &self.workers { - self.job_distributor.send(JobType::Nothing).unwrap(); - } - for worker in &mut self.workers { - if let Some(thread) = worker.thread.take() { - thread.join().unwrap() - } - } - } -} - -pub mod utils { - const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - use rand::distributions::{Alphanumeric, Standard}; - use std::collections::HashSet; - use std::collections::TryReserveError; - - /// Generate a random UTF-8 string - pub fn ran_string(len: usize, rand: impl rand::Rng) -> String { - let rand_string: String = rand - .sample_iter(&Alphanumeric) - .take(len) - .map(char::from) - .collect(); - rand_string - } - /// Generate a vector of random bytes - pub fn ran_bytes(len: usize, rand: impl rand::Rng) -> Vec { - rand.sample_iter(&Standard).take(len).collect() - } - /// Generate multiple vectors of random bytes - pub fn generate_random_byte_vector( - count: usize, - size: usize, - mut rng: impl rand::Rng, - unique: bool, - ) -> Result>, TryReserveError> { - if unique { - let mut keys = HashSet::new(); - keys.try_reserve(size)?; - (0..count).into_iter().for_each(|_| { - let mut ran = ran_bytes(size, &mut rng); - while keys.contains(&ran) { - ran = ran_bytes(size, &mut rng); - } - keys.insert(ran); - }); - Ok(keys.into_iter().collect()) - } else { - let mut keys = Vec::new(); - keys.try_reserve_exact(size)?; - let ran_byte_key = ran_bytes(size, &mut rng); - (0..count).for_each(|_| keys.push(ran_byte_key.clone())); - Ok(keys) - } - } - /// Generate a vector of random UTF-8 valid strings - pub fn generate_random_string_vector( - count: usize, - size: usize, - mut rng: impl rand::Rng, - unique: bool, - ) -> Result, TryReserveError> { - if unique { - let mut keys = HashSet::new(); - keys.try_reserve(size)?; - (0..count).into_iter().for_each(|_| { - let mut ran = ran_string(size, &mut rng); - while keys.contains(&ran) { - ran = ran_string(size, &mut rng); - } - keys.insert(ran); - }); - Ok(keys.into_iter().collect()) - } else { - let mut keys = Vec::new(); - keys.try_reserve_exact(size)?; - (0..count) - .into_iter() - .map(|_| ran_string(size, &mut rng)) - .for_each(|bytes| keys.push(bytes)); - Ok(keys) - } - } - pub fn rand_alphastring(len: usize, rng: &mut impl rand::Rng) -> String { - (0..len) - .map(|_| { - let idx = rng.gen_range(0..CHARSET.len()); - CHARSET[idx] as char - }) - .collect() - } -} diff --git a/libstress/src/traits.rs b/libstress/src/traits.rs deleted file mode 100644 index 12d7c343..00000000 --- a/libstress/src/traits.rs +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Created on Fri Jun 18 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use std::fmt; - -/// A trait for aggresive erroring -pub trait ExitError { - /// Abort the process if the type errors with an error code or - /// return the type - fn exit_error(self, msg: Ms) -> T - where - Ms: ToString; -} - -impl ExitError for Result -where - E: fmt::Display, -{ - fn exit_error(self, msg: Ms) -> T - where - Ms: ToString, - { - match self { - Self::Err(e) => { - log::error!("{} : '{}'", msg.to_string(), e); - std::process::exit(0x01); - } - Self::Ok(v) => v, - } - } -} - -impl ExitError for Option { - fn exit_error(self, msg: Ms) -> T - where - Ms: ToString, - { - match self { - Self::None => { - log::error!("{}", msg.to_string()); - std::process::exit(0x01); - } - Self::Some(v) => v, - } - } -} diff --git a/pkg/common/skyd.service b/pkg/common/skyd.service index 3990c09f..8182822d 100644 --- a/pkg/common/skyd.service +++ b/pkg/common/skyd.service @@ -8,7 +8,7 @@ Type=simple Restart=always RestartSec=1 User=skytable -ExecStart=/usr/bin/skyd --noart +ExecStart=/usr/bin/skyd --config=/var/lib/skytable/config.yaml WorkingDirectory=/var/lib/skytable [Install] diff --git a/pkg/debian/description.txt b/pkg/debian/description.txt index 15a8dca7..ac8d681f 100644 --- a/pkg/debian/description.txt +++ b/pkg/debian/description.txt @@ -1,5 +1,5 @@ Skytable is a free and open-source NoSQL database that aims to provide flexibility in data modeling at scale. The `skytable` package contains the database server (`skyd`), -an interactive command-line client (`skysh`), a benchmarking -tool (`sky-bench`) and a migration tool (`sky-migrate`). \ No newline at end of file +an interactive command-line client (`skysh`) and a benchmarking +tool (`sky-bench`). \ No newline at end of file diff --git a/pkg/debian/postinst b/pkg/debian/postinst old mode 100644 new mode 100755 index 15592134..1bf534ea --- a/pkg/debian/postinst +++ b/pkg/debian/postinst @@ -1,15 +1,35 @@ #!/bin/sh -e +SKY_DIR=/var/lib/skytable systemctl daemon-reload -if [ $1 = "install" ]; then +echo "Doing '$1'" +if [ "$1" = "configure" ]; then + # Enable and start skyd on fresh install systemctl enable skyd -fi + systemctl start skyd + echo "Generating password and configuration" -if [ $1 = "upgrade" ]; then + if [ -f /var/lib/skytable/config.yaml ]; then + echo "Configuration already exists. Not updating configuration." + else + mv /var/lib/skytable/config.yaml.tmp /var/lib/skytable/config.yaml + # Generate and set password + if [ ! -f "$SKY_DIR/config.yaml" ]; then + echo "Error: The file $SKY_DIR/config.yaml does not exist." + exit 1 # Exit with an error code + fi + PASSWORD=$(tr -dc A-Za-z0-9 /dev/null; then + deluser --system skytable + fi + if getent group skytable > /dev/null; then + delgroup skytable + fi + echo "Removing the configuration file ..." + rm /var/lib/skytable/config.yaml + echo "Cleanup complete." + ;; + *) + # No action required for other cases (upgrade, failed-upgrade, etc.) + ;; +esac + +#DEBHELPER# diff --git a/pkg/debian/preinst b/pkg/debian/preinst index 158b1666..106070c2 100755 --- a/pkg/debian/preinst +++ b/pkg/debian/preinst @@ -2,19 +2,22 @@ SKY_DIR=/var/lib/skytable -# create the data directory -if [ ! -e $SKY_DIR ]; then - mkdir $SKY_DIR -elif [ ! -d $SKY_DIR ]; then - echo "ERROR: /var/lib/skytable exists but it is not a directory" 1>&2 - return 1 +# Create the data directory if it doesn't exist +if [ ! -e "$SKY_DIR" ]; then + mkdir -p "$SKY_DIR" + echo "Created directory $SKY_DIR" +elif [ ! -d "$SKY_DIR" ]; then + echo "ERROR: $SKY_DIR exists but it is not a directory" 1>&2 + exit 1 fi -if [ $1 = "install" ]; then - # add the `skytable` user - adduser --system --group skytable - # change ownership - chown skytable:skytable /var/lib/skytable +# On initial install, add the `skytable` user +if [ "$1" = "install" ]; then + echo "Creating user 'skytable'" + if ! getent passwd skytable > /dev/null; then + adduser --system --group --no-create-home skytable + fi + chown -R skytable:skytable "$SKY_DIR" + echo "Created user 'skytable'" fi - -#DEBHELPER# \ No newline at end of file +#DEBHELPER# diff --git a/pkg/debian/prerm b/pkg/debian/prerm new file mode 100755 index 00000000..8fe8f64d --- /dev/null +++ b/pkg/debian/prerm @@ -0,0 +1,6 @@ +#!/bin/sh -e + +echo "Stopping processes" +systemctl stop skyd +systemctl disable skyd +echo "Stopped processes" diff --git a/pkg/docker/start-server.sh b/pkg/docker/start-server.sh new file mode 100644 index 00000000..6f16923f --- /dev/null +++ b/pkg/docker/start-server.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +CONFIG_FILE="/var/lib/skytable/config.yaml" +PASSWORD_MARKER="rootpass" +IP_MARKER="127.0.0.1" + +generate_password() { + uuidgen | cut -c -16 +} + +sed -i "s/$IP_MARKER/0.0.0.0/g" "$CONFIG_FILE" + +if grep -q "$PASSWORD_MARKER" "$CONFIG_FILE"; then + # Password not set, generate a new one + PASSWORD=$(generate_password) + sed -i "s/$PASSWORD_MARKER/$PASSWORD/g" "$CONFIG_FILE" + echo "Generated Password: $PASSWORD" +else + echo "Using existing password in config file" +fi + +exec skyd --config "$CONFIG_FILE" diff --git a/scripts/unicode.pl b/scripts/unicode.pl deleted file mode 100644 index 9457f1e1..00000000 --- a/scripts/unicode.pl +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/perl -w -=pod -All credits for the random unicode string generation logic go to Paul Sarena who released -the original version here: https://github.com/bits/UTF-8-Unicode-Test-Documents and released -it under the BSD 3-Clause "New" or "Revised" License -=cut -use strict; -use warnings qw( FATAL utf8 ); -use utf8; # tell Perl parser there are non-ASCII characters in this lexical scope -use open qw( :encoding(UTF-8) :std ); # Declare that anything that opens a filehandles within this lexical scope is to assume that that stream is encoded in UTF-8 unless you tell it otherwise - -use Encode; -use HTML::Entities; - -my $html_pre = q| - - - - UTF-8 Codepoint Sequence - -|; - -my $html_post = q| -|; - -my $output_directory = './utf8/'; - -my $utf8_seq; - -# 0000–​FFFF Plane 0: Basic Multilingual Plane -# 10000–​1FFFF Plane 1: Supplementary Multilingual Plane -# 20000–​2FFFF Plane 2: Supplementary Ideographic Plane -# 30000–​DFFFF Planes 3–13: Unassigned -# E0000–​EFFFF Plane 14: Supplement­ary Special-purpose Plane -# F0000–​10FFFF Planes 15–16: Supplement­ary Private Use Area - -foreach my $separator ('', ' ') { - foreach my $end (0xFF, 0xFFF, 0xFFFF, 0x1FFFF, 0x2FFFF, 0x10FFFF) { - - # UTF-8 codepoint sequence of assigned, printable codepoints - $utf8_seq = gen_seq({ - start => 0x00, - end => $end, - separator => $separator, - skip_unprintable => 1, - replace_unprintable => 1, - skip_unassigned => 1, - writefiles => ($separator ? 'txt,html' : 'txt') - }); - - - # UTF-8 codepoint sequence of assigned, printable and unprintable codepoints as-is - $utf8_seq = gen_seq({ - start => 0x00, - end => $end, - separator => $separator, - skip_unprintable => 0, - replace_unprintable => 0, - skip_unassigned => 1, - writefiles => ($separator ? 'txt,html' : 'txt') - }); - # UTF-8 codepoint sequence of assigned, printable and unprintable codepoints replaced - $utf8_seq = gen_seq({ - start => 0x00, - end => $end, - separator => $separator, - skip_unprintable => 0, - replace_unprintable => 1, - skip_unassigned => 1, - writefiles => ($separator ? 'txt,html' : 'txt') - }); - - - # UTF-8 codepoint sequence of assinged and unassigned, printable and unprintable codepoints as-is - $utf8_seq = gen_seq({ - start => 0x00, - end => $end, - separator => $separator, - skip_unprintable => 0, - replace_unprintable => 0, - skip_unassigned => 0, - writefiles => ($separator ? 'txt,html' : 'txt') - }); - # UTF-8 codepoint sequence of assinged and unassigned, printable and unprintable codepoints replaced - $utf8_seq = gen_seq({ - start => 0x00, - end => $end, - separator => $separator, - skip_unprintable => 0, - replace_unprintable => 1, - skip_unassigned => 0, - writefiles => ($separator ? 'txt,html' : 'txt') - }); - - } -} - -# print Encode::encode('UTF-8', $utf8_seq), "\n"; - - - -sub gen_seq{ - my $config = shift; - - $config->{start} = 0x00 unless defined $config->{start}; - $config->{end} = 0x10FFFF unless defined $config->{end}; - $config->{skip_unassigned} = 1 unless defined $config->{skip_unassigned}; - $config->{skip_unprintable} = 1 unless defined $config->{skip_unprintable}; - $config->{replace_unprintable} = 1 unless defined $config->{replace_unprintable}; - $config->{separator} = ' ' unless defined $config->{separator}; - $config->{newlines_every} = 50 unless defined $config->{newlines_every}; - $config->{writefiles} = 'text,html' unless defined $config->{writefiles}; - - my $utf8_seq; - my $codepoints_this_line = 0; - my $codepoints_printed = 0; - - for my $i ($config->{start} .. $config->{end}) { - - next if ($i >= 0xD800 && $i <= 0xDFFF); # high and low surrogate halves used by UTF-16 (U+D800 through U+DFFF) are not legal Unicode values, and the UTF-8 encoding of them is an invalid byte sequence - next if ($i >= 0xFDD0 && $i <= 0xFDEF); # Non-characters - next if ( # Non-characters - $i == 0xFFFE || $i == 0xFFFF || - $i == 0x1FFFE || $i == 0x1FFFF || - $i == 0x2FFFE || $i == 0x2FFFF || - $i == 0x3FFFE || $i == 0x3FFFF || - $i == 0x4FFFE || $i == 0x4FFFF || - $i == 0x5FFFE || $i == 0x5FFFF || - $i == 0x6FFFE || $i == 0x6FFFF || - $i == 0x7FFFE || $i == 0x7FFFF || - $i == 0x8FFFE || $i == 0x8FFFF || - $i == 0x9FFFE || $i == 0x9FFFF || - $i == 0xaFFFE || $i == 0xAFFFF || - $i == 0xbFFFE || $i == 0xBFFFF || - $i == 0xcFFFE || $i == 0xCFFFF || - $i == 0xdFFFE || $i == 0xDFFFF || - $i == 0xeFFFE || $i == 0xEFFFF || - $i == 0xfFFFE || $i == 0xFFFFF || - $i == 0x10FFFE || $i == 0x10FFFF - ); - - my $codepoint = chr($i); - - # skip unassiggned codepoints - next if $config->{skip_unassigned} && $codepoint !~ /^\p{Assigned}/o; - - if ( $codepoint =~ /^\p{IsPrint}/o ) { - $utf8_seq .= $codepoint; - } else { # not printable - next if $config->{skip_unprintable}; - # include unprintable or replace it - $utf8_seq .= $config->{replace_unprintable} ? '�' : $codepoint; - } - - $codepoints_printed++; - - if ($config->{separator}) { - if ($config->{newlines_every} && $codepoints_this_line++ == $config->{newlines_every}) { - $utf8_seq .= "\n"; - $codepoints_this_line = 0; - } else { - $utf8_seq .= $config->{separator}; - } - } - } - - utf8::upgrade($utf8_seq); - - - if ($config->{writefiles}) { - - my $filebasename = 'utf8_sequence_' . - (sprintf '%#x', $config->{start}) . - '-' . - (sprintf '%#x', $config->{end}) . - ($config->{skip_unassigned} ? '_assigned' : '_including-unassigned') . - ($config->{skip_unprintable} ? '_printable' : '_including-unprintable') . - (!$config->{skip_unprintable} ? - ($config->{replace_unprintable} ? '-replaced' : '-asis') : - '' - ) . - ($config->{separator} ? - ($config->{newlines_every} ? '' : '_without-newlines') : - '_unseparated' - ); - - - my $title = 'UTF-8 codepoint sequence' . - ($config->{skip_unassigned} ? ' of assigned' : ' of assinged and unassigned') . - ($config->{skip_unprintable} ? ', printable' : ', with unprintable') . - (!$config->{skip_unprintable} ? - ($config->{replace_unprintable} ? ' codepoints replaced' : ' codepoints as-is') : - ' codepoints' - ) . - ' in the range ' . - (sprintf '%#x', $config->{start}) . - '-' . - (sprintf '%#x', $config->{end}) . - ($config->{newlines_every} ? '' : ', as a long string without newlines'); - - my $html_pre_custom = $html_pre; - $html_pre_custom =~ s|UTF\-8 codepoint sequence|$title|; - - - my $filename = ${output_directory} . ($config->{separator} ? '' : 'un') . 'separated/' . ${filebasename}; - - if ($config->{writefiles} =~ /te?xt/) { - open FH, ">${filename}.txt" or die "cannot open $filename: $!"; - print FH $utf8_seq; - close FH; - } - - if ($config->{writefiles} =~ /html/) { - open FH, ">${filename}_unescaped.html" or die "cannot open $filename: $!"; - print FH $html_pre_custom, $utf8_seq, $html_post; - close FH; - } - - # open FH, ">${output_directory}${filebasename}_escaped.html"; - # print FH $html_pre_custom, HTML::Entities::encode_entities($utf8_seq), $html_post; - # close FH; - - print "Output $title ($codepoints_printed codepoints)\n"; - } - - return $utf8_seq; -} \ No newline at end of file diff --git a/server/Cargo.toml b/server/Cargo.toml index 2a5eab9d..064203ed 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,9 +1,10 @@ [package] authors = ["Sayan Nandan "] -build = "build.rs" edition = "2021" name = "skyd" version = "0.8.0" +description = "Skytable is a modern NoSQL database powered by BlueQL that aims to deliver performance, scalability and flexibility with data" +license = "AGPL-3.0" [dependencies] # internal deps @@ -11,48 +12,39 @@ libsky = { path = "../libsky" } sky_macros = { path = "../sky-macros" } rcrypt = "0.4.0" # external deps -ahash = "0.8.2" -bytes = "1.3.0" -chrono = "0.4.23" -clap = { version = "2", features = ["yaml"] } -env_logger = "0.10.0" -hashbrown = { version = "0.13.1", features = ["raw"] } -log = "0.4.17" -openssl = { version = "0.10.45", features = ["vendored"] } +bytes = "1.5.0" +env_logger = "0.10.1" +log = "0.4.20" +openssl = { version = "0.10.61", features = ["vendored"] } +crossbeam-epoch = { version = "0.9.15" } parking_lot = "0.12.1" -regex = "1.7.1" -serde = { version = "1.0.152", features = ["derive"] } -tokio = { version = "1.24.1", features = ["full"] } +serde = { version = "1.0.193", features = ["derive"] } +tokio = { version = "1.34.0", features = ["full"] } tokio-openssl = "0.6.3" -toml = "0.5.10" -base64 = "0.13.1" +uuid = { version = "1.6.1", features = ["v4", "fast-rng", "macro-diagnostics"] } +crc = "3.0.1" +serde_yaml = "0.9.27" [target.'cfg(all(not(target_env = "msvc"), not(miri)))'.dependencies] # external deps -jemallocator = "0.5.0" +jemallocator = "0.5.4" [target.'cfg(target_os = "windows")'.dependencies] # external deps -winapi = { version = "0.3.9", features = ["fileapi"] } +winapi = { version = "0.3.9", features = [ + "fileapi", + "sysinfoapi", + "minwinbase", +] } [target.'cfg(unix)'.dependencies] # external deps -libc = "0.2.139" - -[target.'cfg(unix)'.build-dependencies] -# external deps -cc = "1.0.78" +libc = "0.2.150" [dev-dependencies] -# internal deps -libstress = { path = "../libstress" } -skytable = { git = "https://github.com/skytable/client-rust", features = [ - "aio", - "aio-ssl", -], default-features = false, branch = "next" } # external deps -bincode = "1.3.3" rand = "0.8.5" -tokio = { version = "1.24.1", features = ["test-util"] } +tokio = { version = "1.34.0", features = ["test-util"] } +skytable = { branch = "octave", git = "https://github.com/skytable/client-rust.git" } [features] nightly = [] @@ -71,19 +63,24 @@ priority = "optional" assets = [ [ "target/release/skyd", - "usr/bin/", + "/usr/bin/skyd", "755", ], [ "target/release/skysh", - "usr/bin/", + "/usr/bin/skysh", "755", ], [ "target/release/sky-bench", - "usr/bin/", + "/usr/bin/sky-bench", "755", ], + [ + "../examples/config-files/dpkg/config.yaml", + "/var/lib/skytable/config.yaml.tmp", + "644" + ], [ "../pkg/common/skyd.service", "/etc/systemd/system/skyd.service", diff --git a/server/build.rs b/server/build.rs deleted file mode 100644 index 666a620b..00000000 --- a/server/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -fn main() { - #[cfg(unix)] - { - println!("cargo:rerun-if-changed=native/flock-posix.c"); - cc::Build::new() - .file("native/flock-posix.c") - .compile("libflock-posix.a"); - } -} diff --git a/server/src/actions/del.rs b/server/src/actions/del.rs deleted file mode 100644 index 617c8152..00000000 --- a/server/src/actions/del.rs +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Created on Wed Aug 19 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # `DEL` queries -//! This module provides functions to work with `DEL` queries - -use crate::{ - corestore::table::DataModel, dbnet::prelude::*, - kvengine::encoding::ENCODING_LUT_ITER, util::compiler, -}; - -action!( - /// Run a `DEL` query - /// - /// Do note that this function is blocking since it acquires a write lock. - /// It will write an entire datagroup, for this `del` action - fn del(handle: &Corestore, con: &mut Connection, act: ActionIter<'a>) { - ensure_length::

(act.len(), |size| size != 0)?; - let table = get_tbl_ref!(handle, con); - macro_rules! remove { - ($engine:expr) => {{ - let encoding_is_okay = ENCODING_LUT_ITER[$engine.is_key_encoded()](act.as_ref()); - if compiler::likely(encoding_is_okay) { - let done_howmany: Option; - { - if registry::state_okay() { - let mut many = 0; - act.for_each(|key| { - many += $engine.remove_unchecked(key) as usize; - }); - done_howmany = Some(many); - } else { - done_howmany = None; - } - } - if let Some(done_howmany) = done_howmany { - con.write_usize(done_howmany).await?; - } else { - con._write_raw(P::RCODE_SERVER_ERR).await?; - } - } else { - return util::err(P::RCODE_ENCODING_ERROR); - } - }}; - } - match table.get_model_ref() { - DataModel::KV(kve) => { - remove!(kve) - } - DataModel::KVExtListmap(kvlmap) => { - remove!(kvlmap) - } - #[allow(unreachable_patterns)] - _ => return util::err(P::RSTRING_WRONG_MODEL), - } - Ok(()) - } -); diff --git a/server/src/actions/exists.rs b/server/src/actions/exists.rs deleted file mode 100644 index cf974fab..00000000 --- a/server/src/actions/exists.rs +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Created on Wed Aug 19 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # `EXISTS` queries -//! This module provides functions to work with `EXISTS` queries - -use crate::{ - corestore::table::DataModel, dbnet::prelude::*, - kvengine::encoding::ENCODING_LUT_ITER, queryengine::ActionIter, util::compiler, -}; - -action!( - /// Run an `EXISTS` query - fn exists(handle: &Corestore, con: &mut Connection, act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len != 0)?; - let mut how_many_of_them_exist = 0usize; - macro_rules! exists { - ($engine:expr) => {{ - let encoding_is_okay = ENCODING_LUT_ITER[$engine.is_key_encoded()](act.as_ref()); - if compiler::likely(encoding_is_okay) { - act.for_each(|key| { - how_many_of_them_exist += $engine.exists_unchecked(key) as usize; - }); - con.write_usize(how_many_of_them_exist).await?; - } else { - return util::err(P::RCODE_ENCODING_ERROR); - } - }}; - } - let tbl = get_tbl_ref!(handle, con); - match tbl.get_model_ref() { - DataModel::KV(kve) => exists!(kve), - DataModel::KVExtListmap(kve) => exists!(kve), - #[allow(unreachable_patterns)] - _ => return util::err(P::RSTRING_WRONG_MODEL), - } - Ok(()) - } -); diff --git a/server/src/actions/flushdb.rs b/server/src/actions/flushdb.rs deleted file mode 100644 index 7bb9e06c..00000000 --- a/server/src/actions/flushdb.rs +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Created on Thu Sep 24 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{dbnet::prelude::*, queryengine::ActionIter}; - -action!( - /// Delete all the keys in the database - fn flushdb(handle: &Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len < 2)?; - if registry::state_okay() { - if act.is_empty() { - // flush the current table - get_tbl_ref!(handle, con).truncate_table(); - } else { - // flush the entity - let raw_entity = unsafe { act.next_unchecked() }; - let entity = handle_entity!(con, raw_entity); - get_tbl!(&entity, handle, con).truncate_table(); - } - con._write_raw(P::RCODE_OKAY).await?; - } else { - con._write_raw(P::RCODE_SERVER_ERR).await?; - } - Ok(()) - } -); diff --git a/server/src/actions/get.rs b/server/src/actions/get.rs deleted file mode 100644 index 868e39fb..00000000 --- a/server/src/actions/get.rs +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Created on Fri Aug 14 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # `GET` queries -//! This module provides functions to work with `GET` queries - -use crate::{dbnet::prelude::*, util::compiler}; - -action!( - /// Run a `GET` query - fn get( - handle: &crate::corestore::Corestore, - con: &mut Connection, - mut act: ActionIter<'a>, - ) { - ensure_length::

(act.len(), |len| len == 1)?; - let kve = handle.get_table_with::()?; - unsafe { - match kve.get_cloned(act.next_unchecked()) { - Ok(Some(val)) => { - con.write_mono_length_prefixed_with_tsymbol(&val, kve.get_value_tsymbol()) - .await? - } - Err(_) => compiler::cold_err(con._write_raw(P::RCODE_ENCODING_ERROR)).await?, - Ok(_) => con._write_raw(P::RCODE_NIL).await?, - } - } - Ok(()) - } -); diff --git a/server/src/actions/keylen.rs b/server/src/actions/keylen.rs deleted file mode 100644 index d5b87170..00000000 --- a/server/src/actions/keylen.rs +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Created on Sun Sep 27 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::dbnet::prelude::*; - -action!( - /// Run a `KEYLEN` query - /// - /// At this moment, `keylen` only supports a single key - fn keylen(handle: &crate::corestore::Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len == 1)?; - let res: Option = { - let reader = handle.get_table_with::()?; - unsafe { - // UNSAFE(@ohsayan): this is completely safe as we've already checked - // the number of arguments is one - match reader.get(act.next_unchecked()) { - Ok(v) => v.map(|b| b.len()), - Err(_) => None, - } - } - }; - if let Some(value) = res { - // Good, we got the key's length, write it off to the stream - con.write_usize(value).await?; - } else { - // Ah, couldn't find that key - con._write_raw(P::RCODE_NIL).await?; - } - Ok(()) - } -); diff --git a/server/src/actions/lists/lget.rs b/server/src/actions/lists/lget.rs deleted file mode 100644 index 6c2fc5ee..00000000 --- a/server/src/actions/lists/lget.rs +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Created on Fri Sep 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{corestore::SharedSlice, dbnet::prelude::*}; - -const LEN: &[u8] = "LEN".as_bytes(); -const LIMIT: &[u8] = "LIMIT".as_bytes(); -const VALUEAT: &[u8] = "VALUEAT".as_bytes(); -const LAST: &[u8] = "LAST".as_bytes(); -const FIRST: &[u8] = "FIRST".as_bytes(); -const RANGE: &[u8] = "RANGE".as_bytes(); - -struct Range { - start: usize, - stop: Option, -} - -impl Range { - pub fn new(start: usize) -> Self { - Self { start, stop: None } - } - pub fn set_stop(&mut self, stop: usize) { - self.stop = Some(stop); - } - pub fn into_vec(self, slice: &[SharedSlice]) -> Option> { - slice - .get(self.start..self.stop.unwrap_or(slice.len())) - .map(|slc| slc.to_vec()) - } -} - -action! { - /// Handle an `LGET` query for the list model (KVExt) - /// ## Syntax - /// - `LGET ` will return the full list - /// - `LGET LEN` will return the length of the list - /// - `LGET LIMIT ` will return a maximum of `limit` elements - /// - `LGET VALUEAT ` will return the value at the provided index - /// - `LGET FIRST` will return the first item - /// - `LGET LAST` will return the last item - /// if it exists - fn lget(handle: &Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len != 0)?; - let listmap = handle.get_table_with::()?; - // get the list name - let listname = unsafe { act.next_unchecked() }; - // now let us see what we need to do - macro_rules! get_numeric_count { - () => { - match unsafe { String::from_utf8_lossy(act.next_unchecked()) }.parse::() { - Ok(int) => int, - Err(_) => return util::err(P::RCODE_WRONGTYPE_ERR), - } - }; - } - match act.next_uppercase().as_ref() { - None => { - // just return everything in the list - let items = match listmap.list_cloned_full(listname) { - Ok(Some(list)) => list, - Ok(None) => return Err(P::RCODE_NIL.into()), - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - }; - writelist!(con, listmap, items); - } - Some(subaction) => { - match subaction.as_ref() { - LEN => { - ensure_length::

(act.len(), |len| len == 0)?; - match listmap.list_len(listname) { - Ok(Some(len)) => con.write_usize(len).await?, - Ok(None) => return Err(P::RCODE_NIL.into()), - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - } - } - LIMIT => { - ensure_length::

(act.len(), |len| len == 1)?; - let count = get_numeric_count!(); - match listmap.list_cloned(listname, count) { - Ok(Some(items)) => writelist!(con, listmap, items), - Ok(None) => return Err(P::RCODE_NIL.into()), - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - } - } - VALUEAT => { - ensure_length::

(act.len(), |len| len == 1)?; - let idx = get_numeric_count!(); - let maybe_value = listmap.get(listname).map(|list| { - list.map(|lst| lst.read().get(idx).cloned()) - }); - match maybe_value { - Ok(v) => match v { - Some(Some(value)) => { - con.write_mono_length_prefixed_with_tsymbol( - &value, listmap.get_value_tsymbol() - ).await?; - } - Some(None) => { - // bad index - return Err(P::RSTRING_LISTMAP_BAD_INDEX.into()); - } - None => { - // not found - return Err(P::RCODE_NIL.into()); - } - } - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - } - } - LAST => { - ensure_length::

(act.len(), |len| len == 0)?; - let maybe_value = listmap.get(listname).map(|list| { - list.map(|lst| lst.read().last().cloned()) - }); - match maybe_value { - Ok(v) => match v { - Some(Some(value)) => { - con.write_mono_length_prefixed_with_tsymbol( - &value, listmap.get_value_tsymbol() - ).await?; - }, - Some(None) => return Err(P::RSTRING_LISTMAP_LIST_IS_EMPTY.into()), - None => return Err(P::RCODE_NIL.into()), - } - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - } - } - FIRST => { - ensure_length::

(act.len(), |len| len == 0)?; - let maybe_value = listmap.get(listname).map(|list| { - list.map(|lst| lst.read().first().cloned()) - }); - match maybe_value { - Ok(v) => match v { - Some(Some(value)) => { - con.write_mono_length_prefixed_with_tsymbol( - &value, listmap.get_value_tsymbol() - ).await?; - }, - Some(None) => return Err(P::RSTRING_LISTMAP_LIST_IS_EMPTY.into()), - None => return Err(P::RCODE_NIL.into()), - } - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - } - } - RANGE => { - match act.next_string_owned() { - Some(start) => { - let start: usize = match start.parse() { - Ok(v) => v, - Err(_) => return util::err(P::RCODE_WRONGTYPE_ERR), - }; - let mut range = Range::new(start); - if let Some(stop) = act.next_string_owned() { - let stop: usize = match stop.parse() { - Ok(v) => v, - Err(_) => return util::err(P::RCODE_WRONGTYPE_ERR), - }; - range.set_stop(stop); - }; - match listmap.get(listname) { - Ok(Some(list)) => { - let ret = range.into_vec(&list.read()); - match ret { - Some(ret) => { - writelist!(con, listmap, ret); - }, - None => return Err(P::RSTRING_LISTMAP_BAD_INDEX.into()), - } - } - Ok(None) => return Err(P::RCODE_NIL.into()), - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - } - } - None => return Err(P::RCODE_ACTION_ERR.into()), - } - } - _ => return Err(P::RCODE_UNKNOWN_ACTION.into()), - } - } - } - Ok(()) - } -} diff --git a/server/src/actions/lists/lmod.rs b/server/src/actions/lists/lmod.rs deleted file mode 100644 index cb728a85..00000000 --- a/server/src/actions/lists/lmod.rs +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Created on Wed Sep 15 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{corestore::SharedSlice, dbnet::prelude::*, util::compiler}; - -const CLEAR: &[u8] = "CLEAR".as_bytes(); -const PUSH: &[u8] = "PUSH".as_bytes(); -const REMOVE: &[u8] = "REMOVE".as_bytes(); -const INSERT: &[u8] = "INSERT".as_bytes(); -const POP: &[u8] = "POP".as_bytes(); - -action! { - /// Handle `LMOD` queries - /// ## Syntax - /// - `LMOD push ` - /// - `LMOD pop ` - /// - `LMOD insert ` - /// - `LMOD remove ` - /// - `LMOD clear` - fn lmod(handle: &Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len > 1)?; - let listmap = handle.get_table_with::()?; - // get the list name - let listname = unsafe { act.next_unchecked() }; - macro_rules! get_numeric_count { - () => { - match unsafe { String::from_utf8_lossy(act.next_unchecked()) }.parse::() { - Ok(int) => int, - Err(_) => return Err(P::RCODE_WRONGTYPE_ERR.into()), - } - }; - } - // now let us see what we need to do - match unsafe { act.next_uppercase_unchecked() }.as_ref() { - CLEAR => { - ensure_length::

(act.len(), |len| len == 0)?; - let list = match listmap.get_inner_ref().get(listname) { - Some(l) => l, - _ => return Err(P::RCODE_NIL.into()), - }; - let okay = if registry::state_okay() { - list.write().clear(); - P::RCODE_OKAY - } else { - P::RCODE_SERVER_ERR - }; - con._write_raw(okay).await? - } - PUSH => { - ensure_boolean_or_aerr::

(!act.is_empty())?; - let list = match listmap.get_inner_ref().get(listname) { - Some(l) => l, - _ => return Err(P::RCODE_NIL.into()), - }; - let venc_ok = listmap.get_val_encoder(); - let ret = if compiler::likely(act.as_ref().all(venc_ok)) { - if registry::state_okay() { - list.write().extend(act.map(SharedSlice::new)); - P::RCODE_OKAY - } else { - P::RCODE_SERVER_ERR - } - } else { - P::RCODE_ENCODING_ERROR - }; - con._write_raw(ret).await? - } - REMOVE => { - ensure_length::

(act.len(), |len| len == 1)?; - let idx_to_remove = get_numeric_count!(); - if registry::state_okay() { - let maybe_value = listmap.get_inner_ref().get(listname).map(|list| { - let mut wlock = list.write(); - if idx_to_remove < wlock.len() { - wlock.remove(idx_to_remove); - true - } else { - false - } - }); - con._write_raw(P::OKAY_BADIDX_NIL_NLUT[maybe_value]).await? - } else { - return Err(P::RCODE_SERVER_ERR.into()); - } - } - INSERT => { - ensure_length::

(act.len(), |len| len == 2)?; - let idx_to_insert_at = get_numeric_count!(); - let bts = unsafe { act.next_unchecked() }; - let ret = if compiler::likely(listmap.is_val_ok(bts)) { - if registry::state_okay() { - // okay state, good to insert - let maybe_insert = match listmap.get(listname) { - Ok(lst) => lst.map(|list| { - let mut wlock = list.write(); - if idx_to_insert_at < wlock.len() { - // we can insert - wlock.insert(idx_to_insert_at, SharedSlice::new(bts)); - true - } else { - // oops, out of bounds - false - } - }), - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - }; - P::OKAY_BADIDX_NIL_NLUT[maybe_insert] - } else { - // flush broken; server err - P::RCODE_SERVER_ERR - } - } else { - // encoding failed, uh - P::RCODE_ENCODING_ERROR - }; - con._write_raw(ret).await? - } - POP => { - ensure_length::

(act.len(), |len| len < 2)?; - let idx = if act.len() == 1 { - // we have an idx - Some(get_numeric_count!()) - } else { - // no idx - None - }; - if registry::state_okay() { - let maybe_pop = match listmap.get(listname) { - Ok(lst) => lst.map(|list| { - let mut wlock = list.write(); - if let Some(idx) = idx { - if idx < wlock.len() { - // so we can pop - Some(wlock.remove(idx)) - } else { - None - } - } else { - wlock.pop() - } - }), - Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()), - }; - match maybe_pop { - Some(Some(val)) => { - con.write_mono_length_prefixed_with_tsymbol( - &val, listmap.get_value_tsymbol() - ).await?; - } - Some(None) => { - con._write_raw(P::RSTRING_LISTMAP_BAD_INDEX).await?; - } - None => con._write_raw(P::RCODE_NIL).await?, - } - } else { - con._write_raw(P::RCODE_SERVER_ERR).await? - } - } - _ => con._write_raw(P::RCODE_UNKNOWN_ACTION).await?, - } - Ok(()) - } -} diff --git a/server/src/actions/lists/mod.rs b/server/src/actions/lists/mod.rs deleted file mode 100644 index c4526918..00000000 --- a/server/src/actions/lists/mod.rs +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Created on Tue Sep 07 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#[macro_use] -mod macros; -// modules -pub mod lget; -pub mod lmod; - -use crate::{corestore::SharedSlice, dbnet::prelude::*, kvengine::LockedVec}; - -action! { - /// Handle an `LSET` query for the list model - /// Syntax: `LSET ` - fn lset(handle: &Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len > 0)?; - let listmap = handle.get_table_with::()?; - let listname = unsafe { act.next_unchecked_bytes() }; - let list = listmap.get_inner_ref(); - if registry::state_okay() { - let did = if let Some(entry) = list.fresh_entry(listname) { - let v: Vec = act.map(SharedSlice::new).collect(); - entry.insert(LockedVec::new(v)); - true - } else { - false - }; - con._write_raw(P::OKAY_OVW_BLUT[did]).await? - } else { - con._write_raw(P::RCODE_SERVER_ERR).await? - } - Ok(()) - } -} diff --git a/server/src/actions/lskeys.rs b/server/src/actions/lskeys.rs deleted file mode 100644 index e13c8d6a..00000000 --- a/server/src/actions/lskeys.rs +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Created on Thu May 13 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{ - corestore::{table::DataModel, SharedSlice}, - dbnet::prelude::*, -}; - -const DEFAULT_COUNT: usize = 10; - -action!( - /// Run an `LSKEYS` query - fn lskeys(handle: &crate::corestore::Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |size| size < 4)?; - let (table, count) = if act.is_empty() { - (get_tbl!(handle, con), DEFAULT_COUNT) - } else if act.len() == 1 { - // two args, could either be count or an entity - let nextret = unsafe { act.next_unchecked() }; - if unsafe { ucidx!(nextret, 0) }.is_ascii_digit() { - // noice, this is a number; let's try to parse it - let count = if let Ok(cnt) = String::from_utf8_lossy(nextret).parse::() { - cnt - } else { - return util::err(P::RCODE_WRONGTYPE_ERR); - }; - (get_tbl!(handle, con), count) - } else { - // sigh, an entity - let entity = handle_entity!(con, nextret); - (get_tbl!(&entity, handle, con), DEFAULT_COUNT) - } - } else { - // an entity and a count, gosh this fella is really trying us - let entity_ret = unsafe { act.next().unsafe_unwrap() }; - let count_ret = unsafe { act.next().unsafe_unwrap() }; - let entity = handle_entity!(con, entity_ret); - let count = if let Ok(cnt) = String::from_utf8_lossy(count_ret).parse::() { - cnt - } else { - return util::err(P::RCODE_WRONGTYPE_ERR); - }; - (get_tbl!(&entity, handle, con), count) - }; - let tsymbol = match table.get_model_ref() { - DataModel::KV(kv) => kv.get_value_tsymbol(), - DataModel::KVExtListmap(kv) => kv.get_value_tsymbol(), - }; - let items: Vec = match table.get_model_ref() { - DataModel::KV(kv) => kv.get_inner_ref().get_keys(count), - DataModel::KVExtListmap(kv) => kv.get_inner_ref().get_keys(count), - }; - con.write_typed_non_null_array_header(items.len(), tsymbol) - .await?; - for key in items { - con.write_typed_non_null_array_element(&key).await?; - } - Ok(()) - } -); diff --git a/server/src/actions/macros.rs b/server/src/actions/macros.rs deleted file mode 100644 index a1998078..00000000 --- a/server/src/actions/macros.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Created on Thu Nov 11 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -/* - Don't modulo because it's an L1 miss and an L2 hit. Use lowbit checks to check for parity -*/ - -#[macro_export] -/// endian independent check to see if the lowbit is set or not. Returns true if the lowbit -/// is set. this is undefined to be applied on signed values on one's complement targets -macro_rules! is_lowbit_set { - ($v:expr) => { - $v & 1 == 1 - }; -} - -#[macro_export] -/// endian independent check to see if the lowbit is unset or not. Returns true if the lowbit -/// is unset. this is undefined to be applied on signed values on one's complement targets -macro_rules! is_lowbit_unset { - ($v:expr) => { - $v & 1 == 0 - }; -} - -#[macro_export] -macro_rules! get_tbl { - ($entity:expr, $store:expr, $con:expr) => {{ - $crate::actions::translate_ddl_error::>( - $store.get_table($entity), - )? - }}; - ($store:expr, $con:expr) => {{ - match $store.get_ctable() { - Some(tbl) => tbl, - None => return $crate::util::err(P::RSTRING_DEFAULT_UNSET), - } - }}; -} - -#[macro_export] -macro_rules! get_tbl_ref { - ($store:expr, $con:expr) => {{ - match $store.get_ctable_ref() { - Some(tbl) => tbl, - None => return $crate::util::err(P::RSTRING_DEFAULT_UNSET), - } - }}; -} - -#[macro_export] -macro_rules! handle_entity { - ($con:expr, $ident:expr) => {{ - match $crate::blueql::util::from_slice_action_result::

(&$ident) { - Ok(e) => e, - Err(e) => return Err(e.into()), - } - }}; -} diff --git a/server/src/actions/mget.rs b/server/src/actions/mget.rs deleted file mode 100644 index 73b0cc40..00000000 --- a/server/src/actions/mget.rs +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Created on Thu Aug 27 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{ - dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER, queryengine::ActionIter, - util::compiler, -}; - -action!( - /// Run an `MGET` query - /// - fn mget(handle: &crate::corestore::Corestore, con: &mut Connection, act: ActionIter<'a>) { - ensure_length::

(act.len(), |size| size != 0)?; - let kve = handle.get_table_with::()?; - let encoding_is_okay = ENCODING_LUT_ITER[kve.is_key_encoded()](act.as_ref()); - if compiler::likely(encoding_is_okay) { - con.write_typed_array_header(act.len(), kve.get_value_tsymbol()) - .await?; - for key in act { - match kve.get_cloned_unchecked(key) { - Some(v) => con.write_typed_array_element(&v).await?, - None => con.write_typed_array_element_null().await?, - } - } - } else { - return util::err(P::RCODE_ENCODING_ERROR); - } - Ok(()) - } -); diff --git a/server/src/actions/mod.rs b/server/src/actions/mod.rs deleted file mode 100644 index 7d7337df..00000000 --- a/server/src/actions/mod.rs +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Created on Wed Aug 19 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # Actions -//! -//! Actions are like shell commands, you provide arguments -- they return output. This module contains a collection -//! of the actions supported by Skytable -//! - -#[macro_use] -mod macros; -pub mod dbsize; -pub mod del; -pub mod exists; -pub mod flushdb; -pub mod get; -pub mod keylen; -pub mod lists; -pub mod lskeys; -pub mod mget; -pub mod mpop; -pub mod mset; -pub mod mupdate; -pub mod pop; -pub mod set; -pub mod strong; -pub mod update; -pub mod uset; -pub mod whereami; -use { - crate::{corestore::memstore::DdlError, protocol::interface::ProtocolSpec, util}, - std::io::Error as IoError, -}; - -/// A generic result for actions -pub type ActionResult = Result; - -/// Errors that can occur while running actions -#[derive(Debug)] -pub enum ActionError { - ActionError(&'static [u8]), - IoError(std::io::Error), -} - -impl PartialEq for ActionError { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::ActionError(a1), Self::ActionError(a2)) => a1 == a2, - (Self::IoError(ioe1), Self::IoError(ioe2)) => ioe1.to_string() == ioe2.to_string(), - _ => false, - } - } -} - -impl From<&'static [u8]> for ActionError { - fn from(e: &'static [u8]) -> Self { - Self::ActionError(e) - } -} - -impl From for ActionError { - fn from(e: IoError) -> Self { - Self::IoError(e) - } -} - -#[cold] -#[inline(never)] -fn map_ddl_error_to_status(e: DdlError) -> ActionError { - let r = match e { - DdlError::AlreadyExists => P::RSTRING_ALREADY_EXISTS, - DdlError::DdlTransactionFailure => P::RSTRING_DDL_TRANSACTIONAL_FAILURE, - DdlError::DefaultNotFound => P::RSTRING_DEFAULT_UNSET, - DdlError::NotEmpty => P::RSTRING_KEYSPACE_NOT_EMPTY, - DdlError::NotReady => P::RSTRING_NOT_READY, - DdlError::ObjectNotFound => P::RSTRING_CONTAINER_NOT_FOUND, - DdlError::ProtectedObject => P::RSTRING_PROTECTED_OBJECT, - DdlError::StillInUse => P::RSTRING_STILL_IN_USE, - DdlError::WrongModel => P::RSTRING_WRONG_MODEL, - }; - ActionError::ActionError(r) -} - -#[inline(always)] -pub fn translate_ddl_error(r: Result) -> Result { - match r { - Ok(r) => Ok(r), - Err(e) => Err(map_ddl_error_to_status::

(e)), - } -} - -pub fn ensure_length(len: usize, is_valid: fn(usize) -> bool) -> ActionResult<()> { - if util::compiler::likely(is_valid(len)) { - Ok(()) - } else { - util::err(P::RCODE_ACTION_ERR) - } -} - -pub fn ensure_boolean_or_aerr(boolean: bool) -> ActionResult<()> { - if util::compiler::likely(boolean) { - Ok(()) - } else { - util::err(P::RCODE_ACTION_ERR) - } -} - -pub mod heya { - //! Respond to `HEYA` queries - use crate::dbnet::prelude::*; - action!( - /// Returns a `HEY!` `Response` - fn heya(_handle: &Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len == 0 || len == 1)?; - if act.len() == 1 { - let raw_byte = unsafe { act.next_unchecked() }; - con.write_mono_length_prefixed_with_tsymbol(raw_byte, b'+') - .await?; - } else { - con._write_raw(P::ELEMRESP_HEYA).await?; - } - Ok(()) - } - ); -} diff --git a/server/src/actions/mpop.rs b/server/src/actions/mpop.rs deleted file mode 100644 index 6bd988ad..00000000 --- a/server/src/actions/mpop.rs +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Created on Wed Aug 11 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{ - corestore, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER, - queryengine::ActionIter, util::compiler, -}; - -action!( - /// Run an MPOP action - fn mpop(handle: &corestore::Corestore, con: &mut Connection, act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len != 0)?; - if registry::state_okay() { - let kve = handle.get_table_with::()?; - let encoding_is_okay = ENCODING_LUT_ITER[kve.is_key_encoded()](act.as_ref()); - if compiler::likely(encoding_is_okay) { - con.write_typed_array_header(act.len(), kve.get_value_tsymbol()) - .await?; - for key in act { - match kve.pop_unchecked(key) { - Some(val) => con.write_typed_array_element(&val).await?, - None => con.write_typed_array_element_null().await?, - } - } - } else { - return util::err(P::RCODE_ENCODING_ERROR); - } - } else { - // don't begin the operation at all if the database is poisoned - return util::err(P::RCODE_SERVER_ERR); - } - Ok(()) - } -); diff --git a/server/src/actions/mset.rs b/server/src/actions/mset.rs deleted file mode 100644 index d4666bdd..00000000 --- a/server/src/actions/mset.rs +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Created on Thu Aug 27 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{ - corestore::SharedSlice, dbnet::prelude::*, - kvengine::encoding::ENCODING_LUT_ITER_PAIR, util::compiler, -}; - -action!( - /// Run an `MSET` query - fn mset(handle: &crate::corestore::Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - let howmany = act.len(); - ensure_length::

(howmany, |size| size & 1 == 0 && size != 0)?; - let kve = handle.get_table_with::()?; - let encoding_is_okay = ENCODING_LUT_ITER_PAIR[kve.get_encoding_tuple()](&act); - if compiler::likely(encoding_is_okay) { - let done_howmany: Option = if registry::state_okay() { - let mut didmany = 0; - while let (Some(key), Some(val)) = (act.next(), act.next()) { - didmany += - kve.set_unchecked(SharedSlice::new(key), SharedSlice::new(val)) as usize; - } - Some(didmany) - } else { - None - }; - if let Some(done_howmany) = done_howmany { - con.write_usize(done_howmany).await?; - } else { - return util::err(P::RCODE_SERVER_ERR); - } - } else { - return util::err(P::RCODE_ENCODING_ERROR); - } - Ok(()) - } -); diff --git a/server/src/actions/mupdate.rs b/server/src/actions/mupdate.rs deleted file mode 100644 index bc97b421..00000000 --- a/server/src/actions/mupdate.rs +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Created on Thu Aug 27 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{ - corestore::SharedSlice, dbnet::prelude::*, - kvengine::encoding::ENCODING_LUT_ITER_PAIR, util::compiler, -}; - -action!( - /// Run an `MUPDATE` query - fn mupdate(handle: &crate::corestore::Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - let howmany = act.len(); - ensure_length::

(howmany, |size| size & 1 == 0 && size != 0)?; - let kve = handle.get_table_with::()?; - let encoding_is_okay = ENCODING_LUT_ITER_PAIR[kve.get_encoding_tuple()](&act); - let done_howmany: Option; - if compiler::likely(encoding_is_okay) { - if registry::state_okay() { - let mut didmany = 0; - while let (Some(key), Some(val)) = (act.next(), act.next()) { - didmany += - kve.update_unchecked(SharedSlice::new(key), SharedSlice::new(val)) as usize; - } - done_howmany = Some(didmany); - } else { - done_howmany = None; - } - if let Some(done_howmany) = done_howmany { - con.write_usize(done_howmany).await?; - } else { - return util::err(P::RCODE_SERVER_ERR); - } - } else { - return util::err(P::RCODE_ENCODING_ERROR); - } - Ok(()) - } -); diff --git a/server/src/actions/pop.rs b/server/src/actions/pop.rs deleted file mode 100644 index a162dd34..00000000 --- a/server/src/actions/pop.rs +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Created on Mon Jun 14 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::dbnet::prelude::*; - -action! { - fn pop(handle: &Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len == 1)?; - let key = unsafe { - // SAFETY: We have checked for there to be one arg - act.next_unchecked() - }; - if registry::state_okay() { - let kve = handle.get_table_with::()?; - match kve.pop(key) { - Ok(Some(val)) => con.write_mono_length_prefixed_with_tsymbol( - &val, kve.get_value_tsymbol() - ).await?, - Ok(None) => return util::err(P::RCODE_NIL), - Err(()) => return util::err(P::RCODE_ENCODING_ERROR), - } - } else { - return util::err(P::RCODE_SERVER_ERR); - } - Ok(()) - } -} diff --git a/server/src/actions/set.rs b/server/src/actions/set.rs deleted file mode 100644 index d4f58cc5..00000000 --- a/server/src/actions/set.rs +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Created on Fri Aug 14 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # `SET` queries -//! This module provides functions to work with `SET` queries - -use crate::{corestore::SharedSlice, dbnet::prelude::*, queryengine::ActionIter}; - -action!( - /// Run a `SET` query - fn set(handle: &crate::corestore::Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len == 2)?; - if registry::state_okay() { - let did_we = { - let writer = handle.get_table_with::()?; - match unsafe { - // UNSAFE(@ohsayan): This is completely safe as we've already checked - // that there are exactly 2 arguments - writer.set( - SharedSlice::new(act.next().unsafe_unwrap()), - SharedSlice::new(act.next().unsafe_unwrap()), - ) - } { - Ok(true) => Some(true), - Ok(false) => Some(false), - Err(()) => None, - } - }; - con._write_raw(P::SET_NLUT[did_we]).await?; - } else { - con._write_raw(P::RCODE_SERVER_ERR).await?; - } - Ok(()) - } -); diff --git a/server/src/actions/strong/mod.rs b/server/src/actions/strong/mod.rs deleted file mode 100644 index 2b7d73d6..00000000 --- a/server/src/actions/strong/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Created on Fri Jul 30 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # Strong Actions -//! Strong actions are like "do all" or "fail all" actions, built specifically for -//! multiple keys. So let's say you used `SSET` instead of `MSET` for setting keys: -//! what'd be the difference? -//! In this case, if all the keys are non-existing, which is a requirement for `MSET`, -//! only then would the keys be set. That is, only if all the keys can be set, will the action -//! run and return code `0` - otherwise the action won't do anything and return an overwrite error. -//! There is no point of using _strong actions_ for a single key/value pair, since it will only -//! slow things down due to the checks performed. -//! Do note that this isn't the same as the gurantees provided by ACID transactions - -pub use self::{sdel::sdel, sset::sset, supdate::supdate}; -mod sdel; -mod sset; -mod supdate; - -#[cfg(test)] -mod tests; - -#[derive(Debug)] -enum StrongActionResult { - /// Internal server error - ServerError, - /// A single value was not found - Nil, - /// An overwrite error for a single value - OverwriteError, - /// An encoding error occurred - EncodingError, - /// Everything worked as expected - Okay, -} - -#[cfg(test)] -impl StrongActionResult { - pub const fn is_ok(&self) -> bool { - matches!(self, StrongActionResult::Okay) - } -} diff --git a/server/src/actions/strong/sdel.rs b/server/src/actions/strong/sdel.rs deleted file mode 100644 index 27ae991a..00000000 --- a/server/src/actions/strong/sdel.rs +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Created on Fri Jul 30 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - actions::strong::StrongActionResult, - dbnet::prelude::*, - kvengine::{KVEStandard, SingleEncoder}, - protocol::iter::DerefUnsafeSlice, - util::compiler, - }, - core::slice::Iter, -}; - -action! { - /// Run an `SDEL` query - /// - /// This either returns `Okay` if all the keys were `del`eted, or it returns a - /// `Nil`, which is code `1` - fn sdel(handle: &crate::corestore::Corestore, con: &mut Connection, act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len != 0)?; - let kve = handle.get_table_with::()?; - if registry::state_okay() { - // guarantee one check: consistency - let key_encoder = kve.get_key_encoder(); - let outcome = unsafe { - // UNSAFE(@ohsayan): The lifetime of `act` ensures that the - // pointers are still valid - self::snapshot_and_del(kve, key_encoder, act.into_inner()) - }; - match outcome { - StrongActionResult::Okay => con._write_raw(P::RCODE_OKAY).await?, - StrongActionResult::Nil => { - // good, it failed because some key didn't exist - return util::err(P::RCODE_NIL); - }, - StrongActionResult::ServerError => return util::err(P::RCODE_SERVER_ERR), - StrongActionResult::EncodingError => { - // error we love to hate: encoding error, ugh - return util::err(P::RCODE_ENCODING_ERROR); - }, - StrongActionResult::OverwriteError => unsafe { - // SAFETY check: never the case - impossible!() - } - } - } else { - return util::err(P::RCODE_SERVER_ERR); - } - Ok(()) - } -} - -/// Snapshot the current status and then delete maintaining concurrency -/// guarantees -pub(super) fn snapshot_and_del<'a, T: 'a + DerefUnsafeSlice>( - kve: &'a KVEStandard, - key_encoder: SingleEncoder, - act: Iter<'a, T>, -) -> StrongActionResult { - let mut snapshots = Vec::with_capacity(act.len()); - let mut err_enc = false; - let iter_stat_ok; - { - iter_stat_ok = act.as_ref().iter().all(|key| { - let key = unsafe { - // UNSAFE(@ohsayan): The caller has passed a slice and they should - // ensure that it is valid - key.deref_slice() - }; - if compiler::likely(key_encoder(key)) { - if let Some(snap) = kve.take_snapshot_unchecked(key) { - snapshots.push(snap); - true - } else { - false - } - } else { - err_enc = true; - false - } - }); - } - cfg_test!({ - // give the caller 10 seconds to do some crap - do_sleep!(10 s); - }); - if compiler::unlikely(err_enc) { - return compiler::cold_err(StrongActionResult::EncodingError); - } - if registry::state_okay() { - // guarantee upholded: consistency - if iter_stat_ok { - // nice, all keys exist; let's plonk 'em - let kve = kve; - let lowtable = kve.get_inner_ref(); - act.zip(snapshots).for_each(|(key, snapshot)| { - let key = unsafe { - // UNSAFE(@ohsayan): The caller has passed a slice and they should - // ensure that it is valid - key.deref_slice() - }; - // the check is very important: some thread may have updated the - // value after we snapshotted it. In that case, let this key - // be whatever the "newer" value is. Since our snapshot is a "happens-before" - // thing, this is absolutely fine - let _ = lowtable.remove_if(key, |_, val| val.eq(&snapshot)); - }); - StrongActionResult::Okay - } else { - StrongActionResult::Nil - } - } else { - StrongActionResult::ServerError - } -} diff --git a/server/src/actions/strong/sset.rs b/server/src/actions/strong/sset.rs deleted file mode 100644 index f8157e5b..00000000 --- a/server/src/actions/strong/sset.rs +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Created on Fri Jul 30 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - actions::strong::StrongActionResult, - corestore::SharedSlice, - dbnet::prelude::*, - kvengine::{DoubleEncoder, KVEStandard}, - protocol::iter::DerefUnsafeSlice, - util::compiler, - }, - core::slice::Iter, -}; - -action! { - /// Run an `SSET` query - /// - /// This either returns `Okay` if all the keys were set, or it returns an - /// `Overwrite Error` or code `2` - fn sset(handle: &crate::corestore::Corestore, con: &mut Connection, act: ActionIter<'a>) { - let howmany = act.len(); - ensure_length::

(howmany, |size| size & 1 == 0 && size != 0)?; - let kve = handle.get_table_with::()?; - if registry::state_okay() { - let encoder = kve.get_double_encoder(); - let outcome = unsafe { - // UNSAFE(@ohsayan): The lifetime of `act` guarantees that the - // pointers remain valid - self::snapshot_and_insert(kve, encoder, act.into_inner()) - }; - match outcome { - StrongActionResult::Okay => con._write_raw(P::RCODE_OKAY).await?, - StrongActionResult::OverwriteError => return util::err(P::RCODE_OVERWRITE_ERR), - StrongActionResult::ServerError => return util::err(P::RCODE_SERVER_ERR), - StrongActionResult::EncodingError => { - // error we love to hate: encoding error, ugh - return util::err(P::RCODE_ENCODING_ERROR); - }, - StrongActionResult::Nil => unsafe { - // SAFETY check: never the case - impossible!() - } - } - } else { - return util::err(P::RCODE_SERVER_ERR); - } - Ok(()) - } -} - -/// Take a consistent snapshot of the database at this current point in time -/// and then mutate the entries, respecting concurrency guarantees -pub(super) fn snapshot_and_insert<'a, T: 'a + DerefUnsafeSlice>( - kve: &'a KVEStandard, - encoder: DoubleEncoder, - mut act: Iter<'a, T>, -) -> StrongActionResult { - let mut enc_err = false; - let lowtable = kve.get_inner_ref(); - let key_iter_stat_ok; - { - key_iter_stat_ok = act.as_ref().chunks_exact(2).all(|kv| unsafe { - let key = ucidx!(kv, 0).deref_slice(); - let value = ucidx!(kv, 1).deref_slice(); - if compiler::likely(encoder(key, value)) { - lowtable.get(key).is_none() - } else { - enc_err = true; - false - } - }); - } - cfg_test!({ - // give the caller 10 seconds to do some crap - do_sleep!(10 s); - }); - if compiler::unlikely(enc_err) { - return compiler::cold_err(StrongActionResult::EncodingError); - } - if registry::state_okay() { - if key_iter_stat_ok { - let _kve = kve; - let lowtable = lowtable; - // fine, the keys were non-existent when we looked at them - while let (Some(key), Some(value)) = (act.next(), act.next()) { - unsafe { - if let Some(fresh) = - lowtable.fresh_entry(SharedSlice::new(key.deref_slice())) - { - fresh.insert(SharedSlice::new(value.deref_slice())); - } - // we don't care if some other thread initialized the value we checked - // it. We expected a fresh entry, so that's what we'll check and use - } - } - StrongActionResult::Okay - } else { - StrongActionResult::OverwriteError - } - } else { - StrongActionResult::ServerError - } -} diff --git a/server/src/actions/strong/supdate.rs b/server/src/actions/strong/supdate.rs deleted file mode 100644 index d6f7dd2d..00000000 --- a/server/src/actions/strong/supdate.rs +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Created on Fri Jul 30 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - actions::strong::StrongActionResult, - corestore::SharedSlice, - dbnet::prelude::*, - kvengine::{DoubleEncoder, KVEStandard}, - protocol::iter::DerefUnsafeSlice, - util::compiler, - }, - core::slice::Iter, -}; - -action! { - /// Run an `SUPDATE` query - /// - /// This either returns `Okay` if all the keys were updated, or it returns `Nil` - /// or code `1` - fn supdate(handle: &crate::corestore::Corestore, con: &mut Connection, act: ActionIter<'a>) { - let howmany = act.len(); - ensure_length::

(howmany, |size| size & 1 == 0 && size != 0)?; - let kve = handle.get_table_with::()?; - if registry::state_okay() { - let encoder = kve.get_double_encoder(); - let outcome = unsafe { - // UNSAFE(@ohsayan): the lifetime of `act` ensure ptr validity - self::snapshot_and_update(kve, encoder, act.into_inner()) - }; - match outcome { - StrongActionResult::Okay => con._write_raw(P::RCODE_OKAY).await?, - StrongActionResult::Nil => { - // good, it failed because some key didn't exist - return util::err(P::RCODE_NIL); - }, - StrongActionResult::ServerError => return util::err(P::RCODE_SERVER_ERR), - StrongActionResult::EncodingError => { - // error we love to hate: encoding error, ugh - return util::err(P::RCODE_ENCODING_ERROR); - }, - StrongActionResult::OverwriteError => unsafe { - // SAFETY check: never the case - impossible!() - } - } - } else { - return util::err(P::RCODE_SERVER_ERR); - } - Ok(()) - } -} - -/// Take a consistent snapshot of the database at this point in time. Once snapshotting -/// completes, mutate the entries in place while keeping up with isolation guarantees -/// `(all_okay, enc_err)` -pub(super) fn snapshot_and_update<'a, T: 'a + DerefUnsafeSlice>( - kve: &'a KVEStandard, - encoder: DoubleEncoder, - mut act: Iter<'a, T>, -) -> StrongActionResult { - let mut enc_err = false; - let mut snapshots = Vec::with_capacity(act.len()); - let iter_stat_ok; - { - // snapshot the values at this point in time - iter_stat_ok = act.as_ref().chunks_exact(2).all(|kv| unsafe { - let key = ucidx!(kv, 0).deref_slice(); - let value = ucidx!(kv, 1).deref_slice(); - if compiler::likely(encoder(key, value)) { - if let Some(snapshot) = kve.take_snapshot_unchecked(key) { - snapshots.push(snapshot); - true - } else { - false - } - } else { - enc_err = true; - false - } - }); - } - cfg_test!({ - // give the caller 10 seconds to do some crap - do_sleep!(10 s); - }); - if compiler::unlikely(enc_err) { - return compiler::cold_err(StrongActionResult::EncodingError); - } - if registry::state_okay() { - // uphold consistency - if iter_stat_ok { - let kve = kve; - // good, so all the values existed when we snapshotted them; let's update 'em - let mut snap_cc = snapshots.into_iter(); - let lowtable = kve.get_inner_ref(); - while let (Some(key), Some(value), Some(snapshot)) = - (act.next(), act.next(), snap_cc.next()) - { - unsafe { - // When we snapshotted, we looked at `snapshot`. If the value is still the - // same, then we'll update it. Otherwise, let it be - if let Some(mut mutable) = - lowtable.mut_entry(SharedSlice::new(key.deref_slice())) - { - if mutable.value().eq(&snapshot) { - mutable.insert(SharedSlice::new(value.deref_slice())); - } else { - drop(mutable); - } - } - } - } - StrongActionResult::Okay - } else { - StrongActionResult::Nil - } - } else { - StrongActionResult::ServerError - } -} diff --git a/server/src/actions/strong/tests.rs b/server/src/actions/strong/tests.rs deleted file mode 100644 index 9208fa60..00000000 --- a/server/src/actions/strong/tests.rs +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Created on Fri Jul 30 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -mod sdel_concurrency_tests { - use { - super::super::sdel, - crate::{corestore::SharedSlice, kvengine::KVEStandard}, - std::{sync::Arc, thread}, - }; - - #[test] - fn test_snapshot_okay() { - let kve = KVEStandard::init(true, true); - kve.upsert(SharedSlice::from("k1"), SharedSlice::from("v1")) - .unwrap(); - kve.upsert(SharedSlice::from("k2"), SharedSlice::from("v2")) - .unwrap(); - let encoder = kve.get_key_encoder(); - let it = bi!("k1", "k2"); - let ret = sdel::snapshot_and_del(&kve, encoder, it.as_ref().iter().as_ref().iter()); - assert!(ret.is_ok()); - } - #[test] - fn test_sdel_snapshot_fail_with_t2() { - let kve = Arc::new(KVEStandard::init(true, true)); - let kve1 = kve.clone(); - let encoder = kve.get_key_encoder(); - { - kve.upsert(SharedSlice::from("k1"), SharedSlice::from("v1")) - .unwrap(); - kve.upsert(SharedSlice::from("k2"), SharedSlice::from("v2")) - .unwrap(); - } - let it = bi!("k1", "k2"); - // sdel will wait 10s for us - let t1handle = thread::spawn(move || { - sdel::snapshot_and_del(&kve1, encoder, it.as_ref().iter().as_ref().iter()) - }); - // we have 10s: we sleep 5 to let the snapshot complete (thread spawning takes time) - do_sleep!(5 s); - assert!(kve - .update(SharedSlice::from("k1"), SharedSlice::from("updated-v1")) - .unwrap()); - // let us join t1 - let ret = t1handle.join().unwrap(); - assert!(ret.is_ok()); - // although we told sdel to delete it, it.as_ref().iter() shouldn't because we externally - // updated the value - assert!(kve.exists(&SharedSlice::from("k1")).unwrap()); - } -} - -mod sset_concurrency_tests { - use { - super::super::sset, - crate::{corestore::SharedSlice, kvengine::KVEStandard}, - std::{sync::Arc, thread}, - }; - - #[test] - fn test_snapshot_okay() { - let kve = KVEStandard::init(true, true); - let encoder = kve.get_double_encoder(); - let it = bi!("k1", "v1", "k2", "v2"); - let ret = sset::snapshot_and_insert(&kve, encoder, it.as_ref().iter()); - assert!(ret.is_ok()); - } - #[test] - fn test_sset_snapshot_fail_with_t2() { - let kve = Arc::new(KVEStandard::init(true, true)); - let kve1 = kve.clone(); - let encoder = kve.get_double_encoder(); - let it = bi!("k1", "v1", "k2", "v2"); - // sset will wait 10s for us - let t1handle = - thread::spawn(move || sset::snapshot_and_insert(&kve1, encoder, it.as_ref().iter())); - // we have 10s: we sleep 5 to let the snapshot complete (thread spawning takes time) - do_sleep!(5 s); - // update the value externally - assert!(kve - .set(SharedSlice::from("k1"), SharedSlice::from("updated-v1")) - .unwrap()); - // let us join t1 - let ret = t1handle.join().unwrap(); - // but set won't fail because someone set it before it did; this is totally - // acceptable because we only wanted to set it if it matches the status when - // we created a snapshot - assert!(ret.is_ok()); - // although we told sset to set a key, but it shouldn't because we updated it - assert_eq!( - kve.get(&SharedSlice::from("k1")).unwrap().unwrap().clone(), - SharedSlice::from("updated-v1") - ); - } -} - -mod supdate_concurrency_tests { - use { - super::super::supdate, - crate::{corestore::SharedSlice, kvengine::KVEStandard}, - std::{sync::Arc, thread}, - }; - - #[test] - fn test_snapshot_okay() { - let kve = KVEStandard::init(true, true); - kve.upsert(SharedSlice::from("k1"), SharedSlice::from("v1")) - .unwrap(); - kve.upsert(SharedSlice::from("k2"), SharedSlice::from("v2")) - .unwrap(); - let encoder = kve.get_double_encoder(); - let it = bi!("k1", "v1", "k2", "v2"); - let ret = supdate::snapshot_and_update(&kve, encoder, it.as_ref().iter()); - assert!(ret.is_ok()); - } - #[test] - fn test_supdate_snapshot_fail_with_t2() { - let kve = Arc::new(KVEStandard::init(true, true)); - kve.upsert(SharedSlice::from("k1"), SharedSlice::from("v1")) - .unwrap(); - kve.upsert(SharedSlice::from("k2"), SharedSlice::from("v2")) - .unwrap(); - let kve1 = kve.clone(); - let encoder = kve.get_double_encoder(); - let it = bi!("k1", "v1", "k2", "v2"); - // supdate will wait 10s for us - let t1handle = - thread::spawn(move || supdate::snapshot_and_update(&kve1, encoder, it.as_ref().iter())); - // we have 10s: we sleep 5 to let the snapshot complete (thread spawning takes time) - do_sleep!(5 s); - // lets update the value externally - assert!(kve - .update(SharedSlice::from("k1"), SharedSlice::from("updated-v1")) - .unwrap()); - // let us join t1 - let ret = t1handle.join().unwrap(); - assert!(ret.is_ok()); - // although we told supdate to update the key, it.as_ref().iter() shouldn't because we updated it - // externally; hence our `updated-v1` value should persist - assert_eq!( - kve.get(&SharedSlice::from("k1")).unwrap().unwrap().clone(), - SharedSlice::from("updated-v1") - ); - } -} diff --git a/server/src/actions/update.rs b/server/src/actions/update.rs deleted file mode 100644 index dd08dc47..00000000 --- a/server/src/actions/update.rs +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Created on Mon Aug 17 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # `UPDATE` queries -//! This module provides functions to work with `UPDATE` queries -//! - -use crate::{corestore::SharedSlice, dbnet::prelude::*}; - -action!( - /// Run an `UPDATE` query - fn update(handle: &Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len == 2)?; - if registry::state_okay() { - let did_we = { - let writer = handle.get_table_with::()?; - match unsafe { - // UNSAFE(@ohsayan): This is completely safe as we've already checked - // that there are exactly 2 arguments - writer.update( - SharedSlice::new(act.next_unchecked()), - SharedSlice::new(act.next_unchecked()), - ) - } { - Ok(true) => Some(true), - Ok(false) => Some(false), - Err(()) => None, - } - }; - con._write_raw(P::UPDATE_NLUT[did_we]).await?; - } else { - return util::err(P::RCODE_SERVER_ERR); - } - Ok(()) - } -); diff --git a/server/src/actions/uset.rs b/server/src/actions/uset.rs deleted file mode 100644 index d881e435..00000000 --- a/server/src/actions/uset.rs +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Created on Fri Sep 25 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{ - corestore::SharedSlice, dbnet::prelude::*, - kvengine::encoding::ENCODING_LUT_ITER_PAIR, queryengine::ActionIter, util::compiler, -}; - -action!( - /// Run an `USET` query - /// - /// This is like "INSERT or UPDATE" - fn uset(handle: &crate::corestore::Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - let howmany = act.len(); - ensure_length::

(howmany, |size| size & 1 == 0 && size != 0)?; - let kve = handle.get_table_with::()?; - let encoding_is_okay = ENCODING_LUT_ITER_PAIR[kve.get_encoding_tuple()](&act); - if compiler::likely(encoding_is_okay) { - if registry::state_okay() { - while let (Some(key), Some(val)) = (act.next(), act.next()) { - kve.upsert_unchecked(SharedSlice::new(key), SharedSlice::new(val)); - } - con.write_usize(howmany / 2).await?; - } else { - return util::err(P::RCODE_SERVER_ERR); - } - } else { - return util::err(P::RCODE_ENCODING_ERROR); - } - Ok(()) - } -); diff --git a/server/src/admin/mksnap.rs b/server/src/admin/mksnap.rs deleted file mode 100644 index c4521d24..00000000 --- a/server/src/admin/mksnap.rs +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Created on Tue Oct 13 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - dbnet::prelude::*, kvengine::encoding, - storage::v1::sengine::SnapshotActionResult, - }, - core::str, - std::path::{Component, PathBuf}, -}; - -action!( - /// Create a snapshot - /// - fn mksnap(handle: &crate::corestore::Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - let engine = handle.get_engine(); - if act.is_empty() { - // traditional mksnap - match engine.mksnap(handle.clone_store()).await { - SnapshotActionResult::Ok => con._write_raw(P::RCODE_OKAY).await?, - SnapshotActionResult::Failure => return util::err(P::RCODE_SERVER_ERR), - SnapshotActionResult::Disabled => return util::err(P::RSTRING_SNAPSHOT_DISABLED), - SnapshotActionResult::Busy => return util::err(P::RSTRING_SNAPSHOT_BUSY), - _ => unsafe { impossible!() }, - } - } else if act.len() == 1 { - // remote snapshot, let's see what we've got - let name = unsafe { - // SAFETY: We have already checked that there is one item - act.next_unchecked_bytes() - }; - if !encoding::is_utf8(&name) { - return util::err(P::RCODE_ENCODING_ERROR); - } - - // SECURITY: Check for directory traversal syntax - let st = unsafe { - // SAFETY: We have already checked for UTF-8 validity - str::from_utf8_unchecked(&name) - }; - let path = PathBuf::from(st); - let illegal_snapshot = path - .components() - .filter(|dir| { - // Sanitize snapshot name, to avoid directory traversal attacks - // If the snapshot name has any root directory or parent directory, then - // we'll allow it to pass through this adaptor. - // As a result, this iterator will give us a count of the 'bad' components - dir == &Component::RootDir || dir == &Component::ParentDir - }) - .count() - != 0; - if illegal_snapshot { - return util::err(P::RSTRING_SNAPSHOT_ILLEGAL_NAME); - } - - // now make the snapshot - match engine.mkrsnap(&name, handle.clone_store()).await { - SnapshotActionResult::Ok => con._write_raw(P::RCODE_OKAY).await?, - SnapshotActionResult::Failure => return util::err(P::RCODE_SERVER_ERR), - SnapshotActionResult::Busy => return util::err(P::RSTRING_SNAPSHOT_BUSY), - SnapshotActionResult::AlreadyExists => { - return util::err(P::RSTRING_SNAPSHOT_DUPLICATE) - } - _ => unsafe { impossible!() }, - } - } else { - return util::err(P::RCODE_ACTION_ERR); - } - Ok(()) - } -); diff --git a/server/src/admin/sys.rs b/server/src/admin/sys.rs deleted file mode 100644 index d35d0135..00000000 --- a/server/src/admin/sys.rs +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Created on Tue Mar 29 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - corestore::booltable::BoolTable, dbnet::prelude::*, - storage::v1::interface::DIR_ROOT, - }, - libsky::VERSION, -}; - -const INFO: &[u8] = b"info"; -const METRIC: &[u8] = b"metric"; -const INFO_PROTOCOL: &[u8] = b"protocol"; -const INFO_PROTOVER: &[u8] = b"protover"; -const INFO_VERSION: &[u8] = b"version"; -const METRIC_HEALTH: &[u8] = b"health"; -const METRIC_STORAGE_USAGE: &[u8] = b"storage"; -const ERR_UNKNOWN_PROPERTY: &[u8] = b"!16\nunknown-property\n"; -const ERR_UNKNOWN_METRIC: &[u8] = b"!14\nunknown-metric\n"; - -const HEALTH_TABLE: BoolTable<&str> = BoolTable::new("good", "critical"); - -action! { - fn sys(_handle: &Corestore, con: &mut Connection, iter: ActionIter<'_>) { - let mut iter = iter; - ensure_boolean_or_aerr::

(iter.len() == 2)?; - match unsafe { iter.next_lowercase_unchecked() }.as_ref() { - INFO => sys_info(con, &mut iter).await, - METRIC => sys_metric(con, &mut iter).await, - _ => util::err(P::RCODE_UNKNOWN_ACTION), - } - } - fn sys_info(con: &mut Connection, iter: &mut ActionIter<'_>) { - match unsafe { iter.next_lowercase_unchecked() }.as_ref() { - INFO_PROTOCOL => con.write_string(P::PROTOCOL_VERSIONSTRING).await?, - INFO_PROTOVER => con.write_float(P::PROTOCOL_VERSION).await?, - INFO_VERSION => con.write_string(VERSION).await?, - _ => return util::err(ERR_UNKNOWN_PROPERTY), - } - Ok(()) - } - fn sys_metric(con: &mut Connection, iter: &mut ActionIter<'_>) { - match unsafe { iter.next_lowercase_unchecked() }.as_ref() { - METRIC_HEALTH => { - con.write_string(HEALTH_TABLE[registry::state_okay()]).await? - } - METRIC_STORAGE_USAGE => { - match util::os::dirsize(DIR_ROOT) { - Ok(size) => con.write_int64(size).await?, - Err(e) => { - log::error!("Failed to get storage usage with: {e}"); - return util::err(P::RCODE_SERVER_ERR); - }, - } - } - _ => return util::err(ERR_UNKNOWN_METRIC), - } - Ok(()) - } -} diff --git a/server/src/arbiter.rs b/server/src/arbiter.rs deleted file mode 100644 index 53ab967b..00000000 --- a/server/src/arbiter.rs +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Created on Sat Jun 26 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - auth::AuthProvider, - config::{ConfigurationSet, SnapshotConfig, SnapshotPref}, - corestore::Corestore, - dbnet, - diskstore::flock::FileLock, - services, - storage::v1::sengine::SnapshotEngine, - util::{ - error::{Error, SkyResult}, - os::TerminationSignal, - }, - }, - std::{sync::Arc, thread::sleep}, - tokio::{ - sync::{ - broadcast, - mpsc::{self, Sender}, - }, - task::{self, JoinHandle}, - time::Duration, - }, -}; - -const TERMSIG_THRESHOLD: usize = 3; - -/// Start the server waiting for incoming connections or a termsig -pub async fn run( - ConfigurationSet { - ports, - bgsave, - snapshot, - maxcon, - auth, - protocol, - .. - }: ConfigurationSet, - restore_filepath: Option, -) -> SkyResult { - // Intialize the broadcast channel - let (signal, _) = broadcast::channel(1); - let engine = match &snapshot { - SnapshotConfig::Enabled(SnapshotPref { atmost, .. }) => SnapshotEngine::new(*atmost), - SnapshotConfig::Disabled => SnapshotEngine::new_disabled(), - }; - let engine = Arc::new(engine); - // restore data - services::restore_data(restore_filepath) - .map_err(|e| Error::ioerror_extra(e, "restoring data from backup"))?; - // init the store - let db = Corestore::init_with_snapcfg(engine.clone())?; - // refresh the snapshotengine state - engine.parse_dir()?; - let auth_provider = match auth.origin_key { - Some(key) => { - let authref = db.get_store().setup_auth(); - AuthProvider::new(authref, Some(key.into_inner())) - } - None => AuthProvider::new_disabled(), - }; - - // initialize the background services - let bgsave_handle = tokio::spawn(services::bgsave::bgsave_scheduler( - db.clone(), - bgsave, - signal.subscribe(), - )); - let snapshot_handle = tokio::spawn(services::snapshot::snapshot_service( - engine, - db.clone(), - snapshot, - signal.subscribe(), - )); - - // bind to signals - let termsig = - TerminationSignal::init().map_err(|e| Error::ioerror_extra(e, "binding to signals"))?; - // start the server (single or multiple listeners) - let mut server = dbnet::connect( - ports, - protocol, - maxcon, - db.clone(), - auth_provider, - signal.clone(), - ) - .await?; - - tokio::select! { - _ = server.run_server() => {}, - _ = termsig => {} - } - - log::info!("Signalling all workers to shut down"); - // drop the signal and let others exit - drop(signal); - server.finish_with_termsig().await; - - // wait for the background services to terminate - let _ = snapshot_handle.await; - let _ = bgsave_handle.await; - Ok(db) -} - -fn spawn_task(tx: Sender, db: Corestore, do_sleep: bool) -> JoinHandle<()> { - task::spawn_blocking(move || { - if do_sleep { - log::info!("Waiting for 10 seconds before retrying ..."); - sleep(Duration::from_secs(10)); - } - let ret = match crate::services::bgsave::run_bgsave(&db) { - Ok(()) => { - log::info!("Save before termination successful"); - true - } - Err(e) => { - log::error!("Failed to run save on termination: {e}"); - false - } - }; - tx.blocking_send(ret).expect("Receiver dropped"); - }) -} - -pub fn finalize_shutdown(corestore: Corestore, pid_file: FileLock) { - assert_eq!( - corestore.strong_count(), - 1, - "Correctness error. finalize_shutdown called before dropping server runtime" - ); - let rt = tokio::runtime::Builder::new_multi_thread() - .thread_name("server-final") - .enable_all() - .build() - .unwrap(); - let dbc = corestore.clone(); - let mut okay: bool = rt.block_on(async move { - let db = dbc; - let (tx, mut rx) = mpsc::channel::(1); - spawn_task(tx.clone(), db.clone(), false); - let spawn_again = || { - // we failed, so we better sleep - // now spawn it again to see the state - spawn_task(tx.clone(), db.clone(), true) - }; - let mut threshold = TERMSIG_THRESHOLD; - loop { - let termsig = match TerminationSignal::init().map_err(|e| e.to_string()) { - Ok(sig) => sig, - Err(e) => { - log::error!("Failed to bind to signal with error: {e}"); - crate::exit_error(); - } - }; - tokio::select! { - ret = rx.recv() => { - if ret.unwrap() { - // that's good to go - break true; - } else { - spawn_again(); - } - } - _ = termsig => { - threshold -= 1; - if threshold == 0 { - log::error!("Termination signal received but failed to flush data. Quitting because threshold exceeded"); - break false; - } else { - log::error!("Termination signal received but failed to flush data. Threshold is at {threshold}"); - continue; - } - }, - } - } - }); - okay &= services::pre_shutdown_cleanup(pid_file, Some(corestore.get_store())); - if okay { - log::info!("Goodbye :)"); - } else { - log::error!("Didn't terminate successfully"); - crate::exit_error(); - } -} diff --git a/server/src/auth/keys.rs b/server/src/auth/keys.rs deleted file mode 100644 index f766914f..00000000 --- a/server/src/auth/keys.rs +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Created on Mon Feb 21 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::provider::{Authkey, AUTHKEY_SIZE}, - crate::corestore::array::Array, -}; - -type AuthkeyArray = Array; -const RAN_BYTES_SIZE: usize = 40; - -/// Return a "human readable key" and the "authbytes" that can be stored -/// safely. To do this: -/// - Generate 64 random bytes -/// - Encode that into base64. This is the client key -/// - Hash the key using rcrypt. This is the server key that -/// will be stored -pub fn generate_full() -> (String, Authkey) { - let mut bytes: [u8; RAN_BYTES_SIZE] = [0u8; RAN_BYTES_SIZE]; - openssl::rand::rand_bytes(&mut bytes).unwrap(); - let ret = base64::encode_config(bytes, base64::BCRYPT); - let hash = rcrypt::hash(&ret, rcrypt::DEFAULT_COST).unwrap(); - let store_in_db = unsafe { - let mut array = AuthkeyArray::new(); - // we guarantee that the size is equal to 40 - array.extend_from_slice_unchecked(&hash); - array.into_array_unchecked() - }; - (ret, store_in_db) -} - -/// Verify a "human readable key" against the provided "authbytes" -pub fn verify_key(input: &[u8], hash: &[u8]) -> Option { - rcrypt::verify(input, hash).ok() -} diff --git a/server/src/auth/mod.rs b/server/src/auth/mod.rs deleted file mode 100644 index c60cdc8c..00000000 --- a/server/src/auth/mod.rs +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Created on Mon Feb 21 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -/* - * For our authn/authz, we have two important keys: - * - The origin key: This is the key saved in the configuration file that can also be - * used as the "recovery key" in the event the "root key" is lost. To claim the root - * account, one needs this key. This is a fixed width key with 40 characters - * - The root key: This is the superuser key that can be used to create/deny other - * accounts. On claiming the root account, this key is issued - * - * When the root account is claimed, it can be used to create "standard users". standard - * users have access to everything but the ability to create/revoke other users -*/ - -mod keys; -pub mod provider; -pub use provider::{AuthProvider, Authmap}; - -#[cfg(test)] -mod tests; - -use crate::dbnet::prelude::*; - -const AUTH_CLAIM: &[u8] = b"claim"; -const AUTH_LOGIN: &[u8] = b"login"; -const AUTH_LOGOUT: &[u8] = b"logout"; -const AUTH_ADDUSER: &[u8] = b"adduser"; -const AUTH_DELUSER: &[u8] = b"deluser"; -const AUTH_RESTORE: &[u8] = b"restore"; -const AUTH_LISTUSER: &[u8] = b"listuser"; -const AUTH_WHOAMI: &[u8] = b"whoami"; - -action! { - /// Handle auth. Should have passed the `auth` token - fn auth( - con: &mut Connection, - auth: &mut AuthProviderHandle, - iter: ActionIter<'_> - ) { - let mut iter = iter; - match iter.next_lowercase().unwrap_or_aerr::

()?.as_ref() { - AUTH_LOGIN => self::_auth_login(con, auth, &mut iter).await, - AUTH_CLAIM => self::_auth_claim(con, auth, &mut iter).await, - AUTH_ADDUSER => { - ensure_boolean_or_aerr::

(iter.len() == 1)?; // just the username - let username = unsafe { iter.next_unchecked() }; - let key = auth.provider_mut().claim_user::

(username)?; - con.write_string(&key).await?; - Ok(()) - } - AUTH_LOGOUT => { - ensure_boolean_or_aerr::

(iter.is_empty())?; // nothing else - auth.provider_mut().logout::

()?; - auth.set_unauth(); - con._write_raw(P::RCODE_OKAY).await?; - Ok(()) - } - AUTH_DELUSER => { - ensure_boolean_or_aerr::

(iter.len() == 1)?; // just the username - auth.provider_mut().delete_user::

(unsafe { iter.next_unchecked() })?; - con._write_raw(P::RCODE_OKAY).await?; - Ok(()) - } - AUTH_RESTORE => self::auth_restore(con, auth, &mut iter).await, - AUTH_LISTUSER => self::auth_listuser(con, auth, &mut iter).await, - AUTH_WHOAMI => self::auth_whoami(con, auth, &mut iter).await, - _ => util::err(P::RCODE_UNKNOWN_ACTION), - } - } - fn auth_whoami(con: &mut Connection, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) { - ensure_boolean_or_aerr::

(ActionIter::is_empty(iter))?; - con.write_string(&auth.provider().whoami::

()?).await?; - Ok(()) - } - fn auth_listuser(con: &mut Connection, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) { - ensure_boolean_or_aerr::

(ActionIter::is_empty(iter))?; - let usernames = auth.provider().collect_usernames::

()?; - con.write_typed_non_null_array_header(usernames.len(), b'+').await?; - for username in usernames { - con.write_typed_non_null_array_element(username.as_bytes()).await?; - } - Ok(()) - } - fn auth_restore(con: &mut Connection, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) { - let newkey = match iter.len() { - 1 => { - // so this fella thinks they're root - auth.provider().regenerate::

( - unsafe { iter.next_unchecked() } - )? - } - 2 => { - // so this fella is giving us the origin key - let origin = unsafe { iter.next_unchecked() }; - let id = unsafe { iter.next_unchecked() }; - auth.provider().regenerate_using_origin::

(origin, id)? - } - _ => return util::err(P::RCODE_ACTION_ERR), - }; - con.write_string(&newkey).await?; - Ok(()) - } - fn _auth_claim(con: &mut Connection, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) { - ensure_boolean_or_aerr::

(iter.len() == 1)?; // just the origin key - let origin_key = unsafe { iter.next_unchecked() }; - let key = auth.provider_mut().claim_root::

(origin_key)?; - auth.set_auth(); - con.write_string(&key).await?; - Ok(()) - } - /// Handle a login operation only. The **`login` token is expected to be present** - fn auth_login_only( - con: &mut Connection, - auth: &mut AuthProviderHandle, - iter: ActionIter<'_> - ) { - let mut iter = iter; - match iter.next_lowercase().unwrap_or_aerr::

()?.as_ref() { - AUTH_LOGIN => self::_auth_login(con, auth, &mut iter).await, - AUTH_CLAIM => self::_auth_claim(con, auth, &mut iter).await, - AUTH_RESTORE => self::auth_restore(con, auth, &mut iter).await, - AUTH_WHOAMI => self::auth_whoami(con, auth, &mut iter).await, - _ => util::err(P::AUTH_CODE_PERMS), - } - } - fn _auth_login(con: &mut Connection, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) { - // sweet, where's our username and password - ensure_boolean_or_aerr::

(iter.len() == 2)?; // just the uname and pass - let (username, password) = unsafe { (iter.next_unchecked(), iter.next_unchecked()) }; - auth.provider_mut().login::

(username, password)?; - auth.set_auth(); - con._write_raw(P::RCODE_OKAY).await?; - Ok(()) - } -} diff --git a/server/src/auth/provider.rs b/server/src/auth/provider.rs deleted file mode 100644 index 19495138..00000000 --- a/server/src/auth/provider.rs +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Created on Sun Mar 06 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::keys, - crate::{ - actions::{ActionError, ActionResult}, - corestore::{array::Array, htable::Coremap}, - protocol::interface::ProtocolSpec, - util::err, - }, - std::sync::Arc, -}; - -// constants -/// Size of an authn key in bytes -pub const AUTHKEY_SIZE: usize = 40; -/// Size of an authn ID in bytes -pub const AUTHID_SIZE: usize = 40; - -pub mod testsuite_data { - #![allow(unused)] - //! Temporary users created by the testsuite in debug mode - pub const TESTSUITE_ROOT_USER: &str = "root"; - pub const TESTSUITE_TEST_USER: &str = "testuser"; - pub const TESTSUITE_ROOT_TOKEN: &str = "XUOdVKhEONnnGwNwT7WeLqbspDgVtKex0/nwFwBSW7XJxioHwpg6H."; - pub const TESTSUITE_TEST_TOKEN: &str = "mpobAB7EY8vnBs70d/..h1VvfinKIeEJgt1rg4wUkwF6aWCvGGR9le"; -} - -uninit_array! { - const USER_ROOT_ARRAY: [u8; 40] = [b'r', b'o', b'o', b't']; -} -/// The root user -const USER_ROOT: AuthID = unsafe { AuthID::from_const(USER_ROOT_ARRAY, 4) }; - -/// An authn ID -type AuthID = Array; -/// An authn key -pub type Authkey = [u8; AUTHKEY_SIZE]; -/// Authmap -pub type Authmap = Arc>; - -/// The authn/authz provider -/// -pub struct AuthProvider { - origin: Option, - /// the current user - whoami: Option, - /// a map of users - authmap: Authmap, -} - -impl AuthProvider { - fn _new(authmap: Authmap, whoami: Option, origin: Option) -> Self { - Self { - authmap, - whoami, - origin, - } - } - /// New provider with no origin-key - pub fn new_disabled() -> Self { - Self::_new(Default::default(), None, None) - } - /// New provider with zero users - #[cfg(test)] - pub fn new_blank(origin: Option) -> Self { - Self::_new(Default::default(), None, origin) - } - /// New provider with users from the provided map - /// - /// ## Test suite - /// The testsuite creates users `root` and `testuser`; this **does not** apply to - /// release mode - pub fn new(authmap: Arc>, origin: Option) -> Self { - let slf = Self::_new(authmap, None, origin); - #[cfg(debug_assertions)] - { - // 'root' user in test mode - slf.authmap.true_if_insert( - AuthID::try_from_slice(testsuite_data::TESTSUITE_ROOT_USER).unwrap(), - [ - 172, 143, 117, 169, 158, 156, 33, 106, 139, 107, 20, 106, 91, 219, 34, 157, 98, - 147, 142, 91, 222, 238, 205, 120, 72, 171, 90, 218, 147, 2, 75, 67, 44, 108, - 185, 124, 55, 40, 156, 252, - ], - ); - // 'testuser' user in test mode - slf.authmap.true_if_insert( - AuthID::try_from_slice(testsuite_data::TESTSUITE_TEST_USER).unwrap(), - [ - 172, 183, 60, 221, 53, 240, 231, 217, 113, 112, 98, 16, 109, 62, 235, 95, 184, - 107, 130, 139, 43, 197, 40, 31, 176, 127, 185, 22, 172, 124, 39, 225, 124, 71, - 193, 115, 176, 162, 239, 93, - ], - ); - } - slf - } - pub const fn is_enabled(&self) -> bool { - matches!(self.origin, Some(_)) - } - pub fn claim_root(&mut self, origin_key: &[u8]) -> ActionResult { - self.verify_origin::

(origin_key)?; - // the origin key was good, let's try claiming root - let (key, store) = keys::generate_full(); - if self.authmap.true_if_insert(USER_ROOT, store) { - // claimed, sweet, log them in - self.whoami = Some(USER_ROOT); - Ok(key) - } else { - err(P::AUTH_ERROR_ALREADYCLAIMED) - } - } - fn are_you_root(&self) -> ActionResult { - self.ensure_enabled::

()?; - match self.whoami.as_ref().map(|v| v.eq(&USER_ROOT)) { - Some(v) => Ok(v), - None => err(P::AUTH_CODE_PERMS), - } - } - pub fn claim_user(&self, claimant: &[u8]) -> ActionResult { - self.ensure_root::

()?; - self._claim_user::

(claimant) - } - pub fn _claim_user(&self, claimant: &[u8]) -> ActionResult { - let (key, store) = keys::generate_full(); - if self - .authmap - .true_if_insert(Self::try_auth_id::

(claimant)?, store) - { - Ok(key) - } else { - err(P::AUTH_ERROR_ALREADYCLAIMED) - } - } - pub fn login(&mut self, account: &[u8], token: &[u8]) -> ActionResult<()> { - self.ensure_enabled::

()?; - match self - .authmap - .get(account) - .map(|token_hash| keys::verify_key(token, token_hash.as_slice())) - { - Some(Some(true)) => { - // great, authenticated - self.whoami = Some(Self::try_auth_id::

(account)?); - Ok(()) - } - _ => { - // either the password was wrong, or the username was wrong - err(P::AUTH_CODE_BAD_CREDENTIALS) - } - } - } - pub fn regenerate_using_origin( - &self, - origin: &[u8], - account: &[u8], - ) -> ActionResult { - self.verify_origin::

(origin)?; - self._regenerate::

(account) - } - pub fn regenerate(&self, account: &[u8]) -> ActionResult { - self.ensure_root::

()?; - self._regenerate::

(account) - } - /// Regenerate the token for the given user. This returns a new token - fn _regenerate(&self, account: &[u8]) -> ActionResult { - let id = Self::try_auth_id::

(account)?; - let (key, store) = keys::generate_full(); - if self.authmap.true_if_update(id, store) { - Ok(key) - } else { - err(P::AUTH_CODE_BAD_CREDENTIALS) - } - } - fn try_auth_id(authid: &[u8]) -> ActionResult { - if authid.is_ascii() && authid.len() <= AUTHID_SIZE { - Ok(unsafe { - // We just verified the length - AuthID::from_slice(authid) - }) - } else { - err(P::AUTH_ERROR_ILLEGAL_USERNAME) - } - } - pub fn logout(&mut self) -> ActionResult<()> { - self.ensure_enabled::

()?; - self.whoami - .take() - .map(|_| ()) - .ok_or(ActionError::ActionError(P::AUTH_CODE_PERMS)) - } - fn ensure_enabled(&self) -> ActionResult<()> { - self.origin - .as_ref() - .map(|_| ()) - .ok_or(ActionError::ActionError(P::AUTH_ERROR_DISABLED)) - } - pub fn verify_origin(&self, origin: &[u8]) -> ActionResult<()> { - if self.get_origin::

()?.eq(origin) { - Ok(()) - } else { - err(P::AUTH_CODE_BAD_CREDENTIALS) - } - } - fn get_origin(&self) -> ActionResult<&Authkey> { - match self.origin.as_ref() { - Some(key) => Ok(key), - None => err(P::AUTH_ERROR_DISABLED), - } - } - fn ensure_root(&self) -> ActionResult<()> { - if self.are_you_root::

()? { - Ok(()) - } else { - err(P::AUTH_CODE_PERMS) - } - } - pub fn delete_user(&self, user: &[u8]) -> ActionResult<()> { - self.ensure_root::

()?; - if user.eq(&USER_ROOT) { - // can't delete root! - err(P::AUTH_ERROR_FAILED_TO_DELETE_USER) - } else if self.authmap.true_if_removed(user) { - Ok(()) - } else { - err(P::AUTH_CODE_BAD_CREDENTIALS) - } - } - /// List all the users - pub fn collect_usernames(&self) -> ActionResult> { - self.ensure_root::

()?; - Ok(self - .authmap - .iter() - .map(|kv| String::from_utf8_lossy(kv.key()).to_string()) - .collect()) - } - /// Return the AuthID of the current user - pub fn whoami(&self) -> ActionResult { - self.ensure_enabled::

()?; - self.whoami - .as_ref() - .map(|v| String::from_utf8_lossy(v).to_string()) - .ok_or(ActionError::ActionError(P::AUTH_CODE_PERMS)) - } -} - -impl Clone for AuthProvider { - fn clone(&self) -> Self { - Self { - authmap: self.authmap.clone(), - whoami: None, - origin: self.origin, - } - } -} diff --git a/server/src/auth/tests.rs b/server/src/auth/tests.rs deleted file mode 100644 index 1c773e4d..00000000 --- a/server/src/auth/tests.rs +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Created on Tue Feb 22 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -mod keys { - use super::super::keys::{generate_full, verify_key}; - - #[test] - fn test_verify_key() { - let (key, store) = generate_full(); - assert!(verify_key(key.as_bytes(), &store).unwrap()); - } -} - -mod authn { - use crate::{ - actions::ActionError, - auth::AuthProvider, - protocol::{interface::ProtocolSpec, Skyhash2}, - }; - - const ORIG: &[u8; 40] = b"c4299d190fb9a00626797fcc138c56eae9971664"; - - #[test] - fn claim_root_okay() { - let mut provider = AuthProvider::new_blank(Some(*ORIG)); - let _ = provider.claim_root::(ORIG).unwrap(); - } - #[test] - fn claim_root_wrongkey() { - let mut provider = AuthProvider::new_blank(Some(*ORIG)); - let claim_err = provider.claim_root::(&ORIG[1..]).unwrap_err(); - assert_eq!( - claim_err, - ActionError::ActionError(Skyhash2::AUTH_CODE_BAD_CREDENTIALS) - ); - } - #[test] - fn claim_root_disabled() { - let mut provider = AuthProvider::new_disabled(); - assert_eq!( - provider.claim_root::(b"abcd").unwrap_err(), - ActionError::ActionError(Skyhash2::AUTH_ERROR_DISABLED) - ); - } - #[test] - fn claim_root_already_claimed() { - let mut provider = AuthProvider::new_blank(Some(*ORIG)); - let _ = provider.claim_root::(ORIG).unwrap(); - assert_eq!( - provider.claim_root::(ORIG).unwrap_err(), - ActionError::ActionError(Skyhash2::AUTH_ERROR_ALREADYCLAIMED) - ); - } - #[test] - fn claim_user_okay_with_login() { - let mut provider = AuthProvider::new_blank(Some(*ORIG)); - // claim root - let rootkey = provider.claim_root::(ORIG).unwrap(); - // login as root - provider - .login::(b"root", rootkey.as_bytes()) - .unwrap(); - // claim user - let _ = provider.claim_user::(b"sayan").unwrap(); - } - - #[test] - fn claim_user_fail_not_root_with_login() { - let mut provider = AuthProvider::new_blank(Some(*ORIG)); - // claim root - let rootkey = provider.claim_root::(ORIG).unwrap(); - // login as root - provider - .login::(b"root", rootkey.as_bytes()) - .unwrap(); - // claim user - let userkey = provider.claim_user::(b"user").unwrap(); - // login as user - provider - .login::(b"user", userkey.as_bytes()) - .unwrap(); - // now try to claim an user being a non-root account - assert_eq!( - provider.claim_user::(b"otheruser").unwrap_err(), - ActionError::ActionError(Skyhash2::AUTH_CODE_PERMS) - ); - } - #[test] - fn claim_user_fail_anonymous() { - let mut provider = AuthProvider::new_blank(Some(*ORIG)); - // claim root - let _ = provider.claim_root::(ORIG).unwrap(); - // logout - provider.logout::().unwrap(); - // try to claim as an anonymous user - assert_eq!( - provider.claim_user::(b"newuser").unwrap_err(), - ActionError::ActionError(Skyhash2::AUTH_CODE_PERMS) - ); - } -} diff --git a/server/src/blueql/ast.rs b/server/src/blueql/ast.rs deleted file mode 100644 index 5b8422c7..00000000 --- a/server/src/blueql/ast.rs +++ /dev/null @@ -1,485 +0,0 @@ -/* - * Created on Tue Jun 14 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ - error::{LangError, LangResult}, - lexer::{Keyword, Lexer, Token, Type, TypeExpression}, - RawSlice, - }, - crate::util::{compiler, Life}, - core::{marker::PhantomData, mem::transmute, ptr}, -}; - -#[derive(Debug)] -#[cfg_attr(test, derive(PartialEq, Eq))] -#[repr(u8)] -/// A statement that can be executed -pub enum Statement { - /// Create a new space with the provided ID - CreateSpace(RawSlice), - /// Create a new model with the provided configuration - CreateModel { - entity: Entity, - model: FieldConfig, - volatile: bool, - }, - /// Drop the given model - DropModel { entity: Entity, force: bool }, - /// Drop the given space - DropSpace { entity: RawSlice, force: bool }, - /// Inspect the given space - InspectSpace(Option), - /// Inspect the given model - InspectModel(Option), - /// Inspect all the spaces in the database - InspectSpaces, - /// Switch to the given entity - Use(Entity), -} - -pub type StatementLT<'a> = Life<'a, Statement>; - -#[derive(Debug)] -#[cfg_attr(test, derive(PartialEq, Eq))] -pub enum Entity { - Current(RawSlice), - Full(RawSlice, RawSlice), -} - -impl Entity { - const MAX_LENGTH_EX: usize = 65; - pub fn from_slice(slice: &[u8]) -> LangResult { - Compiler::new(&Lexer::lex(slice)?).parse_entity_name() - } -} - -#[derive(Debug)] -#[cfg_attr(test, derive(PartialEq, Eq))] -/// The field configuration used when declaring the fields for a model -pub struct FieldConfig { - /// the types of the fields - pub types: Vec, - /// the names of the fields - pub names: Vec, -} - -impl FieldConfig { - /// Create an empty field configuration - pub const fn new() -> Self { - Self { - types: Vec::new(), - names: Vec::new(), - } - } - // TODO(@ohsayan): Completely deprecate the model-code based API - pub fn get_model_code(&self) -> LangResult { - let Self { types, names } = self; - let invalid_expr = { - // the model API doesn't support named fields (it's super limited; we need to drop it) - !names.is_empty() - || types.len() != 2 - // the key type cannot be compound - || types[0].0.len() != 1 - // the key type cannot be a list - || types[0].0[0] == Type::List - // the value cannot have a depth more than two - || types[1].0.len() > 2 - // if the value is a string or binary, it cannot have a depth more than 1 - || ((types[1].0[0] == Type::Binary || types[1].0[0] == Type::String) && types[1].0.len() != 1) - // if the value is a list, it must have a depth of two - || (types[1].0[0] == Type::List && types[1].0.len() != 2) - // if the value is a list, the type argument cannot be a list (it's stupid, I know; that's exactly - // why I'll be ditching this API in the next two PRs) - || (types[1].0[0] == Type::List && types[1].0[1] == Type::List) - }; - if compiler::unlikely(invalid_expr) { - // the value type cannot have a depth more than 2 - return Err(LangError::UnsupportedModelDeclaration); - } - let key_expr = &types[0].0; - let value_expr = &types[1].0; - if value_expr[0] == Type::List { - let k_enc = key_expr[0] == Type::String; - let v_enc = value_expr[1] == Type::String; - Ok(((k_enc as u8) << 1) + (v_enc as u8) + 4) - } else { - let k_enc = key_expr[0] == Type::String; - let v_enc = value_expr[0] == Type::String; - let ret = k_enc as u8 + v_enc as u8; - Ok((ret & 1) + ((k_enc as u8) << 1)) - } - } -} - -// expect state -#[derive(Debug)] -#[repr(u8)] -#[derive(PartialEq)] -/// What to expect next -enum Expect { - /// Expect a type - Type = 0, - /// Expect a [`Token::CloseAngular`] - Close = 1, -} - -/// A compiler for BlueQL queries -/// -/// This compiler takes an input stream and evaluates the query using a traditional -/// lexer-parser pipeline -pub struct Compiler<'a> { - cursor: *const Token, - end_ptr: *const Token, - _lt: PhantomData<&'a [u8]>, -} - -impl<'a> Compiler<'a> { - #[inline(always)] - /// Check if we have not exhausted the token stream - fn not_exhausted(&self) -> bool { - self.cursor < self.end_ptr - } - #[inline(always)] - /// Deref the cursor - unsafe fn deref_cursor(&self) -> &Token { - &*self.cursor - } - #[inline(always)] - fn peek_neq(&self, token: &Token) -> bool { - self.not_exhausted() && unsafe { self.deref_cursor() != token } - } - #[inline(always)] - /// Check if the token ahead equals the given token - fn peek_eq(&self, tok: &Token) -> bool { - self.not_exhausted() && unsafe { self.deref_cursor() == tok } - } - #[inline(always)] - /// Check if the token ahead equals the given token, moving the cursor ahead if so - fn next_eq(&mut self, tok: &Token) -> bool { - let next_is_eq = self.not_exhausted() && unsafe { self.deref_cursor() == tok }; - unsafe { self.incr_cursor_if(next_is_eq) }; - next_is_eq - } - #[inline(always)] - /// Increment the cursor if the condition is true - unsafe fn incr_cursor_if(&mut self, cond: bool) { - self.incr_cursor_by(cond as usize) - } - #[inline(always)] - /// Move the cursor ahead by `by` positions - unsafe fn incr_cursor_by(&mut self, by: usize) { - self.cursor = self.cursor.add(by) - } - #[inline(always)] - /// Move the cursor ahead by one - unsafe fn incr_cursor(&mut self) { - self.incr_cursor_by(1) - } - #[inline(always)] - unsafe fn decr_cursor(&mut self) { - self.decr_cursor_by(1) - } - #[inline(always)] - unsafe fn decr_cursor_by(&mut self, by: usize) { - self.cursor = self.cursor.sub(by) - } - #[inline(always)] - /// Read the element ahead if we have not exhausted the token stream. This - /// will forward the cursor - fn next(&mut self) -> Option { - if self.not_exhausted() { - let r = Some(unsafe { ptr::read(self.cursor) }); - unsafe { self.incr_cursor() }; - r - } else { - None - } - } - #[inline(always)] - fn next_result(&mut self) -> LangResult { - if compiler::likely(self.not_exhausted()) { - let r = unsafe { ptr::read(self.cursor) }; - unsafe { self.incr_cursor() }; - Ok(r) - } else { - Err(LangError::UnexpectedEOF) - } - } - #[inline(always)] - fn next_ident(&mut self) -> LangResult { - match self.next() { - Some(Token::Identifier(rws)) => Ok(rws), - Some(_) => Err(LangError::InvalidSyntax), - None => Err(LangError::UnexpectedEOF), - } - } - #[inline(always)] - /// Returns the remaining number of tokens - fn remaining(&self) -> usize { - self.end_ptr as usize - self.cursor as usize - } -} - -impl<'a> Compiler<'a> { - #[inline(always)] - #[cfg(test)] - /// Compile the given BlueQL source - pub fn compile(src: &'a [u8]) -> LangResult> { - Self::compile_with_extra(src, 0) - } - #[inline(always)] - /// Compile the given BlueQL source with optionally supplied extra arguments - /// HACK: Just helps us omit an additional check - pub fn compile_with_extra(src: &'a [u8], len: usize) -> LangResult> { - let tokens = Lexer::lex(src)?; - Self::new(&tokens).eval(len).map(Life::new) - } - #[inline(always)] - pub const fn new(tokens: &[Token]) -> Self { - unsafe { - Self { - cursor: tokens.as_ptr(), - end_ptr: tokens.as_ptr().add(tokens.len()), - _lt: PhantomData, - } - } - } - #[inline(always)] - /// The inner eval method - fn eval(&mut self, extra_len: usize) -> LangResult { - let stmt = match self.next() { - Some(tok) => match tok { - Token::Keyword(Keyword::Create) => self.parse_create0(), - Token::Keyword(Keyword::Drop) => self.parse_drop0(), - Token::Keyword(Keyword::Inspect) => self.parse_inspect0(), - Token::Keyword(Keyword::Use) => self.parse_use0(), - _ => Err(LangError::ExpectedStatement), - }, - None => Err(LangError::UnexpectedEOF), - }; - if compiler::likely(self.remaining() == 0 && extra_len == 0) { - stmt - } else { - Err(LangError::InvalidSyntax) - } - } - #[inline(always)] - fn parse_use0(&mut self) -> LangResult { - Ok(Statement::Use(self.parse_entity_name()?)) - } - #[inline(always)] - /// Parse an inspect statement - fn parse_inspect0(&mut self) -> LangResult { - match self.next_result()? { - Token::Keyword(Keyword::Model) => self.parse_inspect_model0(), - Token::Keyword(Keyword::Space) => self.parse_inspect_space0(), - Token::Identifier(spaces) - if unsafe { spaces.as_slice() }.eq_ignore_ascii_case(b"spaces") => - { - Ok(Statement::InspectSpaces) - } - _ => Err(LangError::InvalidSyntax), - } - } - #[inline(always)] - /// Parse `inspect model ` - fn parse_inspect_model0(&mut self) -> LangResult { - match self.next() { - Some(Token::Identifier(ident)) => Ok(Statement::InspectModel(Some( - self.parse_entity_name_with_start(ident)?, - ))), - Some(_) => Err(LangError::InvalidSyntax), - None => Ok(Statement::InspectModel(None)), - } - } - #[inline(always)] - /// Parse `inspect space ` - fn parse_inspect_space0(&mut self) -> LangResult { - match self.next() { - Some(Token::Identifier(ident)) => Ok(Statement::InspectSpace(Some(ident))), - Some(_) => Err(LangError::InvalidSyntax), - None => Ok(Statement::InspectSpace(None)), - } - } - #[inline(always)] - /// Parse a drop statement - fn parse_drop0(&mut self) -> LangResult { - let (drop_container, drop_id) = (self.next(), self.next()); - match (drop_container, drop_id) { - (Some(Token::Keyword(Keyword::Model)), Some(Token::Identifier(model_name))) => { - Ok(Statement::DropModel { - entity: self.parse_entity_name_with_start(model_name)?, - force: self.next_eq(&Token::Keyword(Keyword::Force)), - }) - } - (Some(Token::Keyword(Keyword::Space)), Some(Token::Identifier(space_name))) => { - Ok(Statement::DropSpace { - entity: space_name, - force: self.next_eq(&Token::Keyword(Keyword::Force)), - }) - } - _ => Err(LangError::InvalidSyntax), - } - } - #[inline(always)] - /// Parse a create statement - fn parse_create0(&mut self) -> LangResult { - match self.next() { - Some(Token::Keyword(Keyword::Model)) => self.parse_create_model0(), - Some(Token::Keyword(Keyword::Space)) => self.parse_create_space0(), - Some(_) => Err(LangError::UnknownCreateQuery), - None => Err(LangError::UnexpectedEOF), - } - } - #[inline(always)] - /// Parse a `create model` statement - fn parse_create_model0(&mut self) -> LangResult { - let entity = self.parse_entity_name()?; - self.parse_create_model1(entity) - } - #[inline(always)] - /// Parse a field expression and return a `Statement::CreateModel` - pub(super) fn parse_create_model1(&mut self, entity: Entity) -> LangResult { - let mut fc = FieldConfig::new(); - let mut is_good_expr = self.next_eq(&Token::OpenParen); - while is_good_expr && self.peek_neq(&Token::CloseParen) { - match self.next() { - Some(Token::Identifier(field_name)) => { - // we have a field name - is_good_expr &= self.next_eq(&Token::Colon); - if let Some(Token::Keyword(Keyword::Type(ty))) = self.next() { - fc.names.push(field_name); - fc.types.push(self.parse_type_expression(ty)?); - } else { - is_good_expr = false; - } - is_good_expr &= self.peek_eq(&Token::CloseParen) || self.next_eq(&Token::Comma); - } - Some(Token::Keyword(Keyword::Type(ty))) => { - // we have a type name - fc.types.push(self.parse_type_expression(ty)?); - is_good_expr &= self.peek_eq(&Token::CloseParen) || self.next_eq(&Token::Comma); - } - _ => is_good_expr = false, - } - } - is_good_expr &= self.next_eq(&Token::CloseParen); - is_good_expr &= fc.types.len() >= 2; - // important; we either have all unnamed fields or all named fields; having some unnamed - // and some named is ambiguous because there's not "straightforward" way to query them - // without introducing some funky naming conventions ($ if you don't have the - // right name sounds like an outrageous idea) - is_good_expr &= fc.names.is_empty() || fc.names.len() == fc.types.len(); - let volatile = self.next_eq(&Token::Keyword(Keyword::Volatile)); - if compiler::likely(is_good_expr) { - Ok(Statement::CreateModel { - entity, - model: fc, - volatile, - }) - } else { - Err(LangError::BadExpression) - } - } - #[inline(always)] - /// Parse a type expression return a `TypeExpression` - fn parse_type_expression(&mut self, first_type: Type) -> LangResult { - let mut expr = Vec::with_capacity(2); - expr.push(first_type); - - // count of open and close brackets - let mut p_open = 0; - let mut p_close = 0; - let mut valid_expr = true; - - // we already have the starting type; next is either nothing or open angular - let mut has_more_args = self.peek_eq(&Token::OpenAngular); - - let mut expect = Expect::Type; - while valid_expr && has_more_args && self.peek_neq(&Token::CloseParen) { - match self.next() { - Some(Token::OpenAngular) => p_open += 1, - Some(Token::Keyword(Keyword::Type(ty))) if expect == Expect::Type => { - expr.push(ty); - let next = self.next(); - let next_is_open = next == Some(Token::OpenAngular); - let next_is_close = next == Some(Token::CloseAngular); - p_open += next_is_open as usize; - p_close += next_is_close as usize; - expect = unsafe { transmute(next_is_close) }; - } - Some(Token::CloseAngular) if expect == Expect::Close => { - p_close += 1; - expect = Expect::Close; - } - Some(Token::Comma) => { - unsafe { self.decr_cursor() } - has_more_args = false - } - _ => valid_expr = false, - } - } - valid_expr &= p_open == p_close; - if compiler::likely(valid_expr) { - Ok(TypeExpression(expr)) - } else { - Err(LangError::InvalidSyntax) - } - } - #[inline(always)] - /// Parse a `create space` statement - fn parse_create_space0(&mut self) -> LangResult { - match self.next() { - Some(Token::Identifier(model_name)) => Ok(Statement::CreateSpace(model_name)), - Some(_) => Err(LangError::InvalidSyntax), - None => Err(LangError::UnexpectedEOF), - } - } - #[inline(always)] - fn parse_entity_name_with_start(&mut self, start: RawSlice) -> LangResult { - if self.peek_eq(&Token::Period) { - unsafe { self.incr_cursor() }; - Ok(Entity::Full(start, self.next_ident()?)) - } else { - Ok(Entity::Current(start)) - } - } - #[inline(always)] - pub(super) fn parse_entity_name(&mut self) -> LangResult { - // let's peek the next token - match self.next_ident()? { - id if self.peek_eq(&Token::Period) - && compiler::likely(id.len() < Entity::MAX_LENGTH_EX) => - { - unsafe { self.incr_cursor() }; - Ok(Entity::Full(id, self.next_ident()?)) - } - id if compiler::likely(id.len() < Entity::MAX_LENGTH_EX) => Ok(Entity::Current(id)), - _ => Err(LangError::InvalidSyntax), - } - } -} diff --git a/server/src/blueql/error.rs b/server/src/blueql/error.rs deleted file mode 100644 index ba065143..00000000 --- a/server/src/blueql/error.rs +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Created on Tue Jun 14 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{ - actions::{ActionError, ActionResult}, - protocol::interface::ProtocolSpec, -}; - -#[derive(Debug, PartialEq, Eq)] -#[repr(u8)] -/// BlueQL errors -pub enum LangError { - /// Invalid syntax - InvalidSyntax, - /// Invalid numeric literal - InvalidNumericLiteral, - /// Unexpected end-of-statement - UnexpectedEOF, - /// Expected a statement but found some other token - ExpectedStatement, - /// Got an unknown create query - UnknownCreateQuery, - /// Bad expression - BadExpression, - /// An invalid string literal - InvalidStringLiteral, - /// Unsupported model declaration - UnsupportedModelDeclaration, - /// Unexpected character - UnexpectedChar, -} - -/// Results for BlueQL -pub type LangResult = Result; - -#[inline(never)] -#[cold] -pub(super) const fn cold_err(e: LangError) -> &'static [u8] { - match e { - LangError::BadExpression => P::BQL_BAD_EXPRESSION, - LangError::ExpectedStatement => P::BQL_EXPECTED_STMT, - LangError::InvalidNumericLiteral => P::BQL_INVALID_NUMERIC_LITERAL, - LangError::InvalidStringLiteral => P::BQL_INVALID_STRING_LITERAL, - LangError::InvalidSyntax => P::BQL_INVALID_SYNTAX, - LangError::UnexpectedEOF => P::BQL_UNEXPECTED_EOF, - LangError::UnknownCreateQuery => P::BQL_UNKNOWN_CREATE_QUERY, - LangError::UnsupportedModelDeclaration => P::BQL_UNSUPPORTED_MODEL_DECL, - LangError::UnexpectedChar => P::BQL_UNEXPECTED_CHAR, - } -} - -#[inline(always)] -pub fn map_ql_err_to_resp(e: LangResult) -> ActionResult { - match e { - Ok(v) => Ok(v), - Err(e) => Err(ActionError::ActionError(cold_err::

(e))), - } -} diff --git a/server/src/blueql/executor.rs b/server/src/blueql/executor.rs deleted file mode 100644 index 9dac3aa6..00000000 --- a/server/src/blueql/executor.rs +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Created on Wed Jun 15 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::dbnet::BufferedSocketStream; - -use { - super::{ - ast::{Statement, StatementLT}, - error, - }, - crate::{ - actions::{self, ActionError, ActionResult}, - blueql, - corestore::memstore::ObjectID, - dbnet::prelude::*, - }, -}; - -pub async fn execute<'a, P, C>( - handle: &'a mut Corestore, - con: &mut Connection, - maybe_statement: &[u8], - extra: usize, -) -> ActionResult<()> -where - P: ProtocolSpec, - C: BufferedSocketStream, -{ - let statement = - error::map_ql_err_to_resp::(blueql::compile(maybe_statement, extra))?; - let system_health_okay = registry::state_okay(); - let result = match statement.as_ref() { - Statement::Use(entity) => handle.swap_entity(entity), - Statement::CreateSpace(space_name) if system_health_okay => { - // ret okay - handle.create_keyspace(unsafe { ObjectID::from_slice(space_name.as_slice()) }) - } - Statement::DropSpace { entity, force } if system_health_okay => { - // ret okay - let entity = unsafe { ObjectID::from_slice(entity.as_slice()) }; - if *force { - handle.force_drop_keyspace(entity) - } else { - handle.drop_keyspace(entity) - } - } - Statement::DropModel { entity, force } if system_health_okay => { - // ret okay - handle.drop_table(entity, *force) - } - Statement::CreateModel { - entity, - model, - volatile, - } if system_health_okay => { - match model.get_model_code() { - // ret okay - Ok(code) => handle.create_table(entity, code, *volatile), - Err(e) => return Err(ActionError::ActionError(error::cold_err::

(e))), - } - } - Statement::InspectSpaces => { - // ret directly - con.write_typed_non_null_array(&handle.get_store().list_keyspaces(), b'+') - .await?; - return Ok(()); - } - Statement::InspectSpace(space) => { - // ret directly - con.write_typed_non_null_array( - handle.list_tables::

(space.as_ref().map(|v| unsafe { v.as_slice() }))?, - b'+', - ) - .await?; - return Ok(()); - } - Statement::InspectModel(model) => { - // ret directly - con.write_string(&handle.describe_table::

(model)?) - .await?; - return Ok(()); - } - _ => { - // the server is broken - con._write_raw(P::RCODE_SERVER_ERR).await?; - return Ok(()); - } - }; - actions::translate_ddl_error::(result)?; - con._write_raw(P::RCODE_OKAY).await?; - Ok(()) -} diff --git a/server/src/blueql/lexer.rs b/server/src/blueql/lexer.rs deleted file mode 100644 index 188aa378..00000000 --- a/server/src/blueql/lexer.rs +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Created on Tue Jun 14 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ - error::{LangError, LangResult}, - RawSlice, - }, - crate::util::compiler, - core::{marker::PhantomData, slice, str}, -}; - -#[derive(Debug, PartialEq, Eq)] -#[repr(u8)] -/// BQL tokens -pub enum Token { - OpenParen, // ( - CloseParen, // ) - OpenAngular, // < - CloseAngular, // > - Comma, // , - Colon, // : - Period, // . - QuotedString(String), - Identifier(RawSlice), - Number(u64), - Keyword(Keyword), -} - -impl From for Token { - fn from(kw: Keyword) -> Self { - Self::Keyword(kw) - } -} - -#[cfg(test)] -impl From<&'static str> for Token { - fn from(sl: &'static str) -> Self { - Self::Identifier(sl.into()) - } -} - -impl From for Token { - fn from(num: u64) -> Self { - Self::Number(num) - } -} - -impl From for Token { - fn from(ty: Type) -> Self { - Self::Keyword(Keyword::Type(ty)) - } -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -#[repr(u8)] -/// BlueQL keywords -pub enum Keyword { - Create, - Use, - Drop, - Inspect, - Model, - Space, - Volatile, - Force, - Type(Type), -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(u8)] -/// BlueQL types -pub enum Type { - String, - Binary, - List, -} - -#[derive(Debug, PartialEq, Eq)] -/// Type expression (ty>) -pub struct TypeExpression(pub Vec); - -impl Keyword { - /// Attempt to parse a keyword from the given slice - #[inline(always)] - pub fn try_from_slice(slice: &[u8]) -> Option { - let r = match slice.to_ascii_lowercase().as_slice() { - b"create" => Keyword::Create, - b"drop" => Keyword::Drop, - b"inspect" => Keyword::Inspect, - b"model" => Keyword::Model, - b"space" => Keyword::Space, - b"volatile" => Keyword::Volatile, - b"string" => Keyword::Type(Type::String), - b"binary" => Keyword::Type(Type::Binary), - b"list" => Keyword::Type(Type::List), - b"force" => Keyword::Force, - b"use" => Keyword::Use, - _ => return None, - }; - Some(r) - } -} - -#[inline(always)] -/// Find the distance between two pointers -fn find_ptr_distance(start: *const u8, stop: *const u8) -> usize { - stop as usize - start as usize -} - -/// A `Lexer` for BlueQL tokens -pub struct Lexer<'a> { - cursor: *const u8, - end_ptr: *const u8, - _lt: PhantomData<&'a [u8]>, - last_error: Option, - tokens: Vec, -} - -const _ENSURE_EQ_SIZE: () = - assert!(std::mem::size_of::>() == std::mem::size_of::()); - -impl<'a> Lexer<'a> { - #[inline(always)] - /// Create a new `Lexer` - pub const fn new(buf: &'a [u8]) -> Self { - unsafe { - Self { - cursor: buf.as_ptr(), - end_ptr: buf.as_ptr().add(buf.len()), - last_error: None, - tokens: Vec::new(), - _lt: PhantomData, - } - } - } -} - -impl<'a> Lexer<'a> { - #[inline(always)] - /// Returns the cursor - const fn cursor(&self) -> *const u8 { - self.cursor - } - #[inline(always)] - /// Returns the end ptr - const fn end_ptr(&self) -> *const u8 { - self.end_ptr - } - #[inline(always)] - /// Increments the cursor by 1 - unsafe fn incr_cursor(&mut self) { - self.incr_cursor_by(1) - } - /// Increments the cursor by 1 if `cond` is true - #[inline(always)] - unsafe fn incr_cursor_if(&mut self, cond: bool) { - self.incr_cursor_by(cond as usize) - } - #[inline(always)] - /// Increments the cursor by `by` positions - unsafe fn incr_cursor_by(&mut self, by: usize) { - self.cursor = self.cursor.add(by) - } - #[inline(always)] - /// Derefs the cursor - unsafe fn deref_cursor(&self) -> u8 { - *self.cursor() - } - #[inline(always)] - /// Checks if we have reached EOA - fn not_exhausted(&self) -> bool { - self.cursor() < self.end_ptr() - } - #[inline(always)] - /// Returns true if we have reached EOA - fn exhausted(&self) -> bool { - self.cursor() >= self.end_ptr() - } - #[inline(always)] - /// Check if the peeked value matches the predicate. Returns false if EOA - fn peek_is(&self, predicate: impl Fn(u8) -> bool) -> bool { - self.not_exhausted() && unsafe { predicate(self.deref_cursor()) } - } - #[inline(always)] - /// Check if the byte ahead is equal to the provided byte. Returns false - /// if reached EOA - fn peek_eq(&self, eq: u8) -> bool { - self.not_exhausted() && unsafe { self.deref_cursor() == eq } - } - #[inline(always)] - /// Check if the byte ahead is not equal to the provided byte. Returns false - /// if reached EOA - fn peek_neq(&self, neq: u8) -> bool { - self.not_exhausted() && unsafe { self.deref_cursor() != neq } - } - #[inline(always)] - /// Same as `peek_eq`, but forwards the cursor if the byte is matched - fn peek_eq_and_forward(&mut self, eq: u8) -> bool { - let did_peek = self.peek_eq(eq); - unsafe { self.incr_cursor_if(did_peek) }; - did_peek - } - #[inline(always)] - /// Same as `peek_eq_or_eof` but forwards the cursor on match - fn peek_eq_or_eof_and_forward(&mut self, eq: u8) -> bool { - let did_forward = self.peek_eq_and_forward(eq); - unsafe { self.incr_cursor_if(did_forward) }; - did_forward | self.exhausted() - } - #[inline(always)] - /// Trim the whitespace ahead - fn trim_ahead(&mut self) { - while self.peek_eq_and_forward(b' ') {} - } - #[inline(always)] - unsafe fn check_escaped(&mut self, escape_what: u8) -> bool { - debug_assert!(self.not_exhausted()); - self.deref_cursor() == b'\\' && { - self.not_exhausted() && self.deref_cursor() == escape_what - } - } - #[inline(always)] - fn push_token(&mut self, token: impl Into) { - self.tokens.push(token.into()) - } -} - -impl<'a> Lexer<'a> { - #[inline(always)] - /// Attempt to scan a number - fn scan_number(&mut self) { - let start = self.cursor(); - while self.peek_is(|byte| byte.is_ascii_digit()) { - unsafe { self.incr_cursor() } - } - let slice = unsafe { - str::from_utf8_unchecked(slice::from_raw_parts( - start, - find_ptr_distance(start, self.cursor()), - )) - }; - let next_is_ws_or_eof = self.peek_eq_or_eof_and_forward(b' '); - match slice.parse() { - Ok(num) if compiler::likely(next_is_ws_or_eof) => { - // this is a good number; push it in - self.push_token(Token::Number(num)); - } - _ => { - // that breaks the state - self.last_error = Some(LangError::InvalidNumericLiteral); - } - } - } - #[inline(always)] - /// Attempt to scan an ident - fn scan_ident(&mut self) -> RawSlice { - let start = self.cursor(); - while self.peek_is(|byte| (byte.is_ascii_alphanumeric() || byte == b'_')) { - unsafe { self.incr_cursor() } - } - let len = find_ptr_distance(start, self.cursor()); - unsafe { RawSlice::new(start, len) } - } - #[inline(always)] - fn scan_ident_or_keyword(&mut self) { - let ident = self.scan_ident(); - match Keyword::try_from_slice(unsafe { - // UNSAFE(@ohsayan): The source buffer's presence guarantees that this is correct - ident.as_slice() - }) { - Some(kw) => self.push_token(kw), - None => self.push_token(Token::Identifier(ident)), - } - } - #[inline(always)] - /// Scan a quoted string - fn scan_quoted_string(&mut self, quote_style: u8) { - unsafe { self.incr_cursor() } - // a quoted string with the given quote style - let mut stringbuf = Vec::new(); - // should start with '"' - let mut is_okay = true; - while is_okay && self.peek_neq(quote_style) { - let is_escaped_backslash = unsafe { - // UNSAFE(@ohsayan): The qp is not exhausted, so this is fine - self.check_escaped(b'\\') - }; - let is_escaped_quote = unsafe { - // UNSAFE(@ohsayan): The qp is not exhausted, so this is fine - self.check_escaped(b'"') - }; - unsafe { - // UNSAFE(@ohsayan): If either is true, then it is correct to do this - self.incr_cursor_if(is_escaped_backslash | is_escaped_quote) - }; - unsafe { - // UNSAFE(@ohsayan): if not escaped, this is fine. if escaped, this is still - // fine because the escaped byte was checked - stringbuf.push(self.deref_cursor()); - } - unsafe { - // UNSAFE(@ohsayan): if escaped we have moved ahead by one but the escaped char - // is still one more so we go ahead. if not, then business as usual - self.incr_cursor() - }; - } - // should be terminated by a '"' - is_okay &= self.peek_eq_and_forward(quote_style); - match String::from_utf8(stringbuf) { - Ok(s) if compiler::likely(is_okay) => { - // valid string literal - self.push_token(Token::QuotedString(s)); - } - _ => { - // state broken - self.last_error = Some(LangError::InvalidStringLiteral) - } - } - } - #[inline(always)] - fn scan_arbitrary_byte(&mut self, byte: u8) { - let r = match byte { - b'<' => Token::OpenAngular, - b'>' => Token::CloseAngular, - b'(' => Token::OpenParen, - b')' => Token::CloseParen, - b',' => Token::Comma, - b':' => Token::Colon, - b'.' => Token::Period, - _ => { - self.last_error = Some(LangError::UnexpectedChar); - return; - } - }; - unsafe { self.incr_cursor() }; - self.push_token(r); - } -} - -impl<'a> Lexer<'a> { - #[inline(always)] - /// Lex the input stream into tokens - pub fn lex(src: &'a [u8]) -> LangResult> { - Self::new(src)._lex() - } - #[inline(always)] - /// The inner lex method - fn _lex(mut self) -> LangResult> { - while self.not_exhausted() && self.last_error.is_none() { - match unsafe { self.deref_cursor() } { - byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(), - byte if byte.is_ascii_digit() => self.scan_number(), - b' ' => self.trim_ahead(), - b'\n' | b'\t' => { - // simply ignore - unsafe { - // UNSAFE(@ohsayan): This is totally fine. We just looked at the byte - self.incr_cursor() - } - } - quote_style @ (b'"' | b'\'') => self.scan_quoted_string(quote_style), - byte => self.scan_arbitrary_byte(byte), - } - } - match self.last_error { - None => Ok(self.tokens), - Some(e) => Err(e), - } - } -} diff --git a/server/src/blueql/mod.rs b/server/src/blueql/mod.rs deleted file mode 100644 index 7dd7fe92..00000000 --- a/server/src/blueql/mod.rs +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Created on Tue Jun 14 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -mod ast; -mod error; -mod executor; -mod lexer; -pub mod util; -// test modules -#[cfg(test)] -mod tests; -// re-export -use { - self::{ast::Statement, error::LangResult}, - crate::util::Life, -}; -pub use {ast::Compiler, ast::Entity, executor::execute}; - -#[cfg(test)] -use core::fmt; -use core::{mem, slice}; - -#[allow(clippy::needless_lifetimes)] -#[inline(always)] -pub fn compile<'a>(src: &'a [u8], extra: usize) -> LangResult> { - Compiler::compile_with_extra(src, extra) -} - -#[cfg_attr(not(test), derive(Debug))] -#[cfg_attr(not(test), derive(PartialEq, Eq))] -pub struct RawSlice { - ptr: *const u8, - len: usize, -} - -unsafe impl Send for RawSlice {} -unsafe impl Sync for RawSlice {} - -#[cfg(test)] -impl fmt::Debug for RawSlice { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", String::from_utf8_lossy(unsafe { self.as_slice() })) - } -} - -#[cfg(test)] -impl PartialEq for RawSlice { - fn eq(&self, other: &Self) -> bool { - unsafe { self.as_slice() == other.as_slice() } - } -} -#[cfg(test)] -impl Eq for RawSlice {} - -impl RawSlice { - const _ENSURE_ALIGN: () = assert!(mem::align_of::() == mem::align_of::<&[u8]>()); - pub const unsafe fn new(ptr: *const u8, len: usize) -> Self { - Self { ptr, len } - } - pub unsafe fn as_slice(&self) -> &[u8] { - slice::from_raw_parts(self.ptr, self.len) - } - pub const fn len(&self) -> usize { - self.len - } -} - -#[cfg(test)] -impl From for RawSlice -where - T: AsRef<[u8]>, -{ - fn from(t: T) -> Self { - let t = t.as_ref(); - unsafe { Self::new(t.as_ptr(), t.len()) } - } -} diff --git a/server/src/blueql/tests.rs b/server/src/blueql/tests.rs deleted file mode 100644 index aa187e80..00000000 --- a/server/src/blueql/tests.rs +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Created on Tue Jun 14 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use super::{ - ast::{Compiler, Entity, FieldConfig, Statement}, - error::LangError, - lexer::{Keyword, Lexer, Token, Type, TypeExpression}, -}; - -macro_rules! src { - ($name:ident, $($src:expr),* $(,)?) => { - const $name: &[&[u8]] = &[$($src.as_bytes()),*]; - }; -} - -mod lexer { - //! Lexer tests - - use super::*; - - #[test] - fn lex_ident() { - let src = b"mytbl"; - assert_eq!( - Lexer::lex(src).unwrap(), - vec![Token::Identifier("mytbl".into())] - ) - } - - #[test] - fn lex_keyword() { - let src = b"create"; - assert_eq!( - Lexer::lex(src).unwrap(), - vec![Token::Keyword(Keyword::Create)] - ) - } - - #[test] - fn lex_number() { - let src = b"123456"; - assert_eq!(Lexer::lex(src).unwrap(), vec![Token::Number(123456)]) - } - - #[test] - fn lex_full() { - let src = b"create model tweet"; - assert_eq!( - Lexer::lex(src).unwrap(), - vec![ - Token::Keyword(Keyword::Create), - Token::Keyword(Keyword::Model), - Token::Identifier("tweet".into()) - ] - ); - } - - #[test] - fn lex_combined_tokens() { - let src = b"create model tweet(name: string, pic: binary, posts: list)"; - assert_eq!( - Lexer::lex(src).unwrap(), - vec![ - Keyword::Create.into(), - Keyword::Model.into(), - "tweet".into(), - Token::OpenParen, - Token::Identifier("name".into()), - Token::Colon, - Type::String.into(), - Token::Comma, - Token::Identifier("pic".into()), - Token::Colon, - Type::Binary.into(), - Token::Comma, - Token::Identifier("posts".into()), - Token::Colon, - Type::List.into(), - Token::OpenAngular, - Type::String.into(), - Token::CloseAngular, - Token::CloseParen - ] - ); - } - - #[test] - fn lex_quoted_string() { - let src_a = "'hello, world🦀!'".as_bytes(); - let src_b = r#" "hello, world🦀!" "#.as_bytes(); - let src_c = r#" "\"hello, world🦀!\"" "#.as_bytes(); - assert_eq!( - Lexer::lex(src_a).unwrap(), - vec![Token::QuotedString("hello, world🦀!".into())] - ); - assert_eq!( - Lexer::lex(src_b).unwrap(), - vec![Token::QuotedString("hello, world🦀!".into())] - ); - assert_eq!( - Lexer::lex(src_c).unwrap(), - vec![Token::QuotedString("\"hello, world🦀!\"".into())] - ) - } - - #[test] - fn lex_fail_unknown_chars() { - const SOURCES: &[&[u8]] = &[ - b"!", b"@", b"#", b"$", b"%", b"^", b"&", b"*", b"[", b"]", b"{", b"}", b"|", b"\\", - b"/", b"~", b"`", b";", b"hello?", - ]; - for source in SOURCES { - assert_eq!(Lexer::lex(source).unwrap_err(), LangError::UnexpectedChar); - } - } - - #[test] - fn lex_fail_unclosed_litstring() { - const SOURCES: &[&[u8]] = &[b"'hello, world", br#""hello, world"#]; - for source in SOURCES { - assert_eq!( - Lexer::lex(source).unwrap_err(), - LangError::InvalidStringLiteral - ); - } - } - - #[test] - fn lex_fail_litnum() { - src!(SOURCES, "12345f", "123!", "123'"); - for source in SOURCES { - assert_eq!( - Lexer::lex(source).unwrap_err(), - LangError::InvalidNumericLiteral - ); - } - } - - #[test] - fn lex_ignore_lf() { - let test_slice = b"create\n"; - assert_eq!( - Lexer::lex(test_slice).unwrap(), - vec![Token::Keyword(Keyword::Create)] - ) - } - - #[test] - fn lex_ignore_tab() { - let test_slice = b"create\t"; - assert_eq!( - Lexer::lex(test_slice).unwrap(), - vec![Token::Keyword(Keyword::Create)] - ) - } -} - -mod ast { - //! AST tests - - #[test] - fn parse_entity_name_test() { - assert_eq!( - Compiler::new(&Lexer::lex(b"hello").unwrap()) - .parse_entity_name() - .unwrap(), - Entity::Current("hello".into()) - ); - assert_eq!( - Compiler::new(&Lexer::lex(b"hello.world").unwrap()) - .parse_entity_name() - .unwrap(), - Entity::Full("hello".into(), "world".into()) - ); - } - - use super::*; - #[cfg(test)] - fn setup_src_stmt() -> (Vec, Statement) { - let src = - b"create model twitter.tweet(username: string, password: binary, posts: list) volatile" - .to_vec(); - let stmt = Statement::CreateModel { - entity: Entity::Full("twitter".into(), "tweet".into()), - model: FieldConfig { - types: vec![ - TypeExpression(vec![Type::String]), - TypeExpression(vec![Type::Binary]), - TypeExpression(vec![Type::List, Type::String]), - ], - names: vec!["username".into(), "password".into(), "posts".into()], - }, - volatile: true, - }; - (src, stmt) - } - - #[test] - fn stmt_create_named_unnamed_mixed() { - let src = b"create model twitter.tweet(username: string, binary,)".to_vec(); - assert_eq!( - Compiler::compile(&src).unwrap_err(), - LangError::BadExpression - ); - } - #[test] - fn stmt_create_unnamed() { - let src = b"create model twitter.passwords(string, binary)".to_vec(); - let expected = Statement::CreateModel { - entity: Entity::Full("twitter".into(), "passwords".into()), - model: FieldConfig { - names: vec![], - types: vec![ - TypeExpression(vec![Type::String]), - TypeExpression(vec![Type::Binary]), - ], - }, - volatile: false, - }; - assert_eq!(Compiler::compile(&src).unwrap(), expected); - } - #[test] - fn stmt_drop_space() { - assert_eq!( - Compiler::compile(b"drop space twitter force").unwrap(), - Statement::DropSpace { - entity: "twitter".into(), - force: true - } - ); - } - #[test] - fn stmt_drop_model() { - assert_eq!( - Compiler::compile(b"drop model twitter.tweet force").unwrap(), - Statement::DropModel { - entity: Entity::Full("twitter".into(), "tweet".into()), - force: true - } - ); - } - #[test] - fn stmt_inspect_space() { - assert_eq!( - Compiler::compile(b"inspect space twitter").unwrap(), - Statement::InspectSpace(Some("twitter".into())) - ); - } - #[test] - fn stmt_inspect_model() { - assert_eq!( - Compiler::compile(b"inspect model twitter.tweet").unwrap(), - Statement::InspectModel(Some(Entity::Full("twitter".into(), "tweet".into()))) - ); - } - #[test] - fn compile_full() { - let (src, stmt) = setup_src_stmt(); - assert_eq!(Compiler::compile(&src).unwrap(), stmt) - } - #[test] - fn bad_model_code() { - let get_model_code = |src| { - let l = Lexer::lex(src).unwrap(); - let stmt = Compiler::new(&l) - .parse_create_model1(Entity::Current("jotsy".into())) - .unwrap_or_else(|_| panic!("Failed for payload: {}", String::from_utf8_lossy(src))); - match stmt { - Statement::CreateModel { model, .. } => model.get_model_code(), - x => panic!("Expected model found {:?}", x), - } - }; - // check rules - // first type cannot be list - src!( - SRC, - // rule: first cannot be list - "(list, string)", - "(list, binary)", - "(list, string)", - "(list, string)", - // rule: cannot have more than two args - "(list, string, string)", - // rule: non-compound types cannot have type arguments - "(string, binary)", - // rule: fields can't be named - "(id: string, posts: list)", - // rule: nested lists are disallowed - "(string, list>)" - ); - for src in SRC { - assert_eq!( - get_model_code(src).unwrap_err(), - LangError::UnsupportedModelDeclaration, - "{}", - String::from_utf8_lossy(src) - ); - } - } -} diff --git a/server/src/cli.yml b/server/src/cli.yml deleted file mode 100644 index c8a3024c..00000000 --- a/server/src/cli.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Skytable Server -version: 0.8.0 -author: Sayan N. -about: The Skytable Database server -args: - - config: - short: c - required: false - long: withconfig - value_name: cfgfile - help: Sets a configuration file to start skyd - takes_value: true - - restore: - short: r - required: false - long: restore - value_name: backupdir - help: Restores data from a previous snapshot made in the provided directory - takes_value: true - - host: - short: h - required: false - long: host - value_name: host - help: Sets the host to which the server will bind - takes_value: true - - port: - short: p - required: false - long: port - value_name: port - help: Sets the port to which the server will bind - takes_value: true - - noart: - required: false - long: noart - help: Disables terminal artwork - takes_value: false - - nosave: - required: false - long: nosave - help: Disables automated background saving - takes_value: false - - saveduration: - required: false - long: saveduration - value_name: duration - short: S - takes_value: true - help: Set the BGSAVE duration - - snapevery: - required: false - long: snapevery - value_name: duration - help: Set the periodic snapshot duration - takes_value: true - - snapkeep: - required: false - long: snapkeep - value_name: count - help: Sets the number of most recent snapshots to keep - takes_value: true - - sslkey: - required: false - long: sslkey - short: k - value_name: key - help: Sets the PEM key file to use for SSL/TLS - takes_value: true - - sslchain: - required: false - long: sslchain - short: z - value_name: chain - help: Sets the PEM chain file to use for SSL/TLS - takes_value: true - - sslonly: - required: false - long: sslonly - takes_value: false - help: Tells the server to only accept SSL connections and disables the non-SSL port - - sslport: - required: false - long: sslport - takes_value: true - value_name: sslport - help: Set a custom SSL port to bind to - - tlspassin: - required: false - long: tlspassin - takes_value: true - value_name: tlspassin - help: Path to the file containing the passphrase for the TLS certificate - - stopwriteonfail: - required: false - long: stop-write-on-fail - takes_value: true - help: Stop accepting writes if any persistence method except BGSAVE fails (defaults to true) - - maxcon: - required: false - long: maxcon - takes_value: true - help: Set the maximum number of connections - value_name: maxcon - - mode: - required: false - long: mode - takes_value: true - short: m - help: Sets the deployment type - value_name: mode - - authkey: - required: false - long: auth-origin-key - takes_value: true - help: Set the authentication origin key - value_name: origin_key - - protover: - required: false - long: protover - takes_value: true - help: Set the protocol version - value_name: protover diff --git a/server/src/config/cfgcli.rs b/server/src/config/cfgcli.rs deleted file mode 100644 index fd03da7e..00000000 --- a/server/src/config/cfgcli.rs +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Created on Fri Jan 28 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ConfigSourceParseResult, Configset, TryFromConfigSource}, - clap::ArgMatches, -}; - -/// A flag. The flag is said to be set if `self.set` is true and unset if `self.set` is false. However, -/// if the flag is set, the value of SWITCH determines what value it is set to -#[derive(Copy, Clone)] -pub(super) struct Flag { - set: bool, -} - -impl Flag { - pub(super) fn new(set: bool) -> Self { - Self { set } - } -} - -impl TryFromConfigSource for Flag { - fn is_present(&self) -> bool { - self.set - } - fn mutate_failed(self, target: &mut bool, trip: &mut bool) -> bool { - if self.set { - *trip = true; - *target = SWITCH; - } - false - } - fn try_parse(self) -> ConfigSourceParseResult { - if self.set { - ConfigSourceParseResult::Okay(SWITCH) - } else { - ConfigSourceParseResult::Absent - } - } -} - -pub(super) fn parse_cli_args(matches: ArgMatches) -> Configset { - let mut defset = Configset::new_cli(); - macro_rules! fcli { - ($fn:ident, $($source:expr, $key:literal),*) => { - defset.$fn( - $( - $source, - $key, - )* - ) - }; - } - // protocol settings - fcli! { - protocol_settings, - matches.value_of("protover"), - "--protover" - }; - // server settings - fcli!( - server_tcp, - matches.value_of("host"), - "--host", - matches.value_of("port"), - "--port" - ); - fcli!( - server_noart, - Flag::::new(matches.is_present("noart")), - "--noart" - ); - fcli!(server_mode, matches.value_of("mode"), "--mode"); - fcli!(server_maxcon, matches.value_of("maxcon"), "--maxcon"); - // bgsave settings - fcli!( - bgsave_settings, - Flag::::new(matches.is_present("nosave")), - "--nosave", - matches.value_of("saveduration"), - "--saveduration" - ); - // snapshot settings - fcli!( - snapshot_settings, - matches.value_of("snapevery"), - "--snapevery", - matches.value_of("snapkeep"), - "--snapkeep", - matches.value_of("stop-write-on-fail"), - "--stop-write-on-fail" - ); - // TLS settings - fcli!( - tls_settings, - matches.value_of("sslkey"), - "--sslkey", - matches.value_of("sslchain"), - "--sslchain", - matches.value_of("sslport"), - "--sslport", - Flag::::new(matches.is_present("sslonly")), - "--sslonly", - matches.value_of("tlspass"), - "--tlspassin" - ); - // auth settings - fcli!( - auth_settings, - matches.value_of("authkey"), - "--auth-origin-key" - ); - defset -} diff --git a/server/src/config/cfgenv.rs b/server/src/config/cfgenv.rs deleted file mode 100644 index 23ead026..00000000 --- a/server/src/config/cfgenv.rs +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Created on Thu Jan 27 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use super::Configset; - -/// Returns the environment configuration -pub(super) fn parse_env_config() -> Configset { - let mut defset = Configset::new_env(); - macro_rules! fenv { - ( - $fn:ident, - $( - $field:ident - ),* - ) => { - defset.$fn( - $( - ::std::env::var(stringify!($field)), - stringify!($field), - )* - ); - }; - } - // protocol settings - fenv!(protocol_settings, SKY_PROTOCOL_VERSION); - // server settings - fenv!(server_tcp, SKY_SYSTEM_HOST, SKY_SYSTEM_PORT); - fenv!(server_noart, SKY_SYSTEM_NOART); - fenv!(server_maxcon, SKY_SYSTEM_MAXCON); - fenv!(server_mode, SKY_DEPLOY_MODE); - // bgsave settings - fenv!(bgsave_settings, SKY_BGSAVE_ENABLED, SKY_BGSAVE_DURATION); - // snapshot settings - fenv!( - snapshot_settings, - SKY_SNAPSHOT_DURATION, - SKY_SNAPSHOT_KEEP, - SKY_SNAPSHOT_FAILSAFE - ); - // TLS settings - fenv!( - tls_settings, - SKY_TLS_KEY, - SKY_TLS_CERT, - SKY_TLS_PORT, - SKY_TLS_ONLY, - SKY_TLS_PASSIN - ); - fenv!(auth_settings, SKY_AUTH_ORIGIN_KEY); - defset -} diff --git a/server/src/config/cfgfile.rs b/server/src/config/cfgfile.rs deleted file mode 100644 index cec8c2de..00000000 --- a/server/src/config/cfgfile.rs +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Created on Sat Oct 02 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ - AuthSettings, ConfigSourceParseResult, Configset, Modeset, OptString, ProtocolVersion, - TryFromConfigSource, - }, - serde::Deserialize, - std::net::IpAddr, -}; - -/// This struct is an _object representation_ used for parsing the TOML file -#[derive(Deserialize, Debug, PartialEq, Eq)] -pub struct Config { - /// The `server` key - pub(super) server: ConfigKeyServer, - /// The `bgsave` key - pub(super) bgsave: Option, - /// The snapshot key - pub(super) snapshot: Option, - /// SSL configuration - pub(super) ssl: Option, - /// auth settings - pub(super) auth: Option, -} - -/// This struct represents the `server` key in the TOML file -#[derive(Deserialize, Debug, PartialEq, Eq)] -pub struct ConfigKeyServer { - /// The host key is any valid IPv4/IPv6 address - pub(super) host: IpAddr, - /// The port key is any valid port - pub(super) port: u16, - /// The noart key is an `Option`al boolean value which is set to true - /// for secure environments to disable terminal artwork - pub(super) noart: Option, - /// The maximum number of clients - pub(super) maxclient: Option, - /// The deployment mode - pub(super) mode: Option, - pub(super) protocol: Option, -} - -/// The BGSAVE section in the config file -#[derive(Deserialize, Debug, PartialEq, Eq)] -pub struct ConfigKeyBGSAVE { - /// Whether BGSAVE is enabled or not - /// - /// If this key is missing, then we can assume that BGSAVE is enabled - pub(super) enabled: Option, - /// The duration after which BGSAVE should start - /// - /// If this is the only key specified, then it is clear that BGSAVE is enabled - /// and the duration is `every` - pub(super) every: Option, -} - -/// The snapshot section in the TOML file -#[derive(Deserialize, Debug, PartialEq, Eq)] -pub struct ConfigKeySnapshot { - /// After how many seconds should the snapshot be created - pub(super) every: u64, - /// The maximum number of snapshots to keep - /// - /// If atmost is set to `0`, then all the snapshots will be kept - pub(super) atmost: usize, - /// Prevent writes to the database if snapshotting fails - pub(super) failsafe: Option, -} - -#[derive(Deserialize, Debug, PartialEq, Eq)] -pub struct KeySslOpts { - pub(super) key: String, - pub(super) chain: String, - pub(super) port: u16, - pub(super) only: Option, - pub(super) passin: Option, -} - -/// A custom non-null type for config files -pub struct NonNull { - val: T, -} - -impl From for NonNull { - fn from(val: T) -> Self { - Self { val } - } -} - -impl TryFromConfigSource for NonNull { - fn is_present(&self) -> bool { - true - } - fn mutate_failed(self, target: &mut T, trip: &mut bool) -> bool { - *target = self.val; - *trip = true; - false - } - fn try_parse(self) -> ConfigSourceParseResult { - ConfigSourceParseResult::Okay(self.val) - } -} - -pub struct Optional { - base: Option, -} - -impl Optional { - pub const fn some(val: T) -> Self { - Self { base: Some(val) } - } -} - -impl From> for Optional { - fn from(base: Option) -> Self { - Self { base } - } -} - -impl TryFromConfigSource for Optional { - fn is_present(&self) -> bool { - self.base.is_some() - } - fn mutate_failed(self, target: &mut T, trip: &mut bool) -> bool { - if let Some(v) = self.base { - *trip = true; - *target = v; - } - false - } - fn try_parse(self) -> ConfigSourceParseResult { - match self.base { - Some(v) => ConfigSourceParseResult::Okay(v), - None => ConfigSourceParseResult::Absent, - } - } -} - -type ConfigFile = Config; - -pub fn from_file(file: ConfigFile) -> Configset { - let mut set = Configset::new_file(); - let ConfigFile { - server, - bgsave, - snapshot, - ssl, - auth, - } = file; - // server settings - set.server_tcp( - Optional::some(server.host), - "server.host", - Optional::some(server.port), - "server.port", - ); - set.protocol_settings(server.protocol, "server.protocol"); - set.server_maxcon(Optional::from(server.maxclient), "server.maxcon"); - set.server_noart(Optional::from(server.noart), "server.noart"); - set.server_mode(Optional::from(server.mode), "server.mode"); - // bgsave settings - if let Some(bgsave) = bgsave { - let ConfigKeyBGSAVE { enabled, every } = bgsave; - set.bgsave_settings( - Optional::from(enabled), - "bgsave.enabled", - Optional::from(every), - "bgsave.every", - ); - } - // snapshot settings - if let Some(snapshot) = snapshot { - let ConfigKeySnapshot { - every, - atmost, - failsafe, - } = snapshot; - set.snapshot_settings( - NonNull::from(every), - "snapshot.every", - NonNull::from(atmost), - "snapshot.atmost", - Optional::from(failsafe), - "snapshot.failsafe", - ); - } - // TLS settings - if let Some(tls) = ssl { - let KeySslOpts { - key, - chain, - port, - only, - passin, - } = tls; - set.tls_settings( - NonNull::from(key), - "ssl.key", - NonNull::from(chain), - "ssl.chain", - NonNull::from(port), - "ssl.port", - Optional::from(only), - "ssl.only", - OptString::from(passin), - "ssl.passin", - ); - } - if let Some(auth) = auth { - let AuthSettings { origin_key } = auth; - set.auth_settings(Optional::from(origin_key), "auth.origin") - } - set -} diff --git a/server/src/config/definitions.rs b/server/src/config/definitions.rs deleted file mode 100644 index b356d990..00000000 --- a/server/src/config/definitions.rs +++ /dev/null @@ -1,504 +0,0 @@ -/* - * Created on Fri Jan 28 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{feedback::WarningStack, DEFAULT_IPV4, DEFAULT_PORT}, - crate::{config::AuthkeyWrapper, dbnet::MAXIMUM_CONNECTION_LIMIT}, - core::{fmt, str::FromStr}, - serde::{ - de::{self, Deserializer, Visitor}, - Deserialize, - }, - std::net::IpAddr, -}; - -/// The BGSAVE configuration -/// -/// If BGSAVE is enabled, then the duration (corresponding to `every`) is wrapped in the `Enabled` -/// variant. Otherwise, the `Disabled` variant is to be used -#[derive(PartialEq, Eq, Debug)] -pub enum BGSave { - Enabled(u64), - Disabled, -} - -impl BGSave { - /// Create a new BGSAVE configuration with all the fields - pub const fn new(enabled: bool, every: u64) -> Self { - if enabled { - BGSave::Enabled(every) - } else { - BGSave::Disabled - } - } - /// The default BGSAVE configuration - /// - /// Defaults: - /// - `enabled`: true - /// - `every`: 120 - pub const fn default() -> Self { - BGSave::new(true, 120) - } - /// Check if BGSAVE is disabled - pub const fn is_disabled(&self) -> bool { - matches!(self, Self::Disabled) - } -} - -#[repr(u8)] -#[derive(Debug, Eq, PartialEq)] -pub enum ProtocolVersion { - V1, - V2, -} - -impl Default for ProtocolVersion { - fn default() -> Self { - Self::V2 - } -} - -impl ToString for ProtocolVersion { - fn to_string(&self) -> String { - match self { - Self::V1 => "Skyhash 1.0".to_owned(), - Self::V2 => "Skyhash 2.0".to_owned(), - } - } -} - -struct ProtocolVersionVisitor; - -impl<'de> Visitor<'de> for ProtocolVersionVisitor { - type Value = ProtocolVersion; - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "a 40 character ASCII string") - } - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - value.parse().map_err(|_| { - E::custom("Invalid value for protocol version. Valid inputs: 1.0, 1.1, 1.2, 2.0") - }) - } -} - -impl<'de> Deserialize<'de> for ProtocolVersion { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_str(ProtocolVersionVisitor) - } -} - -/// A `ConfigurationSet` which can be used by main::check_args_or_connect() to bind -/// to a `TcpListener` and show the corresponding terminal output for the given -/// configuration -#[derive(Debug, PartialEq, Eq)] -pub struct ConfigurationSet { - /// If `noart` is set to true, no terminal artwork should be displayed - pub noart: bool, - /// The BGSAVE configuration - pub bgsave: BGSave, - /// The snapshot configuration - pub snapshot: SnapshotConfig, - /// Port configuration - pub ports: PortConfig, - /// The maximum number of connections - pub maxcon: usize, - /// The deployment mode - pub mode: Modeset, - /// The auth settings - pub auth: AuthSettings, - /// The protocol version - pub protocol: ProtocolVersion, -} - -impl ConfigurationSet { - #[allow(clippy::too_many_arguments)] - pub const fn new( - noart: bool, - bgsave: BGSave, - snapshot: SnapshotConfig, - ports: PortConfig, - maxcon: usize, - mode: Modeset, - auth: AuthSettings, - protocol: ProtocolVersion, - ) -> Self { - Self { - noart, - bgsave, - snapshot, - ports, - maxcon, - mode, - auth, - protocol, - } - } - /// Create a default `ConfigurationSet` with the following setup defaults: - /// - `host`: 127.0.0.1 - /// - `port` : 2003 - /// - `noart` : false - /// - `bgsave_enabled` : true - /// - `bgsave_duration` : 120 - /// - `ssl` : disabled - pub const fn default() -> Self { - Self::new( - false, - BGSave::default(), - SnapshotConfig::default(), - PortConfig::new_insecure_only(DEFAULT_IPV4, 2003), - MAXIMUM_CONNECTION_LIMIT, - Modeset::Dev, - AuthSettings::default(), - ProtocolVersion::V2, - ) - } - /// Returns `false` if `noart` is enabled. Otherwise it returns `true` - pub const fn is_artful(&self) -> bool { - !self.noart - } -} - -/// Port configuration -/// -/// This enumeration determines whether the ports are: -/// - `Multi`: This means that the database server will be listening to both -/// SSL **and** non-SSL requests -/// - `SecureOnly` : This means that the database server will only accept SSL requests -/// and will not even activate the non-SSL socket -/// - `InsecureOnly` : This indicates that the server would only accept non-SSL connections -/// and will not even activate the SSL socket -#[derive(Debug, PartialEq, Eq)] -pub enum PortConfig { - SecureOnly { - host: IpAddr, - ssl: SslOpts, - }, - Multi { - host: IpAddr, - port: u16, - ssl: SslOpts, - }, - InsecureOnly { - host: IpAddr, - port: u16, - }, -} - -impl Default for PortConfig { - fn default() -> PortConfig { - PortConfig::InsecureOnly { - host: DEFAULT_IPV4, - port: DEFAULT_PORT, - } - } -} - -impl PortConfig { - pub const fn new_secure_only(host: IpAddr, ssl: SslOpts) -> Self { - PortConfig::SecureOnly { host, ssl } - } - pub const fn new_insecure_only(host: IpAddr, port: u16) -> Self { - PortConfig::InsecureOnly { host, port } - } - pub fn get_host(&self) -> IpAddr { - match self { - Self::InsecureOnly { host, .. } - | Self::SecureOnly { host, .. } - | Self::Multi { host, .. } => *host, - } - } - pub fn upgrade_to_tls(&mut self, ssl: SslOpts) { - match self { - Self::InsecureOnly { host, port } => { - *self = Self::Multi { - host: *host, - port: *port, - ssl, - } - } - Self::SecureOnly { .. } | Self::Multi { .. } => { - panic!("Port config is already upgraded to TLS") - } - } - } - pub const fn insecure_only(&self) -> bool { - matches!(self, Self::InsecureOnly { .. }) - } - pub const fn secure_only(&self) -> bool { - matches!(self, Self::SecureOnly { .. }) - } - pub fn get_description(&self) -> String { - match self { - Self::Multi { host, port, ssl } => { - format!( - "skyhash://{host}:{port} and skyhash-secure://{host}:{tlsport}", - tlsport = ssl.get_port(), - ) - } - Self::SecureOnly { - host, - ssl: SslOpts { port, .. }, - } => format!("skyhash-secure://{host}:{port}"), - Self::InsecureOnly { host, port } => format!("skyhash://{host}:{port}",), - } - } -} - -#[derive(Deserialize, Debug, PartialEq, Eq)] -pub struct SslOpts { - pub key: String, - pub chain: String, - pub port: u16, - pub passfile: Option, -} - -impl SslOpts { - pub const fn new(key: String, chain: String, port: u16, passfile: Option) -> Self { - SslOpts { - key, - chain, - port, - passfile, - } - } - pub const fn get_port(&self) -> u16 { - self.port - } -} - -#[derive(Debug, PartialEq, Eq)] -/// The snapshot configuration -/// -pub struct SnapshotPref { - /// Capture a snapshot `every` seconds - pub every: u64, - /// The maximum numeber of snapshots to be kept - pub atmost: usize, - /// Lock writes if snapshotting fails - pub poison: bool, -} - -impl SnapshotPref { - /// Create a new a new `SnapshotPref` instance - pub const fn new(every: u64, atmost: usize, poison: bool) -> Self { - SnapshotPref { - every, - atmost, - poison, - } - } - /// Returns `every,almost` as a tuple for pattern matching - pub const fn decompose(self) -> (u64, usize, bool) { - (self.every, self.atmost, self.poison) - } -} - -#[derive(Debug, PartialEq, Eq)] -/// Snapshotting configuration -/// -/// The variant `Enabled` directly carries a `ConfigKeySnapshot` object that -/// is parsed from the configuration file, The variant `Disabled` is a ZST, and doesn't -/// hold any data -pub enum SnapshotConfig { - /// Snapshotting is enabled: this variant wraps around a `SnapshotPref` - /// object - Enabled(SnapshotPref), - /// Snapshotting is disabled - Disabled, -} - -impl SnapshotConfig { - /// Snapshots are disabled by default, so `SnapshotConfig::Disabled` is the - /// default configuration - pub const fn default() -> Self { - SnapshotConfig::Disabled - } -} - -type RestoreFile = Option; - -#[derive(Debug, PartialEq, Eq)] -/// The type of configuration: -/// - The default configuration -/// - A custom supplied configuration -pub struct ConfigType { - pub(super) config: ConfigurationSet, - restore: RestoreFile, - is_custom: bool, - warnings: Option, -} - -impl ConfigType { - fn _new( - config: ConfigurationSet, - restore: RestoreFile, - is_custom: bool, - warnings: Option, - ) -> Self { - Self { - config, - restore, - is_custom, - warnings, - } - } - pub fn print_warnings(&self) { - if let Some(warnings) = self.warnings.as_ref() { - warnings.print_warnings() - } - } - pub fn finish(self) -> (ConfigurationSet, Option) { - (self.config, self.restore) - } - pub fn is_custom(&self) -> bool { - self.is_custom - } - pub fn is_artful(&self) -> bool { - self.config.is_artful() - } - pub fn new_custom( - config: ConfigurationSet, - restore: RestoreFile, - warnings: WarningStack, - ) -> Self { - Self::_new(config, restore, true, Some(warnings)) - } - pub fn new_default(restore: RestoreFile) -> Self { - Self::_new(ConfigurationSet::default(), restore, false, None) - } - /// Check if the current deploy mode is prod - pub const fn is_prod_mode(&self) -> bool { - matches!(self.config.mode, Modeset::Prod) - } - pub fn wpush(&mut self, w: impl ToString) { - match self.warnings.as_mut() { - Some(stack) => stack.push(w), - None => { - self.warnings = { - let mut wstack = WarningStack::new(""); - wstack.push(w); - Some(wstack) - }; - } - } - } -} - -#[derive(Debug, PartialEq, Eq)] -pub enum Modeset { - Dev, - Prod, -} - -impl FromStr for Modeset { - type Err = (); - fn from_str(st: &str) -> Result { - match st { - "dev" => Ok(Modeset::Dev), - "prod" => Ok(Modeset::Prod), - _ => Err(()), - } - } -} - -struct ModesetVisitor; - -impl<'de> Visitor<'de> for ModesetVisitor { - type Value = Modeset; - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Expecting a string with the deployment mode") - } - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - match value { - "dev" => Ok(Modeset::Dev), - "prod" => Ok(Modeset::Prod), - _ => Err(E::custom(format!("Bad value `{value}` for modeset"))), - } - } -} - -impl<'de> Deserialize<'de> for Modeset { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_str(ModesetVisitor) - } -} - -#[derive(Debug, PartialEq, Eq, Deserialize)] -pub struct AuthSettings { - pub origin_key: Option, -} - -impl AuthSettings { - pub const fn default() -> Self { - Self { origin_key: None } - } - #[cfg(test)] - pub fn new(origin: AuthkeyWrapper) -> Self { - Self { - origin_key: Some(origin), - } - } -} - -struct AuthSettingsVisitor; - -impl<'de> Visitor<'de> for AuthSettingsVisitor { - type Value = AuthkeyWrapper; - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "a 40 character ASCII string") - } - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - AuthkeyWrapper::try_new(value).ok_or_else(|| { - E::custom( - "Invalid value for authkey. must be 40 ASCII characters with nonzero first char", - ) - }) - } -} - -impl<'de> Deserialize<'de> for AuthkeyWrapper { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_str(AuthSettingsVisitor) - } -} diff --git a/server/src/config/feedback.rs b/server/src/config/feedback.rs deleted file mode 100644 index 5c51863e..00000000 --- a/server/src/config/feedback.rs +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Created on Tue Jan 25 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#[cfg(unix)] -use crate::util::os::ResourceLimit; -use { - super::{ConfigurationSet, SnapshotConfig, SnapshotPref}, - core::{fmt, ops}, - std::io::Error as IoError, - toml::de::Error as TomlError, -}; - -#[cfg(test)] -const EMSG_ENV: &str = "Environment"; -const EMSG_PROD: &str = "Production mode"; -const TAB: &str = " "; - -#[derive(Debug, PartialEq, Eq)] -pub struct FeedbackStack { - stack: Vec, - feedback_type: &'static str, - feedback_source: &'static str, -} - -impl FeedbackStack { - fn new(feedback_source: &'static str, feedback_type: &'static str) -> Self { - Self { - stack: Vec::new(), - feedback_type, - feedback_source, - } - } - pub fn source(&self) -> &'static str { - self.feedback_source - } - pub fn push(&mut self, f: impl ToString) { - self.stack.push(f.to_string()) - } -} - -impl fmt::Display for FeedbackStack { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if !self.is_empty() { - if self.len() == 1 { - write!( - f, - "{} {}: {}", - self.feedback_source, self.feedback_type, self.stack[0] - )?; - } else { - write!(f, "{} {}:", self.feedback_source, self.feedback_type)?; - for err in self.stack.iter() { - write!(f, "\n{}- {}", TAB, err)?; - } - } - } - Ok(()) - } -} - -impl ops::Deref for FeedbackStack { - type Target = Vec; - fn deref(&self) -> &Self::Target { - &self.stack - } -} -impl ops::DerefMut for FeedbackStack { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.stack - } -} - -#[derive(Debug, PartialEq, Eq)] -pub struct ErrorStack { - feedback: FeedbackStack, -} - -impl ErrorStack { - pub fn new(err_source: &'static str) -> Self { - Self { - feedback: FeedbackStack::new(err_source, "errors"), - } - } -} - -impl fmt::Display for ErrorStack { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.feedback) - } -} - -impl ops::Deref for ErrorStack { - type Target = FeedbackStack; - fn deref(&self) -> &Self::Target { - &self.feedback - } -} - -impl ops::DerefMut for ErrorStack { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.feedback - } -} - -#[derive(Debug, PartialEq, Eq)] -pub struct WarningStack { - feedback: FeedbackStack, -} - -impl WarningStack { - pub fn new(warning_source: &'static str) -> Self { - Self { - feedback: FeedbackStack::new(warning_source, "warnings"), - } - } - pub fn print_warnings(&self) { - if !self.feedback.is_empty() { - log::warn!("{}", self); - } - } -} - -impl ops::Deref for WarningStack { - type Target = FeedbackStack; - fn deref(&self) -> &Self::Target { - &self.feedback - } -} - -impl ops::DerefMut for WarningStack { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.feedback - } -} - -impl fmt::Display for WarningStack { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.feedback) - } -} - -#[derive(Debug)] -pub enum ConfigError { - OSError(IoError), - CfgError(ErrorStack), - ConfigFileParseError(TomlError), - Conflict, - ProdError(ErrorStack), -} - -impl PartialEq for ConfigError { - fn eq(&self, oth: &Self) -> bool { - match (self, oth) { - (Self::OSError(lhs), Self::OSError(rhs)) => lhs.to_string() == rhs.to_string(), - (Self::CfgError(lhs), Self::CfgError(rhs)) => lhs == rhs, - (Self::ConfigFileParseError(lhs), Self::ConfigFileParseError(rhs)) => lhs == rhs, - (Self::Conflict, Self::Conflict) => true, - (Self::ProdError(lhs), Self::ProdError(rhs)) => lhs == rhs, - _ => false, - } - } -} - -impl From for ConfigError { - fn from(e: IoError) -> Self { - Self::OSError(e) - } -} - -impl From for ConfigError { - fn from(e: toml::de::Error) -> Self { - Self::ConfigFileParseError(e) - } -} - -impl fmt::Display for ConfigError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::ConfigFileParseError(e) => write!(f, "Configuration file parse failed: {}", e), - Self::OSError(e) => write!(f, "OS Error: {}", e), - Self::CfgError(e) => write!(f, "{}", e), - Self::Conflict => write!( - f, - "Conflict error: Either provide CLI args, environment variables or a config file for configuration" - ), - Self::ProdError(e) => write!(f, "You have invalid configuration for production mode. {}", e) - } - } -} - -#[cfg(unix)] -fn check_rlimit_or_err(current: usize, estack: &mut ErrorStack) -> Result<(), ConfigError> { - let rlim = ResourceLimit::get()?; - if rlim.is_over_limit(current) { - estack.push( - "The value for maximum connections exceeds available resources to the server process", - ); - estack.push( - format!( - "The current process is set to a resource limit of {current} and can be set to a maximum limit of {max} in the OS", - current=rlim.current(),max=rlim.max() - )); - } - Ok(()) -} - -#[cfg(not(unix))] -fn check_rlimit_or_err(_: usize, _: &mut ErrorStack) -> Result<(), ConfigError> { - Ok(()) -} - -/// Check if the settings are suitable for use in production mode -pub(super) fn evaluate_prod_settings(cfg: &ConfigurationSet) -> Result<(), ConfigError> { - let mut estack = ErrorStack::new(EMSG_PROD); - // check `noart` - if cfg.is_artful() { - estack.push("Terminal artwork should be disabled"); - } - // first check BGSAVE - if cfg.bgsave.is_disabled() { - estack.push("BGSAVE must be enabled"); - } - // now check snapshot settings (failsafe) - if let SnapshotConfig::Enabled(SnapshotPref { poison, .. }) = cfg.snapshot { - if !poison { - estack.push("Snapshots must be failsafe"); - } - } - // now check TLS settings - if cfg.ports.insecure_only() { - estack.push("Either multi-socket (TCP and TLS) or TLS only must be enabled"); - } - if cfg.auth.origin_key.is_some() && !cfg.ports.secure_only() { - estack.push("When authn+authz is enabled, TLS-only mode must be enabled"); - } - check_rlimit_or_err(cfg.maxcon, &mut estack)?; - if estack.is_empty() { - Ok(()) - } else { - Err(ConfigError::ProdError(estack)) - } -} - -#[cfg(test)] -mod test { - use super::{ConfigError, ErrorStack, WarningStack, EMSG_ENV, EMSG_PROD}; - - #[test] - fn errorstack_fmt() { - const EXPECTED: &str = "\ -Environment errors: Invalid value for `SKY_SYSTEM_PORT`. Expected a 16-bit integer\ -"; - let mut estk = ErrorStack::new(EMSG_ENV); - estk.push("Invalid value for `SKY_SYSTEM_PORT`. Expected a 16-bit integer"); - let fmt = format!("{}", estk); - assert_eq!(fmt, EXPECTED); - } - - #[test] - fn warningstack_fmt() { - const EXPECTED: &str = "\ -Environment warnings: - - BGSAVE is disabled. You may lose data if the host crashes - - The setting for `maxcon` is too high\ - "; - let mut wstk = WarningStack::new(EMSG_ENV); - wstk.push("BGSAVE is disabled. You may lose data if the host crashes"); - wstk.push("The setting for `maxcon` is too high"); - let fmt = format!("{}", wstk); - assert_eq!(fmt, EXPECTED); - } - #[test] - fn prod_mode_error_fmt() { - let mut estack = ErrorStack::new(EMSG_PROD); - estack.push("BGSAVE must be enabled"); - estack.push("Snapshots must be failsafe"); - estack.push("Either multi-socket (TCP and TLS) or TLS-only mode must be enabled"); - estack.push( - "The value for maximum connections exceeds available resources to the server process", - ); - let e = ConfigError::ProdError(estack); - const EXPECTED: &str = "\ -You have invalid configuration for production mode. Production mode errors: - - BGSAVE must be enabled - - Snapshots must be failsafe - - Either multi-socket (TCP and TLS) or TLS-only mode must be enabled - - The value for maximum connections exceeds available resources to the server process\ -"; - assert_eq!(format!("{}", e), EXPECTED); - } -} diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs deleted file mode 100644 index 52fa9406..00000000 --- a/server/src/config/mod.rs +++ /dev/null @@ -1,703 +0,0 @@ -/* - * Created on Thu Jan 27 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::auth::provider::Authkey, - clap::{load_yaml, App}, - core::str::FromStr, - std::{ - env::VarError, - fs, - net::{IpAddr, Ipv4Addr}, - }, -}; - -// internal modules -mod cfgcli; -mod cfgenv; -mod cfgfile; -mod definitions; -mod feedback; -#[cfg(test)] -mod tests; - -// internal imports -use self::cfgfile::Config as ConfigFile; -pub use self::definitions::*; -use self::feedback::{ConfigError, ErrorStack, WarningStack}; -use crate::dbnet::MAXIMUM_CONNECTION_LIMIT; - -// server defaults -const DEFAULT_IPV4: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); -const DEFAULT_PORT: u16 = 2003; -// bgsave defaults -const DEFAULT_BGSAVE_DURATION: u64 = 120; -// snapshot defaults -const DEFAULT_SNAPSHOT_FAILSAFE: bool = true; -// TLS defaults -const DEFAULT_SSL_PORT: u16 = 2004; - -type StaticStr = &'static str; - -#[derive(Debug, PartialEq, Eq)] -pub struct AuthkeyWrapper(pub Authkey); - -impl AuthkeyWrapper { - pub const fn empty() -> Self { - Self([0u8; 40]) - } - pub fn try_new(slf: &str) -> Option { - let valid = slf.len() == 40 // must have 40 chars - && slf.chars().all(|ch| char::is_ascii_alphanumeric(&ch)); // must have ascii alpha - if valid { - let mut ret = Self::empty(); - slf.bytes().enumerate().for_each(|(idx, byte)| { - ret.0[idx] = byte; - }); - Some(ret) - } else { - None - } - } - pub fn into_inner(self) -> Authkey { - self.0 - } -} - -#[derive(Debug)] -/// An enum representing the outcome of a parse operation for a specific configuration item from a -/// specific configuration source -pub enum ConfigSourceParseResult { - Okay(T), - Absent, - ParseFailure, -} - -/// A trait for configuration sources. Any type implementing this trait is considered to be a valid -/// source for configuration -pub trait TryFromConfigSource: Sized { - /// Check if the value is present - fn is_present(&self) -> bool; - /// Attempt to mutate the value if present. If any error occurs - /// while parsing the value, return true. Else return false if all went well. - /// Here: - /// - `target_value`: is a mutable reference to the target var - /// - `trip`: is a mutable ref to a bool that will be set to true if a value is present - /// (whether parseable or not) - fn mutate_failed(self, target_value: &mut T, trip: &mut bool) -> bool; - /// Attempt to parse the value into the target type - fn try_parse(self) -> ConfigSourceParseResult; -} - -impl<'a, T: FromStr + 'a> TryFromConfigSource for Option<&'a str> { - fn is_present(&self) -> bool { - self.is_some() - } - fn mutate_failed(self, target_value: &mut T, trip: &mut bool) -> bool { - self.map(|slf| { - *trip = true; - match slf.parse() { - Ok(p) => { - *target_value = p; - false - } - Err(_) => true, - } - }) - .unwrap_or(false) - } - fn try_parse(self) -> ConfigSourceParseResult { - self.map(|s| { - s.parse() - .map(|ret| ConfigSourceParseResult::Okay(ret)) - .unwrap_or(ConfigSourceParseResult::ParseFailure) - }) - .unwrap_or(ConfigSourceParseResult::Absent) - } -} - -impl FromStr for AuthkeyWrapper { - type Err = (); - fn from_str(slf: &str) -> Result { - Self::try_new(slf).ok_or(()) - } -} - -impl<'a, T: FromStr + 'a> TryFromConfigSource for Result { - fn is_present(&self) -> bool { - !matches!(self, Err(VarError::NotPresent)) - } - fn mutate_failed(self, target_value: &mut T, trip: &mut bool) -> bool { - match self { - Ok(s) => { - *trip = true; - s.parse() - .map(|v| { - *target_value = v; - false - }) - .unwrap_or(true) - } - Err(e) => { - if matches!(e, VarError::NotPresent) { - false - } else { - // yes, we got the var but failed to parse it into unicode; so trip - *trip = true; - true - } - } - } - } - fn try_parse(self) -> ConfigSourceParseResult { - match self { - Ok(s) => s - .parse() - .map(|v| ConfigSourceParseResult::Okay(v)) - .unwrap_or(ConfigSourceParseResult::ParseFailure), - Err(e) => match e { - VarError::NotPresent => ConfigSourceParseResult::Absent, - VarError::NotUnicode(_) => ConfigSourceParseResult::ParseFailure, - }, - } - } -} - -#[derive(Debug, PartialEq, Eq, Default)] -/// Since we have conflicting trait implementations, we define a custom `Option` type -pub struct OptString { - base: Option, -} - -impl OptString { - pub const fn new_null() -> Self { - Self { base: None } - } -} - -impl From> for OptString { - fn from(base: Option) -> Self { - Self { base } - } -} - -impl FromStr for OptString { - type Err = (); - fn from_str(st: &str) -> Result { - Ok(Self { - base: Some(st.to_string()), - }) - } -} - -impl FromStr for ProtocolVersion { - type Err = (); - fn from_str(st: &str) -> Result { - match st { - "1" | "1.0" | "1.1" | "1.2" => Ok(Self::V1), - "2" | "2.0" => Ok(Self::V2), - _ => Err(()), - } - } -} - -impl TryFromConfigSource for Option { - fn is_present(&self) -> bool { - self.is_some() - } - fn mutate_failed(self, target: &mut ProtocolVersion, trip: &mut bool) -> bool { - if let Some(v) = self { - *target = v; - *trip = true; - } - false - } - fn try_parse(self) -> ConfigSourceParseResult { - self.map(ConfigSourceParseResult::Okay) - .unwrap_or(ConfigSourceParseResult::Absent) - } -} - -impl TryFromConfigSource for OptString { - fn is_present(&self) -> bool { - self.base.is_some() - } - fn mutate_failed(self, target: &mut OptString, trip: &mut bool) -> bool { - if let Some(v) = self.base { - target.base = Some(v); - *trip = true; - } - false - } - fn try_parse(self) -> ConfigSourceParseResult { - self.base - .map(|v| ConfigSourceParseResult::Okay(OptString { base: Some(v) })) - .unwrap_or(ConfigSourceParseResult::Absent) - } -} - -#[derive(Debug)] -/// A high-level configuration set that automatically handles errors, warnings and provides a convenient [`Result`] -/// type that can be used -pub struct Configset { - did_mutate: bool, - cfg: ConfigurationSet, - estack: ErrorStack, - wstack: WarningStack, -} - -impl Configset { - const EMSG_ENV: StaticStr = "Environment"; - const EMSG_CLI: StaticStr = "CLI"; - const EMSG_FILE: StaticStr = "Configuration file"; - - /// Internal ctor for a given feedback source. We do not want to expose this to avoid - /// erroneous feedback source names - fn _new(feedback_source: StaticStr) -> Self { - Self { - did_mutate: false, - cfg: ConfigurationSet::default(), - estack: ErrorStack::new(feedback_source), - wstack: WarningStack::new(feedback_source), - } - } - /// Create a new configset for environment variables - pub fn new_env() -> Self { - Self::_new(Self::EMSG_ENV) - } - /// Create a new configset for CLI args - pub fn new_cli() -> Self { - Self::_new(Self::EMSG_CLI) - } - /// Create a new configset for config files - pub fn new_file() -> Self { - Self { - did_mutate: true, - cfg: ConfigurationSet::default(), - estack: ErrorStack::new(Self::EMSG_FILE), - wstack: WarningStack::new(Self::EMSG_FILE), - } - } - /// Mark the configset mutated - fn mutated(&mut self) { - self.did_mutate = true; - } - /// Push an error onto the error stack - fn epush(&mut self, field_key: StaticStr, expected: StaticStr) { - self.estack - .push(format!("Bad value for `{field_key}`. Expected {expected}",)) - } - /// Check if no errors have occurred - pub fn is_okay(&self) -> bool { - self.estack.is_empty() - } - /// Check if the configset was mutated - pub fn is_mutated(&self) -> bool { - self.did_mutate - } - /// Attempt to mutate with a target `TryFromConfigSource` type, and push in any error that occurs - /// using the given diagnostic info - fn try_mutate( - &mut self, - new: impl TryFromConfigSource, - target: &mut T, - field_key: StaticStr, - expected: StaticStr, - ) { - if new.mutate_failed(target, &mut self.did_mutate) { - self.epush(field_key, expected) - } - } - /// Attempt to mutate with a target `TryFromConfigSource` type, and push in any error that occurs - /// using the given diagnostic info while checking the correctly parsed type using the provided validation - /// closure for any additional validation check that goes beyond type correctness - fn try_mutate_with_condcheck( - &mut self, - new: impl TryFromConfigSource, - target: &mut T, - field_key: StaticStr, - expected: StaticStr, - validation_fn: F, - ) where - F: Fn(&T) -> bool, - { - let mut needs_error = false; - match new.try_parse() { - ConfigSourceParseResult::Okay(ok) => { - self.mutated(); - needs_error = !validation_fn(&ok); - *target = ok; - } - ConfigSourceParseResult::ParseFailure => { - self.mutated(); - needs_error = true - } - ConfigSourceParseResult::Absent => {} - } - if needs_error { - self.epush(field_key, expected) - } - } - /// This method can be used to chain configurations to ultimately return the first modified configuration - /// that occurs. For example: `cfg_file.and_then(cfg_cli).and_then(cfg_env)`; it will return the first - /// modified Configset - /// - /// ## Panics - /// This method will panic if both the provided sets are mutated. Hence, you need to check beforehand that - /// there is no conflict - pub fn and_then(self, other: Self) -> Self { - if self.is_mutated() { - if other.is_mutated() { - panic!( - "Double mutation: {env_a} and {env_b}", - env_a = self.estack.source(), - env_b = other.estack.source() - ); - } - self - } else { - other - } - } - /// Turns self into a Result that can be used by config::get_config() - pub fn into_result(self, restore_file: Option) -> Result { - let mut target = if self.is_okay() { - // no errors, sweet - if self.is_mutated() { - let Self { cfg, wstack, .. } = self; - ConfigType::new_custom(cfg, restore_file, wstack) - } else { - ConfigType::new_default(restore_file) - } - } else { - return Err(ConfigError::CfgError(self.estack)); - }; - if target.config.protocol != ProtocolVersion::default() { - target.wpush(format!( - "{} is deprecated. Switch to {}", - target.config.protocol.to_string(), - ProtocolVersion::default().to_string() - )); - } - if target.is_prod_mode() { - self::feedback::evaluate_prod_settings(&target.config).map(|_| target) - } else { - target.wpush("Running in `user` mode. Set mode to `prod` in production"); - Ok(target) - } - } -} - -// protocol settings -impl Configset { - pub fn protocol_settings( - &mut self, - nproto: impl TryFromConfigSource, - nproto_key: StaticStr, - ) { - let mut proto = ProtocolVersion::default(); - self.try_mutate( - nproto, - &mut proto, - nproto_key, - "a protocol version like 2.0 or 1.0", - ); - self.cfg.protocol = proto; - } -} - -// server settings -impl Configset { - pub fn server_tcp( - &mut self, - nhost: impl TryFromConfigSource, - nhost_key: StaticStr, - nport: impl TryFromConfigSource, - nport_key: StaticStr, - ) { - let mut host = DEFAULT_IPV4; - let mut port = DEFAULT_PORT; - self.try_mutate(nhost, &mut host, nhost_key, "an IPv4/IPv6 address"); - self.try_mutate(nport, &mut port, nport_key, "a 16-bit positive integer"); - self.cfg.ports = PortConfig::new_insecure_only(host, port); - } - pub fn server_noart(&mut self, nart: impl TryFromConfigSource, nart_key: StaticStr) { - let mut noart = false; - self.try_mutate(nart, &mut noart, nart_key, "true/false"); - self.cfg.noart = noart; - } - pub fn server_maxcon( - &mut self, - nmaxcon: impl TryFromConfigSource, - nmaxcon_key: StaticStr, - ) { - let mut maxcon = MAXIMUM_CONNECTION_LIMIT; - self.try_mutate_with_condcheck( - nmaxcon, - &mut maxcon, - nmaxcon_key, - "a positive integer greater than zero", - |max| *max > 0, - ); - self.cfg.maxcon = maxcon; - } - pub fn server_mode(&mut self, nmode: impl TryFromConfigSource, nmode_key: StaticStr) { - let mut modeset = Modeset::Dev; - self.try_mutate( - nmode, - &mut modeset, - nmode_key, - "a string with 'user' or 'prod'", - ); - self.cfg.mode = modeset; - } -} - -// bgsave settings -impl Configset { - pub fn bgsave_settings( - &mut self, - nenabled: impl TryFromConfigSource, - nenabled_key: StaticStr, - nduration: impl TryFromConfigSource, - nduration_key: StaticStr, - ) { - let mut enabled = true; - let mut duration = DEFAULT_BGSAVE_DURATION; - let has_custom_duration = nduration.is_present(); - self.try_mutate(nenabled, &mut enabled, nenabled_key, "true/false"); - self.try_mutate_with_condcheck( - nduration, - &mut duration, - nduration_key, - "a positive integer greater than zero", - |dur| *dur > 0, - ); - if enabled { - self.cfg.bgsave = BGSave::Enabled(duration); - } else { - if has_custom_duration { - self.wstack.push(format!( - "Specifying `{nduration_key}` is useless when BGSAVE is disabled" - )); - } - self.wstack - .push("BGSAVE is disabled. You may lose data if the host crashes"); - } - } -} - -// snapshot settings -impl Configset { - pub fn snapshot_settings( - &mut self, - nevery: impl TryFromConfigSource, - nevery_key: StaticStr, - natmost: impl TryFromConfigSource, - natmost_key: StaticStr, - nfailsafe: impl TryFromConfigSource, - nfailsafe_key: StaticStr, - ) { - match (nevery.is_present(), natmost.is_present()) { - (false, false) => { - // noice, disabled - if nfailsafe.is_present() { - // this mutation is pointless, but it is just for the sake of making sure - // that the `failsafe` key has a proper boolean, no matter if it is pointless - let mut _failsafe = DEFAULT_SNAPSHOT_FAILSAFE; - self.try_mutate(nfailsafe, &mut _failsafe, nfailsafe_key, "true/false"); - self.wstack.push(format!( - "Specifying `{nfailsafe_key}` is usless when snapshots are disabled" - )); - } - } - (true, true) => { - let mut every = 0; - let mut atmost = 0; - let mut failsafe = DEFAULT_SNAPSHOT_FAILSAFE; - self.try_mutate_with_condcheck( - nevery, - &mut every, - nevery_key, - "an integer greater than 0", - |dur| *dur > 0, - ); - self.try_mutate( - natmost, - &mut atmost, - natmost_key, - "a positive integer. 0 indicates that all snapshots will be kept", - ); - self.try_mutate(nfailsafe, &mut failsafe, nfailsafe_key, "true/false"); - self.cfg.snapshot = - SnapshotConfig::Enabled(SnapshotPref::new(every, atmost, failsafe)); - } - (false, true) | (true, false) => { - // no changes, but still attempted to change - self.mutated(); - self.estack.push(format!( - "To use snapshots, pass values for both `{nevery_key}` and `{natmost_key}`" - )) - } - } - } -} - -// TLS settings -#[allow(clippy::too_many_arguments)] -impl Configset { - pub fn tls_settings( - &mut self, - nkey: impl TryFromConfigSource, - nkey_key: StaticStr, - ncert: impl TryFromConfigSource, - ncert_key: StaticStr, - nport: impl TryFromConfigSource, - nport_key: StaticStr, - nonly: impl TryFromConfigSource, - nonly_key: StaticStr, - npass: impl TryFromConfigSource, - npass_key: StaticStr, - ) { - match (nkey.is_present(), ncert.is_present()) { - (true, true) => { - // get the cert details - let mut key = String::new(); - let mut cert = String::new(); - self.try_mutate(nkey, &mut key, nkey_key, "path to private key file"); - self.try_mutate(ncert, &mut cert, ncert_key, "path to TLS certificate file"); - - // now get port info - let mut port = DEFAULT_SSL_PORT; - self.try_mutate(nport, &mut port, nport_key, "a positive 16-bit integer"); - - // now check if TLS only - let mut tls_only = false; - self.try_mutate(nonly, &mut tls_only, nonly_key, "true/false"); - - // check if we have a TLS cert - let mut tls_pass = OptString::new_null(); - self.try_mutate( - npass, - &mut tls_pass, - npass_key, - "path to TLS cert passphrase", - ); - - let sslopts = SslOpts::new(key, cert, port, tls_pass.base); - // now check if TLS only - if tls_only { - let host = self.cfg.ports.get_host(); - self.cfg.ports = PortConfig::new_secure_only(host, sslopts) - } else { - // multi. go and upgrade existing - self.cfg.ports.upgrade_to_tls(sslopts); - } - } - (true, false) | (false, true) => { - self.mutated(); - self.estack.push(format!( - "To use TLS, pass values for both `{nkey_key}` and `{ncert_key}`" - )); - } - (false, false) => { - if nport.is_present() { - self.mutated(); - self.wstack.push(format!( - "Specifying `{nport_key}` is pointless when TLS is disabled" - )); - } - if nonly.is_present() { - self.mutated(); - self.wstack.push(format!( - "Specifying `{nonly_key}` is pointless when TLS is disabled" - )); - } - if npass.is_present() { - self.mutated(); - self.wstack.push(format!( - "Specifying `{npass_key}` is pointless when TLS is disabled" - )); - } - } - } - } -} - -// Auth settings -impl Configset { - pub fn auth_settings( - &mut self, - nauth: impl TryFromConfigSource, - nauth_key: StaticStr, - ) { - let mut def = AuthkeyWrapper::empty(); - self.try_mutate(nauth, &mut def, nauth_key, "A 40-byte long ASCII string"); - if def != AuthkeyWrapper::empty() { - self.cfg.auth = AuthSettings { - origin_key: Some(def), - }; - } - } -} - -pub fn get_config() -> Result { - // initialize clap because that will let us check for CLI/file configs - let cfg_layout = load_yaml!("../cli.yml"); - let matches = App::from_yaml(cfg_layout).get_matches(); - let restore_file = matches.value_of("restore").map(|v| v.to_string()); - - // get config from file - let cfg_from_file = if let Some(file) = matches.value_of("config") { - let file = fs::read(file)?; - let cfg_file: ConfigFile = toml::from_slice(&file)?; - Some(cfgfile::from_file(cfg_file)) - } else { - None - }; - - // get config from CLI - let cfg_from_cli = cfgcli::parse_cli_args(matches); - // get config from env - let cfg_from_env = cfgenv::parse_env_config(); - // calculate the number of config sources - let cfg_degree = cfg_from_cli.is_mutated() as u8 - + cfg_from_env.is_mutated() as u8 - + cfg_from_file.is_some() as u8; - // if degree is more than 1, there is a conflict - let has_conflict = cfg_degree > 1; - if has_conflict { - return Err(ConfigError::Conflict); - } - if cfg_degree == 0 { - // no configuration, use default - Ok(ConfigType::new_default(restore_file)) - } else { - cfg_from_file - .unwrap_or_else(|| cfg_from_env.and_then(cfg_from_cli)) - .into_result(restore_file) - } -} diff --git a/server/src/config/tests.rs b/server/src/config/tests.rs deleted file mode 100644 index 91d00603..00000000 --- a/server/src/config/tests.rs +++ /dev/null @@ -1,837 +0,0 @@ -/* - * Created on Thu Sep 23 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{BGSave, Configset, PortConfig, SnapshotConfig, SnapshotPref, SslOpts, DEFAULT_IPV4}, - crate::ROOT_DIR, - std::fs, -}; - -// server tests -// TCP -#[test] -fn server_tcp() { - let mut cfgset = Configset::new_env(); - cfgset.server_tcp( - Some("127.0.0.1"), - "SKY_SERVER_HOST", - Some("2004"), - "SKY_SERVER_PORT", - ); - assert_eq!( - cfgset.cfg.ports, - PortConfig::new_insecure_only(DEFAULT_IPV4, 2004) - ); - assert!(cfgset.is_mutated()); - assert!(cfgset.is_okay()); -} - -#[test] -fn server_tcp_fail_host() { - let mut cfgset = Configset::new_env(); - cfgset.server_tcp( - Some("?127.0.0.1"), - "SKY_SERVER_HOST", - Some("2004"), - "SKY_SERVER_PORT", - ); - assert_eq!( - cfgset.cfg.ports, - PortConfig::new_insecure_only(DEFAULT_IPV4, 2004) - ); - assert!(cfgset.is_mutated()); - assert!(!cfgset.is_okay()); - assert_eq!( - cfgset.estack[0], - "Bad value for `SKY_SERVER_HOST`. Expected an IPv4/IPv6 address" - ); -} - -#[test] -fn server_tcp_fail_port() { - let mut cfgset = Configset::new_env(); - cfgset.server_tcp( - Some("127.0.0.1"), - "SKY_SERVER_HOST", - Some("65537"), - "SKY_SERVER_PORT", - ); - assert_eq!( - cfgset.cfg.ports, - PortConfig::new_insecure_only(DEFAULT_IPV4, 2003) - ); - assert!(cfgset.is_mutated()); - assert!(!cfgset.is_okay()); - assert_eq!( - cfgset.estack[0], - "Bad value for `SKY_SERVER_PORT`. Expected a 16-bit positive integer" - ); -} - -#[test] -fn server_tcp_fail_both() { - let mut cfgset = Configset::new_env(); - cfgset.server_tcp( - Some("?127.0.0.1"), - "SKY_SERVER_HOST", - Some("65537"), - "SKY_SERVER_PORT", - ); - assert_eq!( - cfgset.cfg.ports, - PortConfig::new_insecure_only(DEFAULT_IPV4, 2003) - ); - assert!(cfgset.is_mutated()); - assert!(!cfgset.is_okay()); - assert_eq!( - cfgset.estack[0], - "Bad value for `SKY_SERVER_HOST`. Expected an IPv4/IPv6 address" - ); - assert_eq!( - cfgset.estack[1], - "Bad value for `SKY_SERVER_PORT`. Expected a 16-bit positive integer" - ); -} - -// noart -#[test] -fn server_noart_okay() { - let mut cfgset = Configset::new_env(); - cfgset.server_noart(Some("true"), "SKY_SYSTEM_NOART"); - assert!(!cfgset.cfg.is_artful()); - assert!(cfgset.is_okay()); - assert!(cfgset.is_mutated()); -} - -#[test] -fn server_noart_fail() { - let mut cfgset = Configset::new_env(); - cfgset.server_noart(Some("truee"), "SKY_SYSTEM_NOART"); - assert!(cfgset.cfg.is_artful()); - assert!(!cfgset.is_okay()); - assert_eq!( - cfgset.estack[0], - "Bad value for `SKY_SYSTEM_NOART`. Expected true/false" - ); - assert!(cfgset.is_mutated()); -} - -#[test] -fn server_maxcon_okay() { - let mut cfgset = Configset::new_env(); - cfgset.server_maxcon(Some("12345"), "SKY_SYSTEM_MAXCON"); - assert!(cfgset.is_mutated()); - assert!(cfgset.is_okay()); - assert_eq!(cfgset.cfg.maxcon, 12345); -} - -#[test] -fn server_maxcon_fail() { - let mut cfgset = Configset::new_env(); - cfgset.server_maxcon(Some("12345A"), "SKY_SYSTEM_MAXCON"); - assert!(cfgset.is_mutated()); - assert!(!cfgset.is_okay()); - assert_eq!( - cfgset.estack[0], - "Bad value for `SKY_SYSTEM_MAXCON`. Expected a positive integer greater than zero" - ); - assert_eq!(cfgset.cfg.maxcon, 50000); -} - -// bgsave settings -#[test] -fn bgsave_okay() { - let mut cfgset = Configset::new_env(); - cfgset.bgsave_settings( - Some("true"), - "SKY_BGSAVE_ENABLED", - Some("128"), - "SKY_BGSAVE_DURATION", - ); - assert!(cfgset.is_mutated()); - assert!(cfgset.is_okay()); - assert_eq!(cfgset.cfg.bgsave, BGSave::Enabled(128)); -} - -#[test] -fn bgsave_fail() { - let mut cfgset = Configset::new_env(); - cfgset.bgsave_settings( - Some("truee"), - "SKY_BGSAVE_ENABLED", - Some("128"), - "SKY_BGSAVE_DURATION", - ); - assert!(cfgset.is_mutated()); - assert!(!cfgset.is_okay()); - assert_eq!( - cfgset.estack[0], - "Bad value for `SKY_BGSAVE_ENABLED`. Expected true/false" - ); - assert_eq!(cfgset.cfg.bgsave, BGSave::Enabled(128)); -} - -// snapshot settings -#[test] -fn snapshot_okay() { - let mut cfgset = Configset::new_env(); - cfgset.snapshot_settings( - Some("3600"), - "SKY_SNAPSHOT_EVERY", - Some("0"), - "SKY_SNAPSHOT_ATMOST", - Some("false"), - "SKY_SNAPSHOT_FAILSAFE", - ); - assert!(cfgset.is_mutated()); - assert!(cfgset.is_okay()); - assert_eq!( - cfgset.cfg.snapshot, - SnapshotConfig::Enabled(SnapshotPref::new(3600, 0, false)) - ); -} - -#[test] -fn snapshot_fail() { - let mut cfgset = Configset::new_env(); - cfgset.snapshot_settings( - Some("3600"), - "SKY_SNAPSHOT_EVERY", - Some("0"), - "SKY_SNAPSHOT_ATMOST", - Some("falsee"), - "SKY_SNAPSHOT_FAILSAFE", - ); - assert!(cfgset.is_mutated()); - assert!(!cfgset.is_okay()); - assert_eq!( - cfgset.estack[0], - "Bad value for `SKY_SNAPSHOT_FAILSAFE`. Expected true/false" - ); - assert_eq!( - cfgset.cfg.snapshot, - SnapshotConfig::Enabled(SnapshotPref::new(3600, 0, true)) - ); -} - -#[test] -fn snapshot_fail_with_missing_required_values() { - let mut cfgset = Configset::new_env(); - cfgset.snapshot_settings( - Some("3600"), - "SKY_SNAPSHOT_EVERY", - None, - "SKY_SNAPSHOT_ATMOST", - None, - "SKY_SNAPSHOT_FAILSAFE", - ); - assert!(cfgset.is_mutated()); - assert!(!cfgset.is_okay()); - assert_eq!( - cfgset.estack[0], - "To use snapshots, pass values for both `SKY_SNAPSHOT_EVERY` and `SKY_SNAPSHOT_ATMOST`" - ); - assert_eq!(cfgset.cfg.snapshot, SnapshotConfig::Disabled); -} - -// TLS settings -#[test] -fn tls_settings_okay() { - let mut cfg = Configset::new_env(); - cfg.tls_settings( - Some("key.pem"), - "SKY_TLS_KEY", - Some("cert.pem"), - "SKY_TLS_CERT", - Some("2005"), - "SKY_TLS_PORT", - Some("false"), - "SKY_TLS_ONLY", - None, - "SKY_TLS_PASSIN", - ); - assert!(cfg.is_mutated()); - assert!(cfg.is_okay()); - assert_eq!(cfg.cfg.ports, { - let mut pf = PortConfig::default(); - pf.upgrade_to_tls(SslOpts::new( - "key.pem".to_owned(), - "cert.pem".to_owned(), - 2005, - None, - )); - pf - }); -} - -#[test] -fn tls_settings_fail() { - let mut cfg = Configset::new_env(); - cfg.tls_settings( - Some("key.pem"), - "SKY_TLS_KEY", - Some("cert.pem"), - "SKY_TLS_CERT", - Some("A2005"), - "SKY_TLS_PORT", - Some("false"), - "SKY_TLS_ONLY", - None, - "SKY_TLS_PASSIN", - ); - assert!(cfg.is_mutated()); - assert!(!cfg.is_okay()); - assert_eq!(cfg.cfg.ports, { - let mut pf = PortConfig::default(); - pf.upgrade_to_tls(SslOpts::new( - "key.pem".to_owned(), - "cert.pem".to_owned(), - 2004, - None, - )); - pf - }); -} - -#[test] -fn tls_settings_fail_with_missing_required_values() { - let mut cfg = Configset::new_env(); - cfg.tls_settings( - Some("key.pem"), - "SKY_TLS_KEY", - None, - "SKY_TLS_CERT", - Some("2005"), - "SKY_TLS_PORT", - Some("false"), - "SKY_TLS_ONLY", - None, - "SKY_TLS_PASSIN", - ); - assert!(cfg.is_mutated()); - assert!(!cfg.is_okay()); - assert_eq!(cfg.cfg.ports, PortConfig::default()); -} - -/// Gets a `toml` file from `WORKSPACEROOT/examples/config-files` -fn get_toml_from_examples_dir(filename: &str) -> String { - let path = format!("{ROOT_DIR}examples/config-files/{filename}"); - fs::read_to_string(path).unwrap() -} - -mod cfg_file_tests { - use super::get_toml_from_examples_dir; - use crate::config::AuthkeyWrapper; - use crate::config::{ - cfgfile, AuthSettings, BGSave, Configset, ConfigurationSet, Modeset, PortConfig, - ProtocolVersion, SnapshotConfig, SnapshotPref, SslOpts, DEFAULT_IPV4, DEFAULT_PORT, - }; - use crate::dbnet::MAXIMUM_CONNECTION_LIMIT; - use std::net::{IpAddr, Ipv6Addr}; - - fn cfgset_from_toml_str(file: String) -> Result { - let toml = toml::from_str(&file)?; - Ok(cfgfile::from_file(toml)) - } - - #[test] - fn config_file_okay() { - let file = get_toml_from_examples_dir("template.toml"); - let toml = toml::from_str(&file).unwrap(); - let cfg_from_file = cfgfile::from_file(toml); - assert!(cfg_from_file.is_mutated()); - assert!(cfg_from_file.is_okay()); - // expected - let mut expected = ConfigurationSet::default(); - expected.snapshot = SnapshotConfig::Enabled(SnapshotPref::new(3600, 4, true)); - expected.ports = PortConfig::new_secure_only( - crate::config::DEFAULT_IPV4, - SslOpts::new( - "/path/to/keyfile.pem".to_owned(), - "/path/to/chain.pem".to_owned(), - 2004, - Some("/path/to/cert/passphrase.txt".to_owned()), - ), - ); - expected.auth.origin_key = - Some(AuthkeyWrapper::try_new(crate::TEST_AUTH_ORIGIN_KEY).unwrap()); - // check - assert_eq!(cfg_from_file.cfg, expected); - } - - #[test] - fn test_config_file_ok() { - let file = get_toml_from_examples_dir("skyd.toml"); - let cfg = cfgset_from_toml_str(file).unwrap(); - assert_eq!(cfg.cfg, ConfigurationSet::default()); - } - - #[test] - fn test_config_file_noart() { - let file = get_toml_from_examples_dir("secure-noart.toml"); - let cfg = cfgset_from_toml_str(file).unwrap(); - assert_eq!( - cfg.cfg, - ConfigurationSet { - noart: true, - bgsave: BGSave::default(), - snapshot: SnapshotConfig::default(), - ports: PortConfig::default(), - maxcon: MAXIMUM_CONNECTION_LIMIT, - mode: Modeset::Dev, - auth: AuthSettings::default(), - protocol: ProtocolVersion::default(), - } - ); - } - - #[test] - fn test_config_file_ipv6() { - let file = get_toml_from_examples_dir("ipv6.toml"); - let cfg = cfgset_from_toml_str(file).unwrap(); - assert_eq!( - cfg.cfg, - ConfigurationSet { - noart: false, - bgsave: BGSave::default(), - snapshot: SnapshotConfig::default(), - ports: PortConfig::new_insecure_only( - IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0x1)), - DEFAULT_PORT - ), - maxcon: MAXIMUM_CONNECTION_LIMIT, - mode: Modeset::Dev, - auth: AuthSettings::default(), - protocol: ProtocolVersion::default(), - } - ); - } - - #[test] - fn test_config_file_template() { - let file = get_toml_from_examples_dir("template.toml"); - let cfg = cfgset_from_toml_str(file).unwrap(); - assert_eq!( - cfg.cfg, - ConfigurationSet::new( - false, - BGSave::default(), - SnapshotConfig::Enabled(SnapshotPref::new(3600, 4, true)), - PortConfig::new_secure_only( - DEFAULT_IPV4, - SslOpts::new( - "/path/to/keyfile.pem".into(), - "/path/to/chain.pem".into(), - 2004, - Some("/path/to/cert/passphrase.txt".to_owned()) - ) - ), - MAXIMUM_CONNECTION_LIMIT, - Modeset::Dev, - AuthSettings::new(AuthkeyWrapper::try_new(crate::TEST_AUTH_ORIGIN_KEY).unwrap()), - ProtocolVersion::default() - ) - ); - } - - #[test] - fn test_config_file_bad_bgsave_section() { - let file = get_toml_from_examples_dir("badcfg2.toml"); - let cfg = cfgset_from_toml_str(file); - assert!(cfg.is_err()); - } - - #[test] - fn test_config_file_custom_bgsave() { - let file = get_toml_from_examples_dir("withcustombgsave.toml"); - let cfg = cfgset_from_toml_str(file).unwrap(); - assert_eq!( - cfg.cfg, - ConfigurationSet { - noart: false, - bgsave: BGSave::new(true, 600), - snapshot: SnapshotConfig::default(), - ports: PortConfig::default(), - maxcon: MAXIMUM_CONNECTION_LIMIT, - mode: Modeset::Dev, - auth: AuthSettings::default(), - protocol: ProtocolVersion::default(), - } - ); - } - - #[test] - fn test_config_file_bgsave_enabled_only() { - /* - * This test demonstrates a case where the user just said that BGSAVE is enabled. - * In that case, we will default to the 120 second duration - */ - let file = get_toml_from_examples_dir("bgsave-justenabled.toml"); - let cfg = cfgset_from_toml_str(file).unwrap(); - assert_eq!( - cfg.cfg, - ConfigurationSet { - noart: false, - bgsave: BGSave::default(), - snapshot: SnapshotConfig::default(), - ports: PortConfig::default(), - maxcon: MAXIMUM_CONNECTION_LIMIT, - mode: Modeset::Dev, - auth: AuthSettings::default(), - protocol: ProtocolVersion::default(), - } - ) - } - - #[test] - fn test_config_file_bgsave_every_only() { - /* - * This test demonstrates a case where the user just gave the value for every - * In that case, it means BGSAVE is enabled and set to `every` seconds - */ - let file = get_toml_from_examples_dir("bgsave-justevery.toml"); - let cfg = cfgset_from_toml_str(file).unwrap(); - assert_eq!( - cfg.cfg, - ConfigurationSet { - noart: false, - bgsave: BGSave::new(true, 600), - snapshot: SnapshotConfig::default(), - ports: PortConfig::default(), - maxcon: MAXIMUM_CONNECTION_LIMIT, - mode: Modeset::Dev, - auth: AuthSettings::default(), - protocol: ProtocolVersion::default(), - } - ) - } - - #[test] - fn test_config_file_snapshot() { - let file = get_toml_from_examples_dir("snapshot.toml"); - let cfg = cfgset_from_toml_str(file).unwrap(); - assert_eq!( - cfg.cfg, - ConfigurationSet { - snapshot: SnapshotConfig::Enabled(SnapshotPref::new(3600, 4, true)), - bgsave: BGSave::default(), - noart: false, - ports: PortConfig::default(), - maxcon: MAXIMUM_CONNECTION_LIMIT, - mode: Modeset::Dev, - auth: AuthSettings::default(), - protocol: ProtocolVersion::default(), - } - ); - } -} - -mod cli_arg_tests { - use crate::config::{cfgcli, PortConfig}; - use clap::{load_yaml, App}; - #[test] - fn cli_args_okay() { - let cfg_layout = load_yaml!("../cli.yml"); - let cli_args = ["skyd", "--host", "127.0.0.2"]; - let matches = App::from_yaml(cfg_layout).get_matches_from(cli_args); - let ret = cfgcli::parse_cli_args(matches); - assert_eq!( - ret.cfg.ports, - PortConfig::new_insecure_only("127.0.0.2".parse().unwrap(), 2003) - ); - assert!(ret.is_mutated()); - assert!(ret.is_okay()); - } - #[test] - fn cli_args_okay_no_mut() { - let cfg_layout = load_yaml!("../cli.yml"); - let cli_args = ["skyd", "--restore", "/some/restore/path"]; - let matches = App::from_yaml(cfg_layout).get_matches_from(cli_args); - let ret = cfgcli::parse_cli_args(matches); - assert!(!ret.is_mutated()); - assert!(ret.is_okay()); - } - #[test] - fn cli_args_fail() { - let cfg_layout = load_yaml!("../cli.yml"); - let cli_args = ["skyd", "--port", "port2003"]; - let matches = App::from_yaml(cfg_layout).get_matches_from(cli_args); - let ret = cfgcli::parse_cli_args(matches); - assert!(ret.is_mutated()); - assert!(!ret.is_okay()); - assert_eq!( - ret.estack[0], - "Bad value for `--port`. Expected a 16-bit positive integer" - ); - } -} - -mod try_from_config_source_impls { - use crate::config::{cfgcli::Flag, cfgfile::Optional, TryFromConfigSource, DEFAULT_IPV4}; - use std::env::{set_var, var}; - use std::fmt::Debug; - - const EXPECT_TRUE: bool = true; - const EXPECT_FALSE: bool = false; - const MUTATED: bool = true; - const NOT_MUTATED: bool = false; - const IS_PRESENT: bool = true; - const IS_ABSENT: bool = false; - const MUTATION_FAILURE: bool = true; - const NO_MUTATION_FAILURE: bool = false; - - fn _mut_base_test_expected( - new: impl TryFromConfigSource, - expected: T, - is_present: bool, - mutate_failed: bool, - has_mutated: bool, - ) { - let mut default = Default::default(); - let mut mutated = false; - assert_eq!(new.is_present(), is_present); - assert_eq!(new.mutate_failed(&mut default, &mut mutated), mutate_failed); - assert_eq!(mutated, has_mutated); - assert_eq!(default, expected); - } - - fn _mut_base_test( - new: impl TryFromConfigSource, - mut default: T, - is_present: bool, - mutate_failed: bool, - has_mutated: bool, - ) { - let mut mutated = false; - dbg!(new.is_present(), is_present); - assert_eq!(new.is_present(), is_present); - assert_eq!(new.mutate_failed(&mut default, &mut mutated), mutate_failed); - assert_eq!(mutated, has_mutated); - } - - fn mut_test_pass(new: impl TryFromConfigSource, default: T) { - _mut_base_test(new, default, IS_PRESENT, NO_MUTATION_FAILURE, MUTATED) - } - - fn mut_test_fail(new: impl TryFromConfigSource, default: T) { - _mut_base_test(new, default, IS_PRESENT, MUTATION_FAILURE, MUTATED) - } - - mod env_var { - use super::*; - - // test for Result - #[test] - fn env_okay_ipv4() { - set_var("TEST_SKY_SYSTEM_HOST", "127.0.0.1"); - mut_test_pass(var("TEST_SKY_SYSTEM_HOST"), DEFAULT_IPV4); - } - - #[test] - fn env_fail_ipv4() { - set_var("TEST_SKY_SYSTEM_HOST2", "127.0.0.1A"); - mut_test_fail(var("TEST_SKY_SYSTEM_HOST2"), DEFAULT_IPV4); - } - } - - mod option_str { - use super::*; - - // test for Option<&str> (as in CLI) - #[test] - fn option_str_okay_ipv4() { - let ip = Some("127.0.0.1"); - mut_test_pass(ip, DEFAULT_IPV4); - } - - #[test] - fn option_str_fail_ipv4() { - let ip = Some("127.0.0.1A"); - mut_test_fail(ip, DEFAULT_IPV4); - } - - #[test] - fn option_str_nomut() { - let ip = None; - _mut_base_test( - ip, - DEFAULT_IPV4, - IS_ABSENT, - NO_MUTATION_FAILURE, - NOT_MUTATED, - ); - } - } - - mod cfgcli_flag { - use super::*; - - #[test] - fn flag_true_if_set_okay_set() { - // this is true if flag is present - let flag = Flag::::new(true); - // we expect true - _mut_base_test_expected(flag, EXPECT_TRUE, IS_PRESENT, NO_MUTATION_FAILURE, MUTATED); - } - - #[test] - fn flag_true_if_set_okay_unset() { - // this is true if flag is present, but the flag here is not present - let flag = Flag::::new(false); - // we expect no mutation because the flag was not set - _mut_base_test( - flag, - EXPECT_FALSE, - IS_ABSENT, - NO_MUTATION_FAILURE, - NOT_MUTATED, - ); - } - - #[test] - fn flag_false_if_set_okay_set() { - // this is false if flag is present - let flag = Flag::::new(true); - // expect mutation to have happened - _mut_base_test_expected(flag, EXPECT_FALSE, IS_PRESENT, NO_MUTATION_FAILURE, MUTATED); - } - - #[test] - fn flag_false_if_set_okay_unset() { - // this is false if flag is present, but the flag is absent - let flag = Flag::::new(false); - // expect no mutation - _mut_base_test( - flag, - EXPECT_FALSE, - IS_ABSENT, - NO_MUTATION_FAILURE, - NOT_MUTATED, - ); - } - } - - mod optional { - use super::*; - - // test for cfg file scenario - #[test] - fn optional_okay_ipv4() { - let ip = Optional::some(DEFAULT_IPV4); - mut_test_pass(ip, DEFAULT_IPV4); - } - - #[test] - fn optional_okay_ipv4_none() { - let ip = Optional::from(None); - _mut_base_test( - ip, - DEFAULT_IPV4, - IS_ABSENT, - NO_MUTATION_FAILURE, - NOT_MUTATED, - ); - } - } - - mod cfgfile_nonull { - use super::*; - use crate::config::cfgfile::NonNull; - - #[test] - fn nonnull_okay() { - let port = NonNull::from(2100); - _mut_base_test_expected(port, 2100, IS_PRESENT, NO_MUTATION_FAILURE, MUTATED); - } - } - - mod optstring { - use super::*; - use crate::config::OptString; - - #[test] - fn optstring_okay() { - let pass = OptString::from(Some("tlspass.txt".to_owned())); - _mut_base_test_expected( - pass, - OptString::from(Some("tlspass.txt".to_owned())), - IS_PRESENT, - NO_MUTATION_FAILURE, - MUTATED, - ); - } - - #[test] - fn optstring_null_okay() { - let pass = OptString::from(None); - _mut_base_test_expected( - pass, - OptString::new_null(), - IS_ABSENT, - NO_MUTATION_FAILURE, - NOT_MUTATED, - ); - } - } -} - -mod modeset_de { - use crate::config::Modeset; - use serde::Deserialize; - - #[derive(Deserialize, Debug)] - struct Example { - mode: Modeset, - } - - #[test] - fn deserialize_modeset_prod_okay() { - #[derive(Deserialize, Debug)] - struct Example { - mode: Modeset, - } - let toml = r#"mode="prod""#; - let x: Example = toml::from_str(toml).unwrap(); - assert_eq!(x.mode, Modeset::Prod); - } - - #[test] - fn deserialize_modeset_user_okay() { - let toml = r#"mode="dev""#; - let x: Example = toml::from_str(toml).unwrap(); - assert_eq!(x.mode, Modeset::Dev); - } - - #[test] - fn deserialize_modeset_fail() { - let toml = r#"mode="superuser""#; - let e = toml::from_str::(toml).unwrap_err(); - assert_eq!( - e.to_string(), - "Bad value `superuser` for modeset for key `mode` at line 1 column 6" - ); - } -} diff --git a/server/src/corestore/array.rs b/server/src/corestore/array.rs deleted file mode 100644 index 199be981..00000000 --- a/server/src/corestore/array.rs +++ /dev/null @@ -1,624 +0,0 @@ -/* - * Created on Tue Jul 06 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use core::{ - any, - borrow::{Borrow, BorrowMut}, - cmp::Ordering, - fmt, - hash::{Hash, Hasher}, - iter::FromIterator, - mem::{ManuallyDrop, MaybeUninit}, - ops, ptr, slice, str, -}; - -/// A compile-time, fixed size array that can have unintialized memory. This array is as -/// efficient as you'd expect a normal array to be, but with the added benefit that you -/// don't have to initialize all the elements. This was inspired by the arrayvec crate. -/// Safe abstractions are made available enabling us to not enter uninitialized space and -/// read the _available_ elements. The array size is limited to 16 bits or 2 bytes to -/// prevent stack overflows. -/// -/// ## Panics -/// To avoid stack corruption among other crazy things, several implementations like [`Extend`] -/// can panic. There are _silently corrupting_ methods too which can be used if you can uphold -/// the guarantees -pub struct Array { - /// the maybe bad stack - stack: [MaybeUninit; N], - /// the initialized length - /// no stack should be more than 16 bytes - init_len: u16, -} - -/// The len scopeguard is like a scopeguard that provides panic safety incase an append-like -/// operation involving iterators causes the iterator to panic. This makes sure that we still -/// set the len on panic -pub struct LenScopeGuard<'a, T: Copy> { - real_ref: &'a mut T, - temp: T, -} - -impl<'a, T: ops::AddAssign + Copy> LenScopeGuard<'a, T> { - pub fn new(real_ref: &'a mut T) -> Self { - let ret = *real_ref; - Self { - real_ref, - temp: ret, - } - } - pub fn incr(&mut self, val: T) { - self.temp += val; - } - pub fn get_temp(&self) -> T { - self.temp - } -} - -impl<'a, T: Copy> Drop for LenScopeGuard<'a, T> { - fn drop(&mut self) { - *self.real_ref = self.temp; - } -} - -macro_rules! impl_zeroed_nm { - ($($ty:ty),* $(,)?) => { - $( - impl Array<$ty, N> { - pub const fn new_zeroed() -> Self { - Self { - stack: [MaybeUninit::new(0); N], - init_len: N as u16, - } - } - } - )* - }; -} - -impl_zeroed_nm! { - u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize -} - -impl Array { - // just some silly hackery here because uninit_array isn't stabilized -- move on - const VALUE: MaybeUninit = MaybeUninit::uninit(); - const ARRAY: [MaybeUninit; N] = [Self::VALUE; N]; - /// Create a new array - pub const fn new() -> Self { - Array { - stack: Self::ARRAY, - init_len: 0, - } - } - /// This is very safe from the ctor point of view, but the correctness of `init_len` - /// may be a bad assumption and might make us read garbage - pub const unsafe fn from_const(array: [MaybeUninit; N], init_len: u16) -> Self { - Self { - stack: array, - init_len, - } - } - pub unsafe fn bump_init_len(&mut self, bump: u16) { - self.init_len += bump - } - /// This literally turns [T; M] into [T; N]. How can you expect it to be safe? - /// This function is extremely unsafe. I mean, I don't even know how to call it safe. - /// There's one way though: make M == N. This will panic in debug mode if M > N. In - /// release mode, good luck - unsafe fn from_const_array(arr: [T; M]) -> Self { - debug_assert!( - N >= M, - "Provided const array exceeds size limit of initialized array" - ); - // do not double-free or destroy the elements - let array = ManuallyDrop::new(arr); - let mut arr = Array::::new(); - // copy it over - let ptr = &*array as *const [T; M] as *const [MaybeUninit; N]; - ptr.copy_to_nonoverlapping(&mut arr.stack as *mut [MaybeUninit; N], 1); - arr.set_len(N); - arr - } - /// Get the apparent length of the array - pub const fn len(&self) -> usize { - self.init_len as usize - } - /// Get the capacity of the array - pub const fn capacity(&self) -> usize { - N - } - /// Check if the array is full - pub const fn is_full(&self) -> bool { - N == self.len() - } - /// Get the remaining capacity of the array - pub const fn remaining_cap(&self) -> usize { - self.capacity() - self.len() - } - /// Set the length of the array - /// - /// ## Safety - /// This is one of those, use to leak memory functions. If you change the length, - /// you'll be reading random garbage from the memory and doing a double-free on drop - pub unsafe fn set_len(&mut self, len: usize) { - self.init_len = len as u16; // lossy cast, we maintain all invariants - } - /// Get the array as a mut ptr - unsafe fn as_mut_ptr(&mut self) -> *mut T { - self.stack.as_mut_ptr() as *mut _ - } - /// Get the array as a const ptr - unsafe fn as_ptr(&self) -> *const T { - self.stack.as_ptr() as *const _ - } - /// Push an element into the array **without any bounds checking**. - /// - /// ## Safety - /// This function is **so unsafe** that you possibly don't want to call it, or - /// even think about calling it. You can end up corrupting your own stack or - /// other's valuable data - pub unsafe fn push_unchecked(&mut self, element: T) { - let len = self.len(); - ptr::write(self.as_mut_ptr().add(len), element); - self.set_len(len + 1); - } - /// This is a nice version of a push that does bound checks - pub fn push_panic(&mut self, element: T) -> Result<(), ()> { - if self.len() < N { - // so we can push it in - unsafe { self.push_unchecked(element) }; - Ok(()) - } else { - Err(()) - } - } - /// This is a _panicky_ but safer alternative to `push_unchecked` that panics on - /// incorrect lengths - pub fn push(&mut self, element: T) { - self.push_panic(element).unwrap(); - } - /// Pop an item off the array - pub fn pop(&mut self) -> Option { - if self.len() == 0 { - // nothing here - None - } else { - unsafe { - let new_len = self.len() - 1; - self.set_len(new_len); - // len - 1 == offset - Some(ptr::read(self.as_ptr().add(new_len))) - } - } - } - /// Truncate the array to a given size. This is super safe and doesn't even panic - /// if you provide a silly `new_len`. - pub fn truncate(&mut self, new_len: usize) { - let len = self.len(); - if new_len < len { - // we need to drop off a part of the array - unsafe { - // drop_in_place will handle the ZST invariant for us - ptr::drop_in_place(slice::from_raw_parts_mut( - self.as_mut_ptr().add(new_len), - len - new_len, - )) - } - } - } - /// Empty the internal array - pub fn clear(&mut self) { - self.truncate(0) - } - /// Extend self from a slice - pub fn extend_from_slice(&mut self, slice: &[T]) -> Result<(), ()> - where - T: Copy, - { - if self.remaining_cap() < slice.len() { - // no more space here - return Err(()); - } - unsafe { - self.extend_from_slice_unchecked(slice); - } - Ok(()) - } - /// Extend self from a slice without doing a single check - /// - /// ## Safety - /// This function is just very very and. You can write giant things into your own - /// stack corrupting it, corrupting other people's things and creating undefined - /// behavior like no one else. - pub unsafe fn extend_from_slice_unchecked(&mut self, slice: &[T]) { - let self_len = self.len(); - let other_len = slice.len(); - ptr::copy_nonoverlapping(slice.as_ptr(), self.as_mut_ptr().add(self_len), other_len); - self.set_len(self_len + other_len); - } - /// Returns self as a `[T; N]` array if it is fully initialized. Else it will again return - /// itself - pub fn into_array(self) -> Result<[T; N], Self> { - if self.len() < self.capacity() { - // not fully initialized - Err(self) - } else { - unsafe { Ok(self.into_array_unchecked()) } - } - } - pub unsafe fn into_array_unchecked(self) -> [T; N] { - // make sure we don't do a double free or end up deleting the elements - let _self = ManuallyDrop::new(self); - ptr::read(_self.as_ptr() as *const [T; N]) - } - pub fn try_from_slice(slice: impl AsRef<[T]>) -> Option { - let slice = slice.as_ref(); - if slice.len() > N { - None - } else { - Some(unsafe { Self::from_slice(slice) }) - } - } - /// Extend self from a slice - /// - /// ## Safety - /// The same danger as in from_slice_unchecked - pub unsafe fn from_slice(slice_ref: impl AsRef<[T]>) -> Self { - let mut slf = Self::new(); - slf.extend_from_slice_unchecked(slice_ref.as_ref()); - slf - } - // these operations are incredibly safe because we only pass the initialized part - // of the array - /// Get self as a slice. Super safe because we guarantee that all the other invarians - /// are upheld - pub fn as_slice(&self) -> &[T] { - unsafe { slice::from_raw_parts(self.as_ptr(), self.len()) } - } - /// Get self as a mutable slice. Super safe (see comment above) - fn as_slice_mut(&mut self) -> &mut [T] { - unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len()) } - } -} - -impl Array { - /// This isn't _unsafe_ but it can cause functions expecting pure unicode to - /// crash if the array contains invalid unicode - pub unsafe fn as_str(&self) -> &str { - str::from_utf8_unchecked(self) - } -} - -impl ops::Deref for Array { - type Target = [T]; - fn deref(&self) -> &Self::Target { - self.as_slice() - } -} - -impl ops::DerefMut for Array { - fn deref_mut(&mut self) -> &mut [T] { - self.as_slice_mut() - } -} - -impl From<[T; N]> for Array { - fn from(array: [T; N]) -> Self { - unsafe { Array::from_const_array::(array) } - } -} - -impl Drop for Array { - fn drop(&mut self) { - self.clear() - } -} - -pub struct ArrayIntoIter { - state: usize, - a: Array, -} - -impl Iterator for ArrayIntoIter { - type Item = T; - fn next(&mut self) -> Option { - if self.state == self.a.len() { - // reached end - None - } else { - let idx = self.state; - self.state += 1; - Some(unsafe { ptr::read(self.a.as_ptr().add(idx)) }) - } - } - fn size_hint(&self) -> (usize, Option) { - let l = self.a.len() - self.state; - (l, Some(l)) - } -} - -impl IntoIterator for Array { - type Item = T; - type IntoIter = ArrayIntoIter; - fn into_iter(self) -> Self::IntoIter { - ArrayIntoIter { state: 0, a: self } - } -} - -impl Array { - /// Extend self using an iterator. - /// - /// ## Safety - /// This function can cause undefined damage to your application's stack and/or other's - /// data. Only use if you know what you're doing. If you don't use `extend_from_iter` - /// instead - pub unsafe fn extend_from_iter_unchecked(&mut self, iterable: I) - where - I: IntoIterator, - { - // the ptr to start writing from - let mut ptr = Self::as_mut_ptr(self).add(self.len()); - let mut guard = LenScopeGuard::new(&mut self.init_len); - let mut iter = iterable.into_iter(); - loop { - if let Some(element) = iter.next() { - // write the element - ptr.write(element); - // move to the next location - ptr = ptr.add(1); - // tell the guard to increment - guard.incr(1); - } else { - return; - } - } - } - pub fn extend_from_iter(&mut self, iterable: I) - where - I: IntoIterator, - { - unsafe { - // the ptr to start writing from - let mut ptr = Self::as_mut_ptr(self).add(self.len()); - let end_ptr = Self::as_ptr(self).add(self.capacity()); - let mut guard = LenScopeGuard::new(&mut self.init_len); - let mut iter = iterable.into_iter(); - loop { - if let Some(element) = iter.next() { - // write the element - ptr.write(element); - // move to the next location - ptr = ptr.add(1); - // tell the guard to increment - guard.incr(1); - if end_ptr < ptr { - // our current ptr points to the end of the allocation - // oh no, time for corruption, if the user says so - panic!("Overflowed stack area.") - } - } else { - return; - } - } - } - } -} - -impl Extend for Array { - fn extend>(&mut self, iter: I) { - { - self.extend_from_iter::<_>(iter) - } - } -} - -impl FromIterator for Array { - fn from_iter>(iter: I) -> Self { - let mut arr = Array::new(); - arr.extend(iter); - arr - } -} - -impl Clone for Array -where - T: Clone, -{ - fn clone(&self) -> Self { - self.iter().cloned().collect() - } -} - -impl Hash for Array -where - T: Hash, -{ - fn hash(&self, hasher: &mut H) - where - H: Hasher, - { - Hash::hash(&**self, hasher) - } -} - -impl PartialEq<[u8]> for Array { - fn eq(&self, oth: &[u8]) -> bool { - **self == *oth - } -} - -impl PartialEq> for [u8] { - fn eq(&self, oth: &Array) -> bool { - oth.as_slice() == self - } -} - -impl PartialEq for Array -where - T: PartialEq, -{ - fn eq(&self, other: &Self) -> bool { - **self == **other - } -} - -impl Eq for Array where T: Eq {} - -impl PartialOrd for Array -where - T: PartialOrd, -{ - fn partial_cmp(&self, other: &Self) -> Option { - (**self).partial_cmp(&**other) - } -} - -impl Ord for Array -where - T: Ord, -{ - fn cmp(&self, other: &Self) -> Ordering { - (**self).cmp(&**other) - } -} - -impl Borrow<[T]> for Array { - fn borrow(&self) -> &[T] { - self - } -} - -impl BorrowMut<[T]> for Array { - fn borrow_mut(&mut self) -> &mut [T] { - self - } -} - -impl AsRef<[T]> for Array { - fn as_ref(&self) -> &[T] { - self - } -} - -impl AsMut<[T]> for Array { - fn as_mut(&mut self) -> &mut [T] { - self - } -} - -impl fmt::Debug for Array -where - T: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if any::type_name::().eq(any::type_name::()) { - let slf = unsafe { - // UNSAFE(@ohsayan): Guaranteed by the above invariant - &*(self as *const Array as *const Array) - }; - match String::from_utf8(slf.to_vec()) { - Ok(st) => write!(f, "{:#?}", st), - Err(_) => (**self).fmt(f), - } - } else { - (**self).fmt(f) - } - } -} - -impl Borrow for Array { - fn borrow(&self) -> &str { - unsafe { self.as_str() } - } -} - -unsafe impl Send for Array where T: Send {} -unsafe impl Sync for Array where T: Sync {} - -#[test] -fn test_basic() { - let mut b: Array = Array::new(); - b.extend_from_slice("Hello World".as_bytes()).unwrap(); - assert_eq!( - b, - Array::from([b'H', b'e', b'l', b'l', b'o', b' ', b'W', b'o', b'r', b'l', b'd']) - ); -} - -#[test] -fn test_uninitialized() { - let mut b: Array = Array::new(); - b.push(b'S'); - assert_eq!(b.iter().count(), 1); -} - -#[test] -#[should_panic] -fn test_array_overflow() { - let mut arr: Array = Array::new(); - arr.extend_from_slice("123456".as_bytes()).unwrap(); -} - -#[test] -#[should_panic] -fn test_array_overflow_iter() { - let mut arr: Array = Array::new(); - arr.extend("123456".chars()); -} - -#[test] -fn test_array_clone() { - let mut arr: Array = Array::new(); - arr.extend( - "qHwRsmyBYHbqyHfdShOfVSayVUmeKlEagvJoGuTyvaCqpsfFkZabeuqmVeiKbJxV" - .as_bytes() - .to_owned(), - ); - let myclone = arr.clone(); - assert_eq!(arr, myclone); -} - -#[test] -fn test_array_extend_okay() { - let mut arr: Array = Array::new(); - arr.extend( - "qHwRsmyBYHbqyHfdShOfVSayVUmeKlEagvJoGuTyvaCqpsfFkZabeuqmVeiKbJxV" - .as_bytes() - .to_owned(), - ); -} - -#[test] -#[should_panic] -fn test_array_extend_fail() { - let mut arr: Array = Array::new(); - arr.extend( - "qHwRsmyBYHbqyHfdShOfVSayVUmeKlEagvJoGuTyvaCqpsfFkZabeuqmVeiKbJxV_" - .as_bytes() - .to_owned(), - ); -} diff --git a/server/src/corestore/booltable.rs b/server/src/corestore/booltable.rs deleted file mode 100644 index e605b0c6..00000000 --- a/server/src/corestore/booltable.rs +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Created on Fri Sep 10 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -/* - ⚠⚠⚠⚠ A WORD OF WARNING ⚠⚠⚠⚠ - This module contains some dark stuff (and asumptions) about type layouts and/or representations, - things which can change from time to time. Do not rely on any of this! -*/ - -use core::ops::Index; - -pub type BytesBoolTable = BoolTable<&'static [u8]>; -pub type BytesNicheLUT = NicheLUT<&'static [u8]>; - -/// A two-value boolean LUT -pub struct BoolTable { - base: [T; 2], -} - -impl BoolTable { - /// Supply values in the order: `if_true` and `if_false` - pub const fn new(if_true: T, if_false: T) -> Self { - Self { - base: [if_false, if_true], - } - } -} - -impl Index for BoolTable { - type Output = T; - fn index(&self, index: bool) -> &Self::Output { - unsafe { &*self.base.as_ptr().add(index as usize) } - } -} - -/// A LUT based on niche values, especially built to support the `Option` optimized -/// structure -/// -/// **Warning:** This is a terrible opt and only works on the Rust ABI -pub struct NicheLUT { - base: [T; 3], -} - -impl NicheLUT { - /// Supply values in the following order: [`if_none`, `if_true`, `if_false`] - pub const fn new(if_none: T, if_true: T, if_false: T) -> Self { - Self { - // 0 == S(F); 1 == S(T); 2 == NULL - base: [if_false, if_true, if_none], - } - } -} - -impl Index> for NicheLUT { - type Output = T; - fn index(&self, idx: Option) -> &Self::Output { - unsafe { - &*self - .base - .as_ptr() - .add(*(&idx as *const _ as *const u8) as usize) - } - } -} - -#[test] -fn niche_optim_sanity_test() { - let none: Option = None; - let some_t: Option = Some(true); - let some_f: Option = Some(false); - unsafe { - let r_some_f = &some_f as *const _ as *const u8; - let r_some_t = &some_t as *const _ as *const u8; - let r_none = &none as *const _ as *const u8; - assert_eq!(*r_some_f, 0); - assert_eq!(*r_some_t, 1); - assert_eq!(*r_none, 2); - } -} - -/// A 2-bit indexed boolean LUT -pub struct TwoBitLUT { - base: [T; 4], -} - -type Bit = bool; -type TwoBitIndex = (Bit, Bit); - -impl TwoBitLUT { - /// Supply values in the following order: - /// - 1st unset, 2nd unset - /// - 1st unset, 2nd set - /// - 1st set, 2nd unset - /// - 1st set, 2nd set - pub const fn new(ff: T, ft: T, tf: T, tt: T) -> Self { - Self { - base: [ff, ft, tf, tt], - } - } -} - -impl Index for TwoBitLUT { - type Output = T; - fn index(&self, (bit_a, bit_b): TwoBitIndex) -> &Self::Output { - unsafe { - &*self - .base - .as_ptr() - .add((((bit_a as u8) << 1) + (bit_b as u8)) as usize) - } - } -} - -#[test] -fn test_two_bit_indexed_lut() { - let (bit_a, bit_b) = unsafe { tmut_bool!(0, 0) }; - let twobitlut = TwoBitLUT::new('a', 'b', 'c', 'd'); - // the operators, are just for sanity - assert_eq!('d', twobitlut[(!bit_a, !bit_b)]); - assert_eq!('c', twobitlut[(!bit_a, bit_b)]); - assert_eq!('b', twobitlut[(bit_a, !bit_b)]); - assert_eq!('a', twobitlut[(bit_a, bit_b)]); -} diff --git a/server/src/corestore/buffers.rs b/server/src/corestore/buffers.rs deleted file mode 100644 index 7987de75..00000000 --- a/server/src/corestore/buffers.rs +++ /dev/null @@ -1,413 +0,0 @@ -/* - * Created on Mon Jul 12 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::array::Array, - core::{ops::Deref, str}, -}; - -macro_rules! push_self { - ($self:expr, $what:expr) => { - $self.inner_stack.push_unchecked($what) - }; -} - -macro_rules! lut { - ($e:expr) => { - ucidx!(PAIR_MAP_LUT, $e) - }; -} - -const PAIR_MAP_LUT: [u8; 200] = [ - 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, 0x30, 0x36, 0x30, 0x37, - 0x30, 0x38, 0x30, 0x39, // 0x30 - 0x31, 0x30, 0x31, 0x31, 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, - 0x31, 0x38, 0x31, 0x39, // 0x31 - 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, - 0x32, 0x38, 0x32, 0x39, // 0x32 - 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, 0x33, 0x36, 0x33, 0x37, - 0x33, 0x38, 0x33, 0x39, // 0x33 - 0x34, 0x30, 0x34, 0x31, 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, - 0x34, 0x38, 0x34, 0x39, // 0x34 - 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, - 0x35, 0x38, 0x35, 0x39, // 0x35 - 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, 0x36, 0x36, 0x36, 0x37, - 0x36, 0x38, 0x36, 0x39, // 0x36 - 0x37, 0x30, 0x37, 0x31, 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, - 0x37, 0x38, 0x37, 0x39, // 0x37 - 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, - 0x38, 0x38, 0x38, 0x39, // 0x38 - 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, 0x39, 0x36, 0x39, 0x37, - 0x39, 0x38, 0x39, 0x39, // 0x39 -]; - -#[allow(dead_code)] -/// A 32-bit integer buffer with one extra byte -pub type Integer32Buffer = Integer32BufferRaw<11>; - -#[derive(Debug)] -/// A buffer for unsigned 32-bit integers with one _extra byte_ of memory reserved for -/// adding characters. On initialization (through [`Self::init`]), your integer will be -/// encoded and stored into the _unsafe array_ -pub struct Integer32BufferRaw { - inner_stack: Array, -} - -#[allow(dead_code)] -impl Integer32BufferRaw { - /// Initialize a buffer - pub fn init(integer: u32) -> Self { - let mut slf = Self { - inner_stack: Array::new(), - }; - unsafe { - slf._init_integer(integer); - } - slf - } - /// Initialize an integer. This is unsafe to be called outside because you'll be - /// pushing in another integer and might end up corrupting your own stack as all - /// pushes are unchecked! - unsafe fn _init_integer(&mut self, mut val: u32) { - if val < 10_000 { - let d1 = (val / 100) << 1; - let d2 = (val % 100) << 1; - if val >= 1000 { - push_self!(self, lut!(d1)); - } - if val >= 100 { - push_self!(self, lut!(d1 + 1)); - } - if val >= 10 { - push_self!(self, lut!(d2)); - } - push_self!(self, lut!(d2 + 1)); - } else if val < 100_000_000 { - let b = val / 10000; - let c = val % 10000; - let d1 = (b / 100) << 1; - let d2 = (b % 100) << 1; - let d3 = (c / 100) << 1; - let d4 = (c % 100) << 1; - - if val > 10_000_000 { - push_self!(self, lut!(d1)); - } - if val > 1_000_000 { - push_self!(self, lut!(d1 + 1)); - } - if val > 100_000 { - push_self!(self, lut!(d2)); - } - push_self!(self, lut!(d2 + 1)); - push_self!(self, lut!(d3)); - push_self!(self, lut!(d3 + 1)); - push_self!(self, lut!(d4)); - push_self!(self, lut!(d4 + 1)); - } else { - // worst, 1B or more - let a = val / 100000000; - val %= 100000000; - - if a >= 10 { - let i = a << 1; - push_self!(self, lut!(i)); - push_self!(self, lut!(i + 1)); - } else { - push_self!(self, 0x30); - } - let b = val / 10000; - let c = val % 10000; - let d1 = (b / 100) << 1; - let d2 = (b % 100) << 1; - let d3 = (c / 100) << 1; - let d4 = (c % 100) << 1; - // write back - push_self!(self, lut!(d1)); - push_self!(self, lut!(d1 + 1)); - push_self!(self, lut!(d2)); - push_self!(self, lut!(d2 + 1)); - push_self!(self, lut!(d3)); - push_self!(self, lut!(d3 + 1)); - push_self!(self, lut!(d4)); - push_self!(self, lut!(d4 + 1)); - } - } - /// **This is very unsafe** Only push something when you know that the capacity won't overflow - /// your allowance of 11 bytes. Oh no, there's no panic for you because you'll silently - /// corrupt your own memory (or others' :/) - pub unsafe fn push(&mut self, val: u8) { - push_self!(self, val) - } -} - -impl Deref for Integer32BufferRaw { - type Target = str; - fn deref(&self) -> &Self::Target { - unsafe { str::from_utf8_unchecked(&self.inner_stack) } - } -} - -impl AsRef for Integer32BufferRaw { - fn as_ref(&self) -> &str { - self - } -} - -impl PartialEq for Integer32BufferRaw -where - T: AsRef, -{ - fn eq(&self, other_str: &T) -> bool { - self.as_ref() == other_str.as_ref() - } -} - -#[test] -fn test_int32_buffer() { - let buffer = Integer32Buffer::init(256); - assert_eq!(buffer, 256.to_string()); -} - -#[test] -fn test_int32_buffer_push() { - let mut buffer = Integer32Buffer::init(278); - unsafe { - buffer.push(b'?'); - } - assert_eq!(buffer, "278?"); -} - -/// A 64-bit integer buffer with **no extra byte** -pub type Integer64 = Integer64BufferRaw<20>; - -#[derive(Debug)] -pub struct Integer64BufferRaw { - inner_stack: Array, -} - -const Z_8: u64 = 100_000_000; -const Z_9: u64 = Z_8 * 10; -const Z_10: u64 = Z_9 * 10; -const Z_11: u64 = Z_10 * 10; -const Z_12: u64 = Z_11 * 10; -const Z_13: u64 = Z_12 * 10; -const Z_14: u64 = Z_13 * 10; -const Z_15: u64 = Z_14 * 10; -const Z_16: u64 = Z_15 * 10; - -impl Integer64BufferRaw { - pub fn init(integer: u64) -> Self { - let mut slf = Self { - inner_stack: Array::new(), - }; - unsafe { - slf._init_integer(integer); - } - slf - } - unsafe fn _init_integer(&mut self, mut int: u64) { - if int < Z_8 { - if int < 10_000 { - let d1 = (int / 100) << 1; - let d2 = (int % 100) << 1; - if int >= 1_000 { - push_self!(self, lut!(d1)); - } - if int >= 100 { - push_self!(self, lut!(d1 + 1)); - } - if int >= 10 { - push_self!(self, lut!(d2)); - } - push_self!(self, lut!(d2 + 1)); - } else { - let b = int / 10000; - let c = int % 10000; - let d1 = (b / 100) << 1; - let d2 = (b % 100) << 1; - let d3 = (c / 100) << 1; - let d4 = (c % 100) << 1; - if int >= 10_000_000 { - push_self!(self, lut!(d1)); - } - if int >= 1_000_000 { - push_self!(self, lut!(d1 + 1)); - } - if int >= 100_000 { - push_self!(self, lut!(d2)); - } - push_self!(self, lut!(d2 + 1)); - push_self!(self, lut!(d3)); - push_self!(self, lut!(d3 + 1)); - push_self!(self, lut!(d4)); - push_self!(self, lut!(d4 + 1)); - } - } else if int < Z_16 { - // lets do 8 at a time - let v0 = int / Z_8; - let v1 = int & Z_8; - let b0 = v0 / 10000; - let c0 = v0 % 10000; - let d1 = (b0 / 100) << 1; - let d2 = (b0 % 100) << 1; - let d3 = (c0 / 100) << 1; - let d4 = (c0 % 100) << 1; - let b1 = v1 / 10000; - let c1 = v1 % 10000; - let d5 = (b1 / 100) << 1; - let d6 = (b1 % 100) << 1; - let d7 = (c1 / 100) << 1; - let d8 = (c1 % 100) << 1; - if int >= Z_15 { - push_self!(self, lut!(d1)); - } - if int >= Z_14 { - push_self!(self, lut!(d1 + 1)); - } - if int >= Z_13 { - push_self!(self, lut!(d2)); - } - if int >= Z_12 { - push_self!(self, lut!(d2 + 1)); - } - if int >= Z_11 { - push_self!(self, lut!(d3)); - } - if int >= Z_10 { - push_self!(self, lut!(d3 + 1)); - } - if int >= Z_9 { - push_self!(self, lut!(d4)); - } - push_self!(self, lut!(d4 + 1)); - push_self!(self, lut!(d5)); - push_self!(self, lut!(d5 + 1)); - push_self!(self, lut!(d6)); - push_self!(self, lut!(d6 + 1)); - push_self!(self, lut!(d7)); - push_self!(self, lut!(d7 + 1)); - push_self!(self, lut!(d8)); - push_self!(self, lut!(d8 + 1)); - } else { - let a = int / Z_16; - int %= Z_16; - if a < 10 { - push_self!(self, 0x30 + a as u8); - } else if a < 100 { - let i = a << 1; - push_self!(self, lut!(i)); - push_self!(self, lut!(i + 1)); - } else if a < 1000 { - push_self!(self, 0x30 + (a / 100) as u8); - let i = (a % 100) << 1; - push_self!(self, lut!(i)); - push_self!(self, lut!(i + 1)); - } else { - let i = (a / 100) << 1; - let j = (a % 100) << 1; - push_self!(self, lut!(i)); - push_self!(self, lut!(i + 1)); - push_self!(self, lut!(j)); - push_self!(self, lut!(j + 1)); - } - - let v0 = int / Z_8; - let v1 = int % Z_8; - let b0 = v0 / 10000; - let c0 = v0 % 10000; - let d1 = (b0 / 100) << 1; - let d2 = (b0 % 100) << 1; - let d3 = (c0 / 100) << 1; - let d4 = (c0 % 100) << 1; - let b1 = v1 / 10000; - let c1 = v1 % 10000; - let d5 = (b1 / 100) << 1; - let d6 = (b1 % 100) << 1; - let d7 = (c1 / 100) << 1; - let d8 = (c1 % 100) << 1; - push_self!(self, lut!(d1)); - push_self!(self, lut!(d1 + 1)); - push_self!(self, lut!(d2)); - push_self!(self, lut!(d2 + 1)); - push_self!(self, lut!(d3)); - push_self!(self, lut!(d3 + 1)); - push_self!(self, lut!(d4)); - push_self!(self, lut!(d4 + 1)); - push_self!(self, lut!(d5)); - push_self!(self, lut!(d5 + 1)); - push_self!(self, lut!(d6)); - push_self!(self, lut!(d6 + 1)); - push_self!(self, lut!(d7)); - push_self!(self, lut!(d7 + 1)); - push_self!(self, lut!(d8)); - push_self!(self, lut!(d8 + 1)); - } - } -} - -impl From for Integer64BufferRaw { - fn from(val: usize) -> Self { - Self::init(val as u64) - } -} - -impl From for Integer64BufferRaw { - fn from(val: u64) -> Self { - Self::init(val) - } -} - -impl Deref for Integer64BufferRaw { - type Target = [u8]; - fn deref(&self) -> &Self::Target { - &self.inner_stack - } -} - -impl AsRef for Integer64BufferRaw { - fn as_ref(&self) -> &str { - unsafe { str::from_utf8_unchecked(&self.inner_stack) } - } -} - -impl PartialEq for Integer64BufferRaw -where - T: AsRef, -{ - fn eq(&self, other_str: &T) -> bool { - self.as_ref() == other_str.as_ref() - } -} - -#[test] -fn test_int64_buffer() { - assert_eq!( - 9348910481349849081_u64.to_string(), - Integer64::init(9348910481349849081_u64).as_ref() - ); - assert_eq!(u64::MAX.to_string(), Integer64::init(u64::MAX).as_ref()); -} diff --git a/server/src/corestore/heap_array.rs b/server/src/corestore/heap_array.rs deleted file mode 100644 index cf5f72e3..00000000 --- a/server/src/corestore/heap_array.rs +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Created on Mon Feb 21 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - core::{alloc::Layout, fmt, marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr, slice}, - std::alloc::dealloc, -}; - -/// A heap-allocated array -pub struct HeapArray { - ptr: *const T, - len: usize, - _marker: PhantomData, -} - -pub struct HeapArrayWriter { - base: Vec, -} - -impl HeapArrayWriter { - pub fn with_capacity(cap: usize) -> Self { - Self { - base: Vec::with_capacity(cap), - } - } - /// ## Safety - /// Caller must ensure that `idx <= cap`. If not, you'll corrupt your - /// memory - pub unsafe fn write_to_index(&mut self, idx: usize, element: T) { - debug_assert!(idx <= self.base.capacity()); - ptr::write(self.base.as_mut_ptr().add(idx), element); - self.base.set_len(self.base.len() + 1); - } - /// ## Safety - /// This function can lead to memory unsafety in two ways: - /// - Excess capacity: In that case, it will leak memory - /// - Uninitialized elements: In that case, it will segfault while attempting to call - /// `T`'s dtor - pub unsafe fn finish(self) -> HeapArray { - let base = ManuallyDrop::new(self.base); - HeapArray::new(base.as_ptr(), base.len()) - } -} - -impl HeapArray { - #[cfg(test)] - pub fn new_from_vec(mut v: Vec) -> Self { - v.shrink_to_fit(); - let v = ManuallyDrop::new(v); - unsafe { Self::new(v.as_ptr(), v.len()) } - } - pub unsafe fn new(ptr: *const T, len: usize) -> Self { - Self { - ptr, - len, - _marker: PhantomData, - } - } - pub fn new_writer(cap: usize) -> HeapArrayWriter { - HeapArrayWriter::with_capacity(cap) - } - #[cfg(test)] - pub fn as_slice(&self) -> &[T] { - self - } -} - -impl Drop for HeapArray { - fn drop(&mut self) { - unsafe { - // run dtor - ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr as *mut T, self.len)); - // deallocate - let layout = Layout::array::(self.len).unwrap(); - dealloc(self.ptr as *mut T as *mut u8, layout); - } - } -} - -// totally fine because `u8`s can be safely shared across threads -unsafe impl Send for HeapArray {} -unsafe impl Sync for HeapArray {} - -impl Deref for HeapArray { - type Target = [T]; - fn deref(&self) -> &Self::Target { - unsafe { slice::from_raw_parts(self.ptr, self.len) } - } -} - -impl fmt::Debug for HeapArray { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.iter()).finish() - } -} - -impl PartialEq for HeapArray { - fn eq(&self, other: &Self) -> bool { - self == other - } -} - -#[test] -fn heaparray_impl() { - // basically, this shouldn't segfault - let heap_array = b"notasuperuser".to_vec(); - let heap_array = HeapArray::new_from_vec(heap_array); - assert_eq!(heap_array.as_slice(), b"notasuperuser"); -} diff --git a/server/src/corestore/htable.rs b/server/src/corestore/htable.rs deleted file mode 100644 index b7a6349b..00000000 --- a/server/src/corestore/htable.rs +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Created on Sun May 09 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![allow(unused)] // TODO(@ohsayan): Plonk this - -use { - crate::corestore::map::{ - bref::{Entry, OccupiedEntry, Ref, VacantEntry}, - iter::{BorrowedIter, OwnedIter}, - Skymap, - }, - ahash::RandomState, - std::{borrow::Borrow, hash::Hash, iter::FromIterator, ops::Deref}, -}; - -type HashTable = Skymap; - -#[derive(Debug)] -/// The Coremap contains the actual key/value pairs along with additional fields for data safety -/// and protection -pub struct Coremap { - pub(crate) inner: HashTable, -} - -impl Default for Coremap { - fn default() -> Self { - Coremap { - inner: HashTable::new_ahash(), - } - } -} - -impl Coremap { - /// Create an empty coremap - pub fn new() -> Self { - Self::default() - } - pub fn with_capacity(cap: usize) -> Self { - Coremap { - inner: HashTable::with_capacity(cap), - } - } - pub fn try_with_capacity(cap: usize) -> Result { - if cap > (isize::MAX as usize) { - Err(()) - } else { - Ok(Self::with_capacity(cap)) - } - } - /// Returns the total number of key value pairs - pub fn len(&self) -> usize { - self.inner.len() - } - /// Clears the inner table! - pub fn clear(&self) { - self.inner.clear() - } -} - -impl Coremap -where - K: Eq + Hash, -{ - /// Returns the removed value for key, it it existed - pub fn remove(&self, key: &Q) -> Option<(K, V)> - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - self.inner.remove(key) - } - /// Returns true if an existent key was removed - pub fn true_if_removed(&self, key: &Q) -> bool - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - self.inner.remove(key).is_some() - } - /// Check if a table contains a key - pub fn contains_key(&self, key: &Q) -> bool - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - self.inner.contains_key(key) - } - /// Return a non-consuming iterator - pub fn iter(&self) -> BorrowedIter<'_, K, V, RandomState> { - self.inner.get_iter() - } - /// Get a reference to the value of a key, if it exists - pub fn get(&self, key: &Q) -> Option> - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - self.inner.get(key) - } - /// Returns true if the non-existent key was assigned to a value - pub fn true_if_insert(&self, k: K, v: V) -> bool { - if let Entry::Vacant(ve) = self.inner.entry(k) { - ve.insert(v); - true - } else { - false - } - } - pub fn true_remove_if(&self, key: &Q, exec: impl FnOnce(&K, &V) -> bool) -> bool - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - self.remove_if(key, exec).is_some() - } - pub fn remove_if(&self, key: &Q, exec: impl FnOnce(&K, &V) -> bool) -> Option<(K, V)> - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - self.inner.remove_if(key, exec) - } - /// Update or insert - pub fn upsert(&self, k: K, v: V) { - let _ = self.inner.insert(k, v); - } - /// Returns true if the value was updated - pub fn true_if_update(&self, k: K, v: V) -> bool { - if let Entry::Occupied(mut oe) = self.inner.entry(k) { - oe.insert(v); - true - } else { - false - } - } - pub fn mut_entry(&self, key: K) -> Option> { - if let Entry::Occupied(oe) = self.inner.entry(key) { - Some(oe) - } else { - None - } - } - pub fn fresh_entry(&self, key: K) -> Option> { - if let Entry::Vacant(ve) = self.inner.entry(key) { - Some(ve) - } else { - None - } - } -} - -impl Coremap { - pub fn get_cloned(&self, key: &Q) -> Option - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - self.inner.get_cloned(key) - } -} - -impl Coremap { - /// Returns atleast `count` number of keys from the hashtable - pub fn get_keys(&self, count: usize) -> Vec { - let mut v = Vec::with_capacity(count); - self.iter() - .take(count) - .map(|kv| kv.key().clone()) - .for_each(|key| v.push(key)); - v - } -} - -impl IntoIterator for Coremap { - type Item = (K, V); - type IntoIter = OwnedIter; - fn into_iter(self) -> Self::IntoIter { - self.inner.get_owned_iter() - } -} - -impl FromIterator<(K, V)> for Coremap -where - K: Eq + Hash, -{ - fn from_iter(iter: T) -> Self - where - T: IntoIterator, - { - Coremap { - inner: Skymap::from_iter(iter), - } - } -} diff --git a/server/src/corestore/iarray.rs b/server/src/corestore/iarray.rs deleted file mode 100644 index f081b497..00000000 --- a/server/src/corestore/iarray.rs +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Created on Sun Jul 04 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![allow(dead_code)] // TODO(@ohsayan): Remove this lint or remove offending methods - -use { - crate::corestore::array::LenScopeGuard, - core::{ - alloc::Layout, - borrow::{Borrow, BorrowMut}, - cmp, fmt, - hash::{self, Hash}, - iter::FromIterator, - mem::{self, ManuallyDrop, MaybeUninit}, - ops, - ptr::{self, NonNull}, - slice, - }, - std::alloc as std_alloc, -}; - -pub const fn new_const_iarray() -> IArray<[T; N]> { - IArray { - cap: 0, - store: InlineArray { - stack: ManuallyDrop::new(MaybeUninit::uninit()), - }, - } -} - -/// An arbitrary trait used for identifying something as a contiguous block of memory -pub trait MemoryBlock { - /// The type that will be used for the memory layout - type LayoutItem; - /// The number of _units_ this memory block has - fn size() -> usize; -} - -impl MemoryBlock for [T; N] { - type LayoutItem = T; - fn size() -> usize { - N - } -} - -/// An union that either holds a stack (ptr) or a heap -/// -/// ## Safety -/// If you're trying to access a field without knowing the most recently created one, -/// behavior is undefined. -pub union InlineArray { - /// the stack - stack: ManuallyDrop>, - /// a pointer to the heap allocation and the allocation size - heap_ptr_len: (*mut A::LayoutItem, usize), -} - -impl InlineArray { - /// Get's the stack pointer. This is unsafe because it is not guranteed that the - /// stack pointer field is valid and the caller has to uphold this gurantee - unsafe fn stack_ptr(&self) -> *const A::LayoutItem { - self.stack.as_ptr() as *const _ - } - /// Safe as `stack_ptr`, but returns a mutable pointer - unsafe fn stack_ptr_mut(&mut self) -> *mut A::LayoutItem { - self.stack.as_mut_ptr() as *mut _ - } - /// Create a new union from a stack - fn from_stack(stack: MaybeUninit) -> Self { - Self { - stack: ManuallyDrop::new(stack), - } - } - /// Create a new union from a heap (allocated). - fn from_heap_ptr(start_ptr: *mut A::LayoutItem, len: usize) -> Self { - Self { - heap_ptr_len: (start_ptr, len), - } - } - /// Returns the allocation size of the heap - unsafe fn heap_size(&self) -> usize { - self.heap_ptr_len.1 - } - /// Returns a raw ptr to the heap - unsafe fn heap_ptr(&self) -> *const A::LayoutItem { - self.heap_ptr_len.0 - } - /// Returns a mut ptr to the heap - unsafe fn heap_ptr_mut(&mut self) -> *mut A::LayoutItem { - self.heap_ptr_len.0 as *mut _ - } - /// Returns a mut ref to the heap allocation size - unsafe fn heap_size_mut(&mut self) -> &mut usize { - &mut self.heap_ptr_len.1 - } - /// Returns the entire heap field - unsafe fn heap(&self) -> (*mut A::LayoutItem, usize) { - self.heap_ptr_len - } - /// Returns a mutable reference to the entire heap field - unsafe fn heap_mut(&mut self) -> (*mut A::LayoutItem, &mut usize) { - (self.heap_ptr_mut(), &mut self.heap_ptr_len.1) - } -} - -/// An utility tool for calculating the memory layout for a given `T`. Handles -/// any possible overflows -pub fn calculate_memory_layout(count: usize) -> Result { - let size = mem::size_of::().checked_mul(count).ok_or(())?; - // err is cap overflow - let alignment = mem::align_of::(); - Layout::from_size_align(size, alignment).map_err(|_| ()) -} - -/// Use the global allocator to deallocate the memory block for the given starting ptr -/// upto the given capacity -unsafe fn dealloc(start_ptr: *mut T, capacity: usize) { - std_alloc::dealloc( - start_ptr as *mut u8, - calculate_memory_layout::(capacity).expect("Memory capacity overflow"), - ) -} - -// Break free from Rust's aliasing rules with these typedefs -type DataptrLenptrCapacity = (*const T, usize, usize); -type DataptrLenptrCapacityMut<'a, T> = (*mut T, &'a mut usize, usize); - -/// A stack optimized backing store -/// -/// An [`IArray`] is heavily optimized for storing items on the stack and will -/// not perform very well (but of course will) when the object overflows its -/// stack and is moved to the heap. Optimizations are made to mark overflows -/// as branches that are unlikely to be called. The IArray is like a smallvec, -/// but with extremely aggressive optimizations for items stored on the stack, -/// for example to avoid the maneuvers with speculative execution. -/// This makes the [`IArray`] extremely performant for operations on the stack, -/// but a little expensive when operations are done on the heap -pub struct IArray { - cap: usize, - store: InlineArray, -} - -#[cfg(test)] -impl IArray<[u8; 48]> { - /// Returns a new 48-bit, stack allocated array of bytes - fn new_bytearray() -> Self { - Self::new() - } -} - -impl IArray { - pub fn new() -> IArray { - Self { - cap: 0, - store: InlineArray::from_stack(MaybeUninit::uninit()), - } - } - pub fn from_vec(mut vec: Vec) -> Self { - if vec.capacity() <= Self::stack_capacity() { - let mut store = InlineArray::::from_stack(MaybeUninit::uninit()); - let len = vec.len(); - unsafe { - ptr::copy_nonoverlapping(vec.as_ptr(), store.stack_ptr_mut(), len); - } - // done with the copy - Self { cap: len, store } - } else { - // off to the heap - let (start_ptr, cap, len) = (vec.as_mut_ptr(), vec.capacity(), vec.len()); - // leak the vec - mem::forget(vec); - IArray { - cap, - store: InlineArray::from_heap_ptr(start_ptr, len), - } - } - } - /// Returns the total capacity of the inline stack - fn stack_capacity() -> usize { - if mem::size_of::() > 0 { - // not a ZST, so cap of array - A::size() - } else { - // ZST. Just pile up some garbage and say that we have infinity - usize::MAX - } - } - /// Helper function that returns a ptr to the data, the len and the capacity - fn meta_triple(&self) -> DataptrLenptrCapacity { - unsafe { - if self.went_off_stack() { - let (data_ptr, len_ref) = self.store.heap(); - (data_ptr, len_ref, self.cap) - } else { - // still on stack - (self.store.stack_ptr(), self.cap, Self::stack_capacity()) - } - } - } - /// Mutable version of `meta_triple` - fn meta_triple_mut(&mut self) -> DataptrLenptrCapacityMut { - unsafe { - if self.went_off_stack() { - // get heap - let (data_ptr, len_ref) = self.store.heap_mut(); - (data_ptr, len_ref, self.cap) - } else { - // still on stack - ( - self.store.stack_ptr_mut(), - &mut self.cap, - Self::stack_capacity(), - ) - } - } - } - /// Returns a raw ptr to the data - fn get_data_ptr_mut(&mut self) -> *mut A::LayoutItem { - if self.went_off_stack() { - // get the heap ptr - unsafe { self.store.heap_ptr_mut() } - } else { - // get the stack ptr - unsafe { self.store.stack_ptr_mut() } - } - } - /// Returns true if the allocation is now on the heap - fn went_off_stack(&self) -> bool { - self.cap > Self::stack_capacity() - } - /// Returns the length - pub fn len(&self) -> usize { - if self.went_off_stack() { - // so we're off the stack - unsafe { self.store.heap_size() } - } else { - // still on the stack - self.cap - } - } - /// Returns true if the IArray is empty - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - /// Returns the capacity - fn get_capacity(&self) -> usize { - if self.went_off_stack() { - self.cap - } else { - Self::stack_capacity() - } - } - /// Grow the allocation, if required, to make space for a total of `new_cap` - /// elements - fn grow_block(&mut self, new_cap: usize) { - // infallible - unsafe { - let (data_ptr, &mut len, cap) = self.meta_triple_mut(); - let still_on_stack = !self.went_off_stack(); - assert!(new_cap > len); - if new_cap <= Self::stack_capacity() { - if still_on_stack { - return; - } - self.store = InlineArray::from_stack(MaybeUninit::uninit()); - ptr::copy_nonoverlapping(data_ptr, self.store.stack_ptr_mut(), len); - self.cap = len; - dealloc(data_ptr, cap); - } else if new_cap != cap { - let layout = - calculate_memory_layout::(new_cap).expect("Capacity overflow"); - assert!(layout.size() > 0); - let new_alloc; - if still_on_stack { - new_alloc = NonNull::new(std_alloc::alloc(layout).cast()) - .expect("Allocation error") - .as_ptr(); - ptr::copy_nonoverlapping(data_ptr, new_alloc, len); - } else { - // not on stack - let old_layout = - calculate_memory_layout::(cap).expect("Capacity overflow"); - // realloc the earlier buffer - let new_memory_block_ptr = - std_alloc::realloc(data_ptr as *mut _, old_layout, layout.size()); - new_alloc = NonNull::new(new_memory_block_ptr.cast()) - .expect("Allocation error") - .as_ptr(); - } - self.store = InlineArray::from_heap_ptr(new_alloc, len); - self.cap = new_cap; - } - } - } - /// Reserve space for `additional` elements - fn reserve(&mut self, additional: usize) { - let (_, &mut len, cap) = self.meta_triple_mut(); - if cap - len >= additional { - // already have enough space - return; - } - let new_cap = len - .checked_add(additional) - .map(usize::next_power_of_two) - .expect("Capacity overflow"); - self.grow_block(new_cap) - } - /// Push an element into this IArray - pub fn push(&mut self, val: A::LayoutItem) { - unsafe { - let (mut data_ptr, mut len, cap) = self.meta_triple_mut(); - if (*len).eq(&cap) { - self.reserve(1); - let (heap_ptr, heap_len) = self.store.heap_mut(); - data_ptr = heap_ptr; - len = heap_len; - } - ptr::write(data_ptr.add(*len), val); - *len += 1; - } - } - /// Pop an element off this IArray - pub fn pop(&mut self) -> Option { - unsafe { - let (data_ptr, len_mut, _cap) = self.meta_triple_mut(); - if *len_mut == 0 { - // empty man, what do you want? - None - } else { - // there's something - let last_index = *len_mut - 1; - // we'll say that it's gone - *len_mut = last_index; - // but only read it now from the offset - Some(ptr::read(data_ptr.add(last_index))) - } - } - } - /// This is amazingly dangerous if `idx` doesn't exist. You can potentially - /// corrupt a bunch of things - pub unsafe fn remove(&mut self, idx: usize) -> A::LayoutItem { - let (mut ptr, len_ref, _) = self.meta_triple_mut(); - let len = *len_ref; - *len_ref = len - 1; - ptr = ptr.add(idx); - let item = ptr::read(ptr); - ptr::copy(ptr.add(1), ptr, len - idx - 1); - item - } - /// Shrink this IArray so that it only occupies the required space and not anything - /// more - pub fn shrink(&mut self) { - if self.went_off_stack() { - // it's off the stack, so no chance of moving back to the stack - return; - } - let current_len = self.len(); - if Self::stack_capacity() >= current_len { - // we have a chance of copying this over to our stack - unsafe { - let (data_ptr, len) = self.store.heap(); - self.store = InlineArray::from_stack(MaybeUninit::uninit()); - // copy to stack - ptr::copy_nonoverlapping(data_ptr, self.store.stack_ptr_mut(), len); - // now deallocate the heap - dealloc(data_ptr, self.cap); - self.cap = len; - } - } else if self.get_capacity() > current_len { - // more capacity than current len? so we're on the heap - // grow the block to place it on stack (this will dealloc the heap) - self.grow_block(current_len); - } - } - /// Truncate the IArray to a given length. This **will** call the destructors - pub fn truncate(&mut self, target_len: usize) { - unsafe { - let (data_ptr, len_mut, _cap) = self.meta_triple_mut(); - while target_len < *len_mut { - // get the last index - let last_index = *len_mut - 1; - // drop it - ptr::drop_in_place(data_ptr.add(last_index)); - // update the length - *len_mut = last_index; - } - } - } - /// Clear the internal store - pub fn clear(&mut self) { - // chop off the whole place - self.truncate(0); - } - /// Set the len, **without calling the destructor**. This is the ultimate function - /// to make valgrind unhappy, that is, **you can create memory leaks** if you don't - /// destroy the elements yourself - unsafe fn set_len(&mut self, new_len: usize) { - let (_dataptr, len_mut, _cap) = self.meta_triple_mut(); - *len_mut = new_len; - } -} - -impl IArray -where - A::LayoutItem: Copy, -{ - /// Create an IArray from a slice by copying the elements of the slice into - /// the IArray - pub fn from_slice(slice: &[A::LayoutItem]) -> Self { - // FIXME(@ohsayan): Could we have had this as a From::from() method? - let slice_len = slice.len(); - if slice_len <= Self::stack_capacity() { - // so we can place this thing on the stack - let mut new_stack = MaybeUninit::uninit(); - unsafe { - ptr::copy_nonoverlapping( - slice.as_ptr(), - new_stack.as_mut_ptr() as *mut A::LayoutItem, - slice_len, - ); - } - Self { - cap: slice_len, - store: InlineArray::from_stack(new_stack), - } - } else { - // argggh, on the heap - let mut v = slice.to_vec(); - let (ptr, cap) = (v.as_mut_ptr(), v.capacity()); - // leak it - mem::forget(v); - Self { - cap, - store: InlineArray::from_heap_ptr(ptr, slice_len), - } - } - } - /// Insert a slice at the given index - pub fn insert_slice_at_index(&mut self, slice: &[A::LayoutItem], index: usize) { - self.reserve(slice.len()); - let len = self.len(); - // only catch during tests - debug_assert!(index <= len); - unsafe { - let slice_ptr = slice.as_ptr(); - // we need to add it from the end of the current item - let data_ptr_start = self.get_data_ptr_mut().add(len); - // copy the slice over - ptr::copy(data_ptr_start, data_ptr_start.add(slice.len()), len - index); - ptr::copy_nonoverlapping(slice_ptr, data_ptr_start, slice.len()); - self.set_len(len + slice.len()); - } - } - /// Extend the IArray by using a slice - pub fn extend_from_slice(&mut self, slice: &[A::LayoutItem]) { - // at our len because we're appending it to the end - self.insert_slice_at_index(slice, self.len()) - } - /// Create a new IArray from a pre-defined stack - pub fn from_stack(stack: A) -> Self { - Self { - cap: A::size(), - store: InlineArray::from_stack(MaybeUninit::new(stack)), - } - } -} - -impl ops::Deref for IArray { - type Target = [A::LayoutItem]; - fn deref(&self) -> &Self::Target { - unsafe { - let (start_ptr, len, _) = self.meta_triple(); - slice::from_raw_parts(start_ptr, len) - } - } -} - -impl ops::DerefMut for IArray { - fn deref_mut(&mut self) -> &mut [A::LayoutItem] { - unsafe { - let (start_ptr, &mut len, _) = self.meta_triple_mut(); - slice::from_raw_parts_mut(start_ptr, len) - } - } -} - -impl AsRef<[A::LayoutItem]> for IArray { - fn as_ref(&self) -> &[A::LayoutItem] { - self - } -} - -impl AsMut<[A::LayoutItem]> for IArray { - fn as_mut(&mut self) -> &mut [A::LayoutItem] { - self - } -} - -// we need these for our coremap - -impl Borrow<[A::LayoutItem]> for IArray { - fn borrow(&self) -> &[A::LayoutItem] { - self - } -} - -impl BorrowMut<[A::LayoutItem]> for IArray { - fn borrow_mut(&mut self) -> &mut [A::LayoutItem] { - self - } -} - -impl Drop for IArray { - fn drop(&mut self) { - unsafe { - if self.went_off_stack() { - // free the heap - let (ptr, len) = self.store.heap(); - // let vec's destructor do the work - mem::drop(Vec::from_raw_parts(ptr, len, self.cap)); - } else { - // on stack? get self as a slice and destruct it - ptr::drop_in_place(&mut self[..]); - } - } - } -} - -impl Extend for IArray { - fn extend>(&mut self, iterable: I) { - let mut iter = iterable.into_iter(); - let (lower_bound, _upper_bound) = iter.size_hint(); - // reserve the lower bound; we really want it on the stack - self.reserve(lower_bound); - - unsafe { - let (data_ptr, len_ref, cap) = self.meta_triple_mut(); - let mut len = LenScopeGuard::new(len_ref); - while len.get_temp() < cap { - if let Some(out) = iter.next() { - ptr::write(data_ptr.add(len.get_temp()), out); - len.incr(1); - } else { - return; - } - } - } - // still have something left, probably a heap alloc :( - for elem in iter { - self.push(elem); - } - } -} - -impl fmt::Debug for IArray -where - A::LayoutItem: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - f.debug_list().entries(self.iter()).finish() - } -} - -impl PartialEq> for IArray -where - A::LayoutItem: PartialEq, -{ - fn eq(&self, rhs: &IArray) -> bool { - self[..] == rhs[..] - } -} - -impl Eq for IArray where A::LayoutItem: Eq {} - -impl PartialOrd for IArray -where - A::LayoutItem: PartialOrd, -{ - fn partial_cmp(&self, rhs: &IArray) -> Option { - PartialOrd::partial_cmp(&**self, &**rhs) - } -} - -impl Ord for IArray -where - A::LayoutItem: Ord, -{ - fn cmp(&self, rhs: &IArray) -> cmp::Ordering { - Ord::cmp(&**self, &**rhs) - } -} - -impl Hash for IArray -where - A::LayoutItem: Hash, -{ - fn hash(&self, hasher: &mut H) - where - H: hash::Hasher, - { - (**self).hash(hasher) - } -} - -impl FromIterator for IArray { - fn from_iter>(iter: I) -> Self { - let mut iarray = IArray::new(); - iarray.extend(iter); - iarray - } -} - -impl<'a, A: MemoryBlock> From<&'a [A::LayoutItem]> for IArray -where - A::LayoutItem: Clone, -{ - fn from(slice: &'a [A::LayoutItem]) -> Self { - slice.iter().cloned().collect() - } -} - -unsafe impl Send for IArray where A::LayoutItem: Send {} -unsafe impl Sync for IArray where A::LayoutItem: Sync {} - -#[test] -fn test_equality() { - let mut x = IArray::new_bytearray(); - x.extend_from_slice("AVeryGoodKeyspaceName".as_bytes()); - assert_eq!(x, { - let mut i = IArray::<[u8; 64]>::new(); - "AVeryGoodKeyspaceName" - .chars() - .for_each(|char| i.push(char as u8)); - i - }) -} diff --git a/server/src/corestore/lazy.rs b/server/src/corestore/lazy.rs deleted file mode 100644 index b3f80a05..00000000 --- a/server/src/corestore/lazy.rs +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Created on Sat Jul 03 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::backoff::Backoff, - core::{ - mem, - ops::Deref, - ptr, - sync::atomic::{AtomicBool, AtomicPtr, Ordering}, - }, -}; - -const ORD_ACQ: Ordering = Ordering::Acquire; -const ORD_SEQ: Ordering = Ordering::SeqCst; -const ORD_REL: Ordering = Ordering::Release; -const ORD_RLX: Ordering = Ordering::Relaxed; - -/// A lazily intialized, or _call by need_ value -#[derive(Debug)] -pub struct Lazy { - /// the value (null at first) - value: AtomicPtr, - /// the function that will init the value - init_func: F, - /// is some thread trying to initialize the value - init_state: AtomicBool, -} - -impl Lazy { - pub const fn new(init_func: F) -> Self { - Self { - value: AtomicPtr::new(ptr::null_mut()), - init_func, - init_state: AtomicBool::new(false), - } - } -} - -impl Deref for Lazy -where - F: Fn() -> T, -{ - type Target = T; - fn deref(&self) -> &Self::Target { - let value_ptr = self.value.load(ORD_ACQ); - if !value_ptr.is_null() { - // the value has already been initialized, return - unsafe { - // UNSAFE(@ohsayan): We've just asserted that the value is not null - return &*value_ptr; - } - } - // it's null, so it's useless - - // hold on until someone is trying to init - let backoff = Backoff::new(); - while self - .init_state - .compare_exchange(false, true, ORD_SEQ, ORD_SEQ) - .is_err() - { - // wait until the other thread finishes - backoff.snooze(); - } - /* - see the value before the last store. while we were one the loop, - some other thread could have initialized it already - */ - let value_ptr = self.value.load(ORD_ACQ); - if !value_ptr.is_null() { - // no more init, someone initialized it already - assert!(self.init_state.swap(false, ORD_SEQ)); - unsafe { - // UNSAFE(@ohsayan): We've already loaded the value checked - // that it isn't null - &*value_ptr - } - } else { - // so no one cared to initialize the value in between - // fine, we'll init it - let value = (self.init_func)(); - let value_ptr = Box::into_raw(Box::new(value)); - // now swap out the older value and check it for sanity - assert!(self.value.swap(value_ptr, ORD_SEQ).is_null()); - // set trying to init flag to false - assert!(self.init_state.swap(false, ORD_SEQ)); - unsafe { - // UNSAFE(@ohsayan): We just initialized the value ourselves - // so it is not null! - &*value_ptr - } - } - } -} - -/* - Note on memory leaks: - Suddenly found a possible leak with a static created with the Lazy type? Well, we'll have to - ignore it. That's because destructors aren't called on statics. A thunk leak. So what's to be - done here? Well, the best we can do is implement a destructor but it is never guranteed that - it will be called when used in global scope -*/ - -impl Drop for Lazy { - fn drop(&mut self) { - if mem::needs_drop::() { - // this needs drop - let value_ptr = self.value.load(ORD_ACQ); - if !value_ptr.is_null() { - unsafe { - // UNSAFE(@ohsayan): We've just checked if the value is null or not - mem::drop(Box::from_raw(value_ptr)) - } - } - } - } -} - -cfg_test!( - use crate::corestore::SharedSlice; - use crate::corestore::lazy; - use std::collections::HashMap; - use std::thread; - - #[allow(clippy::type_complexity)] - static LAZY_VALUE: lazy::Lazy, fn() -> HashMap> = lazy::Lazy::new(|| { - #[allow(clippy::mutable_key_type)] - let mut ht = HashMap::new(); - ht.insert("sayan".into(), "is doing something".into()); - ht - }); - - #[test] - fn test_lazy() { - assert_eq!( - LAZY_VALUE.get("sayan".as_bytes()).unwrap().clone(), - SharedSlice::from("is doing something") - ); - } - - #[test] - fn test_two_threads_trying_to_get_at_once() { - let (t1, t2) = ( - thread::spawn(|| { - assert_eq!( - LAZY_VALUE.get("sayan".as_bytes()).unwrap().clone(), - SharedSlice::from("is doing something") - );}), - thread::spawn(|| { - assert_eq!( - LAZY_VALUE.get("sayan".as_bytes()).unwrap().clone(), - SharedSlice::from("is doing something") - ); - }) - ); - { - t1.join().unwrap(); - t2.join().unwrap(); - } - } - - struct WeirdTestStruct(u8); - impl Drop for WeirdTestStruct { - fn drop(&mut self) { - panic!("PANIC ON DROP! THIS IS OKAY!"); - } - } - #[test] - #[should_panic] - fn test_drop() { - // this is only when the lazy is initialized in local scope and not global scope - let x: Lazy WeirdTestStruct> = Lazy::new(|| { - WeirdTestStruct(0) - }); - // just do an useless deref to make the pointer non null - let _deref = &*x; - drop(x); // we should panic right here - } - #[test] - fn test_no_drop_null() { - let x: Lazy WeirdTestStruct> = Lazy::new(|| { - WeirdTestStruct(0) - }); - drop(x); // no panic because it is null - } -); - -/// A "cell" that can be initialized once using a single atomic -pub struct Once { - value: AtomicPtr, -} - -#[allow(dead_code)] // TODO: Remove this -impl Once { - pub const fn new() -> Self { - Self { - value: AtomicPtr::new(ptr::null_mut()), - } - } - pub fn with_value(val: T) -> Self { - Self { - value: AtomicPtr::new(Box::into_raw(Box::new(val))), - } - } - pub fn get(&self) -> Option<&T> { - // synchronizes with the store for value ptr - let ptr = self.value.load(ORD_ACQ); - if ptr.is_null() { - None - } else { - unsafe { Some(&*self.value.load(ORD_ACQ)) } - } - } - pub fn set(&self, val: T) -> bool { - // synchronizes with the store for set - let snapshot = self.value.load(ORD_ACQ); - if snapshot.is_null() { - // let's try to init this - let vptr = Box::into_raw(Box::new(val)); - // if malloc fails, that's fine because there will be no cas - let r = self.value.compare_exchange( - snapshot, vptr, // we must use release ordering to sync with the acq - ORD_REL, - // on failure simply use relaxed because we don't use the value anyways - // -- so why bother stressing out the processor? - ORD_RLX, - ); - r.is_ok() - } else { - false - } - } -} - -impl Drop for Once { - fn drop(&mut self) { - let snapshot = self.value.load(ORD_ACQ); - if !snapshot.is_null() { - unsafe { mem::drop(Box::from_raw(snapshot)) } - } - } -} - -impl From> for Once { - fn from(v: Option) -> Self { - match v { - Some(v) => Self::with_value(v), - None => Self::new(), - } - } -} - -#[test] -fn once_get_none() { - let once: Once = Once::new(); - assert_eq!(once.get(), None); -} - -cfg_test! { - use std::sync::Arc; - use std::time::Duration; -} - -#[test] -fn once_set_get_some() { - let once: Arc> = Arc::new(Once::new()); - let t1 = once.clone(); - let t2 = once.clone(); - let t3 = once.clone(); - let hdl1 = thread::spawn(move || { - assert!(t1.set(10)); - }); - thread::sleep(Duration::from_secs(3)); - let hdl2 = thread::spawn(move || { - assert!(!t2.set(10)); - }); - let hdl3 = thread::spawn(move || { - assert_eq!(*t3.get().unwrap(), 10); - }); - assert_eq!(*once.get().unwrap(), 10); - hdl1.join().unwrap(); - hdl2.join().unwrap(); - hdl3.join().unwrap(); -} diff --git a/server/src/corestore/lock.rs b/server/src/corestore/lock.rs deleted file mode 100644 index dbcaf7f5..00000000 --- a/server/src/corestore/lock.rs +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Created on Tue Jun 29 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # Locks -//! -//! In several scenarios, we may find `std`'s or other crates' implementations of synchronization -//! primitives to be either _too sophisticated_ or _not what we want_. For these cases, we use -//! the primitives that are defined here -//! - -use { - super::backoff::Backoff, - std::{ - cell::UnsafeCell, - ops::{Deref, DerefMut}, - sync::atomic::{AtomicBool, Ordering}, - }, -}; - -const ORD_ACQUIRE: Ordering = Ordering::Acquire; -const ORD_RELEASE: Ordering = Ordering::Release; - -/// An extremely simple lock without the extra fuss: just the raw data and an atomic bool -#[derive(Debug)] -pub struct QuickLock { - rawdata: UnsafeCell, - lock_state: AtomicBool, -} - -unsafe impl Sync for QuickLock {} -unsafe impl Send for QuickLock {} - -/// A lock guard created by [`QuickLock`] -pub struct QLGuard<'a, T> { - lck: &'a QuickLock, -} - -impl<'a, T> QLGuard<'a, T> { - const fn init(lck: &'a QuickLock) -> Self { - Self { lck } - } -} - -/* - * Acq/Rel semantics don't emit fences on Intel platforms, but on weakly ordered targets - * things may look different. -*/ - -impl QuickLock { - pub const fn new(rawdata: T) -> Self { - Self { - lock_state: AtomicBool::new(false), - rawdata: UnsafeCell::new(rawdata), - } - } - /// Try to acquire a lock - pub fn try_lock(&self) -> Option> { - let ret = self - .lock_state - .compare_exchange(false, true, ORD_ACQUIRE, ORD_ACQUIRE); - if ret.is_ok() { - Some(QLGuard::init(self)) - } else { - None - } - } - /// Enter a _busy loop_ waiting to get an unlock. Behold, this is blocking! - pub fn lock(&self) -> QLGuard<'_, T> { - let backoff = Backoff::new(); - loop { - let ret = self.lock_state.compare_exchange_weak( - false, - true, - Ordering::SeqCst, - Ordering::Relaxed, - ); - match ret { - Ok(_) => break QLGuard::init(self), - Err(is_locked) => { - if !is_locked { - break QLGuard::init(self); - } - } - } - backoff.snooze(); - } - } -} - -impl<'a, T> Drop for QLGuard<'a, T> { - fn drop(&mut self) { - #[cfg(test)] - assert!(self.lck.lock_state.swap(false, ORD_RELEASE)); - #[cfg(not(test))] - let _ = self.lck.lock_state.swap(false, ORD_RELEASE); - } -} - -impl<'a, T> Deref for QLGuard<'a, T> { - type Target = T; - fn deref(&self) -> &Self::Target { - unsafe { - // UNSAFE(@ohsayan): Who doesn't like raw pointers anyway? (rustc: sigh) - &*self.lck.rawdata.get() - } - } -} - -impl<'a, T> DerefMut for QLGuard<'a, T> { - fn deref_mut(&mut self) -> &mut T { - unsafe { - // UNSAFE(@ohsayan): Who doesn't like raw pointers anyway? (rustc: sigh) - &mut *self.lck.rawdata.get() - } - } -} - -#[test] -fn test_already_locked() { - let lck = QuickLock::new(200); - let _our_lock = lck.lock(); - assert!(lck.try_lock().is_none()); -} - -#[cfg(test)] -use std::{ - sync::{mpsc, Arc}, - thread, - time::Duration, -}; - -#[cfg(test)] -fn panic_timeout(dur: Duration, f: F) -> T -where - T: Send + 'static, - F: (FnOnce() -> T) + Send + 'static, -{ - let (tx, rx) = mpsc::channel::<()>(); - let handle = thread::spawn(move || { - let val = f(); - let _ = tx.send(()); - val - }); - match rx.recv_timeout(dur) { - Ok(_) => handle.join().expect("Thread panicked"), - Err(_) => panic!("Thread passed timeout"), - } -} - -#[test] -#[should_panic] -fn test_two_lock_timeout() { - let lck = Arc::new(QuickLock::new(1u8)); - let child_lock = lck.clone(); - let _lock = lck.lock(); - panic_timeout(Duration::from_secs(1), move || { - let lck = child_lock; - let _ret = lck.lock(); - }); -} diff --git a/server/src/corestore/map/bref.rs b/server/src/corestore/map/bref.rs deleted file mode 100644 index 7dbd3fcb..00000000 --- a/server/src/corestore/map/bref.rs +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Created on Mon Aug 09 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::LowMap, - crate::util::{compiler, Unwrappable}, - core::{ - hash::{BuildHasher, Hash}, - mem, - ops::{Deref, DerefMut}, - }, - parking_lot::{RwLockReadGuard, RwLockWriteGuard}, - std::{collections::hash_map::RandomState, sync::Arc}, -}; - -/// A read-only reference to a bucket -pub struct Ref<'a, K, V> { - _g: RwLockReadGuard<'a, LowMap>, - k: &'a K, - v: &'a V, -} - -impl<'a, K, V> Ref<'a, K, V> { - /// Create a new reference - pub(super) const fn new(_g: RwLockReadGuard<'a, LowMap>, k: &'a K, v: &'a V) -> Self { - Self { _g, k, v } - } - /// Get a ref to the key - pub const fn key(&self) -> &K { - self.k - } - /// Get a ref to the value - pub const fn value(&self) -> &V { - self.v - } -} - -impl<'a, K, V> Deref for Ref<'a, K, V> { - type Target = V; - fn deref(&self) -> &Self::Target { - self.value() - } -} - -unsafe impl<'a, K: Send, V: Send> Send for Ref<'a, K, V> {} -unsafe impl<'a, K: Sync, V: Sync> Sync for Ref<'a, K, V> {} - -/// A r/w ref to a bucket -pub struct RefMut<'a, K, V> { - _g: RwLockWriteGuard<'a, LowMap>, - _k: &'a K, - v: &'a mut V, -} - -impl<'a, K, V> RefMut<'a, K, V> { - /// Create a new ref - pub(super) fn new(_g: RwLockWriteGuard<'a, LowMap>, k: &'a K, v: &'a mut V) -> Self { - Self { _g, _k: k, v } - } - /// Get a ref to the value - pub const fn value(&self) -> &V { - self.v - } - /// Get a mutable ref to the value - pub fn value_mut(&mut self) -> &mut V { - self.v - } -} - -impl<'a, K, V> Deref for RefMut<'a, K, V> { - type Target = V; - fn deref(&self) -> &Self::Target { - self.value() - } -} - -impl<'a, K, V> DerefMut for RefMut<'a, K, V> { - fn deref_mut(&mut self) -> &mut V { - self.value_mut() - } -} - -unsafe impl<'a, K: Send, V: Send> Send for RefMut<'a, K, V> {} -unsafe impl<'a, K: Sync, V: Sync> Sync for RefMut<'a, K, V> {} - -/// A reference to an occupied entry -pub struct OccupiedEntry<'a, K, V, S> { - guard: RwLockWriteGuard<'a, LowMap>, - elem: (&'a K, &'a mut V), - key: K, - hasher: S, -} - -impl<'a, K: Hash + Eq, V, S: BuildHasher> OccupiedEntry<'a, K, V, S> { - /// Create a new occupied entry ref - pub(super) fn new( - guard: RwLockWriteGuard<'a, LowMap>, - key: K, - elem: (&'a K, &'a mut V), - hasher: S, - ) -> Self { - Self { - guard, - elem, - key, - hasher, - } - } - /// Get a ref to the value - pub fn value(&self) -> &V { - self.elem.1 - } - /// Insert a value into this bucket - pub fn insert(&mut self, other: V) -> V { - mem::replace(self.elem.1, other) - } - /// Remove this element from the map - pub fn remove(mut self) -> V { - let hash = super::make_hash::(&self.hasher, &self.key); - unsafe { - self.guard - .remove_entry(hash, super::ceq(self.elem.0)) - .unsafe_unwrap() - } - .1 - } -} - -unsafe impl<'a, K: Send, V: Send, S> Send for OccupiedEntry<'a, K, V, S> {} -unsafe impl<'a, K: Sync, V: Sync, S> Sync for OccupiedEntry<'a, K, V, S> {} - -/// A ref to a vacant entry -pub struct VacantEntry<'a, K, V, S> { - guard: RwLockWriteGuard<'a, LowMap>, - key: K, - hasher: S, -} - -impl<'a, K: Hash + Eq, V, S: BuildHasher> VacantEntry<'a, K, V, S> { - /// Create a vacant entry ref - pub(super) fn new(guard: RwLockWriteGuard<'a, LowMap>, key: K, hasher: S) -> Self { - Self { guard, key, hasher } - } - /// Insert a value into this bucket - pub fn insert(mut self, value: V) -> RefMut<'a, K, V> { - unsafe { - let hash = super::make_insert_hash::(&self.hasher, &self.key); - let &mut (ref mut k, ref mut v) = self.guard.insert_entry( - hash, - (self.key, value), - super::make_hasher::(&self.hasher), - ); - let kptr = compiler::extend_lifetime(k); - let vptr = compiler::extend_lifetime_mut(v); - RefMut::new(self.guard, kptr, vptr) - } - } -} - -/// An entry, either occupied or vacant -pub enum Entry<'a, K, V, S = RandomState> { - Occupied(OccupiedEntry<'a, K, V, S>), - Vacant(VacantEntry<'a, K, V, S>), -} - -#[cfg(test)] -impl<'a, K, V, S> Entry<'a, K, V, S> { - pub fn is_occupied(&self) -> bool { - matches!(self, Self::Occupied(_)) - } - pub fn is_vacant(&self) -> bool { - matches!(self, Self::Vacant(_)) - } -} - -/// A shared ref to a key -pub struct RefMulti<'a, K, V> { - _g: Arc>>, - k: &'a K, - v: &'a V, -} - -impl<'a, K, V> RefMulti<'a, K, V> { - /// Create a new shared ref - pub const fn new(_g: Arc>>, k: &'a K, v: &'a V) -> Self { - Self { _g, k, v } - } - /// Get a ref to the key - pub const fn key(&self) -> &K { - self.k - } - /// Get a ref to the value - pub const fn value(&self) -> &V { - self.v - } -} - -impl<'a, K, V> Deref for RefMulti<'a, K, V> { - type Target = V; - fn deref(&self) -> &Self::Target { - self.value() - } -} - -unsafe impl<'a, K: Sync, V: Sync> Sync for RefMulti<'a, K, V> {} -unsafe impl<'a, K: Send, V: Send> Send for RefMulti<'a, K, V> {} - -/// A shared r/w ref to a bucket -pub struct RefMultiMut<'a, K, V> { - _g: Arc>>, - _k: &'a K, - v: &'a mut V, -} - -impl<'a, K, V> RefMultiMut<'a, K, V> { - /// Create a new shared r/w ref - pub fn new(_g: Arc>>, k: &'a K, v: &'a mut V) -> Self { - Self { _g, _k: k, v } - } - /// Get a ref to the value - pub const fn value(&self) -> &V { - self.v - } - /// Get a mutable ref to the value - pub fn value_mut(&mut self) -> &mut V { - self.v - } -} - -impl<'a, K, V> Deref for RefMultiMut<'a, K, V> { - type Target = V; - fn deref(&self) -> &Self::Target { - self.value() - } -} - -impl<'a, K, V> DerefMut for RefMultiMut<'a, K, V> { - fn deref_mut(&mut self) -> &mut V { - self.value_mut() - } -} - -unsafe impl<'a, K: Sync, V: Sync> Sync for RefMultiMut<'a, K, V> {} -unsafe impl<'a, K: Send, V: Send> Send for RefMultiMut<'a, K, V> {} diff --git a/server/src/corestore/map/iter.rs b/server/src/corestore/map/iter.rs deleted file mode 100644 index ca013d4f..00000000 --- a/server/src/corestore/map/iter.rs +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Created on Tue Aug 10 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{bref::RefMulti, LowMap, Skymap}, - core::mem, - hashbrown::raw::{RawIntoIter, RawIter}, - parking_lot::RwLockReadGuard, - std::{collections::hash_map::RandomState, sync::Arc}, -}; - -/// An owned iterator for a [`Skymap`] -pub struct OwnedIter { - map: Skymap, - cs: usize, - current: Option>, -} - -impl OwnedIter { - pub fn new(map: Skymap) -> Self { - Self { - map, - cs: 0usize, - current: None, - } - } -} - -impl Iterator for OwnedIter { - type Item = (K, V); - fn next(&mut self) -> Option { - loop { - if let Some(current) = self.current.as_mut() { - if let Some(bucket) = current.next() { - return Some(bucket); - } - } - if self.cs == self.map.shards().len() { - return None; - } - let mut wshard = unsafe { self.map.get_wshard_unchecked(self.cs) }; - // get the next map's iterator - let current_map = mem::replace(&mut *wshard, LowMap::new()); - drop(wshard); - let iter = current_map.into_iter(); - self.current = Some(iter); - self.cs += 1; - } - } -} - -unsafe impl Send for OwnedIter {} -unsafe impl Sync for OwnedIter {} - -type BorrowedIterGroup<'a, K, V> = (RawIter<(K, V)>, Arc>>); - -/// A borrowed iterator for a [`Skymap`] -pub struct BorrowedIter<'a, K, V, S = ahash::RandomState> { - map: &'a Skymap, - cs: usize, - citer: Option>, -} - -impl<'a, K, V, S> BorrowedIter<'a, K, V, S> { - pub const fn new(map: &'a Skymap) -> Self { - Self { - map, - cs: 0usize, - citer: None, - } - } -} - -impl<'a, K, V, S> Iterator for BorrowedIter<'a, K, V, S> { - type Item = RefMulti<'a, K, V>; - fn next(&mut self) -> Option { - loop { - if let Some(current) = self.citer.as_mut() { - if let Some(bucket) = current.0.next() { - let (kptr, vptr) = unsafe { - // we know that this is valid, and this guarantee is - // provided to us by the lifetime - bucket.as_ref() - }; - let guard = current.1.clone(); - return Some(RefMulti::new(guard, kptr, vptr)); - } - } - if self.cs == self.map.shards().len() { - // end of shards - return None; - } - // warning: the rawiter allows us to terribly violate conditions - // you can mutate! - let rshard = unsafe { self.map.get_rshard_unchecked(self.cs) }; - let iter = unsafe { - // same thing: our lt params ensure validity - rshard.iter() - }; - self.citer = Some((iter, Arc::new(rshard))); - self.cs += 1; - } - } -} - -unsafe impl<'a, K: Send, V: Send, S> Send for BorrowedIter<'a, K, V, S> {} -unsafe impl<'a, K: Sync, V: Sync, S> Sync for BorrowedIter<'a, K, V, S> {} - -#[test] -fn test_iter() { - let map = Skymap::default(); - map.insert("hello1", "world"); - map.insert("hello2", "world"); - map.insert("hello3", "world"); - let collected: Vec<(&str, &str)> = map.get_owned_iter().collect(); - assert!(collected.len() == 3); - assert!(collected.contains(&("hello1", "world"))); - assert!(collected.contains(&("hello2", "world"))); - assert!(collected.contains(&("hello3", "world"))); -} diff --git a/server/src/corestore/map/mod.rs b/server/src/corestore/map/mod.rs deleted file mode 100644 index a7c0f600..00000000 --- a/server/src/corestore/map/mod.rs +++ /dev/null @@ -1,439 +0,0 @@ -/* - * Created on Mon Aug 09 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![allow(clippy::manual_map)] // avoid LLVM bloat -#![allow(unused)] // TODO(@ohsayan): Plonk this - -use { - self::{ - bref::{Entry, OccupiedEntry, Ref, RefMut, VacantEntry}, - iter::{BorrowedIter, OwnedIter}, - }, - crate::util::compiler, - core::{ - borrow::Borrow, - fmt, - hash::{BuildHasher, Hash, Hasher}, - iter::FromIterator, - mem, - num::NonZeroUsize, - }, - parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}, - std::{collections::hash_map::RandomState, thread::available_parallelism}, -}; - -pub mod bref; -pub mod iter; - -type LowMap = hashbrown::raw::RawTable<(K, V)>; -type ShardSlice = [RwLock>]; -type SRlock<'a, K, V> = RwLockReadGuard<'a, hashbrown::raw::RawTable<(K, V)>>; -type SWlock<'a, K, V> = RwLockWriteGuard<'a, hashbrown::raw::RawTable<(K, V)>>; -const BITS_IN_USIZE: usize = mem::size_of::() * 8; -const DEFAULT_CAP: usize = 128; - -fn make_hash(hash_builder: &S, val: &Q) -> u64 -where - K: Borrow, - Q: Hash + ?Sized, - S: BuildHasher, -{ - let mut state = hash_builder.build_hasher(); - val.hash(&mut state); - state.finish() -} - -fn make_insert_hash(hash_builder: &S, val: &K) -> u64 -where - K: Hash, - S: BuildHasher, -{ - let mut state = hash_builder.build_hasher(); - val.hash(&mut state); - state.finish() -} - -fn make_hasher(hash_builder: &S) -> impl Fn(&(Q, V)) -> u64 + '_ -where - K: Borrow, - Q: Hash, - S: BuildHasher, -{ - move |val| make_hash::(hash_builder, &val.0) -} - -fn ceq(k: &Q) -> impl Fn(&(K, V)) -> bool + '_ -where - K: Borrow, - Q: ?Sized + Eq, -{ - move |x| k.eq(x.0.borrow()) -} - -fn get_shard_count() -> usize { - (available_parallelism().map_or(1, usize::from) * 16).next_power_of_two() -} - -const fn cttz(amount: usize) -> usize { - amount.trailing_zeros() as usize -} - -/// A striped in-memory map -pub struct Skymap { - shards: Box>, - hasher: S, - shift: usize, -} - -impl Default for Skymap { - fn default() -> Self { - Self::with_hasher(RandomState::default()) - } -} - -impl fmt::Debug for Skymap { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut map = f.debug_map(); - for s in self.get_iter() { - map.entry(s.key(), s.value()); - } - map.finish() - } -} - -impl FromIterator<(K, V)> for Skymap -where - K: Eq + Hash, - S: BuildHasher + Default + Clone, -{ - fn from_iter(iter: T) -> Self - where - T: IntoIterator, - { - let map = Skymap::new(); - iter.into_iter().for_each(|(k, v)| { - let _ = map.insert(k, v); - }); - map - } -} - -impl Skymap { - /// Get a Skymap with the ahash hasher - pub fn new_ahash() -> Self { - Skymap::new() - } -} - -// basic impls -impl Skymap -where - S: BuildHasher + Default, -{ - /// Create a new Skymap with the default state (or seed) of the hasher - pub fn new() -> Self { - Self::with_hasher(S::default()) - } - /// Create a new Skymap with the provided capacity - pub fn with_capacity(cap: usize) -> Self { - Self::with_capacity_and_hasher(cap, S::default()) - } - /// Create a new Skymap with the provided cap and hasher - pub fn with_capacity_and_hasher(mut cap: usize, hasher: S) -> Self { - let shard_count = get_shard_count(); - let shift = BITS_IN_USIZE - cttz(shard_count); - if cap != 0 { - cap = (cap + (shard_count - 1)) & !(shard_count - 1); - } - - let cap_per_shard = cap / shard_count; - Self { - shards: (0..shard_count) - .map(|_| RwLock::new(LowMap::with_capacity(cap_per_shard))) - .collect(), - hasher, - shift, - } - } - /// Create a new Skymap with the provided hasher - pub fn with_hasher(hasher: S) -> Self { - Self::with_capacity_and_hasher(DEFAULT_CAP, hasher) - } - /// Get the len of the Skymap - pub fn len(&self) -> usize { - self.shards.iter().map(|s| s.read().len()).sum() - } - /// Get the capacity of the Skymap - pub fn capacity(&self) -> usize { - self.shards.iter().map(|s| s.read().capacity()).sum() - } - /// Check if the Skymap is empty - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - /// Get a borrowed iterator for the Skymap. Bound to the lifetime - pub fn get_iter(&self) -> BorrowedIter { - BorrowedIter::new(self) - } - /// Get an owned iterator to the Skymap - pub fn get_owned_iter(self) -> OwnedIter { - OwnedIter::new(self) - } -} - -// const impls -impl Skymap { - /// Get a ref to the stripes - const fn shards(&self) -> &ShardSlice { - &self.shards - } - /// Determine the shard - const fn determine_shard(&self, hash: usize) -> usize { - // the idea of the shift was inspired by Joel's idea - (hash << 7) >> self.shift - } - /// Get a ref to the underlying hasher - const fn h(&self) -> &S { - &self.hasher - } -} - -// insert/get/remove impls - -impl Skymap -where - K: Eq + Hash, - S: BuildHasher + Clone, -{ - /// Insert a key/value into the Skymap - pub fn insert(&self, k: K, v: V) -> Option { - let hash = make_insert_hash::(&self.hasher, &k); - let idx = self.determine_shard(hash as usize); - unsafe { - // begin critical section - let mut lowtable = self.get_wshard_unchecked(idx); - if let Some((_, item)) = lowtable.get_mut(hash, ceq(&k)) { - Some(mem::replace(item, v)) - } else { - lowtable.insert(hash, (k, v), make_hasher::(self.h())); - None - } - // end critical section - } - } - /// Remove a key/value from the Skymap - pub fn remove(&self, k: &Q) -> Option<(K, V)> - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - let hash = make_hash::(self.h(), k); - let idx = self.determine_shard(hash as usize); - unsafe { - // begin critical section - let mut lowtable = self.get_wshard_unchecked(idx); - match lowtable.remove_entry(hash, ceq(k)) { - Some(kv) => Some(kv), - None => None, - } - // end critical section - } - } - /// Remove a key/value from the Skymap if it satisfies a certain condition - pub fn remove_if(&self, k: &Q, f: impl FnOnce(&K, &V) -> bool) -> Option<(K, V)> - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - let hash = make_hash::(self.h(), k); - let idx = self.determine_shard(hash as usize); - unsafe { - // begin critical section - let mut lowtable = self.get_wshard_unchecked(idx); - match lowtable.find(hash, ceq(k)) { - Some(bucket) => { - let (kptr, vptr) = bucket.as_ref(); - if f(kptr, vptr) { - Some(lowtable.remove(bucket)) - } else { - None - } - } - None => None, - } - // end critical section - } - } -} - -// lt impls -impl<'a, K: 'a + Hash + Eq, V: 'a, S: BuildHasher + Clone> Skymap { - /// Get a ref to an entry in the Skymap - pub fn get(&'a self, k: &Q) -> Option> - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - let hash = make_hash::(self.h(), k); - let idx = self.determine_shard(hash as usize); - unsafe { - // begin critical section - let lowtable = self.get_rshard_unchecked(idx); - match lowtable.get(hash, ceq(k)) { - Some((ref kptr, ref vptr)) => { - let kptr = compiler::extend_lifetime(kptr); - let vptr = compiler::extend_lifetime(vptr); - Some(Ref::new(lowtable, kptr, vptr)) - } - None => None, - } - // end critical section - } - } - - /// Get a mutable ref to an entry in the Skymap - pub fn get_mut(&'a self, k: &Q) -> Option> - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - let hash = make_hash::(self.h(), k); - let idx = self.determine_shard(hash as usize); - unsafe { - // begin critical section - let mut lowtable = self.get_wshard_unchecked(idx); - match lowtable.get_mut(hash, ceq(k)) { - Some(&mut (ref kptr, ref mut vptr)) => { - let kptr = compiler::extend_lifetime(kptr); - let vptr = compiler::extend_lifetime_mut(vptr); - Some(RefMut::new(lowtable, kptr, vptr)) - } - None => None, - } - // end critical section - } - } - /// Get an entry for in-place mutation - pub fn entry(&'a self, key: K) -> Entry<'a, K, V, S> { - let hash = make_insert_hash::(self.h(), &key); - let idx = self.determine_shard(hash as usize); - unsafe { - // begin critical section - let lowtable = self.get_wshard_unchecked(idx); - if let Some(elem) = lowtable.find(hash, ceq(&key)) { - let (kptr, vptr) = elem.as_mut(); - let kptr = compiler::extend_lifetime(kptr); - let vptr = compiler::extend_lifetime_mut(vptr); - Entry::Occupied(OccupiedEntry::new( - lowtable, - key, - (kptr, vptr), - self.hasher.clone(), - )) - } else { - Entry::Vacant(VacantEntry::new(lowtable, key, self.hasher.clone())) - } - // end critical section - } - } - /// Check if the Skymap contains the provided key - pub fn contains_key(&self, key: &Q) -> bool - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - self.get(key).is_some() - } - /// Clear out all the entries in the Skymap - pub fn clear(&self) { - self.shards().iter().for_each(|shard| shard.write().clear()) - } -} - -// cloned impls -impl<'a, K, V: Clone, S: BuildHasher> Skymap { - pub fn get_cloned(&'a self, k: &Q) -> Option - where - K: Borrow, - Q: Hash + Eq + ?Sized, - { - let hash = make_hash::(self.h(), k); - let idx = self.determine_shard(hash as usize); - unsafe { - // begin critical section - let lowtable = self.get_rshard_unchecked(idx); - match lowtable.get(hash, ceq(k)) { - Some((_kptr, ref vptr)) => Some(vptr.clone()), - None => None, - } - // end critical section - } - } -} - -// inner impls -impl<'a, K: 'a, V: 'a, S> Skymap { - /// Get a rlock to a certain stripe - unsafe fn get_rshard_unchecked(&'a self, shard: usize) -> SRlock<'a, K, V> { - ucidx!(self.shards, shard).read() - } - /// Get a wlock to a certain stripe - unsafe fn get_wshard_unchecked(&'a self, shard: usize) -> SWlock<'a, K, V> { - ucidx!(self.shards, shard).write() - } -} - -#[test] -fn test_insert_remove() { - let map = Skymap::default(); - map.insert("hello", "world"); - assert_eq!(map.remove("hello").unwrap().1, "world"); -} - -#[test] -fn test_remove_if() { - let map = Skymap::default(); - map.insert("hello", "world"); - assert!(map - .remove_if("hello", |_k, v| { (*v).eq("notworld") }) - .is_none()); -} - -#[test] -fn test_insert_get() { - let map = Skymap::default(); - map.insert("sayan", "likes computational dark arts"); - let _ref = map.get("sayan").unwrap(); - assert_eq!(*_ref, "likes computational dark arts") -} - -#[test] -fn test_entry() { - let map = Skymap::default(); - map.insert("hello", "world"); - assert!(map.entry("hello").is_occupied()); - assert!(map.entry("world").is_vacant()); -} diff --git a/server/src/corestore/memstore.rs b/server/src/corestore/memstore.rs deleted file mode 100644 index 9e4d22b6..00000000 --- a/server/src/corestore/memstore.rs +++ /dev/null @@ -1,492 +0,0 @@ -/* - * Created on Fri Jul 02 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # In-memory store -//! -//! This is what things look like: -//! ```text -//! ----------------------------------------------------- -//! | | -//! | |-------------------| |-------------------| | -//! | |-------------------| |-------------------| | -//! | | | TABLE | TABLE | | | | TABLE | TABLE | | | -//! | | |-------|-------| | | |-------|-------| | | -//! | | Keyspace | | Keyspace | | -//! | |-------------------| |-------------------| | -//! | -//! | |-------------------| |-------------------| | -//! | | |-------|-------| | | |-------|-------| | | -//! | | | TABLE | TABLE | | | | TABLE | TABLE | | | -//! | | |-------|-------| | | |-------|-------| | | -//! | | Keyspace | | Keyspace | | -//! | |-------------------| |-------------------| | -//! | | -//! | | -//! | | -//! ----------------------------------------------------- -//! | NODE | -//! |---------------------------------------------------| -//! ``` -//! -//! So, all your data is at the mercy of [`Memstore`]'s constructor -//! and destructor. - -use { - super::KeyspaceResult, - crate::{ - auth::Authmap, - corestore::{ - array::Array, - htable::Coremap, - table::{SystemDataModel, SystemTable, Table}, - }, - registry, - util::Wrapper, - }, - core::{borrow::Borrow, hash::Hash}, - std::sync::Arc, -}; - -uninit_array! { - const DEFAULT_ARRAY: [u8; 64] = [b'd', b'e', b'f', b'a', b'u', b'l', b't']; - const SYSTEM_ARRAY: [u8; 64] = [b's', b'y', b's', b't', b'e', b'm']; - const SYSTEM_AUTH_ARRAY: [u8; 64] = [b'a', b'u', b't', b'h']; -} - -/// typedef for the keyspace/table IDs. We don't need too much fancy here, -/// no atomic pointers and all. Just a nice array. With amazing gurantees -pub type ObjectID = Array; - -/// The `DEFAULT` array (with the rest uninit) -pub const DEFAULT: ObjectID = unsafe { - // SAFETY: known init len - Array::from_const(DEFAULT_ARRAY, 7) -}; -pub const SYSTEM: ObjectID = unsafe { - // SAFETY: known init len - Array::from_const(SYSTEM_ARRAY, 6) -}; -pub const AUTH: ObjectID = unsafe { - // SAFETY: known init len - Array::from_const(SYSTEM_AUTH_ARRAY, 4) -}; - -#[test] -fn test_def_macro_sanity() { - // just make sure our macro is working as expected - let mut def = DEFAULT; - def.push(b'?'); - unsafe { - assert_eq!(def.as_str(), "default?"); - let mut sys = SYSTEM; - sys.push(b'?'); - assert_eq!(sys.as_str(), "system?"); - } -} - -mod cluster { - /// This is for the future where every node will be allocated a shard - #[derive(Debug)] - pub enum ClusterShardRange { - SingleNode, - } - - impl Default for ClusterShardRange { - fn default() -> Self { - Self::SingleNode - } - } - - /// This is for the future for determining the replication strategy - #[derive(Debug)] - pub enum ReplicationStrategy { - /// Single node, no replica sets - Default, - } - - impl Default for ReplicationStrategy { - fn default() -> Self { - Self::Default - } - } -} - -#[derive(Debug, PartialEq, Eq)] -/// Errors arising from trying to modify/access containers -#[allow(dead_code)] -pub enum DdlError { - /// The object is still in use - StillInUse, - /// The object couldn't be found - ObjectNotFound, - /// The object is not user-accessible - ProtectedObject, - /// The default object wasn't found - DefaultNotFound, - /// Incorrect data model semantics were used on a data model - WrongModel, - /// The object already exists - AlreadyExists, - /// The target object is not ready - NotReady, - /// The target object is not empty - NotEmpty, - /// The DDL transaction failed - DdlTransactionFailure, -} - -#[derive(Debug)] -/// The core in-memory table -/// -/// This in-memory table that houses all keyspaces along with other node properties. -/// This is the structure that you should clone in an atomic RC wrapper. This object -/// handles no sort of persistence -pub struct Memstore { - /// the keyspaces - pub keyspaces: Coremap>, - /// the system keyspace with the system tables - pub system: SystemKeyspace, -} - -impl Memstore { - /// Create a new empty in-memory table with literally nothing in it - #[cfg(test)] - pub fn new_empty() -> Self { - Self { - keyspaces: Coremap::new(), - system: SystemKeyspace::new(Coremap::new()), - } - } - pub fn init_with_all( - keyspaces: Coremap>, - system: SystemKeyspace, - ) -> Self { - Self { keyspaces, system } - } - /// Create a new in-memory table with the default keyspace and the default - /// tables. So, whenever you're calling this, this is what you get: - /// ```json - /// "YOURNODE": { - /// "KEYSPACES": [ - /// "default" : { - /// "TABLES": ["default"] - /// }, - /// "system": { - /// "TABLES": [] - /// } - /// ] - /// } - /// ``` - /// - /// When you connect a client without any information about the keyspace you're planning to - /// use, you'll be connected to `ks:default/table:default`. The `ks:default/table:_system` is not - /// for you. It's for the system - pub fn new_default() -> Self { - Self { - keyspaces: { - let n = Coremap::new(); - n.true_if_insert(DEFAULT, Arc::new(Keyspace::empty_default())); - // HACK(@ohsayan): This just ensures that the record is in the preload - n.true_if_insert(SYSTEM, Arc::new(Keyspace::empty())); - n - }, - system: SystemKeyspace::new(Coremap::new()), - } - } - pub fn setup_auth(&self) -> Authmap { - match self.system.tables.fresh_entry(AUTH) { - Some(fresh) => { - // created afresh, fine - let r = Authmap::default(); - fresh.insert(Wrapper::new(SystemTable::new_auth(r.clone()))); - r - } - None => match self.system.tables.get(&AUTH).unwrap().data { - SystemDataModel::Auth(ref am) => am.clone(), - #[allow(unreachable_patterns)] - _ => unsafe { impossible!() }, - }, - } - } - /// Get an atomic reference to a keyspace - pub fn get_keyspace_atomic_ref(&self, keyspace_identifier: &Q) -> Option> - where - ObjectID: Borrow, - Q: Hash + Eq + ?Sized, - { - self.keyspaces.get(keyspace_identifier).map(|ns| ns.clone()) - } - /// Returns true if a new keyspace was created - pub fn create_keyspace(&self, keyspace_identifier: ObjectID) -> bool { - self.keyspaces - .true_if_insert(keyspace_identifier, Arc::new(Keyspace::empty())) - } - /// Drop a keyspace only if it is empty and has no clients connected to it - /// - /// The invariants maintained here are: - /// 1. The keyspace is not referenced to - /// 2. There are no tables in the keyspace - /// - /// **Trip switch handled:** Yes - pub fn drop_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> { - if ksid.eq(&SYSTEM) || ksid.eq(&DEFAULT) { - Err(DdlError::ProtectedObject) - } else if !self.keyspaces.contains_key(&ksid) { - Err(DdlError::ObjectNotFound) - } else { - let removed_keyspace = self.keyspaces.mut_entry(ksid); - match removed_keyspace { - Some(ks) => { - let no_one_is_using_keyspace = Arc::strong_count(ks.value()) == 1; - let no_tables_are_in_keyspace = ks.value().table_count() == 0; - if no_one_is_using_keyspace && no_tables_are_in_keyspace { - // we are free to drop this - ks.remove(); - // trip the preload switch - registry::get_preload_tripswitch().trip(); - // trip the cleanup switch - registry::get_cleanup_tripswitch().trip(); - Ok(()) - } else if !no_tables_are_in_keyspace { - // not empty; may be referenced to or not referenced to - Err(DdlError::NotEmpty) - } else { - // still referenced to; but is empty - Err(DdlError::StillInUse) - } - } - None => Err(DdlError::ObjectNotFound), - } - } - } - /// Force remove a keyspace along with all its tables. This force however only - /// removes tables if they aren't in use and iff the keyspace is not currently - /// in use to avoid the problem of having "ghost tables" - /// - /// The invariants maintained here are: - /// 1. The keyspace is not referenced to - /// 2. The tables in the keyspace are not referenced to - /// - /// **Trip switch handled:** Yes - pub fn force_drop_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> { - if ksid.eq(&SYSTEM) || ksid.eq(&DEFAULT) { - Err(DdlError::ProtectedObject) - } else if !self.keyspaces.contains_key(&ksid) { - Err(DdlError::ObjectNotFound) - } else { - let removed_keyspace = self.keyspaces.mut_entry(ksid); - match removed_keyspace { - Some(keyspace) => { - // force remove drops tables, but only if they aren't in use - // we can potentially have "ghost" tables if we don't do this - // check. Similarly, if we don't check the strong count of the - // keyspace, we might end up with "ghost" keyspaces. A good - // question would be: why not just check the number of tables -- - // well, the force drop is specifically built to address the problem - // of having not empty keyspaces. The invariant that `drop keyspace force` - // maintains is that the keyspace or any of its objects are never - // referenced to -- only then it is safe to delete the keyspace - let no_tables_in_use = Arc::strong_count(keyspace.value()) == 1 - && keyspace - .value() - .tables - .iter() - .all(|table| Arc::strong_count(table.value()) == 1); - if no_tables_in_use { - keyspace.remove(); - // trip the preload switch - registry::get_preload_tripswitch().trip(); - // trip the cleanup switch - registry::get_cleanup_tripswitch().trip(); - Ok(()) - } else { - Err(DdlError::StillInUse) - } - } - None => Err(DdlError::ObjectNotFound), - } - } - } - pub fn list_keyspaces(&self) -> Vec { - self.keyspaces.iter().map(|kv| kv.key().clone()).collect() - } -} - -/// System keyspace -#[derive(Debug)] -pub struct SystemKeyspace { - pub tables: Coremap>, -} - -impl SystemKeyspace { - pub const fn new(tables: Coremap>) -> Self { - Self { tables } - } -} - -#[derive(Debug)] -/// A keyspace houses all the other tables -pub struct Keyspace { - /// the tables - pub tables: Coremap>, - /// the replication strategy for this keyspace - #[allow(dead_code)] // TODO: Remove this once we're ready with replication - replication_strategy: cluster::ReplicationStrategy, -} - -#[cfg(test)] -macro_rules! unsafe_objectid_from_slice { - ($slice:expr) => {{ - unsafe { ObjectID::from_slice($slice) } - }}; -} - -impl Keyspace { - /// Create a new empty keyspace with the default tables: a `default` table - pub fn empty_default() -> Self { - Self { - tables: { - let ht = Coremap::new(); - // add the default table - ht.true_if_insert(DEFAULT, Arc::new(Table::new_default_kve())); - ht - }, - replication_strategy: cluster::ReplicationStrategy::default(), - } - } - pub fn init_with_all_def_strategy(tables: Coremap>) -> Self { - Self { - tables, - replication_strategy: cluster::ReplicationStrategy::default(), - } - } - /// Create a new empty keyspace with zero tables - pub fn empty() -> Self { - Self { - tables: Coremap::new(), - replication_strategy: cluster::ReplicationStrategy::default(), - } - } - pub fn table_count(&self) -> usize { - self.tables.len() - } - /// Get an atomic reference to a table in this keyspace if it exists - pub fn get_table_atomic_ref(&self, table_identifier: &Q) -> Option> - where - ObjectID: Borrow, - Q: Hash + Eq + PartialEq + ?Sized, - { - self.tables.get(table_identifier).map(|v| v.clone()) - } - /// Create a new table - pub fn create_table(&self, tableid: ObjectID, table: Table) -> bool { - self.tables.true_if_insert(tableid, Arc::new(table)) - } - /// Drop a table if it exists, if it is not forbidden and if no one references - /// back to it. We don't want any looming table references i.e table gets deleted - /// for the current connection and newer connections, but older instances still - /// refer to the table. - // FIXME(@ohsayan): Should we actually care? - /// - /// **Trip switch handled:** Yes - fn drop_table_inner(&self, table_identifier: &Q, should_force: bool) -> KeyspaceResult<()> - where - ObjectID: Borrow, - Q: Hash + Eq + PartialEq + ?Sized, - { - if table_identifier.eq(&DEFAULT) { - Err(DdlError::ProtectedObject) - } else if !self.tables.contains_key(table_identifier) { - Err(DdlError::ObjectNotFound) - } else { - // has table - let did_remove = - self.tables - .true_remove_if(table_identifier, |_table_id, table_atomic_ref| { - // 1 because this should just be us, the one instance - Arc::strong_count(table_atomic_ref) == 1 - && (table_atomic_ref.is_empty() || should_force) - }); - if did_remove { - // we need to re-init tree; so trip - registry::get_preload_tripswitch().trip(); - // we need to cleanup tree; so trip - registry::get_cleanup_tripswitch().trip(); - Ok(()) - } else { - Err(DdlError::StillInUse) - } - } - } - pub fn drop_table(&self, tblid: &Q, force: bool) -> KeyspaceResult<()> - where - ObjectID: Borrow, - Q: Hash + Eq + PartialEq + ?Sized, - { - self.drop_table_inner(tblid, force) - } -} - -#[test] -fn test_keyspace_drop_no_atomic_ref() { - let our_keyspace = Keyspace::empty_default(); - assert!(our_keyspace.create_table( - unsafe_objectid_from_slice!("apps"), - Table::new_default_kve() - )); - assert!(our_keyspace - .drop_table(&unsafe_objectid_from_slice!("apps"), false) - .is_ok()); -} - -#[test] -fn test_keyspace_drop_fail_with_atomic_ref() { - let our_keyspace = Keyspace::empty_default(); - assert!(our_keyspace.create_table( - unsafe_objectid_from_slice!("apps"), - Table::new_default_kve() - )); - let _atomic_tbl_ref = our_keyspace - .get_table_atomic_ref(&unsafe_objectid_from_slice!("apps")) - .unwrap(); - assert_eq!( - our_keyspace - .drop_table(&unsafe_objectid_from_slice!("apps"), false) - .unwrap_err(), - DdlError::StillInUse - ); -} - -#[test] -fn test_keyspace_try_delete_protected_table() { - let our_keyspace = Keyspace::empty_default(); - assert_eq!( - our_keyspace - .drop_table(&unsafe_objectid_from_slice!("default"), false) - .unwrap_err(), - DdlError::ProtectedObject - ); -} diff --git a/server/src/corestore/mod.rs b/server/src/corestore/mod.rs deleted file mode 100644 index 18e7dd59..00000000 --- a/server/src/corestore/mod.rs +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Created on Tue Jul 20 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - actions::{translate_ddl_error, ActionResult}, - blueql::Entity, - corestore::{ - memstore::{DdlError, Keyspace, Memstore, ObjectID, DEFAULT}, - table::{DescribeTable, Table}, - }, - protocol::interface::ProtocolSpec, - registry, - storage::{ - self, - v1::{error::StorageEngineResult, sengine::SnapshotEngine}, - }, - util::{self, Unwrappable}, - }, - core::{borrow::Borrow, hash::Hash}, - std::sync::Arc, -}; - -pub mod array; -pub mod backoff; -pub mod booltable; -pub mod buffers; -pub mod heap_array; -pub mod htable; -pub mod iarray; -pub mod lazy; -pub mod lock; -pub mod map; -pub mod memstore; -pub mod rc; -pub mod table; -#[cfg(test)] -mod tests; - -pub use self::rc::SharedSlice; - -pub(super) type KeyspaceResult = Result; - -#[derive(Debug, Clone)] -struct ConnectionEntityState { - /// the current table for a connection - table: Option<(ObjectID, Arc)>, - /// the current keyspace for a connection - ks: Option<(ObjectID, Arc)>, -} - -impl ConnectionEntityState { - fn default(ks: Arc, tbl: Arc
) -> Self { - Self { - table: Some((DEFAULT, tbl)), - ks: Some((DEFAULT, ks)), - } - } - fn set_ks(&mut self, ks: Arc, ksid: ObjectID) { - self.ks = Some((ksid, ks)); - self.table = None; - } - fn set_table(&mut self, ks: Arc, ksid: ObjectID, tbl: Arc
, tblid: ObjectID) { - self.ks = Some((ksid, ks)); - self.table = Some((tblid, tbl)); - } - fn get_id_pack(&self) -> (Option<&ObjectID>, Option<&ObjectID>) { - ( - self.ks.as_ref().map(|(id, _)| id), - self.table.as_ref().map(|(id, _)| id), - ) - } -} - -/// The top level abstraction for the in-memory store. This is free to be shared across -/// threads, cloned and well, whatever. Most importantly, clones have an independent container -/// state that is the state of one connection and its container state preferences are never -/// synced across instances. This is important (see the impl for more info) -#[derive(Debug, Clone)] -pub struct Corestore { - estate: ConnectionEntityState, - /// an atomic reference to the actual backing storage - store: Arc, - /// the snapshot engine - sengine: Arc, -} - -impl Corestore { - /// This is the only function you'll ever need to either create a new database instance - /// or restore from an earlier instance - pub fn init_with_snapcfg(sengine: Arc) -> StorageEngineResult { - let store = storage::unflush::read_full()?; - Ok(Self::default_with_store(store, sengine)) - } - pub fn clone_store(&self) -> Arc { - self.store.clone() - } - pub fn default_with_store(store: Memstore, sengine: Arc) -> Self { - let cks = unsafe { store.get_keyspace_atomic_ref(&DEFAULT).unsafe_unwrap() }; - let ctable = unsafe { cks.get_table_atomic_ref(&DEFAULT).unsafe_unwrap() }; - Self { - estate: ConnectionEntityState::default(cks, ctable), - store: Arc::new(store), - sengine, - } - } - pub fn get_engine(&self) -> &SnapshotEngine { - &self.sengine - } - pub fn get_store(&self) -> &Memstore { - &self.store - } - /// Swap out the current table with a different one - /// - /// If the table is non-existent or the default keyspace was unset, then - /// false is returned. Else true is returned - pub fn swap_entity(&mut self, entity: &Entity) -> KeyspaceResult<()> { - match entity { - // Switch to the provided keyspace - Entity::Current(ks) => { - match self.store.get_keyspace_atomic_ref(unsafe { ks.as_slice() }) { - Some(ksref) => self - .estate - .set_ks(ksref, unsafe { ObjectID::from_slice(ks.as_slice()) }), - None => return Err(DdlError::ObjectNotFound), - } - } - // Switch to the provided table in the given keyspace - Entity::Full(ks, tbl) => { - match self.store.get_keyspace_atomic_ref(unsafe { ks.as_slice() }) { - Some(kspace) => match kspace.get_table_atomic_ref(unsafe { tbl.as_slice() }) { - Some(tblref) => unsafe { - self.estate.set_table( - kspace, - ObjectID::from_slice(ks.as_slice()), - tblref, - ObjectID::from_slice(tbl.as_slice()), - ) - }, - None => return Err(DdlError::ObjectNotFound), - }, - None => return Err(DdlError::ObjectNotFound), - } - } - } - Ok(()) - } - /// Returns the current keyspace, if set - pub fn get_cks(&self) -> KeyspaceResult<&Keyspace> { - match self.estate.ks { - Some((_, ref cks)) => Ok(cks), - _ => Err(DdlError::DefaultNotFound), - } - } - /// Returns the current table, if set - pub fn get_ctable_result(&self) -> KeyspaceResult<&Table> { - match self.estate.table { - Some((_, ref tbl)) => Ok(tbl), - _ => Err(DdlError::DefaultNotFound), - } - } - pub fn get_keyspace(&self, ksid: &Q) -> Option> - where - ObjectID: Borrow, - Q: Hash + Eq + ?Sized, - { - self.store.get_keyspace_atomic_ref(ksid) - } - /// Get an atomic reference to a table - pub fn get_table(&self, entity: &Entity) -> KeyspaceResult> { - match entity { - Entity::Full(ksid, table) => { - match self - .store - .get_keyspace_atomic_ref(unsafe { ksid.as_slice() }) - { - Some(ks) => match ks.get_table_atomic_ref(unsafe { table.as_slice() }) { - Some(tbl) => Ok(tbl), - None => Err(DdlError::ObjectNotFound), - }, - None => Err(DdlError::ObjectNotFound), - } - } - Entity::Current(tbl) => match &self.estate.ks { - Some((_, ks)) => match ks.get_table_atomic_ref(unsafe { tbl.as_slice() }) { - Some(tbl) => Ok(tbl), - None => Err(DdlError::ObjectNotFound), - }, - None => Err(DdlError::DefaultNotFound), - }, - } - } - pub fn get_ctable(&self) -> Option> { - self.estate.table.as_ref().map(|(_, tbl)| tbl.clone()) - } - pub fn get_ctable_ref(&self) -> Option<&Table> { - self.estate.table.as_ref().map(|(_, tbl)| tbl.as_ref()) - } - /// Returns a table with the provided specification - pub fn get_table_with(&self) -> ActionResult<&T::Table> { - T::get::

(self) - } - /// Create a table: in-memory; **no transactional guarantees**. Two tables can be created - /// simultaneously, but are never flushed unless we are very lucky. If the global flush - /// system is close to a flush cycle -- then we are in luck: we pause the flush cycle - /// through a global flush lock and then allow it to resume once we're done adding the table. - /// This enables the flush routine to permanently write the table to disk. But it's all about - /// luck -- the next mutual access may be yielded to the next `create table` command - /// - /// **Trip switch handled:** Yes - pub fn create_table( - &self, - entity: &Entity, - modelcode: u8, - volatile: bool, - ) -> KeyspaceResult<()> { - // first lock the global flush state - let flush_lock = registry::lock_flush_state(); - let ret = match entity { - // Important: create table is only ks - Entity::Current(tblid) => { - match &self.estate.ks { - Some((_, ks)) => { - let tbl = Table::from_model_code(modelcode, volatile); - if let Some(tbl) = tbl { - if ks.create_table( - unsafe { ObjectID::from_slice(tblid.as_slice()) }, - tbl, - ) { - // we need to re-init tree; so trip - registry::get_preload_tripswitch().trip(); - Ok(()) - } else { - Err(DdlError::AlreadyExists) - } - } else { - Err(DdlError::WrongModel) - } - } - None => Err(DdlError::DefaultNotFound), - } - } - Entity::Full(ksid, tblid) => { - match self - .store - .get_keyspace_atomic_ref(unsafe { ksid.as_slice() }) - { - Some(kspace) => { - let tbl = Table::from_model_code(modelcode, volatile); - if let Some(tbl) = tbl { - if kspace.create_table( - unsafe { ObjectID::from_slice(tblid.as_slice()) }, - tbl, - ) { - // trip the preload switch - registry::get_preload_tripswitch().trip(); - Ok(()) - } else { - Err(DdlError::AlreadyExists) - } - } else { - Err(DdlError::WrongModel) - } - } - None => Err(DdlError::ObjectNotFound), - } - } - }; - // free the global flush lock - drop(flush_lock); - ret - } - - /// Drop a table - pub fn drop_table(&self, entity: &Entity, force: bool) -> KeyspaceResult<()> { - match entity { - Entity::Current(tblid) => match &self.estate.ks { - Some((_, ks)) => ks.drop_table(unsafe { tblid.as_slice() }, force), - None => Err(DdlError::DefaultNotFound), - }, - Entity::Full(ksid, tblid) => { - match self - .store - .get_keyspace_atomic_ref(unsafe { ksid.as_slice() }) - { - Some(ks) => ks.drop_table(unsafe { tblid.as_slice() }, force), - None => Err(DdlError::ObjectNotFound), - } - } - } - } - - /// Create a keyspace **without any transactional guarantees** - /// - /// **Trip switch handled:** Yes - pub fn create_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> { - // lock the global flush lock (see comment in create_table to know why) - let flush_lock = registry::lock_flush_state(); - let ret = if self.store.create_keyspace(ksid) { - // woo, created - // trip the preload switch - registry::get_preload_tripswitch().trip(); - Ok(()) - } else { - // ugh, already exists - Err(DdlError::AlreadyExists) - }; - drop(flush_lock); - ret - } - - /// Drop a keyspace - pub fn drop_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> { - // trip switch is handled by memstore here - self.store.drop_keyspace(ksid) - } - - /// Force drop a keyspace - pub fn force_drop_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> { - // trip switch is handled by memstore here - self.store.force_drop_keyspace(ksid) - } - pub fn strong_count(&self) -> usize { - Arc::strong_count(&self.store) - } - pub fn get_ids(&self) -> (Option<&ObjectID>, Option<&ObjectID>) { - self.estate.get_id_pack() - } - pub fn list_tables(&self, ksid: Option<&[u8]>) -> ActionResult> { - Ok(match ksid { - Some(keyspace_name) => { - // inspect the provided keyspace - let ksid = if keyspace_name.len() > 64 { - return util::err(P::RSTRING_BAD_CONTAINER_NAME); - } else { - keyspace_name - }; - let ks = match self.get_keyspace(ksid) { - Some(kspace) => kspace, - None => return util::err(P::RSTRING_CONTAINER_NOT_FOUND), - }; - ks.tables.iter().map(|kv| kv.key().clone()).collect() - } - None => { - // inspect the current keyspace - let cks = translate_ddl_error::(self.get_cks())?; - cks.tables.iter().map(|kv| kv.key().clone()).collect() - } - }) - } - pub fn describe_table(&self, table: &Option) -> ActionResult { - let r = match table { - Some(tbl) => translate_ddl_error::>(self.get_table(tbl))?.describe_self(), - None => translate_ddl_error::(self.get_ctable_result())?.describe_self(), - }; - Ok(r.to_owned()) - } -} diff --git a/server/src/corestore/rc.rs b/server/src/corestore/rc.rs deleted file mode 100644 index 080ececa..00000000 --- a/server/src/corestore/rc.rs +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Created on Mon Aug 15 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use std::{ - alloc::{alloc, dealloc, Layout}, - borrow::Borrow, - fmt::Debug, - hash::Hash, - ops::Deref, - ptr::{self, NonNull}, - slice, - sync::atomic::{self, AtomicUsize, Ordering}, -}; - -/// A [`SharedSlice`] is a dynamically sized, heap allocated slice that can be safely shared across threads. This -/// type can be cheaply cloned and the only major cost is initialization that does a memcpy from the source into -/// a new heap allocation. Once init is complete, cloning only increments an atomic counter and when no more owners -/// of this data exists, i.e the object is orphaned, it will call its destructor and clean up the heap allocation. -/// Do note that two heap allocations are made: -/// - One for the actual data -/// - One for the shared state -pub struct SharedSlice { - inner: NonNull, -} - -impl Debug for SharedSlice { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SharedSlice") - .field("data", &self.as_slice()) - .finish() - } -} - -// UNSAFE(@ohsayan): This is completely safe because our impl guarantees this -unsafe impl Send for SharedSlice {} -unsafe impl Sync for SharedSlice {} - -impl SharedSlice { - #[inline(always)] - /// Create a new [`SharedSlice`] using the given local slice - pub fn new(slice: &[u8]) -> Self { - Self { - inner: unsafe { - NonNull::new_unchecked(Box::leak(Box::new(SharedSliceInner::new(slice)))) - }, - } - } - #[inline(always)] - /// Returns a reference to te inner heap allocation for shared state - fn inner(&self) -> &SharedSliceInner { - unsafe { &*self.inner.as_ptr() } - } - #[inline(never)] - /// A slow-path to deallocating all the heap allocations - unsafe fn slow_drop(&self) { - if self.len() != 0 { - // IMPORTANT: Do not use the aligned pointer as a sentinel - let inner = self.inner(); - // heap array dtor - ptr::drop_in_place(slice::from_raw_parts_mut(inner.data as *mut u8, inner.len)); - // dealloc heap array - dealloc( - inner.data as *mut u8, - Layout::array::(inner.len).unwrap(), - ) - } - // destroy shared state alloc - drop(Box::from_raw(self.inner.as_ptr())) - } - /// Returns a local slice for the shared slice - #[inline(always)] - pub fn as_slice(&self) -> &[u8] { - unsafe { - /* - UNSAFE(@ohsayan): The dtor guarantees that: - 1. we will never end up shooting ourselves in the foot - 2. the ptr is either valid, or invalid but well aligned. this upholds the raw_parts contract - 3. the len is either valid, or zero - */ - let inner = self.inner(); - slice::from_raw_parts(inner.data, inner.len) - } - } -} - -impl Clone for SharedSlice { - #[inline(always)] - fn clone(&self) -> Self { - // relaxed is fine. the fencing in the dtor decr ensures we don't mess things up - let _new_refcount = self.inner().rc.fetch_add(1, Ordering::Relaxed); - Self { inner: self.inner } - } -} - -impl Drop for SharedSlice { - #[inline(always)] - fn drop(&mut self) { - if self.inner().rc.fetch_sub(1, Ordering::Release) != 1 { - // not the last owner; return - return; - } - // use fence for sync with stores - atomic::fence(Ordering::Acquire); - unsafe { - // UNSAFE(@ohsayan): At this point, we can be sure that no one else is using the data - self.slow_drop(); - } - } -} - -// trait impls -impl Hash for SharedSlice { - #[inline(always)] - fn hash(&self, state: &mut H) { - self.as_slice().hash(state); - } -} - -impl> PartialEq for SharedSlice { - #[inline(always)] - fn eq(&self, other: &T) -> bool { - self.as_slice() == other.as_ref() - } -} - -impl PartialEq for SharedSlice { - #[inline(always)] - fn eq(&self, other: &str) -> bool { - self.as_slice() == other.as_bytes() - } -} - -impl AsRef<[u8]> for SharedSlice { - #[inline(always)] - fn as_ref(&self) -> &[u8] { - self.as_slice() - } -} - -impl Borrow<[u8]> for SharedSlice { - #[inline(always)] - fn borrow(&self) -> &[u8] { - self.as_slice().borrow() - } -} - -impl Deref for SharedSlice { - type Target = [u8]; - #[inline(always)] - fn deref(&self) -> &Self::Target { - self.as_slice() - } -} - -impl<'a> From<&'a [u8]> for SharedSlice { - #[inline(always)] - fn from(s: &'a [u8]) -> Self { - Self::new(s) - } -} - -impl<'a> From<&'a str> for SharedSlice { - #[inline(always)] - fn from(s: &'a str) -> Self { - Self::new(s.as_bytes()) - } -} - -impl From for SharedSlice { - #[inline(always)] - fn from(s: String) -> Self { - Self::new(s.as_bytes()) - } -} - -impl From> for SharedSlice { - #[inline(always)] - fn from(v: Vec) -> Self { - Self::new(v.as_slice()) - } -} - -impl Eq for SharedSlice {} - -/// The shared state structure -struct SharedSliceInner { - /// data ptr - data: *const u8, - /// data len - len: usize, - /// ref count - rc: AtomicUsize, -} - -impl SharedSliceInner { - #[inline(always)] - fn new(slice: &[u8]) -> Self { - let layout = Layout::array::(slice.len()).unwrap(); - let data = unsafe { - if slice.is_empty() { - // HACK(@ohsayan): Just ensure that the address is aligned for this - layout.align() as *mut u8 - } else { - // UNSAFE(@ohsayan): Come on, just a malloc and memcpy - let array_ptr = alloc(layout); - ptr::copy_nonoverlapping(slice.as_ptr(), array_ptr, slice.len()); - array_ptr - } - }; - Self { - data, - len: slice.len(), - rc: AtomicUsize::new(1), - } - } -} - -#[test] -fn basic() { - let slice = SharedSlice::from("hello"); - assert_eq!(slice, b"hello"); -} - -#[test] -fn basic_cloned() { - let slice_a = SharedSlice::from("hello"); - let slice_a_clone = slice_a.clone(); - drop(slice_a); - assert_eq!(slice_a_clone, b"hello"); -} - -#[test] -fn basic_cloned_across_threads() { - use std::thread; - const ST: &str = "world"; - const THREADS: usize = 8; - let slice = SharedSlice::from(ST); - let mut handles = Vec::with_capacity(THREADS); - for _ in 0..THREADS { - let clone = slice.clone(); - handles.push(thread::spawn(move || assert_eq!(clone, ST))) - } - handles.into_iter().for_each(|h| h.join().unwrap()); - assert_eq!(slice, ST); -} diff --git a/server/src/corestore/table.rs b/server/src/corestore/table.rs deleted file mode 100644 index 9b068925..00000000 --- a/server/src/corestore/table.rs +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Created on Sat Jul 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#[cfg(test)] -use crate::corestore::{memstore::DdlError, KeyspaceResult}; -use crate::{ - actions::ActionResult, - auth::Authmap, - corestore::{htable::Coremap, SharedSlice}, - dbnet::prelude::Corestore, - kvengine::{KVEListmap, KVEStandard, LockedVec}, - protocol::interface::ProtocolSpec, - util, -}; - -pub trait DescribeTable { - type Table; - fn try_get(table: &Table) -> Option<&Self::Table>; - fn get(store: &Corestore) -> ActionResult<&Self::Table> { - match store.estate.table { - Some((_, ref table)) => { - // so we do have a table - match Self::try_get(table) { - Some(tbl) => Ok(tbl), - None => util::err(P::RSTRING_WRONG_MODEL), - } - } - None => util::err(P::RSTRING_DEFAULT_UNSET), - } - } -} - -pub struct KVEBlob; - -impl DescribeTable for KVEBlob { - type Table = KVEStandard; - fn try_get(table: &Table) -> Option<&Self::Table> { - if let DataModel::KV(ref kve) = table.model_store { - Some(kve) - } else { - None - } - } -} - -pub struct KVEList; - -impl DescribeTable for KVEList { - type Table = KVEListmap; - fn try_get(table: &Table) -> Option<&Self::Table> { - if let DataModel::KVExtListmap(ref kvl) = table.model_store { - Some(kvl) - } else { - None - } - } -} - -#[derive(Debug)] -pub enum SystemDataModel { - Auth(Authmap), -} - -#[derive(Debug)] -pub struct SystemTable { - /// data storage - pub data: SystemDataModel, -} - -impl SystemTable { - pub const fn get_model_ref(&self) -> &SystemDataModel { - &self.data - } - pub fn new(data: SystemDataModel) -> Self { - Self { data } - } - pub fn new_auth(authmap: Authmap) -> Self { - Self::new(SystemDataModel::Auth(authmap)) - } -} - -#[derive(Debug)] -pub enum DataModel { - KV(KVEStandard), - KVExtListmap(KVEListmap), -} - -// same 8 byte ptrs; any chance of optimizations? - -#[derive(Debug)] -/// The underlying table type. This is the place for the other data models (soon!) -pub struct Table { - /// a key/value store - model_store: DataModel, - /// is the table volatile - volatile: bool, -} - -impl Table { - #[cfg(test)] - pub const fn from_kve(kve: KVEStandard, volatile: bool) -> Self { - Self { - model_store: DataModel::KV(kve), - volatile, - } - } - #[cfg(test)] - pub const fn from_kve_listmap(kve: KVEListmap, volatile: bool) -> Self { - Self { - model_store: DataModel::KVExtListmap(kve), - volatile, - } - } - /// Get the key/value store if the table is a key/value store - #[cfg(test)] - pub const fn get_kvstore(&self) -> KeyspaceResult<&KVEStandard> { - #[allow(irrefutable_let_patterns)] - if let DataModel::KV(kvs) = &self.model_store { - Ok(kvs) - } else { - Err(DdlError::WrongModel) - } - } - pub fn count(&self) -> usize { - match &self.model_store { - DataModel::KV(kv) => kv.len(), - DataModel::KVExtListmap(kv) => kv.len(), - } - } - /// Returns this table's _description_ - pub fn describe_self(&self) -> &'static str { - match self.get_model_code() { - // pure KV - 0 if self.is_volatile() => "Keymap { data:(binstr,binstr), volatile:true }", - 0 if !self.is_volatile() => "Keymap { data:(binstr,binstr), volatile:false }", - 1 if self.is_volatile() => "Keymap { data:(binstr,str), volatile:true }", - 1 if !self.is_volatile() => "Keymap { data:(binstr,str), volatile:false }", - 2 if self.is_volatile() => "Keymap { data:(str,str), volatile:true }", - 2 if !self.is_volatile() => "Keymap { data:(str,str), volatile:false }", - 3 if self.is_volatile() => "Keymap { data:(str,binstr), volatile:true }", - 3 if !self.is_volatile() => "Keymap { data:(str,binstr), volatile:false }", - // KVext => list - 4 if self.is_volatile() => "Keymap { data:(binstr,list), volatile:true }", - 4 if !self.is_volatile() => "Keymap { data:(binstr,list), volatile:false }", - 5 if self.is_volatile() => "Keymap { data:(binstr,list), volatile:true }", - 5 if !self.is_volatile() => "Keymap { data:(binstr,list), volatile:false }", - 6 if self.is_volatile() => "Keymap { data:(str,list), volatile:true }", - 6 if !self.is_volatile() => "Keymap { data:(str,list), volatile:false }", - 7 if self.is_volatile() => "Keymap { data:(str,list), volatile:true }", - 7 if !self.is_volatile() => "Keymap { data:(str,list), volatile:false }", - _ => unsafe { impossible!() }, - } - } - pub fn truncate_table(&self) { - match self.model_store { - DataModel::KV(ref kv) => kv.truncate_table(), - DataModel::KVExtListmap(ref kv) => kv.truncate_table(), - } - } - pub fn is_empty(&self) -> bool { - self.count() == 0 - } - /// Returns the storage type as an 8-bit uint - pub const fn storage_type(&self) -> u8 { - self.volatile as u8 - } - /// Returns the volatility of the table - pub const fn is_volatile(&self) -> bool { - self.volatile - } - /// Create a new KVEBlob Table with the provided settings - pub fn new_pure_kve_with_data( - data: Coremap, - volatile: bool, - k_enc: bool, - v_enc: bool, - ) -> Self { - Self { - volatile, - model_store: DataModel::KV(KVEStandard::new(k_enc, v_enc, data)), - } - } - pub fn new_kve_listmap_with_data( - data: Coremap, - volatile: bool, - k_enc: bool, - payload_enc: bool, - ) -> Self { - Self { - volatile, - model_store: DataModel::KVExtListmap(KVEListmap::new(k_enc, payload_enc, data)), - } - } - pub fn from_model_code(code: u8, volatile: bool) -> Option { - macro_rules! pkve { - ($kenc:expr, $venc:expr) => { - Self::new_pure_kve_with_data(Coremap::new(), volatile, $kenc, $venc) - }; - } - macro_rules! listmap { - ($kenc:expr, $penc:expr) => { - Self::new_kve_listmap_with_data(Coremap::new(), volatile, $kenc, $penc) - }; - } - let ret = match code { - // pure kve - 0 => pkve!(false, false), - 1 => pkve!(false, true), - 2 => pkve!(true, true), - 3 => pkve!(true, false), - // kvext: listmap - 4 => listmap!(false, false), - 5 => listmap!(false, true), - 6 => listmap!(true, false), - 7 => listmap!(true, true), - _ => return None, - }; - Some(ret) - } - /// Create a new kve with default settings but with provided volatile configuration - #[cfg(test)] - pub fn new_kve_with_volatile(volatile: bool) -> Self { - Self::new_pure_kve_with_data(Coremap::new(), volatile, false, false) - } - /// Returns the default kve: - /// - `k_enc`: `false` - /// - `v_enc`: `false` - /// - `volatile`: `false` - pub fn new_default_kve() -> Self { - Self::new_pure_kve_with_data(Coremap::new(), false, false, false) - } - /// Returns the model code. See [`bytemarks`] for more info - pub fn get_model_code(&self) -> u8 { - match self.model_store { - DataModel::KV(ref kvs) => { - /* - bin,bin => 0 - bin,str => 1 - str,str => 2 - str,bin => 3 - */ - let (kenc, venc) = kvs.get_encoding_tuple(); - let ret = kenc as u8 + venc as u8; - // a little bitmagic goes a long way - (ret & 1) + ((kenc as u8) << 1) - } - DataModel::KVExtListmap(ref kvlistmap) => { - /* - bin,list => 4, - bin,list => 5, - str,list => 6, - str,list => 7 - */ - let (kenc, venc) = kvlistmap.get_encoding_tuple(); - ((kenc as u8) << 1) + (venc as u8) + 4 - } - } - } - /// Returns the inner data model - pub fn get_model_ref(&self) -> &DataModel { - &self.model_store - } -} diff --git a/server/src/corestore/tests.rs b/server/src/corestore/tests.rs deleted file mode 100644 index 2b334482..00000000 --- a/server/src/corestore/tests.rs +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Created on Fri Jul 30 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -mod memstore_keyspace_tests { - use super::super::{memstore::*, table::Table}; - - #[test] - fn test_drop_keyspace_empty() { - let ms = Memstore::new_empty(); - let obj = unsafe { ObjectID::from_slice("myks") }; - ms.create_keyspace(obj.clone()); - assert!(ms.drop_keyspace(obj).is_ok()); - } - - #[test] - fn test_drop_keyspace_still_accessed() { - let ms = Memstore::new_empty(); - let obj = unsafe { ObjectID::from_slice("myks") }; - ms.create_keyspace(obj.clone()); - let _ks_ref = ms.get_keyspace_atomic_ref(&obj); - assert_eq!(ms.drop_keyspace(obj).unwrap_err(), DdlError::StillInUse); - } - - #[test] - fn test_drop_keyspace_not_empty() { - let ms = Memstore::new_empty(); - let obj = unsafe { ObjectID::from_slice("myks") }; - ms.create_keyspace(obj.clone()); - let ks_ref = ms.get_keyspace_atomic_ref(&obj).unwrap(); - ks_ref.create_table( - unsafe { ObjectID::from_slice("mytbl") }, - Table::new_default_kve(), - ); - assert_eq!(ms.drop_keyspace(obj).unwrap_err(), DdlError::NotEmpty); - } - - #[test] - fn test_force_drop_keyspace_empty() { - let ms = Memstore::new_empty(); - let obj = unsafe { ObjectID::from_slice("myks") }; - ms.create_keyspace(obj.clone()); - assert!(ms.force_drop_keyspace(obj).is_ok()); - } - - #[test] - fn test_force_drop_keyspace_still_accessed() { - let ms = Memstore::new_empty(); - let obj = unsafe { ObjectID::from_slice("myks") }; - ms.create_keyspace(obj.clone()); - let _ks_ref = ms.get_keyspace_atomic_ref(&obj); - assert_eq!( - ms.force_drop_keyspace(obj).unwrap_err(), - DdlError::StillInUse - ); - } - - #[test] - fn test_force_drop_keyspace_table_referenced() { - // the check here is to see if all the tables are not in active use - let ms = Memstore::new_empty(); - let obj = unsafe { ObjectID::from_slice("myks") }; - let tblid = unsafe { ObjectID::from_slice("mytbl") }; - // create the ks - ms.create_keyspace(obj.clone()); - // get an atomic ref to the keyspace - let ks_ref = ms.get_keyspace_atomic_ref(&obj).unwrap(); - // create a table - ks_ref.create_table(tblid.clone(), Table::new_default_kve()); - // ref to the table - let _tbl_ref = ks_ref.get_table_atomic_ref(&tblid).unwrap(); - // drop ks ref - drop(ks_ref); - assert_eq!( - ms.force_drop_keyspace(obj).unwrap_err(), - DdlError::StillInUse - ); - } - - #[test] - fn test_force_drop_keyspace_nonempty_okay() { - // the check here is to see if drop succeeds, provided that no - // tables are in active use - let ms = Memstore::new_empty(); - let obj = unsafe { ObjectID::from_slice("myks") }; - let tblid = unsafe { ObjectID::from_slice("mytbl") }; - // create the ks - ms.create_keyspace(obj.clone()); - // get an atomic ref to the keyspace - let ks_ref = ms.get_keyspace_atomic_ref(&obj).unwrap(); - // create a table - ks_ref.create_table(tblid, Table::new_default_kve()); - // drop ks ref - drop(ks_ref); - // should succeed because the keyspace is non-empty, but no table is referenced to - assert!(ms.force_drop_keyspace(obj).is_ok()); - } -} - -mod modelcode_tests { - use { - super::super::table::Table, - crate::kvengine::{KVEListmap, KVEngine}, - }; - - #[test] - fn test_model_code_pure_kve() { - // binstr, binstr - let kv1 = KVEngine::init(false, false); - // binstr, str - let kv2 = KVEngine::init(false, true); - // str, str - let kv3 = KVEngine::init(true, true); - // str, binstr - let kv4 = KVEngine::init(true, false); - - // now check - let tbl1 = Table::from_kve(kv1, false); - assert_eq!(tbl1.get_model_code(), 0); - let tbl2 = Table::from_kve(kv2, false); - assert_eq!(tbl2.get_model_code(), 1); - let tbl3 = Table::from_kve(kv3, false); - assert_eq!(tbl3.get_model_code(), 2); - let tbl4 = Table::from_kve(kv4, false); - assert_eq!(tbl4.get_model_code(), 3); - } - #[test] - fn test_model_code_kvext_listmap() { - // binstr, list - let l1 = KVEListmap::init(false, false); - // binstr, list - let l2 = KVEListmap::init(false, true); - // str, list - let l3 = KVEListmap::init(true, false); - // str, list - let l4 = KVEListmap::init(true, true); - - // now check - let tbl1 = Table::from_kve_listmap(l1, false); - assert_eq!(tbl1.get_model_code(), 4); - let tbl2 = Table::from_kve_listmap(l2, false); - assert_eq!(tbl2.get_model_code(), 5); - let tbl3 = Table::from_kve_listmap(l3, false); - assert_eq!(tbl3.get_model_code(), 6); - let tbl4 = Table::from_kve_listmap(l4, false); - assert_eq!(tbl4.get_model_code(), 7); - } -} diff --git a/server/src/dbnet/connection.rs b/server/src/dbnet/connection.rs deleted file mode 100644 index 2ddcd7da..00000000 --- a/server/src/dbnet/connection.rs +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Created on Sun Aug 21 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{BufferedSocketStream, QueryResult}, - crate::{ - corestore::buffers::Integer64, - protocol::{interface::ProtocolSpec, ParseError}, - IoResult, - }, - bytes::BytesMut, - std::{ - io::{Error as IoError, ErrorKind}, - marker::PhantomData, - }, - tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter}, -}; - -const BUF_WRITE_CAP: usize = 8192; -const BUF_READ_CAP: usize = 8192; - -/// A generic connection type -/// -/// The generic connection type allows you to choose: -/// 1. A stream (TCP, TLS(TCP), UDS, ...) -/// 2. A protocol (one that implements [`ProtocolSpec`]) -pub struct Connection { - pub(super) stream: BufWriter, - pub(super) buffer: BytesMut, - _marker: PhantomData

, -} - -impl Connection { - pub fn new(stream: T) -> Self { - Connection { - stream: BufWriter::with_capacity(BUF_WRITE_CAP, stream), - buffer: BytesMut::with_capacity(BUF_READ_CAP), - _marker: PhantomData, - } - } -} - -// protocol read -impl Connection { - /// Attempt to read a query - pub(super) async fn read_query(&mut self) -> IoResult { - loop { - match self.stream.read_buf(&mut self.buffer).await { - Ok(0) => { - if self.buffer.is_empty() { - // buffer is empty, and the remote pulled off (simple disconnection) - return Ok(QueryResult::Disconnected); - } else { - // wrote something, and then died. nope, that's an error - return Err(IoError::from(ErrorKind::ConnectionReset)); - } - } - Ok(_) => {} - Err(e) => return Err(e), - } - // see if we have buffered enough data to run anything - match P::decode_packet(self.buffer.as_ref()) { - Ok(query_with_advance) => return Ok(QueryResult::Q(query_with_advance)), - Err(ParseError::NotEnough) => {} - Err(e) => { - self.write_error(P::SKYHASH_PARSE_ERROR_LUT[e as usize - 1]) - .await?; - return Ok(QueryResult::NextLoop); - } - } - } - } -} - -// protocol write (metaframe) -impl Connection { - /// Write a simple query header to the stream - pub(super) async fn write_simple_query_header(&mut self) -> IoResult<()> { - self.stream.write_all(P::SIMPLE_QUERY_HEADER).await - } - - /// Write the pipeline query header - pub(super) async fn write_pipelined_query_header(&mut self, count: usize) -> IoResult<()> { - // write pipeline first byte - self.stream.write_u8(P::PIPELINED_QUERY_FIRST_BYTE).await?; - // write pipeline query count - self.stream.write_all(&Integer64::from(count)).await?; - // write the LF - self.stream.write_u8(P::LF).await - } -} - -// protocol write (helpers) -impl Connection { - /// Write an error to the stream (just used to differentiate between "normal" and "errored" writes) - pub(super) async fn write_error(&mut self, error: &[u8]) -> IoResult<()> { - self.stream.write_all(error).await?; - self.stream.flush().await - } - /// Write something "raw" to the stream (intentional underscore to avoid misuse) - pub async fn _write_raw(&mut self, raw: &[u8]) -> IoResult<()> { - self.stream.write_all(raw).await - } -} - -// protocol write (dataframe) -impl Connection { - // monoelements - /// Encode and write a length-prefixed monoelement - pub async fn write_mono_length_prefixed_with_tsymbol( - &mut self, - data: &[u8], - tsymbol: u8, - ) -> IoResult<()> { - // first write the tsymbol - self.stream.write_u8(tsymbol).await?; - // now write length - self.stream.write_all(&Integer64::from(data.len())).await?; - // now write LF - self.stream.write_u8(P::LF).await?; - // now write the actual body - self.stream.write_all(data).await?; - if P::NEEDS_TERMINAL_LF { - self.stream.write_u8(P::LF).await - } else { - Ok(()) - } - } - /// Encode and write a mon element (**without** length-prefixing) - pub async fn write_mono_with_tsymbol(&mut self, data: &[u8], tsymbol: u8) -> IoResult<()> { - // first write the tsymbol - self.stream.write_u8(tsymbol).await?; - // now write the actual body - self.stream.write_all(data).await?; - self.stream.write_u8(P::LF).await - } - /// Encode and write an unicode string - pub async fn write_string(&mut self, string: &str) -> IoResult<()> { - self.write_mono_length_prefixed_with_tsymbol(string.as_bytes(), P::TSYMBOL_STRING) - .await - } - /// Encode and write a blob - #[allow(unused)] - pub async fn write_binary(&mut self, binary: &[u8]) -> IoResult<()> { - self.write_mono_length_prefixed_with_tsymbol(binary, P::TSYMBOL_BINARY) - .await - } - /// Encode and write an `usize` - pub async fn write_usize(&mut self, size: usize) -> IoResult<()> { - self.write_mono_with_tsymbol(&Integer64::from(size), P::TSYMBOL_INT64) - .await - } - /// Encode and write an `u64` - pub async fn write_int64(&mut self, int: u64) -> IoResult<()> { - self.write_mono_with_tsymbol(&Integer64::from(int), P::TSYMBOL_INT64) - .await - } - /// Encode and write an `f32` - pub async fn write_float(&mut self, float: f32) -> IoResult<()> { - self.write_mono_with_tsymbol(float.to_string().as_bytes(), P::TSYMBOL_FLOAT) - .await - } - - // typed array - /// Write a typed array header (including type information and size) - pub async fn write_typed_array_header(&mut self, len: usize, tsymbol: u8) -> IoResult<()> { - self.stream - .write_all(&[P::TSYMBOL_TYPED_ARRAY, tsymbol]) - .await?; - self.stream.write_all(&Integer64::from(len)).await?; - self.stream.write_u8(P::LF).await - } - /// Encode and write a null element for a typed array - pub async fn write_typed_array_element_null(&mut self) -> IoResult<()> { - self.stream - .write_all(P::TYPE_TYPED_ARRAY_ELEMENT_NULL) - .await - } - /// Encode and write a typed array element - pub async fn write_typed_array_element(&mut self, element: &[u8]) -> IoResult<()> { - self.stream - .write_all(&Integer64::from(element.len())) - .await?; - self.stream.write_u8(P::LF).await?; - self.stream.write_all(element).await?; - if P::NEEDS_TERMINAL_LF { - self.stream.write_u8(P::LF).await - } else { - Ok(()) - } - } - - // typed non-null array - /// write typed non-null array header - pub async fn write_typed_non_null_array_header( - &mut self, - len: usize, - tsymbol: u8, - ) -> IoResult<()> { - self.stream - .write_all(&[P::TSYMBOL_TYPED_NON_NULL_ARRAY, tsymbol]) - .await?; - self.stream.write_all(&Integer64::from(len)).await?; - self.stream.write_all(&[P::LF]).await - } - /// Encode and write typed non-null array element - pub async fn write_typed_non_null_array_element(&mut self, element: &[u8]) -> IoResult<()> { - self.write_typed_array_element(element).await - } - /// Encode and write a typed non-null array - pub async fn write_typed_non_null_array(&mut self, body: B, tsymbol: u8) -> IoResult<()> - where - B: AsRef<[A]>, - A: AsRef<[u8]>, - { - let body = body.as_ref(); - self.write_typed_non_null_array_header(body.len(), tsymbol) - .await?; - for element in body { - self.write_typed_non_null_array_element(element.as_ref()) - .await?; - } - Ok(()) - } -} diff --git a/server/src/dbnet/listener.rs b/server/src/dbnet/listener.rs deleted file mode 100644 index ef96e50e..00000000 --- a/server/src/dbnet/listener.rs +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Created on Sun Aug 21 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ - tcp::{Listener, ListenerV1}, - tls::{SslListener, SslListenerV1}, - }, - crate::{ - auth::AuthProvider, - config::{PortConfig, ProtocolVersion, SslOpts}, - corestore::Corestore, - util::error::{Error, SkyResult}, - IoResult, - }, - core::future::Future, - std::{net::IpAddr, sync::Arc}, - tokio::{ - net::TcpListener, - sync::{broadcast, mpsc, Semaphore}, - }, -}; - -/// The base TCP listener -pub struct BaseListener { - /// An atomic reference to the coretable - pub db: Corestore, - /// The auth provider - pub auth: AuthProvider, - /// The incoming connection listener (binding) - pub listener: TcpListener, - /// The maximum number of connections - pub climit: Arc, - /// The shutdown broadcaster - pub signal: broadcast::Sender<()>, - // When all `Sender`s are dropped - the `Receiver` gets a `None` value - // We send a clone of `terminate_tx` to each `CHandler` - pub terminate_tx: mpsc::Sender<()>, - pub terminate_rx: mpsc::Receiver<()>, -} - -impl BaseListener { - pub async fn init( - db: &Corestore, - auth: AuthProvider, - host: IpAddr, - port: u16, - semaphore: Arc, - signal: broadcast::Sender<()>, - ) -> SkyResult { - let (terminate_tx, terminate_rx) = mpsc::channel(1); - let listener = TcpListener::bind((host, port)) - .await - .map_err(|e| Error::ioerror_extra(e, format!("binding to port {port}")))?; - Ok(Self { - db: db.clone(), - auth, - listener, - climit: semaphore, - signal, - terminate_tx, - terminate_rx, - }) - } - pub async fn release_self(self) { - let Self { - mut terminate_rx, - terminate_tx, - signal, - .. - } = self; - drop(signal); - drop(terminate_tx); - let _ = terminate_rx.recv().await; - } -} - -/// Multiple Listener Interface -/// -/// A `MultiListener` is an abstraction over an `SslListener` or a `Listener` to facilitate -/// easier asynchronous listening on multiple ports. -/// -/// - The `SecureOnly` variant holds an `SslListener` -/// - The `InsecureOnly` variant holds a `Listener` -/// - The `Multi` variant holds both an `SslListener` and a `Listener` -/// This variant enables listening to both secure and insecure sockets at the same time -/// asynchronously -#[allow(clippy::large_enum_variant)] -pub enum MultiListener { - SecureOnly(SslListener), - SecureOnlyV1(SslListenerV1), - InsecureOnly(Listener), - InsecureOnlyV1(ListenerV1), - Multi(Listener, SslListener), - MultiV1(ListenerV1, SslListenerV1), -} - -async fn wait_on_port_futures( - a: impl Future>, - b: impl Future>, -) -> IoResult<()> { - let (e1, e2) = tokio::join!(a, b); - if let Err(e) = e1 { - log::error!("Insecure listener failed with: {}", e); - } - if let Err(e) = e2 { - log::error!("Secure listener failed with: {}", e); - } - Ok(()) -} - -impl MultiListener { - /// Create a new `InsecureOnly` listener - pub fn new_insecure_only(base: BaseListener, protocol: ProtocolVersion) -> Self { - match protocol { - ProtocolVersion::V2 => MultiListener::InsecureOnly(Listener::new(base)), - ProtocolVersion::V1 => MultiListener::InsecureOnlyV1(ListenerV1::new(base)), - } - } - /// Create a new `SecureOnly` listener - pub fn new_secure_only( - base: BaseListener, - ssl: SslOpts, - protocol: ProtocolVersion, - ) -> SkyResult { - let listener = match protocol { - ProtocolVersion::V2 => { - let listener = SslListener::new_pem_based_ssl_connection( - ssl.key, - ssl.chain, - base, - ssl.passfile, - )?; - MultiListener::SecureOnly(listener) - } - ProtocolVersion::V1 => { - let listener = SslListenerV1::new_pem_based_ssl_connection( - ssl.key, - ssl.chain, - base, - ssl.passfile, - )?; - MultiListener::SecureOnlyV1(listener) - } - }; - Ok(listener) - } - /// Create a new `Multi` listener that has both a secure and an insecure listener - pub async fn new_multi( - ssl_base_listener: BaseListener, - tcp_base_listener: BaseListener, - ssl: SslOpts, - protocol: ProtocolVersion, - ) -> SkyResult { - let mls = match protocol { - ProtocolVersion::V2 => { - let secure_listener = SslListener::new_pem_based_ssl_connection( - ssl.key, - ssl.chain, - ssl_base_listener, - ssl.passfile, - )?; - let insecure_listener = Listener::new(tcp_base_listener); - MultiListener::Multi(insecure_listener, secure_listener) - } - ProtocolVersion::V1 => { - let secure_listener = SslListenerV1::new_pem_based_ssl_connection( - ssl.key, - ssl.chain, - ssl_base_listener, - ssl.passfile, - )?; - let insecure_listener = ListenerV1::new(tcp_base_listener); - MultiListener::MultiV1(insecure_listener, secure_listener) - } - }; - Ok(mls) - } - /// Start the server - /// - /// The running of single and/or parallel listeners is handled by this function by - /// exploiting the working of async functions - pub async fn run_server(&mut self) -> IoResult<()> { - match self { - MultiListener::SecureOnly(secure_listener) => secure_listener.run().await, - MultiListener::SecureOnlyV1(secure_listener) => secure_listener.run().await, - MultiListener::InsecureOnly(insecure_listener) => insecure_listener.run().await, - MultiListener::InsecureOnlyV1(insecure_listener) => insecure_listener.run().await, - MultiListener::Multi(insecure_listener, secure_listener) => { - wait_on_port_futures(insecure_listener.run(), secure_listener.run()).await - } - MultiListener::MultiV1(insecure_listener, secure_listener) => { - wait_on_port_futures(insecure_listener.run(), secure_listener.run()).await - } - } - } - /// Signal the ports to shut down and only return after they have shut down - /// - /// **Do note:** This function doesn't flush the `Corestore` object! The **caller has to - /// make sure that the data is saved!** - pub async fn finish_with_termsig(self) { - match self { - MultiListener::InsecureOnly(Listener { base, .. }) - | MultiListener::SecureOnly(SslListener { base, .. }) - | MultiListener::InsecureOnlyV1(ListenerV1 { base, .. }) - | MultiListener::SecureOnlyV1(SslListenerV1 { base, .. }) => base.release_self().await, - MultiListener::Multi(insecure, secure) => { - insecure.base.release_self().await; - secure.base.release_self().await; - } - MultiListener::MultiV1(insecure, secure) => { - insecure.base.release_self().await; - secure.base.release_self().await; - } - } - } -} - -/// Initialize the database networking -pub async fn connect( - ports: PortConfig, - protocol: ProtocolVersion, - maxcon: usize, - db: Corestore, - auth: AuthProvider, - signal: broadcast::Sender<()>, -) -> SkyResult { - let climit = Arc::new(Semaphore::new(maxcon)); - let base_listener_init = |host, port| { - BaseListener::init( - &db, - auth.clone(), - host, - port, - climit.clone(), - signal.clone(), - ) - }; - let description = ports.get_description(); - let server = match ports { - PortConfig::InsecureOnly { host, port } => { - MultiListener::new_insecure_only(base_listener_init(host, port).await?, protocol) - } - PortConfig::SecureOnly { host, ssl } => MultiListener::new_secure_only( - base_listener_init(host, ssl.port).await?, - ssl, - protocol, - )?, - PortConfig::Multi { host, port, ssl } => { - let secure_listener = base_listener_init(host, ssl.port).await?; - let insecure_listener = base_listener_init(host, port).await?; - MultiListener::new_multi(secure_listener, insecure_listener, ssl, protocol).await? - } - }; - log::info!("Server started on {description}"); - Ok(server) -} diff --git a/server/src/dbnet/mod.rs b/server/src/dbnet/mod.rs deleted file mode 100644 index f54dc432..00000000 --- a/server/src/dbnet/mod.rs +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Created on Sun Aug 21 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - self::connection::Connection, - crate::{ - actions::{ActionError, ActionResult}, - auth::AuthProvider, - corestore::Corestore, - protocol::{interface::ProtocolSpec, Query}, - util::compiler, - IoResult, - }, - bytes::Buf, - std::{cell::Cell, sync::Arc, time::Duration}, - tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - sync::{ - broadcast::{self}, - mpsc::{self}, - Semaphore, - }, - time, - }, -}; - -pub type QueryWithAdvance = (Query, usize); -pub const MAXIMUM_CONNECTION_LIMIT: usize = 50000; -use crate::queryengine; - -pub use self::listener::connect; - -mod connection; -#[macro_use] -mod macros; -mod listener; -pub mod prelude; -mod tcp; -mod tls; - -/// This is a "marker trait" that ensures that no silly types are -/// passed into the [`Connection`] type -pub trait BufferedSocketStream: AsyncWriteExt + AsyncReadExt + Unpin {} - -/// Result of [`Connection::read_query`] -enum QueryResult { - /// A [`Query`] read to be run - Q(QueryWithAdvance), - /// Simply proceed to the next run loop iter - NextLoop, - /// The client disconnected - Disconnected, -} - -/// A backoff implementation that is meant to be used in connection loops -pub(self) struct NetBackoff { - c: Cell, -} - -impl NetBackoff { - /// The maximum backoff duration - const MAX_BACKOFF: u8 = 64; - /// Create a new [`NetBackoff`] instance - pub const fn new() -> Self { - Self { c: Cell::new(1) } - } - /// Wait for the current backoff duration - pub async fn spin(&self) { - time::sleep(Duration::from_secs(self.c.get() as _)).await; - self.c.set(self.c.get() << 1); - } - /// Should we disconnect the stream? - pub fn should_disconnect(&self) -> bool { - self.c.get() > Self::MAX_BACKOFF - } -} - -pub struct AuthProviderHandle { - /// the source authentication provider - provider: AuthProvider, - /// authenticated - auth_good: bool, -} - -impl AuthProviderHandle { - pub fn new(provider: AuthProvider) -> Self { - let auth_good = !provider.is_enabled(); - Self { - provider, - auth_good, - } - } - /// This returns `true` if: - /// 1. Authn is disabled - /// 2. The connection is authenticated - pub const fn authenticated(&self) -> bool { - self.auth_good - } - pub fn set_auth(&mut self) { - self.auth_good = true; - } - pub fn set_unauth(&mut self) { - self.auth_good = false; - } - pub fn provider_mut(&mut self) -> &mut AuthProvider { - &mut self.provider - } - pub fn provider(&self) -> &AuthProvider { - &self.provider - } -} - -/// A generic connection handler. You have two choices: -/// 1. Choose the connection kind -/// 2. Choose the protocol implementation -pub struct ConnectionHandler { - /// an atomic reference to the shared in-memory engine - db: Corestore, - /// the connection - con: Connection, - /// the semaphore used to impose limits on number of connections - climit: Arc, - /// the authentication handle - auth: AuthProviderHandle, - /// check for termination signals - termination_signal: broadcast::Receiver<()>, - /// the sender that we drop when we're done with handling a connection (used for gracefule exit) - _term_sig_tx: mpsc::Sender<()>, -} - -impl ConnectionHandler -where - C: BufferedSocketStream, - P: ProtocolSpec, -{ - /// Create a new connection handler - pub fn new( - db: Corestore, - con: Connection, - auth_data: AuthProvider, - climit: Arc, - termination_signal: broadcast::Receiver<()>, - _term_sig_tx: mpsc::Sender<()>, - ) -> Self { - Self { - db, - con, - climit, - auth: AuthProviderHandle::new(auth_data), - termination_signal, - _term_sig_tx, - } - } - pub async fn run(&mut self) -> IoResult<()> { - loop { - let packet = tokio::select! { - pkt = self.con.read_query() => pkt, - _ = self.termination_signal.recv() => { - return Ok(()); - } - }; - match packet { - Ok(QueryResult::Q((query, advance))) => { - // the mutable reference to self ensures that the buffer is not modified - // hence ensuring that the pointers will remain valid - #[cfg(debug_assertions)] - let len_at_start = self.con.buffer.len(); - #[cfg(debug_assertions)] - let sptr_at_start = self.con.buffer.as_ptr() as usize; - #[cfg(debug_assertions)] - let eptr_at_start = sptr_at_start + len_at_start; - { - // The actual execution (the assertions are just debug build sanity checks) - match self.execute_query(query).await { - Ok(()) => {} - Err(ActionError::ActionError(e)) => self.con.write_error(e).await?, - Err(ActionError::IoError(e)) => return Err(e), - } - } - { - // do these assertions to ensure memory safety (this is just for sanity sake) - #[cfg(debug_assertions)] - // len should be unchanged. no functions should **ever** touch the buffer - debug_assert_eq!(self.con.buffer.len(), len_at_start); - #[cfg(debug_assertions)] - // start of allocation should be unchanged - debug_assert_eq!(self.con.buffer.as_ptr() as usize, sptr_at_start); - #[cfg(debug_assertions)] - // end of allocation should be unchanged. else we're entirely violating - // memory safety guarantees - debug_assert_eq!( - unsafe { - // UNSAFE(@ohsayan): THis is always okay - self.con.buffer.as_ptr().add(len_at_start) - } as usize, - eptr_at_start - ); - // this is only when we clear the buffer. since execute_query is not called - // at this point, it's totally fine (so invalidating ptrs is totally cool) - self.con.buffer.advance(advance); - } - } - Ok(QueryResult::Disconnected) => return Ok(()), - Ok(QueryResult::NextLoop) => {} - Err(e) => return Err(e), - } - } - } - async fn execute_query(&mut self, query: Query) -> ActionResult<()> { - let Self { db, con, auth, .. } = self; - match query { - Query::Simple(q) => { - con.write_simple_query_header().await?; - if compiler::likely(auth.authenticated()) { - queryengine::execute_simple(db, con, auth, q).await?; - } else { - queryengine::execute_simple_noauth(db, con, auth, q).await?; - } - } - Query::Pipelined(p) => { - if compiler::likely(auth.authenticated()) { - con.write_pipelined_query_header(p.len()).await?; - queryengine::execute_pipeline(db, con, auth, p).await?; - } else { - con.write_simple_query_header().await?; - con.write_error(P::AUTH_CODE_BAD_CREDENTIALS).await?; - } - } - } - con.stream.flush().await?; - Ok(()) - } -} - -impl Drop for ConnectionHandler { - fn drop(&mut self) { - // Make sure that the permit is returned to the semaphore - // in the case that there is a panic inside - self.climit.add_permits(1); - } -} diff --git a/server/src/dbnet/prelude.rs b/server/src/dbnet/prelude.rs deleted file mode 100644 index 225aecad..00000000 --- a/server/src/dbnet/prelude.rs +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Created on Sun Aug 21 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! A 'prelude' for imports to interface with the database and the client -//! -//! This module is hollow itself, it only re-exports from `dbnet::con` and `tokio::io` - -pub use { - super::{connection::Connection, AuthProviderHandle}, - crate::{ - actions::{ensure_boolean_or_aerr, ensure_length, translate_ddl_error}, - corestore::{ - table::{KVEBlob, KVEList}, - Corestore, - }, - get_tbl, handle_entity, is_lowbit_set, - protocol::interface::ProtocolSpec, - queryengine::ActionIter, - registry, - util::{self, UnwrapActionError, Unwrappable}, - }, - tokio::io::{AsyncReadExt, AsyncWriteExt}, -}; diff --git a/server/src/dbnet/tcp.rs b/server/src/dbnet/tcp.rs deleted file mode 100644 index 0de3d224..00000000 --- a/server/src/dbnet/tcp.rs +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Created on Mon Apr 26 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -pub use protocol::{ParseResult, Query}; -use { - super::NetBackoff, - crate::{ - dbnet::{listener::BaseListener, BufferedSocketStream, Connection, ConnectionHandler}, - protocol::{self, interface::ProtocolSpec, Skyhash1, Skyhash2}, - IoResult, - }, - std::marker::PhantomData, - tokio::net::TcpStream, -}; - -impl BufferedSocketStream for TcpStream {} - -pub type Listener = RawListener; -pub type ListenerV1 = RawListener; - -/// A listener -pub struct RawListener

{ - pub base: BaseListener, - _marker: PhantomData

, -} - -impl RawListener

{ - pub fn new(base: BaseListener) -> Self { - Self { - base, - _marker: PhantomData, - } - } - /// Accept an incoming connection - async fn accept(&mut self) -> IoResult { - let backoff = NetBackoff::new(); - loop { - match self.base.listener.accept().await { - // We don't need the bindaddr - Ok((stream, _)) => return Ok(stream), - Err(e) => { - if backoff.should_disconnect() { - // Too many retries, goodbye user - return Err(e); - } - } - } - // spin to wait for the backoff duration - backoff.spin().await; - } - } - /// Run the server - pub async fn run(&mut self) -> IoResult<()> { - loop { - // Take the permit first, but we won't use it right now - // that's why we will forget it - self.base.climit.acquire().await.unwrap().forget(); - /* - SECURITY: Ignore any errors that may arise in the accept - loop. If we apply the try operator here, we will immediately - terminate the run loop causing the entire server to go down. - Also, do not log any errors because many connection errors - can arise and it will flood the log and might also result - in a crash - */ - let stream = skip_loop_err!(self.accept().await); - let mut chandle = ConnectionHandler::::new( - self.base.db.clone(), - Connection::new(stream), - self.base.auth.clone(), - self.base.climit.clone(), - self.base.signal.subscribe(), - self.base.terminate_tx.clone(), - ); - tokio::spawn(async move { - if let Err(e) = chandle.run().await { - log::error!("Error: {}", e); - } - }); - } - } -} diff --git a/server/src/dbnet/tls.rs b/server/src/dbnet/tls.rs deleted file mode 100644 index 13e512e5..00000000 --- a/server/src/dbnet/tls.rs +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Created on Fri Dec 18 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - dbnet::{ - listener::BaseListener, BufferedSocketStream, Connection, ConnectionHandler, NetBackoff, - }, - protocol::{interface::ProtocolSpec, Skyhash1, Skyhash2}, - util::error::{Error, SkyResult}, - IoResult, - }, - openssl::{ - pkey::PKey, - rsa::Rsa, - ssl::{Ssl, SslAcceptor, SslFiletype, SslMethod}, - }, - std::{fs, marker::PhantomData, pin::Pin}, - tokio::net::TcpStream, - tokio_openssl::SslStream, -}; - -impl BufferedSocketStream for SslStream {} - -pub type SslListener = SslListenerRaw; -pub type SslListenerV1 = SslListenerRaw; - -pub struct SslListenerRaw

{ - pub base: BaseListener, - acceptor: SslAcceptor, - _marker: PhantomData

, -} - -impl SslListenerRaw

{ - pub fn new_pem_based_ssl_connection( - key_file: String, - chain_file: String, - base: BaseListener, - tls_passfile: Option, - ) -> SkyResult> { - let mut acceptor_builder = SslAcceptor::mozilla_intermediate(SslMethod::tls())?; - // cert is the same for both - acceptor_builder.set_certificate_chain_file(chain_file)?; - if let Some(tls_passfile) = tls_passfile { - // first read in the private key - let tls_private_key = fs::read(key_file) - .map_err(|e| Error::ioerror_extra(e, "reading TLS private key"))?; - // read the passphrase because the passphrase file stream was provided - let tls_keyfile_stream = fs::read(tls_passfile) - .map_err(|e| Error::ioerror_extra(e, "reading TLS password file"))?; - // decrypt the private key - let pkey = Rsa::private_key_from_pem_passphrase(&tls_private_key, &tls_keyfile_stream)?; - let pkey = PKey::from_rsa(pkey)?; - // set the private key for the acceptor - acceptor_builder.set_private_key(&pkey)?; - } else { - // no passphrase, needs interactive - acceptor_builder.set_private_key_file(key_file, SslFiletype::PEM)?; - } - Ok(Self { - acceptor: acceptor_builder.build(), - base, - _marker: PhantomData, - }) - } - async fn accept(&mut self) -> SkyResult> { - let backoff = NetBackoff::new(); - loop { - match self.base.listener.accept().await { - // We don't need the bindaddr - // We get the encrypted stream which we need to decrypt - // by using the acceptor - Ok((stream, _)) => { - let ssl = Ssl::new(self.acceptor.context())?; - let mut stream = SslStream::new(ssl, stream)?; - Pin::new(&mut stream).accept().await?; - return Ok(stream); - } - Err(e) => { - if backoff.should_disconnect() { - // Too many retries, goodbye user - return Err(e.into()); - } - } - } - // Wait for the `backoff` duration - backoff.spin().await; - } - } - pub async fn run(&mut self) -> IoResult<()> { - loop { - // Take the permit first, but we won't use it right now - // that's why we will forget it - self.base.climit.acquire().await.unwrap().forget(); - /* - SECURITY: Ignore any errors that may arise in the accept - loop. If we apply the try operator here, we will immediately - terminate the run loop causing the entire server to go down. - Also, do not log any errors because many connection errors - can arise and it will flood the log and might also result - in a crash - */ - let stream = skip_loop_err!(self.accept().await); - let mut sslhandle = ConnectionHandler::, P>::new( - self.base.db.clone(), - Connection::new(stream), - self.base.auth.clone(), - self.base.climit.clone(), - self.base.signal.subscribe(), - self.base.terminate_tx.clone(), - ); - tokio::spawn(async move { - if let Err(e) = sslhandle.run().await { - log::error!("Error: {}", e); - } - }); - } - } -} diff --git a/server/src/diskstore/flock.rs b/server/src/diskstore/flock.rs deleted file mode 100644 index 3fbc32dd..00000000 --- a/server/src/diskstore/flock.rs +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Created on Fri Apr 16 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # File Locking -//! -//! This module provides the `FileLock` struct that can be used for locking and/or unlocking files on -//! unix-based systems and Windows systems - -#![allow(dead_code)] // TODO(@ohsayan): Remove lint or remove offending methods - -use std::{ - fs::{File, OpenOptions}, - io::{Result, Seek, SeekFrom, Write}, - path::Path, -}; - -#[derive(Debug)] -/// # File Lock -/// A file lock object holds a `std::fs::File` that is used to `lock()` and `unlock()` a file with a given -/// `filename` passed into the `lock()` method. The file lock is **not configured** to drop the file lock when the -/// object is dropped. The `file` field is essentially used to get the raw file descriptor for passing to -/// the platform-specific lock/unlock methods. -/// -/// **Note:** You need to lock a file first using this object before unlocking it! -/// -/// ## Suggestions -/// -/// It is always a good idea to attempt a lock release (unlock) explicitly than leaving it to the operating -/// system. If you manually run unlock, another unlock won't be called to avoid an extra costly (is it?) -/// syscall; this is achieved with the `unlocked` flag (field) which is set to true when the `unlock()` function -/// is called. -/// -pub struct FileLock { - file: File, - unlocked: bool, -} - -impl FileLock { - /// Initialize a new `FileLock` by locking a file - /// - /// This function will create and lock a file if it doesn't exist or it - /// will lock the existing file - /// **This will immediately fail if locking fails, i.e it is non-blocking** - pub fn lock(filename: impl AsRef) -> Result { - let file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(filename.as_ref())?; - Self::_lock(&file)?; - Ok(Self { - file, - unlocked: false, - }) - } - /// The internal lock function - /// - /// This is the function that actually locks the file and is kept separate only for purposes - /// of maintainability - fn _lock(file: &File) -> Result<()> { - __sys::try_lock_ex(file) - } - /// Unlock the file - /// - /// This sets the `unlocked` flag to true - pub fn unlock(&mut self) -> Result<()> { - if !self.unlocked { - __sys::unlock_file(&self.file)?; - self.unlocked = true; - Ok(()) - } else { - Ok(()) - } - } - /// Write something to this file - pub fn write(&mut self, bytes: &[u8]) -> Result<()> { - // empty the file - self.file.set_len(0)?; - // set the cursor to start - self.file.seek(SeekFrom::Start(0))?; - // Now write to the file - self.file.write_all(bytes) - } - /// Sync all metadata and flush buffers before returning - pub fn fsync(&self) -> Result<()> { - self.file.sync_all() - } - #[cfg(test)] - pub fn try_clone(&self) -> Result { - Ok(FileLock { - file: __sys::duplicate(&self.file)?, - unlocked: self.unlocked, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn test_basic_file_lock() { - let mut file = FileLock::lock("datalock.bin").unwrap(); - file.write(&[1, 2, 3]).unwrap(); - file.unlock().unwrap(); - } - #[test] - #[should_panic] - fn test_fail_with_two_flocks() { - let _file = FileLock::lock("data2.bin").unwrap(); - let _file2 = FileLock::lock("data2.bin").unwrap(); - std::fs::remove_file("data2.bin").unwrap(); - } - #[cfg(windows)] - #[test] - fn test_windows_lock_and_then_unlock() { - let mut file = FileLock::lock("data4.bin").unwrap(); - file.unlock().unwrap(); - drop(file); - let mut file2 = FileLock::lock("data4.bin").unwrap(); - file2.unlock().unwrap(); - drop(file2); - } - #[test] - fn test_cloned_lock_writes() { - let mut file = FileLock::lock("data5.bin").unwrap(); - let mut cloned = file.try_clone().unwrap(); - // this writes 1, 2, 3 - file.write(&[1, 2, 3]).unwrap(); - // this will truncate the entire previous file and write 4, 5, 6 - cloned.write(&[4, 5, 6]).unwrap(); - drop(cloned); - // this will again truncate the entire previous file and write 7, 8 - file.write(&[7, 8]).unwrap(); - drop(file); - let res = std::fs::read("data5.bin").unwrap(); - // hence ultimately we'll have 7, 8 - assert_eq!(res, vec![7, 8]); - } -} - -#[cfg(windows)] -mod __sys { - //! # Windows platform-specific file locking - //! This module contains methods used by the `FileLock` object in this module to lock and/or - //! unlock files. - - use { - std::{ - fs::File, - io::{Error, Result}, - mem, - os::windows::io::{AsRawHandle, FromRawHandle}, - ptr, - }, - winapi::{ - shared::minwindef::{BOOL, DWORD}, - um::{ - fileapi::{LockFileEx, UnlockFile}, - handleapi::DuplicateHandle, - minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY}, - processthreadsapi::GetCurrentProcess, - winnt::{DUPLICATE_SAME_ACCESS, MAXDWORD}, - }, - }, - }; - - /// Obtain an exclusive lock and **block** until we acquire it - pub fn lock_ex(file: &File) -> Result<()> { - lock_file(file, LOCKFILE_EXCLUSIVE_LOCK) - } - /// Try to obtain an exclusive lock and **immediately return an error if this is blocking** - pub fn try_lock_ex(file: &File) -> Result<()> { - lock_file(file, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY) - } - /// Use the LockFileEx method from Windows fileapi.h to set flags on a file - /// - /// This is the internal function that is used by `lock_ex` and `try_lock_ex` to lock and/or - /// unlock files on Windows platforms. - fn lock_file(file: &File, flags: DWORD) -> Result<()> { - unsafe { - // UNSAFE(@ohsayan): Interfacing with low-level winapi stuff, and we know what's happening here :D - let mut overlapped = mem::zeroed(); - let ret = LockFileEx( - file.as_raw_handle(), // handle - flags, // flags - 0, // reserved DWORD, has to be 0 - MAXDWORD, // nNumberOfBytesToLockLow; low-order (LOWORD) 32-bits of file range to lock - MAXDWORD, // nNumberOfBytesToLockHigh; high-order (HIWORD) 32-bits of file range to lock - &mut overlapped, - ); - if ret == 0 { - Err(Error::last_os_error()) - } else { - Ok(()) - } - } - } - /// Attempt to unlock a file - pub fn unlock_file(file: &File) -> Result<()> { - let ret = unsafe { - // UNSAFE(@ohsayan): Interfacing with low-level winapi stuff, and we know what's happening here :D - UnlockFile( - file.as_raw_handle(), // handle - 0, // LOWORD of starting byte offset - 0, // HIWORD of starting byte offset - MAXDWORD, // LOWORD of file range to unlock - MAXDWORD, // HIWORD of file range to unlock - ) - }; - if ret == 0 { - Err(Error::last_os_error()) - } else { - Ok(()) - } - } - /// Duplicate a file - /// - /// The most important part is the `DUPLICATE_SAME_ACCESS` DWORD. It ensures that the cloned file - /// has the same permissions as the original file - pub fn duplicate(file: &File) -> Result { - unsafe { - // UNSAFE(@ohsayan): Interfacing with low-level winapi stuff, and we know what's happening here :D - let mut handle = ptr::null_mut(); - let current_process = GetCurrentProcess(); - let ret = DuplicateHandle( - current_process, - file.as_raw_handle(), - current_process, - &mut handle, - 0, - true as BOOL, - DUPLICATE_SAME_ACCESS, - ); - if ret == 0 { - Err(Error::last_os_error()) - } else { - Ok(File::from_raw_handle(handle)) - } - } - } -} - -#[cfg(all(not(target_os = "solaris"), unix))] -mod __sys { - //! # Unix platform-specific file locking - //! This module contains methods used by the `FileLock` object in this module to lock and/or - //! unlock files. - use { - libc::c_int, - std::{ - fs::File, - io::{Error, Result}, - os::unix::io::{AsRawFd, FromRawFd}, - }, - }; - - extern "C" { - /// Block and acquire an exclusive lock with `libc`'s `flock` - fn lock_exclusive(fd: i32) -> c_int; - /// Attempt to acquire an exclusive lock in a non-blocking manner with `libc`'s `flock` - fn try_lock_exclusive(fd: i32) -> c_int; - /// Attempt to unlock a file with `libc`'s flock - fn unlock(fd: i32) -> c_int; - } - /// Obtain an exclusive lock and **block** until we acquire it - pub fn lock_ex(file: &File) -> Result<()> { - let errno = unsafe { - // UNSAFE(@ohsayan): This is completely fine to do as we've already written the function - // ourselves and are very much aware that it is safe - lock_exclusive(file.as_raw_fd()) - }; - match errno { - 0 => Ok(()), - x => Err(Error::from_raw_os_error(x)), - } - } - /// Try to obtain an exclusive lock and **immediately return an error if this is blocking** - pub fn try_lock_ex(file: &File) -> Result<()> { - let errno = unsafe { - // UNSAFE(@ohsayan): Again, we've written the function ourselves and know what is going on! - try_lock_exclusive(file.as_raw_fd()) - }; - match errno { - 0 => Ok(()), - x => Err(Error::from_raw_os_error(x)), - } - } - /// Attempt to unlock a file - pub fn unlock_file(file: &File) -> Result<()> { - let errno = unsafe { - // UNSAFE(@ohsayan): Again, we know what's going on here. Good ol' C stuff - unlock(file.as_raw_fd()) - }; - match errno { - 0 => Ok(()), - x => Err(Error::from_raw_os_error(x)), - } - } - /// Duplicate a file - /// - /// Good ol' libc dup() calls - pub fn duplicate(file: &File) -> Result { - unsafe { - // UNSAFE(@ohsayan): Completely safe, just that this is FFI - let fd = libc::dup(file.as_raw_fd()); - if fd < 0 { - Err(Error::last_os_error()) - } else { - Ok(File::from_raw_fd(fd)) - } - } - } -} - -#[cfg(all(target_os = "solaris", unix))] -mod __sys { - //! Solaris doesn't have flock so we'll have to simulate that using fcntl - use std::{ - fs::File, - io::{Error, Result}, - os::unix::io::{AsRawFd, FromRawFd}, - }; - - fn simulate_flock(file: &File, flag: libc::c_int) -> Result<()> { - let mut fle = libc::flock { - l_whence: 0, - l_start: 0, - l_len: 0, - l_type: 0, - l_pad: [0; 4], - l_pid: 0, - l_sysid: 0, - }; - let (cmd, op) = match flag & libc::LOCK_NB { - 0 => (libc::F_SETLKW, flag), - _ => (libc::F_SETLK, flag & !libc::LOCK_NB), - }; - match op { - libc::LOCK_SH => fle.l_type |= libc::F_RDLCK, - libc::LOCK_EX => fle.l_type |= libc::F_WRLCK, - libc::LOCK_UN => fle.l_type |= libc::F_UNLCK, - _ => return Err(Error::from_raw_os_error(libc::EINVAL)), - } - let ret = unsafe { libc::fcntl(file.as_raw_fd(), cmd, &fle) }; - match ret { - -1 => match Error::last_os_error().raw_os_error() { - Some(libc::EACCES) => { - // this is the 'sort of' solaris equivalent to EWOULDBLOCK - Err(Error::from_raw_os_error(libc::EWOULDBLOCK)) - } - _ => return Err(Error::last_os_error()), - }, - _ => Ok(()), - } - } - pub fn lock_ex(file: &File) -> Result<()> { - simulate_flock(file, libc::LOCK_EX) - } - pub fn try_lock_ex(file: &File) -> Result<()> { - simulate_flock(file, libc::LOCK_EX | libc::LOCK_NB) - } - pub fn unlock_file(file: &File) -> Result<()> { - simulate_flock(file, libc::LOCK_UN) - } - pub fn duplicate(file: &File) -> Result { - unsafe { - let fd = libc::dup(file.as_raw_fd()); - if fd < 0 { - Err(Error::last_os_error()) - } else { - Ok(File::from_raw_fd(fd)) - } - } - } -} diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs new file mode 100644 index 00000000..17c48494 --- /dev/null +++ b/server/src/engine/config.rs @@ -0,0 +1,1222 @@ +/* + * Created on Fri Sep 22 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::{error::RuntimeResult, fractal}, + core::fmt, + serde::Deserialize, + std::{collections::HashMap, fs}, +}; + +/* + misc +*/ + +pub type ParsedRawArgs = std::collections::HashMap>; +pub const ROOT_PASSWORD_MIN_LEN: usize = 16; + +#[derive(Debug, PartialEq)] +pub struct ModifyGuard { + val: T, + modified: bool, +} + +impl ModifyGuard { + pub const fn new(val: T) -> Self { + Self { + val, + modified: false, + } + } +} + +impl core::ops::Deref for ModifyGuard { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.val + } +} + +impl core::ops::DerefMut for ModifyGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.modified = true; + &mut self.val + } +} + +/* + configuration +*/ + +#[derive(Debug, PartialEq)] +/// The final configuration that can be used to start up all services +pub struct Configuration { + pub endpoints: ConfigEndpoint, + pub mode: ConfigMode, + pub system: ConfigSystem, + pub auth: ConfigAuth, +} + +impl Configuration { + #[cfg(test)] + pub fn new( + endpoints: ConfigEndpoint, + mode: ConfigMode, + system: ConfigSystem, + auth: ConfigAuth, + ) -> Self { + Self { + endpoints, + mode, + system, + auth, + } + } + const DEFAULT_HOST: &'static str = "127.0.0.1"; + const DEFAULT_PORT_TCP: u16 = 2003; + pub fn default_dev_mode(auth: DecodedAuth) -> Self { + Self { + endpoints: ConfigEndpoint::Insecure(ConfigEndpointTcp { + host: Self::DEFAULT_HOST.to_owned(), + port: Self::DEFAULT_PORT_TCP, + }), + mode: ConfigMode::Dev, + system: ConfigSystem::new(fractal::GENERAL_EXECUTOR_WINDOW), + auth: ConfigAuth::new(auth.plugin, auth.root_pass), + } + } +} + +// endpoint config + +#[derive(Debug, PartialEq)] +/// Endpoint configuration (TCP/TLS/TCP+TLS) +pub enum ConfigEndpoint { + Insecure(ConfigEndpointTcp), + Secure(ConfigEndpointTls), + Multi(ConfigEndpointTcp, ConfigEndpointTls), +} + +#[derive(Debug, PartialEq, Clone)] +/// TCP endpoint configuration +pub struct ConfigEndpointTcp { + host: String, + port: u16, +} + +impl ConfigEndpointTcp { + #[cfg(test)] + pub fn new(host: String, port: u16) -> Self { + Self { host, port } + } + pub fn host(&self) -> &str { + self.host.as_ref() + } + pub fn port(&self) -> u16 { + self.port + } +} + +#[derive(Debug, PartialEq)] +/// TLS endpoint configuration +pub struct ConfigEndpointTls { + pub tcp: ConfigEndpointTcp, + cert: String, + private_key: String, + pkey_pass: String, +} + +impl ConfigEndpointTls { + #[cfg(test)] + pub fn new( + tcp: ConfigEndpointTcp, + cert: String, + private_key: String, + pkey_pass: String, + ) -> Self { + Self { + tcp, + cert, + private_key, + pkey_pass, + } + } + pub fn tcp(&self) -> &ConfigEndpointTcp { + &self.tcp + } + pub fn cert(&self) -> &str { + self.cert.as_ref() + } + pub fn private_key(&self) -> &str { + self.private_key.as_ref() + } + pub fn pkey_pass(&self) -> &str { + self.pkey_pass.as_ref() + } +} + +/* + config mode +*/ + +#[derive(Debug, PartialEq, Deserialize, Clone, Copy)] +/// The configuration mode +pub enum ConfigMode { + /// In [`ConfigMode::Dev`] we're allowed to be more relaxed with settings + #[serde(rename = "dev")] + Dev, + /// In [`ConfigMode::Prod`] we're more stringent with settings + #[serde(rename = "prod")] + Prod, +} + +/* + config system +*/ + +#[derive(Debug, PartialEq)] +/// System configuration settings +pub struct ConfigSystem { + /// time window in seconds for the reliability system to kick-in automatically + pub reliability_system_window: u64, +} + +impl ConfigSystem { + pub fn new(reliability_system_window: u64) -> Self { + Self { + reliability_system_window, + } + } +} + +/* + config auth +*/ + +#[derive(Debug, PartialEq, Deserialize, Clone, Copy)] +pub enum AuthDriver { + #[serde(rename = "pwd")] + Pwd, +} + +#[derive(Debug, PartialEq, Deserialize, Clone)] +pub struct ConfigAuth { + pub plugin: AuthDriver, + pub root_key: String, +} + +impl ConfigAuth { + pub fn new(plugin: AuthDriver, root_key: String) -> Self { + Self { plugin, root_key } + } +} + +/** + decoded configuration + --- + the "raw" configuration that we got from the user. not validated +*/ +#[derive(Debug, PartialEq, Deserialize)] +pub struct DecodedConfiguration { + system: Option, + endpoints: Option, + auth: Option, +} + +impl Default for DecodedConfiguration { + fn default() -> Self { + Self { + system: Default::default(), + endpoints: Default::default(), + auth: None, + } + } +} + +#[derive(Debug, PartialEq, Deserialize)] +pub struct DecodedAuth { + plugin: AuthDriver, + root_pass: String, +} + +#[derive(Debug, PartialEq, Deserialize)] +/// Decoded system configuration +pub struct DecodedSystemConfig { + mode: Option, + rs_window: Option, +} + +#[derive(Debug, PartialEq, Deserialize)] +/// Decoded endpoint configuration +pub struct DecodedEPConfig { + secure: Option, + insecure: Option, +} + +#[derive(Debug, PartialEq, Deserialize)] +/// Decoded secure port configuration +pub struct DecodedEPSecureConfig { + host: String, + port: u16, + cert: String, + private_key: String, + pkey_passphrase: String, +} + +#[derive(Debug, PartialEq, Deserialize)] +/// Decoded insecure port configuration +pub struct DecodedEPInsecureConfig { + host: String, + port: u16, +} + +impl DecodedEPInsecureConfig { + pub fn new(host: &str, port: u16) -> Self { + Self { + host: host.to_owned(), + port, + } + } +} + +/* + errors and misc +*/ + +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +/// A configuration error (with an optional error origin source) +pub struct ConfigError { + source: Option, + kind: ConfigErrorKind, +} + +impl ConfigError { + /// Init config error + fn _new(source: Option, kind: ConfigErrorKind) -> Self { + Self { kind, source } + } + /// New config error with no source + fn new(kind: ConfigErrorKind) -> Self { + Self::_new(None, kind) + } + /// New config error with the given source + fn with_src(source: ConfigSource, kind: ConfigErrorKind) -> Self { + Self::_new(Some(source), kind) + } +} + +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.source { + Some(src) => write!(f, "config error in {}: ", src.as_str())?, + None => write!(f, "config error: ")?, + } + match &self.kind { + ConfigErrorKind::Conflict => write!( + f, + "conflicting settings. please choose either CLI or ENV or configuration file" + ), + ConfigErrorKind::ErrorString(e) => write!(f, "{e}"), + } + } +} + +#[derive(Debug, PartialEq)] +/// The configuration source +pub enum ConfigSource { + /// Command-line + Cli, + /// Environment variabels + Env, + /// Configuration file + File, +} + +impl ConfigSource { + fn as_str(&self) -> &'static str { + match self { + ConfigSource::Cli => "CLI", + ConfigSource::Env => "ENV", + ConfigSource::File => "config file", + } + } +} + +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +/// Type of configuration error +pub enum ConfigErrorKind { + /// Conflict between different setting modes (more than one of CLI/ENV/FILE was provided) + Conflict, + /// A custom error output + ErrorString(String), +} + +/// A configuration source implementation +pub(super) trait ConfigurationSource { + const KEY_AUTH_DRIVER: &'static str; + const KEY_AUTH_ROOT_PASSWORD: &'static str; + const KEY_TLS_CERT: &'static str; + const KEY_TLS_KEY: &'static str; + const KEY_TLS_PKEY_PASS: &'static str; + const KEY_ENDPOINTS: &'static str; + const KEY_RUN_MODE: &'static str; + const KEY_SERVICE_WINDOW: &'static str; + const SOURCE: ConfigSource; + /// Formats an error `Invalid value for {key}` + fn err_invalid_value_for(key: &str) -> ConfigError { + ConfigError::with_src( + Self::SOURCE, + ConfigErrorKind::ErrorString(format!("Invalid value for {key}")), + ) + } + /// Formats an error `Too many values for {key}` + fn err_too_many_values_for(key: &str) -> ConfigError { + ConfigError::with_src( + Self::SOURCE, + ConfigErrorKind::ErrorString(format!("Too many values for {key}")), + ) + } + /// Formats the custom error directly + fn custom_err(error: String) -> ConfigError { + ConfigError::with_src(Self::SOURCE, ConfigErrorKind::ErrorString(error)) + } +} + +/// Check if there are any duplicate values +fn argck_duplicate_values( + v: &[String], + key: &'static str, +) -> RuntimeResult<()> { + if v.len() != 1 { + return Err(CS::err_too_many_values_for(key).into()); + } + Ok(()) +} + +/* + decode helpers +*/ + +/// Protocol to be used by a given endpoint +enum ConnectionProtocol { + Tcp, + Tls, +} + +/// Parse an endpoint (`protocol@host:port`) +fn parse_endpoint(source: ConfigSource, s: &str) -> RuntimeResult<(ConnectionProtocol, &str, u16)> { + let err = || { + Err(ConfigError::with_src( + source, + ConfigErrorKind::ErrorString(format!( + "invalid endpoint syntax. should be `protocol@hostname:port`" + )), + ) + .into()) + }; + let x = s.split("@").collect::>(); + if x.len() != 2 { + return err(); + } + let [protocol, hostport] = [x[0], x[1]]; + let hostport = hostport.split(":").collect::>(); + if hostport.len() != 2 { + return err(); + } + let [host, port] = [hostport[0], hostport[1]]; + let Ok(port) = port.parse::() else { + return err(); + }; + let protocol = match protocol { + "tcp" => ConnectionProtocol::Tcp, + "tls" => ConnectionProtocol::Tls, + _ => return err(), + }; + Ok((protocol, host, port)) +} + +/// Decode a TLS endpoint (read in cert and private key) +fn decode_tls_ep( + cert_path: &str, + key_path: &str, + pkey_pass: &str, + host: &str, + port: u16, +) -> RuntimeResult { + super::fractal::context::set_dmsg("loading TLS configuration from disk"); + let tls_key = fs::read_to_string(key_path)?; + let tls_cert = fs::read_to_string(cert_path)?; + let tls_priv_key_passphrase = fs::read_to_string(pkey_pass)?; + Ok(DecodedEPSecureConfig { + host: host.into(), + port, + cert: tls_cert, + private_key: tls_key, + pkey_passphrase: tls_priv_key_passphrase, + }) +} + +/// Helper for decoding a TLS endpoint (we read in the cert and private key) +fn arg_decode_tls_endpoint( + args: &mut ParsedRawArgs, + host: &str, + port: u16, +) -> RuntimeResult { + let _cert = args.remove(CS::KEY_TLS_CERT); + let _key = args.remove(CS::KEY_TLS_KEY); + let _passphrase = args.remove(CS::KEY_TLS_PKEY_PASS); + let (tls_cert, tls_key, tls_passphrase) = match (_cert, _key, _passphrase) { + (Some(cert), Some(key), Some(pass)) => (cert, key, pass), + _ => { + return Err(ConfigError::with_src( + ConfigSource::Cli, + ConfigErrorKind::ErrorString(format!( + "must supply values for `{}`, `{}` and `{}` when using TLS", + CS::KEY_TLS_CERT, + CS::KEY_TLS_KEY, + CS::KEY_TLS_PKEY_PASS, + )), + ) + .into()); + } + }; + argck_duplicate_values::(&tls_cert, CS::KEY_TLS_CERT)?; + argck_duplicate_values::(&tls_key, CS::KEY_TLS_KEY)?; + argck_duplicate_values::(&tls_passphrase, CS::KEY_TLS_PKEY_PASS)?; + Ok(decode_tls_ep( + &tls_cert[0], + &tls_key[0], + &tls_passphrase[0], + host, + port, + )?) +} + +/* + decode options +*/ + +fn arg_decode_auth( + src_args: &mut ParsedRawArgs, + config: &mut ModifyGuard, +) -> RuntimeResult<()> { + let (Some(auth_driver), Some(mut root_key)) = ( + src_args.remove(CS::KEY_AUTH_DRIVER), + src_args.remove(CS::KEY_AUTH_ROOT_PASSWORD), + ) else { + return Err(ConfigError::with_src( + CS::SOURCE, + ConfigErrorKind::ErrorString(format!( + "to enable auth, you must provide values for both {} and {}", + CS::KEY_AUTH_DRIVER, + CS::KEY_AUTH_ROOT_PASSWORD + )), + ) + .into()); + }; + argck_duplicate_values::(&auth_driver, CS::KEY_AUTH_DRIVER)?; + argck_duplicate_values::(&root_key, CS::KEY_AUTH_DRIVER)?; + let auth_plugin = match auth_driver[0].as_str() { + "pwd" => AuthDriver::Pwd, + _ => return Err(CS::err_invalid_value_for(CS::KEY_AUTH_DRIVER).into()), + }; + config.auth = Some(DecodedAuth { + plugin: auth_plugin, + root_pass: root_key.remove(0), + }); + Ok(()) +} + +/// Decode the endpoints (`protocol@host:port`) +fn arg_decode_endpoints( + args: &mut ParsedRawArgs, + config: &mut ModifyGuard, +) -> RuntimeResult<()> { + let mut insecure = None; + let mut secure = None; + let Some(endpoints) = args.remove(CS::KEY_ENDPOINTS) else { + return Ok(()); + }; + if endpoints.len() > 2 { + return Err(CS::err_too_many_values_for(CS::KEY_ENDPOINTS).into()); + } + for ep in endpoints { + let (proto, host, port) = parse_endpoint(CS::SOURCE, &ep)?; + match proto { + ConnectionProtocol::Tcp if insecure.is_none() => { + insecure = Some(DecodedEPInsecureConfig::new(host, port)); + } + ConnectionProtocol::Tls if secure.is_none() => { + secure = Some(arg_decode_tls_endpoint::(args, host, port)?); + } + _ => { + return Err(CS::custom_err(format!( + "duplicate endpoints specified in `{}`", + CS::KEY_ENDPOINTS + )) + .into()); + } + } + } + if insecure.is_some() | secure.is_some() { + config.endpoints = Some(DecodedEPConfig { secure, insecure }); + } + Ok(()) +} + +/// Decode the run mode: +/// - Dev OR +/// - Prod +fn arg_decode_mode( + mode: &[String], + config: &mut ModifyGuard, +) -> RuntimeResult<()> { + argck_duplicate_values::(&mode, CS::KEY_RUN_MODE)?; + let mode = match mode[0].as_str() { + "dev" => ConfigMode::Dev, + "prod" => ConfigMode::Prod, + _ => return Err(CS::err_invalid_value_for(CS::KEY_RUN_MODE).into()), + }; + match config.system.as_mut() { + Some(s) => s.mode = Some(mode), + None => { + config.system = Some(DecodedSystemConfig { + mode: Some(mode), + rs_window: None, + }) + } + } + Ok(()) +} + +/// Decode the service time window +fn arg_decode_rs_window( + mode: &[String], + config: &mut ModifyGuard, +) -> RuntimeResult<()> { + argck_duplicate_values::(&mode, CS::KEY_SERVICE_WINDOW)?; + match mode[0].parse::() { + Ok(n) => match config.system.as_mut() { + Some(sys) => sys.rs_window = Some(n), + None => { + config.system = Some(DecodedSystemConfig { + mode: None, + rs_window: Some(n), + }) + } + }, + Err(_) => return Err(CS::err_invalid_value_for(CS::KEY_SERVICE_WINDOW).into()), + } + Ok(()) +} + +/* + CLI args process +*/ + +/// CLI help message +pub(super) const CLI_HELP: &str = "\ +Usage: skyd [OPTION]... + +skyd is the Skytable database server daemon and can be used to serve database requests. + +Flags: + -h, --help Display this help menu and exit. + -v, --version Display the version number and exit. + +Options: + --config Set configuration options using the config file + --tlscert Specify the path to the TLS certificate. + --tlskey Specify the path to the TLS private key. + --endpoint Designate an endpoint. Format: protocol@host:port. + This option can be repeated to define multiple endpoints. + --service-window Establish the time window for the background service in seconds. + --auth Identify the authentication plugin by name. + --mode Set the operational mode. Note: This option is mandatory. + --auth-plugin Set the auth plugin. `pwd` is a supported option + --auth-root-password Set the root password + +Examples: + skyd --mode=dev --auth-root-password \"password12345678\" + +Notes: + - If no `--mode` is provided, we default to `dev` + - You must provide `--auth-root-password` to set the default root password + - To use TLS, you must provide both `--tlscert` and `--tlskey` + +For further assistance, refer to the official documentation here: https://docs.skytable.org +"; + +#[derive(Debug, PartialEq)] +/// Return from parsing CLI configuration +pub enum CLIConfigParseReturn { + /// No changes + Default, + /// Output help menu + Help, + /// Output version + Version, + /// We yielded a config + YieldedConfig(T), +} + +impl CLIConfigParseReturn { + #[cfg(test)] + pub fn into_config(self) -> T { + match self { + Self::YieldedConfig(yc) => yc, + _ => panic!(), + } + } +} + +/// Parse CLI args: +/// - `--{option} {value}` +/// - `--{option}={value}` +pub fn parse_cli_args<'a, T: 'a + AsRef>( + src: impl Iterator, +) -> RuntimeResult> { + let mut args_iter = src.into_iter().skip(1); + let mut cli_args: ParsedRawArgs = HashMap::new(); + while let Some(arg) = args_iter.next() { + let arg = arg.as_ref(); + if arg == "--help" || arg == "-h" { + return Ok(CLIConfigParseReturn::Help); + } + if arg == "--version" || arg == "-v" { + return Ok(CLIConfigParseReturn::Version); + } + if !arg.starts_with("--") { + return Err(ConfigError::with_src( + ConfigSource::Cli, + ConfigErrorKind::ErrorString(format!("unexpected argument `{arg}`")), + ) + .into()); + } + // x=1 + let arg_key; + let arg_val; + let splits_arg_and_value = arg.split("=").collect::>(); + if (splits_arg_and_value.len() == 2) & (arg.len() >= 5) { + // --{n}={x}; good + arg_key = splits_arg_and_value[0]; + arg_val = splits_arg_and_value[1].to_string(); + } else if splits_arg_and_value.len() != 1 { + // that's an invalid argument since the split is either `x=y` or `x` and we don't have any args + // with special characters + return Err(ConfigError::with_src( + ConfigSource::Cli, + ConfigErrorKind::ErrorString(format!("incorrectly formatted argument `{arg}`")), + ) + .into()); + } else { + let Some(value) = args_iter.next() else { + return Err(ConfigError::with_src( + ConfigSource::Cli, + ConfigErrorKind::ErrorString(format!("missing value for option `{arg}`")), + ) + .into()); + }; + arg_key = arg; + arg_val = value.as_ref().to_string(); + } + // merge duplicates into a vec + match cli_args.get_mut(arg_key) { + Some(cli) => { + cli.push(arg_val.to_string()); + } + None => { + cli_args.insert(arg_key.to_string(), vec![arg_val.to_string()]); + } + } + } + if cli_args.is_empty() { + Ok(CLIConfigParseReturn::Default) + } else { + Ok(CLIConfigParseReturn::YieldedConfig(cli_args)) + } +} + +/* + env args process +*/ + +/// Parse environment variables +pub fn parse_env_args() -> RuntimeResult> { + const KEYS: [&str; 8] = [ + CSEnvArgs::KEY_AUTH_DRIVER, + CSEnvArgs::KEY_AUTH_ROOT_PASSWORD, + CSEnvArgs::KEY_ENDPOINTS, + CSEnvArgs::KEY_RUN_MODE, + CSEnvArgs::KEY_SERVICE_WINDOW, + CSEnvArgs::KEY_TLS_CERT, + CSEnvArgs::KEY_TLS_KEY, + CSEnvArgs::KEY_TLS_PKEY_PASS, + ]; + let mut ret = HashMap::new(); + for key in KEYS { + let var = match get_var_from_store(key) { + Ok(v) => v, + Err(e) => match e { + std::env::VarError::NotPresent => continue, + std::env::VarError::NotUnicode(_) => { + return Err(ConfigError::with_src( + ConfigSource::Env, + ConfigErrorKind::ErrorString(format!("invalid value for `{key}`")), + ) + .into()) + } + }, + }; + let splits: Vec<_> = var.split(",").map(ToString::to_string).collect(); + ret.insert(key.into(), splits); + } + if ret.is_empty() { + Ok(None) + } else { + Ok(Some(ret)) + } +} + +/* + apply config changes +*/ + +/// Apply the configuration changes to the given mutable config +fn apply_config_changes( + args: &mut ParsedRawArgs, +) -> RuntimeResult> { + let mut config = ModifyGuard::new(DecodedConfiguration::default()); + enum DecodeKind { + Simple { + key: &'static str, + f: fn(&[String], &mut ModifyGuard) -> RuntimeResult<()>, + }, + Complex { + f: fn(&mut ParsedRawArgs, &mut ModifyGuard) -> RuntimeResult<()>, + }, + } + let decode_tasks = [ + // auth + DecodeKind::Complex { + f: arg_decode_auth::, + }, + // mode + DecodeKind::Simple { + key: CS::KEY_RUN_MODE, + f: arg_decode_mode::, + }, + // service time window + DecodeKind::Simple { + key: CS::KEY_SERVICE_WINDOW, + f: arg_decode_rs_window::, + }, + // endpoints + DecodeKind::Complex { + f: arg_decode_endpoints::, + }, + ]; + for task in decode_tasks { + match task { + DecodeKind::Simple { key, f } => match args.get(key) { + Some(values_for_arg) => { + (f)(&values_for_arg, &mut config)?; + args.remove(key); + } + None => {} + }, + DecodeKind::Complex { f } => (f)(args, &mut config)?, + } + } + if !args.is_empty() { + Err(ConfigError::with_src( + CS::SOURCE, + ConfigErrorKind::ErrorString("found unknown arguments".into()), + ) + .into()) + } else { + Ok(config) + } +} + +/* + config source impls +*/ + +pub struct CSCommandLine; +impl CSCommandLine { + const ARG_CONFIG_FILE: &'static str = "--config"; +} +impl ConfigurationSource for CSCommandLine { + const KEY_AUTH_DRIVER: &'static str = "--auth-plugin"; + const KEY_AUTH_ROOT_PASSWORD: &'static str = "--auth-root-password"; + const KEY_TLS_CERT: &'static str = "--tlscert"; + const KEY_TLS_KEY: &'static str = "--tlskey"; + const KEY_TLS_PKEY_PASS: &'static str = "--tls-passphrase"; + const KEY_ENDPOINTS: &'static str = "--endpoint"; + const KEY_RUN_MODE: &'static str = "--mode"; + const KEY_SERVICE_WINDOW: &'static str = "--service-window"; + const SOURCE: ConfigSource = ConfigSource::Cli; +} + +pub struct CSEnvArgs; +impl ConfigurationSource for CSEnvArgs { + const KEY_AUTH_DRIVER: &'static str = "SKYDB_AUTH_PLUGIN"; + const KEY_AUTH_ROOT_PASSWORD: &'static str = "SKYDB_AUTH_ROOT_PASSWORD"; + const KEY_TLS_CERT: &'static str = "SKYDB_TLS_CERT"; + const KEY_TLS_KEY: &'static str = "SKYDB_TLS_KEY"; + const KEY_TLS_PKEY_PASS: &'static str = "SKYDB_TLS_PRIVATE_KEY_PASSWORD"; + const KEY_ENDPOINTS: &'static str = "SKYDB_ENDPOINTS"; + const KEY_RUN_MODE: &'static str = "SKYDB_RUN_MODE"; + const KEY_SERVICE_WINDOW: &'static str = "SKYDB_SERVICE_WINDOW"; + const SOURCE: ConfigSource = ConfigSource::Env; +} + +pub struct CSConfigFile; +impl ConfigurationSource for CSConfigFile { + const KEY_AUTH_DRIVER: &'static str = "auth.plugin"; + const KEY_AUTH_ROOT_PASSWORD: &'static str = "auth.root_password"; + const KEY_TLS_CERT: &'static str = "endpoints.secure.cert"; + const KEY_TLS_KEY: &'static str = "endpoints.secure.key"; + const KEY_TLS_PKEY_PASS: &'static str = "endpoints.secure.pkey_passphrase"; + const KEY_ENDPOINTS: &'static str = "endpoints"; + const KEY_RUN_MODE: &'static str = "system.mode"; + const KEY_SERVICE_WINDOW: &'static str = "system.service_window"; + const SOURCE: ConfigSource = ConfigSource::File; +} + +/* + validate configuration +*/ + +macro_rules! if_some { + ($target:expr => $then:expr) => { + if let Some(x) = $target { + $then(x); + } + }; +} + +macro_rules! err_if { + ($(if $cond:expr => $error:expr),* $(,)?) => { + $(if $cond { return Err($error) })* + } +} + +/// Validate the configuration, and prepare the final configuration +fn validate_configuration( + DecodedConfiguration { + system, + endpoints, + auth, + }: DecodedConfiguration, +) -> RuntimeResult { + let Some(auth) = auth else { + return Err(ConfigError::with_src( + CS::SOURCE, + ConfigErrorKind::ErrorString(format!( + "root account must be configured with {}", + CS::KEY_AUTH_ROOT_PASSWORD + )), + ) + .into()); + }; + // initialize our default configuration + let mut config = Configuration::default_dev_mode(auth); + // mutate + if_some!( + system => |system: DecodedSystemConfig| { + if_some!(system.mode => |mode| config.mode = mode); + if_some!(system.rs_window => |window| config.system.reliability_system_window = window); + } + ); + if_some!( + endpoints => |ep: DecodedEPConfig| { + let has_insecure = ep.insecure.is_some(); + if_some!(ep.insecure => |insecure: DecodedEPInsecureConfig| { + config.endpoints = ConfigEndpoint::Insecure(ConfigEndpointTcp { host: insecure.host, port: insecure.port }); + }); + if_some!(ep.secure => |secure: DecodedEPSecureConfig| { + let secure_ep = ConfigEndpointTls { + tcp: ConfigEndpointTcp { + host: secure.host, + port: secure.port, + }, + cert: secure.cert, + private_key: secure.private_key, + pkey_pass: secure.pkey_passphrase, + }; + match &config.endpoints { + ConfigEndpoint::Insecure(is) => if has_insecure { + // an insecure EP was defined by the user, so set to multi + config.endpoints = ConfigEndpoint::Multi(is.clone(), secure_ep) + } else { + // only secure EP was defined by the user + config.endpoints = ConfigEndpoint::Secure(secure_ep); + }, + _ => unreachable!() + } + }) + } + ); + // now check a few things + err_if!( + if config.system.reliability_system_window == 0 => ConfigError::with_src( + CS::SOURCE, + ConfigErrorKind::ErrorString("invalid value for service window. must be nonzero".into()), + ).into(), + if config.auth.root_key.len() < ROOT_PASSWORD_MIN_LEN => ConfigError::with_src( + CS::SOURCE, + ConfigErrorKind::ErrorString("the root password must have at least 16 characters".into()), + ).into(), + ); + Ok(config) +} + +/* + actual configuration check and exec +*/ + +/// The return from parsing a configuration file +#[derive(Debug, PartialEq)] +pub enum ConfigReturn { + /// Don't need to do anything. We've output a message and we're good to exit + HelpMessage(String), + /// A configuration that we have fully validated was provided + Config(Configuration), +} + +impl ConfigReturn { + #[cfg(test)] + pub fn into_config(self) -> Configuration { + match self { + Self::Config(c) => c, + _ => panic!(), + } + } +} + +/// Apply the changes and validate the configuration +pub(super) fn apply_and_validate( + mut args: ParsedRawArgs, +) -> RuntimeResult { + let cfg = apply_config_changes::(&mut args)?; + validate_configuration::(cfg.val).map(ConfigReturn::Config) +} + +/* + just some test hacks +*/ + +#[cfg(test)] +thread_local! { + static CLI_SRC: std::cell::RefCell>> = std::cell::RefCell::new(None); + static ENV_SRC: std::cell::RefCell>> = std::cell::RefCell::new(None); + static FILE_SRC: std::cell::RefCell> = std::cell::RefCell::new(None); +} +#[cfg(test)] +pub(super) fn set_cli_src(cli: Vec) { + CLI_SRC.with(|args| *args.borrow_mut() = Some(cli)) +} +#[cfg(test)] +pub(super) fn set_env_src(variables: Vec) { + ENV_SRC.with(|env| { + let variables = variables + .into_iter() + .map(|var| { + var.split("=") + .map(ToString::to_string) + .collect::>() + }) + .map(|mut vars| (vars.remove(0), vars.remove(0))) + .collect(); + *env.borrow_mut() = Some(variables); + }) +} +#[cfg(test)] +pub(super) fn set_file_src(src: &str) { + FILE_SRC.with(|s| { + s.borrow_mut().replace(src.to_string()); + }) +} +fn get_file_from_store(filename: &str) -> RuntimeResult { + let _f = filename; + let f; + #[cfg(test)] + { + f = Ok(FILE_SRC.with(|f| f.borrow().clone().unwrap())); + } + #[cfg(not(test))] + { + super::fractal::context::set_dmsg("loading configuration file from disk"); + f = Ok(fs::read_to_string(filename)?); + } + f +} +fn get_var_from_store(name: &str) -> Result { + let var; + #[cfg(test)] + { + var = ENV_SRC.with(|venv| { + let ret = { + match venv.borrow_mut().as_mut() { + None => return Err(std::env::VarError::NotPresent), + Some(env_store) => match env_store.remove(name) { + Some(var) => Ok(var), + None => Err(std::env::VarError::NotPresent), + }, + } + }; + ret + }); + } + #[cfg(not(test))] + { + var = std::env::var(name); + } + var +} +fn get_cli_from_store() -> Vec { + let src; + #[cfg(test)] + { + src = CLI_SRC + .with(|args| args.borrow_mut().take()) + .unwrap_or(vec![]); + } + #[cfg(not(test))] + { + src = std::env::args().collect(); + } + src +} + +/// Check the configuration. We look through: +/// - CLI args +/// - ENV variables +/// - Config file (if any) +pub fn check_configuration() -> RuntimeResult { + // read in our environment variables + let env_args = parse_env_args()?; + // read in our CLI args (since that can tell us whether we need a configuration file) + let read_cli_args = parse_cli_args(get_cli_from_store().into_iter())?; + let cli_args = match read_cli_args { + CLIConfigParseReturn::Default => { + // no options were provided in the CLI + None + } + CLIConfigParseReturn::Help => return Ok(ConfigReturn::HelpMessage(CLI_HELP.into())), + CLIConfigParseReturn::Version => { + // just output the version + return Ok(ConfigReturn::HelpMessage(format!( + "Skytable Database Server (skyd) v{}", + libsky::VERSION + ))); + } + CLIConfigParseReturn::YieldedConfig(cfg) => Some(cfg), + }; + match cli_args { + Some(cfg_from_cli) => { + // we have some CLI args + match cfg_from_cli.get(CSCommandLine::ARG_CONFIG_FILE) { + Some(cfg_file) => return check_config_file(&cfg_from_cli, &env_args, cfg_file), + None => { + // no config file; check if there is a conflict with environment args + if env_args.is_some() { + // as we feared + return Err(ConfigError::with_src( + ConfigSource::Cli, + ConfigErrorKind::Conflict, + ) + .into()); + } + return apply_and_validate::(cfg_from_cli); + } + } + } + None => { + // no CLI args; but do we have anything from env? + match env_args { + Some(args) => { + return apply_and_validate::(args); + } + None => { + // no env args or cli args; we're running on default + return Err(ConfigError::new(ConfigErrorKind::ErrorString( + "no configuration provided".into(), + )) + .into()); + } + } + } + } +} + +/// Check the configuration file +fn check_config_file( + cfg_from_cli: &ParsedRawArgs, + env_args: &Option, + cfg_file: &Vec, +) -> RuntimeResult { + if cfg_from_cli.len() == 1 && env_args.is_none() { + // yes, we only have the config file + argck_duplicate_values::(&cfg_file, CSCommandLine::ARG_CONFIG_FILE)?; + // read the config file + let file = get_file_from_store(&cfg_file[0])?; + let mut config_from_file: DecodedConfiguration = + serde_yaml::from_str(&file).map_err(|e| { + ConfigError::with_src( + ConfigSource::File, + ConfigErrorKind::ErrorString(format!( + "failed to parse YAML config file with error: `{e}`" + )), + ) + })?; + // read in the TLS certs (if any) + match config_from_file.endpoints.as_mut() { + Some(ep) => match ep.secure.as_mut() { + Some(secure_ep) => { + super::fractal::context::set_dmsg("loading TLS configuration from disk"); + let cert = fs::read_to_string(&secure_ep.cert)?; + let private_key = fs::read_to_string(&secure_ep.private_key)?; + let private_key_passphrase = fs::read_to_string(&secure_ep.pkey_passphrase)?; + secure_ep.cert = cert; + secure_ep.private_key = private_key; + secure_ep.pkey_passphrase = private_key_passphrase; + } + None => {} + }, + None => {} + } + // done here + return validate_configuration::(config_from_file).map(ConfigReturn::Config); + } else { + // so there are more configuration options + a config file? (and maybe even env?) + return Err(ConfigError::with_src(ConfigSource::Cli, ConfigErrorKind::Conflict).into()); + } +} diff --git a/server/src/engine/core/dcl.rs b/server/src/engine/core/dcl.rs new file mode 100644 index 00000000..d3e3476e --- /dev/null +++ b/server/src/engine/core/dcl.rs @@ -0,0 +1,95 @@ +/* + * Created on Fri Nov 10 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + data::{tag::TagClass, DictEntryGeneric}, + error::{QueryError, QueryResult}, + fractal::GlobalInstanceLike, + net::protocol::ClientLocalState, + ql::dcl::{SysctlCommand, UserDecl, UserDel}, +}; + +const KEY_PASSWORD: &str = "password"; + +pub fn exec( + g: G, + current_user: &ClientLocalState, + cmd: SysctlCommand, +) -> QueryResult<()> { + if cmd.needs_root() & !current_user.is_root() { + return Err(QueryError::SysPermissionDenied); + } + match cmd { + SysctlCommand::CreateUser(new) => create_user(&g, new), + SysctlCommand::DropUser(drop) => drop_user(&g, current_user, drop), + SysctlCommand::AlterUser(usermod) => alter_user(&g, current_user, usermod), + SysctlCommand::ReportStatus => Ok(()), + } +} + +fn alter_user( + global: &impl GlobalInstanceLike, + cstate: &ClientLocalState, + user: UserDecl, +) -> QueryResult<()> { + if cstate.is_root() { + // the root password can only be changed by shutting down the server + return Err(QueryError::SysAuthError); + } + let (username, password) = get_user_data(user)?; + global.sys_store().alter_user(username, password) +} + +fn create_user(global: &impl GlobalInstanceLike, user: UserDecl) -> QueryResult<()> { + let (username, password) = get_user_data(user)?; + global.sys_store().create_new_user(username, password) +} + +fn get_user_data(mut user: UserDecl) -> Result<(String, String), QueryError> { + let username = user.username().to_owned(); + let password = match user.options_mut().remove(KEY_PASSWORD) { + Some(DictEntryGeneric::Data(d)) + if d.kind() == TagClass::Str && user.options().is_empty() => + unsafe { d.into_str().unwrap_unchecked() }, + None | Some(_) => { + // invalid properties + return Err(QueryError::QExecDdlInvalidProperties); + } + }; + Ok((username, password)) +} + +fn drop_user( + global: &impl GlobalInstanceLike, + cstate: &ClientLocalState, + user_del: UserDel<'_>, +) -> QueryResult<()> { + if cstate.username() == user_del.username() { + // you can't delete yourself! + return Err(QueryError::SysAuthError); + } + global.sys_store().drop_user(user_del.username()) +} diff --git a/server/src/engine/core/ddl_misc.rs b/server/src/engine/core/ddl_misc.rs new file mode 100644 index 00000000..7d2a9511 --- /dev/null +++ b/server/src/engine/core/ddl_misc.rs @@ -0,0 +1,104 @@ +/* + * Created on Thu Nov 30 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + error::{QueryError, QueryResult}, + fractal::GlobalInstanceLike, + net::protocol::{ClientLocalState, Response, ResponseType}, + ql::ddl::Inspect, +}; + +pub fn inspect( + g: &impl GlobalInstanceLike, + c: &ClientLocalState, + stmt: Inspect, +) -> QueryResult { + let ret = match stmt { + Inspect::Global => { + // collect spaces + let spaces = g.namespace().idx().read(); + let mut spaces_iter = spaces.iter().peekable(); + let mut ret = format!("{{\"spaces\":["); + while let Some((space, _)) = spaces_iter.next() { + ret.push('"'); + ret.push_str(&space); + ret.push('"'); + if spaces_iter.peek().is_some() { + ret.push(','); + } + } + if c.is_root() { + // iff the user is root, show information about other users. if not, just show models and settings + ret.push_str("],\"users\":["); + drop(spaces_iter); + drop(spaces); + // collect users + let users = g.sys_store().system_store().auth_data().read(); + let mut users_iter = users.users().iter().peekable(); + while let Some((user, _)) = users_iter.next() { + ret.push('"'); + ret.push_str(&user); + ret.push('"'); + if users_iter.peek().is_some() { + ret.push(','); + } + } + } + ret.push_str("],\"settings\":{}}"); + ret + } + Inspect::Model(m) => match g.namespace().idx_models().read().get(&m) { + Some(m) => format!( + "{{\"decl\":\"{}\",\"rows\":{},\"properties\":{{}}}}", + m.describe(), + m.primary_index().count() + ), + None => return Err(QueryError::QExecObjectNotFound), + }, + Inspect::Space(s) => match g.namespace().idx().read().get(s.as_str()) { + Some(s) => { + let mut ret = format!("{{\"models\":["); + let mut models_iter = s.models().iter().peekable(); + while let Some(mdl) = models_iter.next() { + ret.push('\"'); + ret.push_str(&mdl); + ret.push('\"'); + if models_iter.peek().is_some() { + ret.push(','); + } + } + ret.push_str("]}}"); + ret + } + None => return Err(QueryError::QExecObjectNotFound), + }, + }; + Ok(Response::Serialized { + ty: ResponseType::String, + size: ret.len(), + data: ret.into_bytes(), + }) +} diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs new file mode 100644 index 00000000..b40d7e64 --- /dev/null +++ b/server/src/engine/core/dml/del.rs @@ -0,0 +1,68 @@ +/* + * Created on Sat May 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::{self, dml::QueryExecMeta, model::delta::DataDeltaKind}, + error::{QueryError, QueryResult}, + fractal::GlobalInstanceLike, + idx::MTIndex, + net::protocol::Response, + ql::dml::del::DeleteStatement, + sync, +}; + +pub fn delete_resp( + global: &impl GlobalInstanceLike, + delete: DeleteStatement, +) -> QueryResult { + self::delete(global, delete).map(|_| Response::Empty) +} + +pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> QueryResult<()> { + core::with_model_for_data_update(global, delete.entity(), |model| { + let g = sync::atm::cpin(); + let delta_state = model.delta_state(); + let _idx_latch = model.primary_index().acquire_cd(); + // create new version + let new_version = delta_state.create_new_data_delta_version(); + match model + .primary_index() + .__raw_index() + .mt_delete_return_entry(&model.resolve_where(delete.clauses_mut())?, &g) + { + Some(row) => { + let dp = delta_state.append_new_data_delta_with( + DataDeltaKind::Delete, + row.clone(), + new_version, + &g, + ); + Ok(QueryExecMeta::new(dp)) + } + None => Err(QueryError::QExecDmlRowNotFound), + } + }) +} diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs new file mode 100644 index 00000000..441dacaf --- /dev/null +++ b/server/src/engine/core/dml/ins.rs @@ -0,0 +1,133 @@ +/* + * Created on Mon May 01 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::{ + self, + dml::QueryExecMeta, + index::{DcFieldIndex, PrimaryIndexKey, Row}, + model::{delta::DataDeltaKind, Model}, + }, + error::{QueryError, QueryResult}, + fractal::GlobalInstanceLike, + idx::{IndexBaseSpec, MTIndex, STIndex, STIndexSeq}, + net::protocol::Response, + ql::dml::ins::{InsertData, InsertStatement}, + sync::atm::cpin, +}; + +pub fn insert_resp( + global: &impl GlobalInstanceLike, + insert: InsertStatement, +) -> QueryResult { + self::insert(global, insert).map(|_| Response::Empty) +} + +pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> QueryResult<()> { + core::with_model_for_data_update(global, insert.entity(), |mdl| { + let (pk, data) = prepare_insert(mdl, insert.data())?; + let _idx_latch = mdl.primary_index().acquire_cd(); + let g = cpin(); + let ds = mdl.delta_state(); + // create new version + let new_version = ds.create_new_data_delta_version(); + let row = Row::new(pk, data, ds.schema_current_version(), new_version); + if mdl.primary_index().__raw_index().mt_insert(row.clone(), &g) { + // append delta for new version + let dp = ds.append_new_data_delta_with(DataDeltaKind::Insert, row, new_version, &g); + Ok(QueryExecMeta::new(dp)) + } else { + Err(QueryError::QExecDmlDuplicate) + } + }) +} + +// TODO(@ohsayan): optimize null case +fn prepare_insert( + model: &Model, + insert: InsertData, +) -> QueryResult<(PrimaryIndexKey, DcFieldIndex)> { + let fields = model.fields(); + let mut okay = fields.len() == insert.column_count(); + let mut prepared_data = DcFieldIndex::idx_init_cap(fields.len()); + match insert { + InsertData::Ordered(tuple) => { + let mut fields = fields.stseq_ord_kv(); + let mut tuple = tuple.into_iter(); + while (tuple.len() != 0) & okay { + let mut data; + let field; + unsafe { + // UNSAFE(@ohsayan): safe because of invariant + data = tuple.next().unwrap_unchecked(); + // UNSAFE(@ohsayan): safe because of flag + field = fields.next().unwrap_unchecked(); + } + let (field_id, field) = field; + okay &= field.vt_data_fpath(&mut data); + okay &= prepared_data.st_insert( + unsafe { + // UNSAFE(@ohsayan): the model is right here, so we're good + field_id.clone() + }, + data, + ); + } + } + InsertData::Map(map) => { + let mut inserted = 0; + let mut iter = fields.st_iter_kv().zip(map.into_iter()); + while (iter.len() != 0) & (okay) { + let ((model_field_key, model_field_spec), (this_field_key, mut this_field_data)) = unsafe { + // UNSAFE(@ohsayan): safe because of loop invariant + iter.next().unwrap_unchecked() + }; + okay &= model_field_spec.vt_data_fpath(&mut this_field_data); + okay &= model_field_key.as_str() == this_field_key.as_str(); + prepared_data.st_insert( + unsafe { + // UNSAFE(@ohsayan): the model is right here. it saves us the work! + model_field_key.clone() + }, + this_field_data, + ); + inserted += 1; + } + okay &= inserted == fields.len(); + } + } + let primary_key = prepared_data.remove(model.p_key()); + okay &= primary_key.is_some(); + if okay { + let primary_key = unsafe { + // UNSAFE(@ohsayan): okay check above + PrimaryIndexKey::new_from_dc(primary_key.unwrap_unchecked()) + }; + Ok((primary_key, prepared_data)) + } else { + Err(QueryError::QExecDmlValidationError) + } +} diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs new file mode 100644 index 00000000..6463bbe0 --- /dev/null +++ b/server/src/engine/core/dml/mod.rs @@ -0,0 +1,88 @@ +/* + * Created on Mon May 01 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod del; +mod ins; +mod sel; +mod upd; + +use crate::{ + engine::{ + core::model::Model, + data::{lit::Lit, tag::DataTag}, + error::{QueryError, QueryResult}, + ql::dml::WhereClause, + }, + util::compiler, +}; + +#[cfg(test)] +pub use { + del::delete, + ins::insert, + sel::{select_all, select_custom}, + upd::{collect_trace_path as update_flow_trace, update}, +}; +pub use { + del::delete_resp, + ins::insert_resp, + sel::{select_all_resp, select_resp}, + upd::update_resp, +}; + +impl Model { + pub(self) fn resolve_where<'a>( + &self, + where_clause: &mut WhereClause<'a>, + ) -> QueryResult> { + match where_clause.clauses_mut().remove(self.p_key().as_bytes()) { + Some(clause) + if clause.filter_hint_none() + & (clause.rhs().kind().tag_unique() == self.p_tag().tag_unique()) => + { + Ok(clause.rhs()) + } + _ => compiler::cold_rerr(QueryError::QExecDmlWhereHasUnindexedColumn), + } + } +} + +#[derive(Debug)] +pub struct QueryExecMeta { + delta_hint: usize, +} + +impl QueryExecMeta { + pub fn new(delta_hint: usize) -> Self { + Self { delta_hint } + } + pub fn zero() -> Self { + Self::new(0) + } + pub fn delta_hint(&self) -> usize { + self.delta_hint + } +} diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs new file mode 100644 index 00000000..c85d73b9 --- /dev/null +++ b/server/src/engine/core/dml/sel.rs @@ -0,0 +1,262 @@ +/* + * Created on Thu May 11 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::{ + index::{ + DcFieldIndex, IndexLatchHandleExclusive, PrimaryIndexKey, Row, RowData, RowDataLck, + }, + model::Model, + }, + data::{ + cell::{Datacell, VirtualDatacell}, + tag::{DataTag, TagClass}, + }, + error::{QueryError, QueryResult}, + fractal::GlobalInstanceLike, + idx::{IndexMTRaw, MTIndexExt, STIndex, STIndexSeq}, + mem::IntegerRepr, + net::protocol::{Response, ResponseType}, + ql::dml::sel::{SelectAllStatement, SelectStatement}, + sync, +}; + +pub fn select_resp( + global: &impl GlobalInstanceLike, + select: SelectStatement, +) -> QueryResult { + let mut data = vec![]; + let mut i = 0usize; + self::select_custom(global, select, |item| { + encode_cell(&mut data, item); + i += 1; + })?; + Ok(Response::Serialized { + ty: ResponseType::Row, + size: i, + data, + }) +} + +pub fn select_all_resp( + global: &impl GlobalInstanceLike, + select: SelectAllStatement, +) -> QueryResult { + let mut ret_buf = Vec::new(); + let i = self::select_all( + global, + select, + &mut ret_buf, + |buf, _, col_c| { + IntegerRepr::scoped(col_c as u64, |repr| buf.extend(repr)); + buf.push(b'\n'); + }, + |buf, data, _| encode_cell(buf, data), + )?; + Ok(Response::Serialized { + ty: ResponseType::MultiRow, + size: i, + data: ret_buf, + }) +} + +pub fn select_all( + global: &impl GlobalInstanceLike, + select: SelectAllStatement, + serialize_target: &mut T, + mut f_mdl: Fm, + mut f: F, +) -> QueryResult +where + Fm: FnMut(&mut T, &Model, usize), + F: FnMut(&mut T, &Datacell, usize), +{ + global.namespace().with_model(select.entity, |mdl| { + let g = sync::atm::cpin(); + let mut i = 0; + if select.wildcard { + f_mdl(serialize_target, mdl, mdl.fields().len()); + for (key, data) in RowIteratorAll::new(&g, mdl, select.limit as usize) { + let vdc = VirtualDatacell::new_pk(key, mdl.p_tag()); + for key in mdl.fields().stseq_ord_key() { + let r = if key.as_str() == mdl.p_key() { + &*vdc + } else { + data.fields().get(key).unwrap() + }; + f(serialize_target, r, mdl.fields().len()); + } + i += 1; + } + } else { + // schema check + if select.fields.len() > mdl.fields().len() + || select + .fields + .iter() + .any(|f| !mdl.fields().st_contains(f.as_str())) + { + return Err(QueryError::QExecUnknownField); + } + f_mdl(serialize_target, mdl, select.fields.len()); + for (key, data) in RowIteratorAll::new(&g, mdl, select.limit as usize) { + let vdc = VirtualDatacell::new_pk(key, mdl.p_tag()); + for key in select.fields.iter() { + let r = if key.as_str() == mdl.p_key() { + &*vdc + } else { + data.fields().st_get(key.as_str()).unwrap() + }; + f(serialize_target, r, select.fields.len()); + } + i += 1; + } + } + Ok(i) + }) +} + +fn encode_cell(resp: &mut Vec, item: &Datacell) { + resp.push((item.tag().tag_selector().value_u8() + 1) * (item.is_init() as u8)); + if item.is_null() { + return; + } + unsafe { + // UNSAFE(@ohsayan): +tagck + match item.tag().tag_class() { + TagClass::Bool => return resp.push(item.read_bool() as _), + TagClass::UnsignedInt => IntegerRepr::scoped(item.read_uint(), |b| resp.extend(b)), + TagClass::SignedInt => IntegerRepr::scoped(item.read_sint(), |b| resp.extend(b)), + TagClass::Float => resp.extend(item.read_float().to_string().as_bytes()), + TagClass::Bin | TagClass::Str => { + let slc = item.read_bin(); + IntegerRepr::scoped(slc.len() as u64, |b| resp.extend(b)); + resp.push(b'\n'); + resp.extend(slc); + return; + } + TagClass::List => { + let list = item.read_list(); + let ls = list.read(); + IntegerRepr::scoped(ls.len() as u64, |b| resp.extend(b)); + resp.push(b'\n'); + for item in ls.iter() { + encode_cell(resp, item); + } + return; + } + } + } + resp.push(b'\n'); +} + +pub fn select_custom( + global: &impl GlobalInstanceLike, + mut select: SelectStatement, + mut cellfn: F, +) -> QueryResult<()> +where + F: FnMut(&Datacell), +{ + global.namespace().with_model(select.entity(), |mdl| { + let target_key = mdl.resolve_where(select.clauses_mut())?; + let pkdc = VirtualDatacell::new(target_key.clone(), mdl.p_tag().tag_unique()); + let g = sync::atm::cpin(); + let mut read_field = |key, fields: &DcFieldIndex| { + match fields.st_get(key) { + Some(dc) => cellfn(dc), + None if key == mdl.p_key() => cellfn(&pkdc), + None => return Err(QueryError::QExecUnknownField), + } + Ok(()) + }; + match mdl.primary_index().select(target_key.clone(), &g) { + Some(row) => { + let r = row.resolve_schema_deltas_and_freeze(mdl.delta_state()); + if select.is_wildcard() { + for key in mdl.fields().stseq_ord_key() { + read_field(key.as_ref(), r.fields())?; + } + } else { + for key in select.into_fields() { + read_field(key.as_str(), r.fields())?; + } + } + } + None => return Err(QueryError::QExecDmlRowNotFound), + } + Ok(()) + }) +} + +struct RowIteratorAll<'g> { + _g: &'g sync::atm::Guard, + mdl: &'g Model, + iter: as MTIndexExt>::IterEntry<'g, 'g, 'g>, + _latch: IndexLatchHandleExclusive<'g>, + limit: usize, +} + +impl<'g> RowIteratorAll<'g> { + fn new(g: &'g sync::atm::Guard, mdl: &'g Model, limit: usize) -> Self { + let idx = mdl.primary_index(); + let latch = idx.acquire_exclusive(); + Self { + _g: g, + mdl, + iter: idx.__raw_index().mt_iter_entry(g), + _latch: latch, + limit, + } + } + fn _next( + &mut self, + ) -> Option<( + &'g PrimaryIndexKey, + parking_lot::RwLockReadGuard<'g, RowData>, + )> { + if self.limit == 0 { + return None; + } + self.limit -= 1; + self.iter.next().map(|row| { + ( + row.d_key(), + row.resolve_schema_deltas_and_freeze(self.mdl.delta_state()), + ) + }) + } +} + +impl<'g> Iterator for RowIteratorAll<'g> { + type Item = ( + &'g PrimaryIndexKey, + parking_lot::RwLockReadGuard<'g, RowData>, + ); + fn next(&mut self) -> Option { + self._next() + } +} diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs new file mode 100644 index 00000000..d665a741 --- /dev/null +++ b/server/src/engine/core/dml/upd.rs @@ -0,0 +1,379 @@ +/* + * Created on Thu May 11 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(test)] +use std::cell::RefCell; + +use { + crate::{ + engine::{ + core::{ + self, dml::QueryExecMeta, model::delta::DataDeltaKind, + query_meta::AssignmentOperator, + }, + data::{ + cell::Datacell, + lit::Lit, + tag::{DataTag, FloatSpec, SIntSpec, TagClass, UIntSpec}, + }, + error::{QueryError, QueryResult}, + fractal::GlobalInstanceLike, + idx::STIndex, + net::protocol::Response, + ql::dml::upd::{AssignmentExpression, UpdateStatement}, + sync, + }, + util::compiler, + }, + std::mem, +}; + +#[inline(always)] +unsafe fn dc_op_fail(_: &Datacell, _: Lit) -> (bool, Datacell) { + (false, Datacell::null()) +} +// bool +unsafe fn dc_op_bool_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) { + (true, Datacell::new_bool(rhs.bool())) +} +// uint +unsafe fn dc_op_uint_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let uint = rhs.uint(); + let kind = UIntSpec::from_full(dc.tag()); + (kind.check(uint), Datacell::new_uint(uint, kind)) +} +unsafe fn dc_op_uint_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let kind = UIntSpec::from_full(dc.tag()); + let (uint, did_of) = dc.uint().overflowing_add(rhs.uint()); + (kind.check(uint) & !did_of, Datacell::new_uint(uint, kind)) +} +unsafe fn dc_op_uint_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let kind = UIntSpec::from_full(dc.tag()); + let (uint, did_of) = dc.uint().overflowing_sub(rhs.uint()); + (kind.check(uint) & !did_of, Datacell::new_uint(uint, kind)) +} +unsafe fn dc_op_uint_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let kind = UIntSpec::from_full(dc.tag()); + let (uint, did_of) = dc.uint().overflowing_mul(rhs.uint()); + (kind.check(uint) & !did_of, Datacell::new_uint(uint, kind)) +} +unsafe fn dc_op_uint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let kind = UIntSpec::from_full(dc.tag()); + let (uint, did_of) = dc.uint().overflowing_div(rhs.uint()); + (kind.check(uint) & !did_of, Datacell::new_uint(uint, kind)) +} +// sint +unsafe fn dc_op_sint_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let sint = rhs.sint(); + let kind = SIntSpec::from_full(dc.tag()); + (kind.check(sint), Datacell::new_sint(sint, kind)) +} +unsafe fn dc_op_sint_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let kind = SIntSpec::from_full(dc.tag()); + let (sint, did_of) = dc.sint().overflowing_add(rhs.sint()); + (kind.check(sint) & !did_of, Datacell::new_sint(sint, kind)) +} +unsafe fn dc_op_sint_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let kind = SIntSpec::from_full(dc.tag()); + let (sint, did_of) = dc.sint().overflowing_sub(rhs.sint()); + (kind.check(sint) & !did_of, Datacell::new_sint(sint, kind)) +} +unsafe fn dc_op_sint_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let kind = SIntSpec::from_full(dc.tag()); + let (sint, did_of) = dc.sint().overflowing_mul(rhs.sint()); + (kind.check(sint) & !did_of, Datacell::new_sint(sint, kind)) +} +unsafe fn dc_op_sint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let kind = SIntSpec::from_full(dc.tag()); + let (sint, did_of) = dc.sint().overflowing_div(rhs.sint()); + (kind.check(sint) & !did_of, Datacell::new_sint(sint, kind)) +} +/* + float + --- + FIXME(@ohsayan): floating point always upsets me now and then, this time its + the silent overflow boom and I think I should implement a strict mode (no MySQL, + not `STRICT_ALL_TABLES` unless we do actually end up going down that route. In + that case, oops) + -- + TODO(@ohsayan): account for float32 overflow +*/ +unsafe fn dc_op_float_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let float = rhs.float(); + let kind = FloatSpec::from_full(dc.tag()); + (kind.check(float), Datacell::new_float(float, kind)) +} +unsafe fn dc_op_float_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let result = dc.read_float() + rhs.float(); + let kind = FloatSpec::from_full(dc.tag()); + (kind.check(result), Datacell::new_float(result, kind)) +} +unsafe fn dc_op_float_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let result = dc.read_float() - rhs.float(); + let kind = FloatSpec::from_full(dc.tag()); + (kind.check(result), Datacell::new_float(result, kind)) +} +unsafe fn dc_op_float_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let result = dc.read_float() * rhs.float(); + let kind = FloatSpec::from_full(dc.tag()); + (kind.check(result), Datacell::new_float(result, kind)) +} +unsafe fn dc_op_float_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let result = dc.read_float() / rhs.float(); + let kind = FloatSpec::from_full(dc.tag()); + (kind.check(result), Datacell::new_float(result, kind)) +} +// binary +unsafe fn dc_op_bin_ass(_dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let new_bin = rhs.bin(); + let mut v = Vec::new(); + if v.try_reserve_exact(new_bin.len()).is_err() { + return dc_op_fail(_dc, rhs); + } + v.extend_from_slice(new_bin); + (true, Datacell::new_bin(v.into_boxed_slice())) +} +unsafe fn dc_op_bin_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let push_into_bin = rhs.bin(); + let mut bin = Vec::new(); + if compiler::unlikely(bin.try_reserve_exact(push_into_bin.len()).is_err()) { + return dc_op_fail(dc, rhs); + } + bin.extend_from_slice(dc.read_bin()); + bin.extend_from_slice(push_into_bin); + (true, Datacell::new_bin(bin.into_boxed_slice())) +} +// string +unsafe fn dc_op_str_ass(_dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let new_str = rhs.str(); + let mut v = String::new(); + if v.try_reserve_exact(new_str.len()).is_err() { + return dc_op_fail(_dc, rhs); + } + v.push_str(new_str); + (true, Datacell::new_str(v.into_boxed_str())) +} +unsafe fn dc_op_str_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) { + let push_into_str = rhs.str(); + let mut str = String::new(); + if compiler::unlikely(str.try_reserve_exact(push_into_str.len()).is_err()) { + return dc_op_fail(dc, rhs); + } + str.push_str(dc.read_str()); + str.push_str(push_into_str); + (true, Datacell::new_str(str.into_boxed_str())) +} + +static OPERATOR: [unsafe fn(&Datacell, Lit) -> (bool, Datacell); { + TagClass::MAX as usize * AssignmentOperator::VARIANTS +}] = [ + // bool + dc_op_bool_ass, + // -- pad: 4 + dc_op_fail, + dc_op_fail, + dc_op_fail, + dc_op_fail, + // uint + dc_op_uint_ass, + dc_op_uint_add, + dc_op_uint_sub, + dc_op_uint_mul, + dc_op_uint_div, + // sint + dc_op_sint_ass, + dc_op_sint_add, + dc_op_sint_sub, + dc_op_sint_mul, + dc_op_sint_div, + // float + dc_op_float_ass, + dc_op_float_add, + dc_op_float_sub, + dc_op_float_mul, + dc_op_float_div, + // bin + dc_op_bin_ass, + dc_op_bin_add, + // -- pad: 3 + dc_op_fail, + dc_op_fail, + dc_op_fail, + // str + dc_op_str_ass, + dc_op_str_add, + // -- pad: 3 + dc_op_fail, + dc_op_fail, + dc_op_fail, +]; + +#[inline(always)] +const fn opc(opr: TagClass, ope: AssignmentOperator) -> usize { + (AssignmentOperator::VARIANTS * opr.value_word()) + ope.value_word() +} + +#[cfg(test)] +thread_local! { + pub(super) static ROUTE_TRACE: RefCell> = RefCell::new(Vec::new()); +} + +#[inline(always)] +fn input_trace(v: &'static str) { + #[cfg(test)] + { + ROUTE_TRACE.with(|rcv| rcv.borrow_mut().push(v)) + } + let _ = v; +} +#[cfg(test)] +pub fn collect_trace_path() -> Vec<&'static str> { + ROUTE_TRACE.with(|v| v.borrow().iter().cloned().collect()) +} + +pub fn update_resp( + global: &impl GlobalInstanceLike, + update: UpdateStatement, +) -> QueryResult { + self::update(global, update).map(|_| Response::Empty) +} + +pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> QueryResult<()> { + core::with_model_for_data_update(global, update.entity(), |mdl| { + let mut ret = Ok(QueryExecMeta::zero()); + // prepare row fetch + let key = mdl.resolve_where(update.clauses_mut())?; + // fetch row + let g = sync::atm::cpin(); + let Some(row) = mdl.primary_index().select(key, &g) else { + return Err(QueryError::QExecDmlRowNotFound); + }; + // lock row + let mut row_data_wl = row.d_data().write(); + // create new version + let ds = mdl.delta_state(); + let new_version = ds.create_new_data_delta_version(); + // process changes + let mut rollback_now = false; + let mut rollback_data = Vec::with_capacity(update.expressions().len()); + let mut assn_expressions = update.into_expressions().into_iter(); + /* + FIXME(@ohsayan): where's my usual magic? I'll do it once we have the SE stabilized + */ + // apply changes + while (assn_expressions.len() != 0) & (!rollback_now) { + let AssignmentExpression { + lhs, + rhs, + operator_fn, + } = unsafe { + // UNSAFE(@ohsayan): pre-loop cond + assn_expressions.next().unwrap_unchecked() + }; + let field_definition; + let field_data; + match ( + mdl.fields().st_get(lhs.as_str()), + row_data_wl.fields_mut().st_get_mut(lhs.as_str()), + ) { + (Some(fdef), Some(fdata)) => { + field_definition = fdef; + field_data = fdata; + } + _ => { + input_trace("fieldnotfound"); + rollback_now = true; + ret = Err(QueryError::QExecUnknownField); + break; + } + } + match ( + field_definition.layers()[0].tag().tag_class(), + rhs.kind().tag_class(), + ) { + (tag_a, tag_b) + if (tag_a == tag_b) & (tag_a < TagClass::List) & field_data.is_init() => + { + let (okay, new) = unsafe { OPERATOR[opc(tag_a, operator_fn)](field_data, rhs) }; + rollback_now &= !okay; + rollback_data.push((lhs.as_str(), mem::replace(field_data, new))); + input_trace("sametag;nonnull"); + } + (tag_a, tag_b) + if (tag_a == tag_b) + & field_data.is_null() + & (operator_fn == AssignmentOperator::Assign) => + { + rollback_data.push((lhs.as_str(), mem::replace(field_data, rhs.into()))); + input_trace("sametag;orignull"); + } + (TagClass::List, tag_b) if operator_fn == AssignmentOperator::AddAssign => { + if field_definition.layers()[1].tag().tag_class() == tag_b { + unsafe { + // UNSAFE(@ohsayan): matched tags + let mut list = field_data.read_list().write(); + if list.try_reserve(1).is_ok() { + input_trace("list;sametag"); + list.push(rhs.into()); + } else { + rollback_now = true; + ret = Err(QueryError::SysOutOfMemory); + break; + } + } + } else { + input_trace("list;badtag"); + rollback_now = true; + ret = Err(QueryError::QExecDmlValidationError); + break; + } + } + _ => { + input_trace("unknown_reason;exitmainloop"); + ret = Err(QueryError::QExecDmlValidationError); + rollback_now = true; + break; + } + } + } + if compiler::unlikely(rollback_now) { + input_trace("rollback"); + rollback_data + .into_iter() + .for_each(|(field_id, restored_data)| { + row_data_wl.fields_mut().st_update(field_id, restored_data); + }); + } else { + // update revised tag + row_data_wl.set_txn_revised(new_version); + // publish delta + let dp = + ds.append_new_data_delta_with(DataDeltaKind::Update, row.clone(), new_version, &g); + ret = Ok(QueryExecMeta::new(dp)) + } + ret + }) +} diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs new file mode 100644 index 00000000..d32c8f62 --- /dev/null +++ b/server/src/engine/core/exec.rs @@ -0,0 +1,259 @@ +/* + * Created on Thu Oct 05 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::{ddl_misc, dml, model::Model, space::Space}, + error::{QueryError, QueryResult}, + fractal::{Global, GlobalInstanceLike}, + net::protocol::{ClientLocalState, Response, ResponseType, SQuery}, + ql::{ + ast::{traits::ASTNode, InplaceData, State}, + ddl::Use, + lex::KeywordStmt, + }, +}; + +/* + --- + trigger warning: disgusting hacks below owing to token lifetimes +*/ + +pub async fn dispatch_to_executor<'a>( + global: &Global, + cstate: &mut ClientLocalState, + query: SQuery<'a>, +) -> QueryResult { + let tokens = + crate::engine::ql::lex::SecureLexer::new_with_segments(query.query(), query.params()) + .lex()?; + let mut state = State::new_inplace(&tokens); + state.set_space_maybe(unsafe { + // UNSAFE(@ohsayan): exclusively used within this scope + core::mem::transmute(cstate.get_cs()) + }); + let stmt = state.try_statement()?; + if stmt.is_blocking() { + run_blocking_stmt(global, cstate, state, stmt).await + } else { + run_nb(global, cstate, state, stmt) + } +} + +fn _callgs_map + core::fmt::Debug, T>( + g: &Global, + state: &mut State<'static, InplaceData>, + f: impl FnOnce(&Global, A) -> Result, + map: impl FnOnce(T) -> Response, +) -> QueryResult { + let cs = ASTNode::parse_from_state_hardened(state)?; + Ok(map(f(&g, cs)?)) +} + +#[inline(always)] +fn _callgs + core::fmt::Debug, T>( + g: &Global, + state: &mut State<'static, InplaceData>, + f: impl FnOnce(&Global, A) -> Result, +) -> QueryResult { + let cs = ASTNode::parse_from_state_hardened(state)?; + f(&g, cs) +} + +#[inline(always)] +fn _callgcs + core::fmt::Debug, T>( + g: &Global, + cstate: &ClientLocalState, + state: &mut State<'static, InplaceData>, + f: impl FnOnce(&Global, &ClientLocalState, A) -> Result, +) -> QueryResult { + let a = ASTNode::parse_from_state_hardened(state)?; + f(&g, cstate, a) +} + +#[inline(always)] +fn translate_ddl_result(x: Option) -> Response { + match x { + Some(b) => Response::Bool(b), + None => Response::Empty, + } +} + +async fn run_blocking_stmt( + global: &Global, + cstate: &mut ClientLocalState, + mut state: State<'_, InplaceData>, + stmt: KeywordStmt, +) -> Result { + if !(cstate.is_root() | (stmt == KeywordStmt::Sysctl)) { + // all the actions here need root permission (but we do an exception for sysctl which allows status to be called by anyone) + return Err(QueryError::SysPermissionDenied); + } + state.ensure_minimum_for_blocking_stmt()?; + /* + IMPORTANT: DDL queries will NOT pick up the currently set space. instead EVERY DDL query must manually fully specify the entity that + they want to manipulate. this prevents a whole set of exciting errors like dropping a model with the same model name from another space + */ + state.unset_space(); + let (a, b) = (&state.current()[0], &state.current()[1]); + let sysctl = stmt == KeywordStmt::Sysctl; + let create = stmt == KeywordStmt::Create; + let alter = stmt == KeywordStmt::Alter; + let drop = stmt == KeywordStmt::Drop; + let last_id = b.is_ident(); + let last_allow = Token![allow].eq(b); + let last_if = Token![if].eq(b); + let c_s = (create & Token![space].eq(a) & (last_id | last_if)) as u8 * 2; + let c_m = (create & Token![model].eq(a) & (last_id | last_if)) as u8 * 3; + let a_s = (alter & Token![space].eq(a) & last_id) as u8 * 4; + let a_m = (alter & Token![model].eq(a) & last_id) as u8 * 5; + let d_s = (drop & Token![space].eq(a) & (last_id | last_allow | last_if)) as u8 * 6; + let d_m = (drop & Token![model].eq(a) & (last_id | last_allow | last_if)) as u8 * 7; + let fc = sysctl as u8 | c_s | c_m | a_s | a_m | d_s | d_m; + state.cursor_ahead_if(!sysctl); + static BLK_EXEC: [fn( + Global, + &ClientLocalState, + &mut State<'static, InplaceData>, + ) -> QueryResult; 8] = [ + |_, _, _| Err(QueryError::QLUnknownStatement), + blocking_exec_sysctl, + |g, _, t| { + _callgs_map( + &g, + t, + Space::transactional_exec_create, + translate_ddl_result, + ) + }, + |g, _, t| { + _callgs_map( + &g, + t, + Model::transactional_exec_create, + translate_ddl_result, + ) + }, + |g, _, t| _callgs_map(&g, t, Space::transactional_exec_alter, |_| Response::Empty), + |g, _, t| _callgs_map(&g, t, Model::transactional_exec_alter, |_| Response::Empty), + |g, _, t| _callgs_map(&g, t, Space::transactional_exec_drop, translate_ddl_result), + |g, _, t| _callgs_map(&g, t, Model::transactional_exec_drop, translate_ddl_result), + ]; + let r = unsafe { + // UNSAFE(@ohsayan): the only await is within this block + let c_glob = global.clone(); + let static_cstate: &'static ClientLocalState = core::mem::transmute(cstate); + let static_state: &'static mut State<'static, InplaceData> = + core::mem::transmute(&mut state); + tokio::task::spawn_blocking(move || { + BLK_EXEC[fc as usize](c_glob, static_cstate, static_state) + }) + .await + }; + r.unwrap() +} + +fn blocking_exec_sysctl( + g: Global, + cstate: &ClientLocalState, + state: &mut State<'static, InplaceData>, +) -> QueryResult { + let r = ASTNode::parse_from_state_hardened(state)?; + super::dcl::exec(g, cstate, r).map(|_| Response::Empty) +} + +/* + nb exec +*/ + +fn cstate_use( + global: &Global, + cstate: &mut ClientLocalState, + state: &mut State<'static, InplaceData>, +) -> QueryResult { + let use_c = Use::parse_from_state_hardened(state)?; + match use_c { + Use::Null => cstate.unset_cs(), + Use::Space(new_space) => { + /* + NB: just like SQL, we don't really care about what this is set to as it's basically a shorthand. + so we do a simple vanity check + */ + if !global.namespace().contains_space(new_space.as_str()) { + return Err(QueryError::QExecObjectNotFound); + } + cstate.set_cs(new_space.boxed_str()); + } + Use::RefreshCurrent => match cstate.get_cs() { + None => return Ok(Response::Null), + Some(space) => { + if !global.namespace().contains_space(space) { + cstate.unset_cs(); + return Err(QueryError::QExecObjectNotFound); + } + return Ok(Response::Serialized { + ty: ResponseType::String, + size: space.len(), + data: space.to_owned().into_bytes(), + }); + } + }, + } + Ok(Response::Empty) +} + +fn run_nb( + global: &Global, + cstate: &mut ClientLocalState, + mut state: State<'_, InplaceData>, + stmt: KeywordStmt, +) -> QueryResult { + let stmt_c = stmt.value_u8() - KeywordStmt::Use.value_u8(); + static F: [fn( + &Global, + &mut ClientLocalState, + &mut State<'static, InplaceData>, + ) -> QueryResult; 9] = [ + cstate_use, // use + |g, c, s| _callgcs(g, c, s, ddl_misc::inspect), + |_, _, _| Err(QueryError::QLUnknownStatement), // describe + |g, _, s| _callgs(g, s, dml::insert_resp), + |g, _, s| _callgs(g, s, dml::select_resp), + |g, _, s| _callgs(g, s, dml::update_resp), + |g, _, s| _callgs(g, s, dml::delete_resp), + |_, _, _| Err(QueryError::QLUnknownStatement), // exists + |g, _, s| _callgs(g, s, dml::select_all_resp), + ]; + { + let n_offset_adjust = (stmt == KeywordStmt::Select) & state.cursor_rounded_eq(Token![all]); + state.cursor_ahead_if(n_offset_adjust); + let corrected_offset = (n_offset_adjust as u8 * 8) | (stmt_c * (!n_offset_adjust as u8)); + let mut state = unsafe { + // UNSAFE(@ohsayan): this is a lifetime issue with the token handle + core::mem::transmute(state) + }; + F[corrected_offset as usize](global, cstate, &mut state) + } +} diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs new file mode 100644 index 00000000..de82d9bc --- /dev/null +++ b/server/src/engine/core/index/key.rs @@ -0,0 +1,363 @@ +/* + * Created on Sun Apr 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::mem::ZERO_BLOCK; +#[cfg(test)] +use crate::util::test_utils; +use { + crate::engine::{ + data::{ + cell::Datacell, + lit::Lit, + tag::{DataTag, TagUnique}, + }, + idx::meta::Comparable, + mem::{self, DwordQN, SpecialPaddedWord, WordIO}, + }, + core::{ + fmt, + hash::{Hash, Hasher}, + mem::ManuallyDrop, + slice, str, + }, +}; + +pub struct PrimaryIndexKey { + tag: TagUnique, + data: SpecialPaddedWord, +} + +impl Clone for PrimaryIndexKey { + fn clone(&self) -> Self { + match self.tag { + TagUnique::SignedInt | TagUnique::UnsignedInt => { + let (qw, nw) = self.data.dwordqn_load_qw_nw(); + unsafe { + let slice = slice::from_raw_parts(nw as *const u8, qw as _); + let mut data = ManuallyDrop::new(slice.to_owned().into_boxed_slice()); + Self { + tag: self.tag, + data: SpecialPaddedWord::new(qw, data.as_mut_ptr() as usize), + } + } + } + TagUnique::Bin | TagUnique::Str => Self { + tag: self.tag, + data: unsafe { core::mem::transmute_copy(&self.data) }, + }, + _ => unreachable!(), + } + } +} + +impl PrimaryIndexKey { + pub fn tag(&self) -> TagUnique { + self.tag + } +} + +impl PrimaryIndexKey { + pub unsafe fn data(&self) -> SpecialPaddedWord { + core::mem::transmute_copy(&self.data) + } + pub unsafe fn read_uint(&self) -> u64 { + self.data.load() + } + pub fn uint(&self) -> Option { + (self.tag == TagUnique::UnsignedInt).then_some(unsafe { + // UNSAFE(@ohsayan): verified tag + self.read_uint() + }) + } + pub unsafe fn read_sint(&self) -> i64 { + self.data.load() + } + pub fn sint(&self) -> Option { + (self.tag == TagUnique::SignedInt).then_some(unsafe { + // UNSAFE(@ohsayan): verified tag + self.read_sint() + }) + } + pub unsafe fn read_bin(&self) -> &[u8] { + self.virtual_block() + } + pub fn bin(&self) -> Option<&[u8]> { + (self.tag == TagUnique::Bin).then(|| unsafe { + // UNSAFE(@ohsayan): verified tag + self.read_bin() + }) + } + pub unsafe fn read_str(&self) -> &str { + str::from_utf8_unchecked(self.virtual_block()) + } + pub fn str(&self) -> Option<&str> { + (self.tag == TagUnique::Str).then(|| unsafe { + // UNSAFE(@ohsayan): verified tag + self.read_str() + }) + } +} + +impl PrimaryIndexKey { + #[cfg(test)] + pub fn try_from_dc(dc: Datacell) -> Option { + Self::check(&dc).then(|| unsafe { Self::new_from_dc(dc) }) + } + /// ## Safety + /// + /// Make sure that the [`Datacell`] is an eligible candidate key (ensuring uniqueness constraints + allocation correctness). + /// + /// If you violate this: + /// - You might leak memory + /// - You might segfault + /// - Even if you escape both, it will produce incorrect results which is something you DO NOT want in an index + pub unsafe fn new_from_dc(dc: Datacell) -> Self { + debug_assert!(Self::check(&dc)); + let tag = dc.tag().tag_unique(); + let dc = ManuallyDrop::new(dc); + let (a, b) = unsafe { + // UNSAFE(@ohsayan): this doesn't do anything "bad" by itself. needs the construction to be broken for it to do something silly + dc.as_raw() + } + .dwordqn_load_qw_nw(); + if cfg!(debug_assertions) && tag < TagUnique::Bin { + assert_eq!(b, mem::ZERO_BLOCK.as_ptr() as usize); + } + Self { + tag, + data: unsafe { + // UNSAFE(@ohsayan): loaded above, writing here + SpecialPaddedWord::new(a, b) + }, + } + } + /// Create a new quadword based primary key + pub unsafe fn new_from_qw(tag: TagUnique, qw: u64) -> Self { + debug_assert!(tag == TagUnique::SignedInt || tag == TagUnique::UnsignedInt); + Self { + tag, + data: unsafe { + // UNSAFE(@ohsayan): manually choosing block + SpecialPaddedWord::new(qw, ZERO_BLOCK.as_ptr() as usize) + }, + } + } + pub unsafe fn new_from_dual(tag: TagUnique, qw: u64, ptr: usize) -> Self { + debug_assert!(tag == TagUnique::Str || tag == TagUnique::Bin); + Self { + tag, + data: unsafe { + // UNSAFE(@ohsayan): manually choosing qw and nw + SpecialPaddedWord::new(qw, ptr) + }, + } + } + pub unsafe fn raw_clone(&self) -> Self { + Self::new(self.tag, { + let (qw, nw) = self.data.dwordqn_load_qw_nw(); + SpecialPaddedWord::new(qw, nw) + }) + } + pub fn check(dc: &Datacell) -> bool { + dc.tag().tag_unique().is_unique() + } + /// ## Safety + /// If you mess up construction, everything will fall apart + pub unsafe fn new(tag: TagUnique, data: SpecialPaddedWord) -> Self { + Self { tag, data } + } + fn __compute_vdata_offset(&self) -> [usize; 2] { + let (len, data) = self.data.dwordqn_load_qw_nw(); + if cfg!(debug_assertions) && self.tag < TagUnique::Bin { + assert_eq!(data, mem::ZERO_BLOCK.as_ptr() as usize); + } + let actual_len = (len as usize) * (self.tag >= TagUnique::Bin) as usize; + [data, actual_len] + } + fn virtual_block(&self) -> &[u8] { + let [data, actual_len] = self.__compute_vdata_offset(); + unsafe { + // UNSAFE(@ohsayan): Safe, due to construction + slice::from_raw_parts(data as *const u8, actual_len) + } + } + fn virtual_block_mut(&mut self) -> &mut [u8] { + let [data, actual_len] = self.__compute_vdata_offset(); + unsafe { + // UNSAFE(@ohsayan): safe due to construction + slice::from_raw_parts_mut(data as *mut u8, actual_len) + } + } +} + +impl Drop for PrimaryIndexKey { + fn drop(&mut self) { + if let TagUnique::Bin | TagUnique::Str = self.tag { + unsafe { + // UNSAFE(@ohsayan): Aliasing, sole owner and correct initialization + let vdata = self.virtual_block_mut(); + mem::dealloc_array(vdata.as_mut_ptr(), vdata.len()); + } + } + } +} + +impl PartialEq for PrimaryIndexKey { + fn eq(&self, other: &Self) -> bool { + let [data_1, data_2]: [u64; 2] = [self.data.load(), other.data.load()]; + ((self.tag == other.tag) & (data_1 == data_2)) + && self.virtual_block() == other.virtual_block() + } +} + +impl Eq for PrimaryIndexKey {} + +impl Hash for PrimaryIndexKey { + fn hash(&self, hasher: &mut H) { + self.tag.hash(hasher); + self.virtual_block().hash(hasher); + } +} + +impl<'a> PartialEq> for PrimaryIndexKey { + fn eq(&self, key: &Lit<'a>) -> bool { + debug_assert!(key.kind().tag_unique().is_unique()); + self.tag == key.kind().tag_unique() && self.virtual_block() == key.__vdata() + } +} + +impl<'a> Comparable> for PrimaryIndexKey { + fn cmp_eq(&self, key: &Lit<'a>) -> bool { + >::eq(self, key) + } +} + +impl<'a> Comparable for Lit<'a> { + fn cmp_eq(&self, key: &PrimaryIndexKey) -> bool { + >::eq(key, self) + } +} + +impl fmt::Debug for PrimaryIndexKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut dbg_struct = f.debug_struct("PrimaryIndexKey"); + dbg_struct.field("tag", &self.tag); + macro_rules! fmt { + ($($mtch:ident => $expr:expr),* $(,)?) => { + match self.tag { + $(TagUnique::$mtch => dbg_struct.field("data", &($expr.unwrap())),)* + TagUnique::Illegal => panic!("found illegal value. check ctor."), + } + }; + } + fmt!( + UnsignedInt => self.uint(), + SignedInt => self.sint(), + Bin => self.bin(), + Str => self.str(), + ); + dbg_struct.finish() + } +} + +#[test] +fn check_pk_wrong_type() { + let data = [ + Datacell::from(false), + Datacell::from(100), + Datacell::from(-100), + Datacell::from(10.11), + Datacell::from("hello"), + Datacell::from("hello".as_bytes()), + Datacell::from([]), + ]; + for datum in data { + let tag = datum.tag(); + let candidate = PrimaryIndexKey::try_from_dc(datum); + if tag.tag_unique() == TagUnique::Illegal { + assert!(candidate.is_none(), "{:?}", &candidate); + } else { + assert!(candidate.is_some(), "{:?}", &candidate); + } + } +} + +#[test] +fn check_pk_eq_hash() { + let state = test_utils::randomstate(); + let data = [ + Datacell::from(100), + Datacell::from(-100), + Datacell::from("binary".as_bytes()), + Datacell::from("string"), + ]; + + for datum in data { + let pk1 = PrimaryIndexKey::try_from_dc(datum.clone()).unwrap(); + let pk2 = PrimaryIndexKey::try_from_dc(datum).unwrap(); + assert_eq!(pk1, pk2); + assert_eq!( + test_utils::hash_rs(&state, &pk1), + test_utils::hash_rs(&state, &pk2) + ); + } +} + +#[test] +fn check_pk_lit_eq_hash() { + let state = test_utils::randomstate(); + let data = [ + Lit::new_uint(100), + Lit::new_sint(-100), + Lit::new_bin(b"binary bro"), + Lit::new_str("string bro"), + ]; + for lit in data { + let pk = PrimaryIndexKey::try_from_dc(Datacell::from(lit.clone())).unwrap(); + assert_eq!(pk, lit); + assert_eq!( + test_utils::hash_rs(&state, &lit), + test_utils::hash_rs(&state, &pk) + ); + } +} + +#[test] +fn check_pk_extremes() { + let state = test_utils::randomstate(); + let d1 = PrimaryIndexKey::try_from_dc(Datacell::new_uint_default(u64::MAX)).unwrap(); + let d2 = PrimaryIndexKey::try_from_dc(Datacell::from(Lit::new_uint(u64::MAX))).unwrap(); + assert_eq!(d1, d2); + assert_eq!(d1.uint().unwrap(), u64::MAX); + assert_eq!(d2.uint().unwrap(), u64::MAX); + assert_eq!( + test_utils::hash_rs(&state, &d1), + test_utils::hash_rs(&state, &d2) + ); + assert_eq!(d1, Lit::new_uint(u64::MAX)); + assert_eq!(d2, Lit::new_uint(u64::MAX)); + assert_eq!(d1.uint().unwrap(), u64::MAX); +} diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs new file mode 100644 index 00000000..a6df689a --- /dev/null +++ b/server/src/engine/core/index/mod.rs @@ -0,0 +1,95 @@ +/* + * Created on Sat Apr 08 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod key; +mod row; + +use crate::engine::{ + data::lit::Lit, + idx::{IndexBaseSpec, IndexMTRaw, MTIndex}, + sync::atm::Guard, +}; + +pub use { + key::PrimaryIndexKey, + row::{DcFieldIndex, Row, RowData}, +}; + +pub type RowDataLck = parking_lot::RwLock; + +#[derive(Debug)] +pub struct PrimaryIndex { + data: IndexMTRaw, + latch: IndexLatch, +} + +impl PrimaryIndex { + pub fn new_empty() -> Self { + Self { + data: IndexMTRaw::idx_init(), + latch: IndexLatch::new(), + } + } + pub fn acquire_cd(&self) -> IndexLatchHandleShared { + self.latch.gl_handle_shared() + } + pub fn acquire_exclusive(&self) -> IndexLatchHandleExclusive { + self.latch.gl_handle_exclusive() + } + pub fn select<'a, 'v, 't: 'v, 'g: 't>(&'t self, key: Lit<'a>, g: &'g Guard) -> Option<&'v Row> { + self.data.mt_get_element(&key, g) + } + pub fn __raw_index(&self) -> &IndexMTRaw { + &self.data + } + pub fn count(&self) -> usize { + self.data.mt_len() + } +} + +#[derive(Debug)] +pub struct IndexLatchHandleShared<'t>(parking_lot::RwLockReadGuard<'t, ()>); +#[derive(Debug)] +pub struct IndexLatchHandleExclusive<'t>(parking_lot::RwLockWriteGuard<'t, ()>); + +#[derive(Debug)] +struct IndexLatch { + glck: parking_lot::RwLock<()>, +} + +impl IndexLatch { + fn new() -> Self { + Self { + glck: parking_lot::RwLock::new(()), + } + } + fn gl_handle_shared(&self) -> IndexLatchHandleShared { + IndexLatchHandleShared(self.glck.read()) + } + fn gl_handle_exclusive(&self) -> IndexLatchHandleExclusive { + IndexLatchHandleExclusive(self.glck.write()) + } +} diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs new file mode 100644 index 00000000..dab50a7b --- /dev/null +++ b/server/src/engine/core/index/row.rs @@ -0,0 +1,230 @@ +/* + * Created on Thu Apr 27 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::key::PrimaryIndexKey, + crate::{ + engine::{ + core::model::{DeltaState, DeltaVersion, SchemaDeltaKind}, + data::cell::Datacell, + idx::{meta::hash::HasherNativeFx, mtchm::meta::TreeElement, IndexST, STIndex}, + mem::RawStr, + sync::smart::RawRC, + }, + util::compiler, + }, + parking_lot::{RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard}, + std::mem::ManuallyDrop, +}; + +pub type DcFieldIndex = IndexST; + +#[derive(Debug)] +pub struct Row { + __pk: ManuallyDrop, + __rc: RawRC>, +} + +#[derive(Debug, PartialEq)] +pub struct RowData { + fields: DcFieldIndex, + txn_revised_data: DeltaVersion, + txn_revised_schema_version: DeltaVersion, + // pretty useless from an operational POV; only used during restore + restore_txn_id: DeltaVersion, +} + +impl RowData { + pub fn fields(&self) -> &DcFieldIndex { + &self.fields + } + pub fn fields_mut(&mut self) -> &mut DcFieldIndex { + &mut self.fields + } + pub fn set_txn_revised(&mut self, new: DeltaVersion) { + self.txn_revised_data = new; + } + pub fn get_txn_revised(&self) -> DeltaVersion { + self.txn_revised_data + } + pub fn get_restored_txn_revised(&self) -> DeltaVersion { + self.restore_txn_id + } +} + +impl TreeElement for Row { + type IKey = PrimaryIndexKey; + type Key = PrimaryIndexKey; + type IValue = DcFieldIndex; + type Value = RwLock; + type VEx1 = DeltaVersion; + type VEx2 = DeltaVersion; + fn key(&self) -> &Self::Key { + self.d_key() + } + fn val(&self) -> &Self::Value { + self.d_data() + } + fn new( + k: Self::Key, + v: Self::IValue, + txn_genesis: DeltaVersion, + txn_revised: DeltaVersion, + ) -> Self { + Self::new(k, v, txn_genesis, txn_revised) + } +} + +impl Row { + pub fn new( + pk: PrimaryIndexKey, + data: DcFieldIndex, + schema_version: DeltaVersion, + txn_revised_data: DeltaVersion, + ) -> Self { + Self::new_restored( + pk, + data, + schema_version, + txn_revised_data, + DeltaVersion::genesis(), + ) + } + pub fn new_restored( + pk: PrimaryIndexKey, + data: DcFieldIndex, + schema_version: DeltaVersion, + txn_revised_data: DeltaVersion, + restore_txn_id: DeltaVersion, + ) -> Self { + Self { + __pk: ManuallyDrop::new(pk), + __rc: unsafe { + // UNSAFE(@ohsayan): we free this up later + RawRC::new(RwLock::new(RowData { + fields: data, + txn_revised_schema_version: schema_version, + txn_revised_data, + // pretty useless here + restore_txn_id, + })) + }, + } + } + pub fn d_key(&self) -> &PrimaryIndexKey { + &self.__pk + } + pub fn d_data(&self) -> &RwLock { + self.__rc.data() + } + #[cfg(test)] + pub fn cloned_data(&self) -> Vec<(Box, Datacell)> { + self.d_data() + .read() + .fields() + .st_iter_kv() + .map(|(id, data)| (id.as_str().to_owned().into_boxed_str(), data.clone())) + .collect() + } +} + +impl Row { + /// Only apply deltas if a certain condition is met + pub fn resolve_schema_deltas_and_freeze_if<'g>( + &'g self, + delta_state: &DeltaState, + iff: impl Fn(&RowData) -> bool, + ) -> RwLockReadGuard<'g, RowData> { + let rwl_ug = self.d_data().upgradable_read(); + if !iff(&rwl_ug) { + return RwLockUpgradableReadGuard::downgrade(rwl_ug); + } + let current_version = delta_state.schema_current_version(); + if compiler::likely(current_version <= rwl_ug.txn_revised_schema_version) { + return RwLockUpgradableReadGuard::downgrade(rwl_ug); + } + // we have deltas to apply + let mut wl = RwLockUpgradableReadGuard::upgrade(rwl_ug); + let mut max_delta = wl.txn_revised_schema_version; + for (delta_id, delta) in delta_state.resolve_iter_since(wl.txn_revised_schema_version) { + match delta.kind() { + SchemaDeltaKind::FieldAdd(f) => { + wl.fields.st_insert( + unsafe { + // UNSAFE(@ohsayan): a row is inside a model and is valid as long as it is in there! + // even if the model was chucked and the row was lying around it won't cause any harm because it + // neither frees anything nor allocates + f.clone() + }, + Datacell::null(), + ); + } + SchemaDeltaKind::FieldRem(f) => { + wl.fields.st_delete(f); + } + } + max_delta = *delta_id; + } + // we've revised upto the most most recent delta version (that we saw at this point) + wl.txn_revised_schema_version = max_delta; + return RwLockWriteGuard::downgrade(wl); + } + pub fn resolve_schema_deltas_and_freeze<'g>( + &'g self, + delta_state: &DeltaState, + ) -> RwLockReadGuard<'g, RowData> { + self.resolve_schema_deltas_and_freeze_if(delta_state, |_| true) + } +} + +impl Clone for Row { + fn clone(&self) -> Self { + let rc = unsafe { + // UNSAFE(@ohsayan): we're calling this in the clone implementation + self.__rc.rc_clone() + }; + Self { + __pk: unsafe { + // UNSAFE(@ohsayan): this is safe because of the refcount + ManuallyDrop::new(self.__pk.raw_clone()) + }, + __rc: rc, + ..*self + } + } +} + +impl Drop for Row { + fn drop(&mut self) { + unsafe { + // UNSAFE(@ohsayan): we call in this the dtor itself + self.__rc.rc_drop(|| { + // UNSAFE(@ohsayan): we rely on the correctness of the rc + ManuallyDrop::drop(&mut self.__pk); + }); + } + } +} diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs new file mode 100644 index 00000000..83a4e8c9 --- /dev/null +++ b/server/src/engine/core/mod.rs @@ -0,0 +1,157 @@ +/* + * Created on Wed Oct 12 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +pub(in crate::engine) mod dcl; +pub(super) mod ddl_misc; +pub(in crate::engine) mod dml; +pub(in crate::engine) mod exec; +pub(in crate::engine) mod index; +pub(in crate::engine) mod model; +pub(in crate::engine) mod query_meta; +pub(in crate::engine) mod space; +// util +mod util; +// test +#[cfg(test)] +pub(super) mod tests; +// re-exports +pub use self::util::{EntityID, EntityIDRef}; +// imports +use { + self::{dml::QueryExecMeta, model::Model}, + super::fractal::GlobalInstanceLike, + crate::engine::{ + core::space::Space, + error::{QueryError, QueryResult}, + idx::IndexST, + }, + parking_lot::RwLock, + std::collections::HashMap, +}; + +/// Use this for now since it substitutes for a file lock (and those syscalls are expensive), +/// but something better is in the offing +type RWLIdx = RwLock>; + +#[cfg_attr(test, derive(Debug))] +pub struct GlobalNS { + idx_mdl: RWLIdx, + idx: RWLIdx, Space>, +} + +impl GlobalNS { + pub fn empty() -> Self { + Self { + idx_mdl: RWLIdx::default(), + idx: RWLIdx::default(), + } + } + pub fn ddl_with_all_mut( + &self, + f: impl FnOnce(&mut HashMap, Space>, &mut HashMap) -> T, + ) -> T { + let mut spaces = self.idx.write(); + let mut models = self.idx_mdl.write(); + f(&mut spaces, &mut models) + } + pub fn ddl_with_spaces_write( + &self, + f: impl FnOnce(&mut HashMap, Space>) -> T, + ) -> T { + let mut spaces = self.idx.write(); + f(&mut spaces) + } + pub fn ddl_with_space_mut( + &self, + space: &str, + f: impl FnOnce(&mut Space) -> QueryResult, + ) -> QueryResult { + let mut spaces = self.idx.write(); + let Some(space) = spaces.get_mut(space) else { + return Err(QueryError::QExecObjectNotFound); + }; + f(space) + } + pub fn with_model_space_mut_for_ddl<'a, T, F>( + &self, + entity: EntityIDRef<'a>, + f: F, + ) -> QueryResult + where + F: FnOnce(&Space, &mut Model) -> QueryResult, + { + let mut mdl_idx = self.idx_mdl.write(); + let Some(model) = mdl_idx.get_mut(&entity) else { + return Err(QueryError::QExecObjectNotFound); + }; + let space_read = self.idx.read(); + let space = space_read.get(entity.space()).unwrap(); + f(space, model) + } + pub fn with_model<'a, T, F>(&self, entity: EntityIDRef<'a>, f: F) -> QueryResult + where + F: FnOnce(&Model) -> QueryResult, + { + let mdl_idx = self.idx_mdl.read(); + let Some(model) = mdl_idx.get(&entity) else { + return Err(QueryError::QExecObjectNotFound); + }; + f(model) + } + pub fn idx_models(&self) -> &RWLIdx { + &self.idx_mdl + } + pub fn idx(&self) -> &RWLIdx, Space> { + &self.idx + } + #[cfg(test)] + pub fn create_empty_test_space(&self, space_name: &str) { + let _ = self + .idx() + .write() + .insert(space_name.into(), Space::new_auto_all().into()); + } + pub fn contains_space(&self, name: &str) -> bool { + self.idx.read().contains_key(name) + } +} + +pub(self) fn with_model_for_data_update<'a, F>( + global: &impl GlobalInstanceLike, + entity: EntityIDRef<'a>, + f: F, +) -> QueryResult<()> +where + F: FnOnce(&Model) -> QueryResult, +{ + let mdl_idx = global.namespace().idx_mdl.read(); + let Some(model) = mdl_idx.get(&entity) else { + return Err(QueryError::QExecObjectNotFound); + }; + let r = f(model)?; + model::DeltaState::guard_delta_overflow(global, entity.space(), entity.entity(), model, r); + Ok(()) +} diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs new file mode 100644 index 00000000..3a8dbabc --- /dev/null +++ b/server/src/engine/core/model/alt.rs @@ -0,0 +1,325 @@ +/* + * Created on Sun Mar 05 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{Field, Layer, Model}, + crate::{ + engine::{ + core::EntityIDRef, + data::{ + tag::{DataTag, TagClass}, + DictEntryGeneric, + }, + error::{QueryError, QueryResult}, + fractal::GlobalInstanceLike, + idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq}, + ql::{ + ddl::{ + alt::{AlterKind, AlterModel}, + syn::{ExpandedField, LayerSpec}, + }, + lex::Ident, + }, + txn::gns as gnstxn, + }, + util, + }, + std::collections::{HashMap, HashSet}, +}; + +#[derive(Debug, PartialEq)] +pub(in crate::engine::core) struct AlterPlan<'a> { + pub(in crate::engine::core) model: EntityIDRef<'a>, + pub(in crate::engine::core) no_lock: bool, + pub(in crate::engine::core) action: AlterAction<'a>, +} + +#[derive(Debug, PartialEq)] +pub(in crate::engine::core) enum AlterAction<'a> { + Ignore, + Add(IndexSTSeqCns, Field>), + Update(IndexST, Field>), + Remove(Box<[Ident<'a>]>), +} + +macro_rules! can_ignore { + (AlterAction::$variant:ident($expr:expr)) => { + if crate::engine::mem::StatelessLen::stateless_empty(&$expr) { + AlterAction::Ignore + } else { + AlterAction::$variant($expr) + } + }; +} + +#[inline(always)] +fn no_field(mr: &Model, new: &str) -> bool { + !mr.fields().st_contains(new) +} + +fn check_nullable(props: &mut HashMap, DictEntryGeneric>) -> QueryResult { + match props.remove("nullable") { + Some(DictEntryGeneric::Data(b)) if b.kind() == TagClass::Bool => Ok(b.bool()), + Some(_) => Err(QueryError::QExecDdlInvalidProperties), + None => Ok(false), + } +} + +impl<'a> AlterPlan<'a> { + pub fn fdeltas( + mdl: &Model, + AlterModel { model, kind }: AlterModel<'a>, + ) -> QueryResult> { + let mut no_lock = true; + let mut okay = true; + let action = match kind { + AlterKind::Remove(r) => { + let mut x = HashSet::new(); + if !r.iter().all(|id| x.insert(id.as_str())) { + return Err(QueryError::QExecDdlModelAlterIllegal); + } + let mut not_found = false; + if r.iter().all(|id| { + let not_pk = mdl.not_pk(id); + let exists = !no_field(mdl, id.as_str()); + not_found = !exists; + not_pk & exists + }) { + can_ignore!(AlterAction::Remove(r)) + } else if not_found { + return Err(QueryError::QExecUnknownField); + } else { + return Err(QueryError::QExecDdlModelAlterIllegal); + } + } + AlterKind::Add(new_fields) => { + let mut fields = util::bx_to_vec(new_fields).into_iter(); + let mut add = IndexSTSeqCns::with_capacity(fields.len()); + while (fields.len() != 0) & okay { + let ExpandedField { + field_name, + layers, + mut props, + } = fields.next().unwrap(); + okay &= no_field(mdl, &field_name) & mdl.not_pk(&field_name); + let is_nullable = check_nullable(&mut props)?; + let layers = Field::parse_layers(layers, is_nullable)?; + okay &= add.st_insert(field_name.as_str().into(), layers); + } + can_ignore!(AlterAction::Add(add)) + } + AlterKind::Update(updated_fields) => { + let updated_fields = util::bx_to_vec::>(updated_fields); + let mut updated_fields = updated_fields.into_iter(); + let mut any_delta = 0; + let mut new_fields = IndexST::new(); + while (updated_fields.len() != 0) & okay { + let ExpandedField { + field_name, + layers, + mut props, + } = updated_fields.next().unwrap(); + // enforce pk + mdl.guard_pk(&field_name)?; + // get the current field + let Some(current_field) = mdl.fields().st_get(field_name.as_str()) else { + return Err(QueryError::QExecUnknownField); + }; + // check props + let is_nullable = check_nullable(&mut props)?; + okay &= props.is_empty(); + // check layers + let (anydelta, new_field) = + Self::ldeltas(current_field, layers, is_nullable, &mut no_lock, &mut okay)?; + any_delta += anydelta as usize; + okay &= new_fields.st_insert(field_name.as_str().into(), new_field); + } + if any_delta == 0 { + AlterAction::Ignore + } else { + AlterAction::Update(new_fields) + } + } + }; + if okay { + Ok(Self { + model, + action, + no_lock, + }) + } else { + Err(QueryError::QExecDdlModelAlterIllegal) + } + } + fn ldeltas( + current: &Field, + layers: Vec>, + nullable: bool, + super_nlck: &mut bool, + super_okay: &mut bool, + ) -> QueryResult<(bool, Field)> { + #[inline(always)] + fn classeq(current: &Layer, new: &Layer, class: TagClass) -> bool { + // KIDDOS, LEARN SOME RELATIONS BEFORE WRITING CODE + (current.tag.tag_class() == new.tag.tag_class()) & (current.tag.tag_class() == class) + } + #[inline(always)] + fn interop(current: &Layer, new: &Layer) -> bool { + classeq(current, new, TagClass::UnsignedInt) + | classeq(current, new, TagClass::SignedInt) + | classeq(current, new, TagClass::Float) + } + if layers.len() > current.layers().len() { + // simply a dumb tomato; ELIMINATE THESE DUMB TOMATOES + return Err(QueryError::QExecDdlModelAlterIllegal); + } + let mut no_lock = !(current.is_nullable() & !nullable); + let mut deltasize = (current.is_nullable() ^ nullable) as usize; + let mut okay = true; + let mut new_field = current.clone(); + new_field.nullable = nullable; + let mut zipped_layers = layers + .into_iter() + .rev() + .zip(current.layers()) + .zip(new_field.layers.iter_mut()); + // check all layers + while (zipped_layers.len() != 0) & okay { + let ((LayerSpec { ty, props }, current_layer), new_layer) = + zipped_layers.next().unwrap(); + // actually parse the new layer + okay &= props.is_empty(); + let Some(new_parsed_layer) = Layer::get_layer(&ty) else { + return Err(QueryError::QExecDdlInvalidTypeDefinition); + }; + match ( + current_layer.tag.tag_selector(), + new_parsed_layer.tag.tag_selector(), + ) { + (current_tag, new_tag) if current_tag == new_tag => { + // no delta + } + (current_selector, new_selector) if interop(current_layer, &new_parsed_layer) => { + // now, we're not sure if we can run this + // FIXME(@ohsayan): look, should we be explicit about this? + no_lock &= new_selector >= current_selector; + deltasize += (new_selector != current_selector) as usize; + } + _ => { + // can't cast this directly + return Err(QueryError::QExecDdlInvalidTypeDefinition); + } + } + *new_layer = new_parsed_layer; + } + *super_nlck &= no_lock; + *super_okay &= okay; + if okay { + Ok((deltasize != 0, new_field)) + } else { + Err(QueryError::QExecDdlModelAlterIllegal) + } + } +} + +impl Model { + pub fn transactional_exec_alter( + global: &G, + alter: AlterModel, + ) -> QueryResult<()> { + let (space_name, model_name) = (alter.model.space(), alter.model.entity()); + global + .namespace() + .with_model_space_mut_for_ddl(alter.model, |space, model| { + // prepare plan + let plan = AlterPlan::fdeltas(model, alter)?; + // we have a legal plan; acquire exclusive if we need it + if !plan.no_lock { + // TODO(@ohsayan): allow this later on, once we define the syntax + return Err(QueryError::QExecNeedLock); + } + // fine, we're good + match plan.action { + AlterAction::Ignore => {} + AlterAction::Add(new_fields) => { + // TODO(@ohsayan): this impacts lockdown duration; fix it + if G::FS_IS_NON_NULL { + // prepare txn + let txn = gnstxn::AlterModelAddTxn::new( + gnstxn::ModelIDRef::new_ref( + &space_name, + &space, + &model_name, + model, + ), + &new_fields, + ); + // commit txn + global.namespace_txn_driver().lock().try_commit(txn)?; + } + let mut mutator = model.model_mutator(); + new_fields + .stseq_ord_kv() + .map(|(x, y)| (x.clone(), y.clone())) + .for_each(|(field_id, field)| { + mutator.add_field(field_id, field); + }); + } + AlterAction::Remove(removed) => { + if G::FS_IS_NON_NULL { + // prepare txn + let txn = gnstxn::AlterModelRemoveTxn::new( + gnstxn::ModelIDRef::new_ref(&space_name, space, &model_name, model), + &removed, + ); + // commit txn + global.namespace_txn_driver().lock().try_commit(txn)?; + } + let mut mutator = model.model_mutator(); + removed.iter().for_each(|field_id| { + mutator.remove_field(field_id.as_str()); + }); + } + AlterAction::Update(updated) => { + if G::FS_IS_NON_NULL { + // prepare txn + let txn = gnstxn::AlterModelUpdateTxn::new( + gnstxn::ModelIDRef::new_ref(&space_name, space, &model_name, model), + &updated, + ); + // commit txn + global.namespace_txn_driver().lock().try_commit(txn)?; + } + let mut mutator = model.model_mutator(); + updated.into_iter().for_each(|(field_id, field)| { + mutator.update_field(field_id.as_ref(), field); + }); + } + } + Ok(()) + }) + } +} diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs new file mode 100644 index 00000000..e902f073 --- /dev/null +++ b/server/src/engine/core/model/delta.rs @@ -0,0 +1,234 @@ +/* + * Created on Sat May 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::Model, + crate::engine::{ + core::{dml::QueryExecMeta, index::Row}, + fractal::{FractalToken, GlobalInstanceLike}, + mem::RawStr, + sync::atm::Guard, + sync::queue::Queue, + }, + std::{ + collections::btree_map::{BTreeMap, Range}, + sync::atomic::{AtomicU64, AtomicUsize, Ordering}, + }, +}; + +#[derive(Debug)] +/// A delta state for the model +pub struct DeltaState { + // schema + schema_current_version: u64, + schema_deltas: BTreeMap, + // data + data_current_version: AtomicU64, + data_deltas: Queue, + data_deltas_size: AtomicUsize, +} + +impl DeltaState { + /// A new, fully resolved delta state with version counters set to 0 + pub fn new_resolved() -> Self { + Self { + schema_current_version: 0, + schema_deltas: BTreeMap::new(), + data_current_version: AtomicU64::new(0), + data_deltas: Queue::new(), + data_deltas_size: AtomicUsize::new(0), + } + } +} + +// data direct +impl DeltaState { + pub(in crate::engine::core) fn guard_delta_overflow( + global: &impl GlobalInstanceLike, + space_name: &str, + model_name: &str, + model: &Model, + hint: QueryExecMeta, + ) { + global.request_batch_resolve_if_cache_full(space_name, model_name, model, hint) + } +} + +// data +impl DeltaState { + pub fn append_new_data_delta_with( + &self, + kind: DataDeltaKind, + row: Row, + data_version: DeltaVersion, + g: &Guard, + ) -> usize { + self.append_new_data_delta(DataDelta::new(data_version, row, kind), g) + } + pub fn append_new_data_delta(&self, delta: DataDelta, g: &Guard) -> usize { + self.data_deltas.blocking_enqueue(delta, g); + self.data_deltas_size.fetch_add(1, Ordering::Release) + 1 + } + pub fn create_new_data_delta_version(&self) -> DeltaVersion { + DeltaVersion(self.__data_delta_step()) + } +} + +impl DeltaState { + fn __data_delta_step(&self) -> u64 { + self.data_current_version.fetch_add(1, Ordering::AcqRel) + } + pub fn __data_delta_dequeue(&self, g: &Guard) -> Option { + self.data_deltas.blocking_try_dequeue(g) + } +} + +// schema +impl DeltaState { + pub fn resolve_iter_since( + &self, + current_version: DeltaVersion, + ) -> Range { + self.schema_deltas.range(current_version.step()..) + } + pub fn schema_current_version(&self) -> DeltaVersion { + DeltaVersion(self.schema_current_version) + } + pub fn unresolved_append_field_add(&mut self, field_name: RawStr) { + self.__schema_append_unresolved_delta(SchemaDeltaPart::field_add(field_name)); + } + pub fn unresolved_append_field_rem(&mut self, field_name: RawStr) { + self.__schema_append_unresolved_delta(SchemaDeltaPart::field_rem(field_name)); + } +} + +impl DeltaState { + fn __schema_delta_step(&mut self) -> DeltaVersion { + let current = self.schema_current_version; + self.schema_current_version += 1; + DeltaVersion(current) + } + fn __schema_append_unresolved_delta(&mut self, part: SchemaDeltaPart) -> DeltaVersion { + let v = self.__schema_delta_step(); + self.schema_deltas.insert(v, part); + v + } +} + +// fractal +impl DeltaState { + pub fn __fractal_take_full_from_data_delta(&self, _token: FractalToken) -> usize { + self.data_deltas_size.swap(0, Ordering::AcqRel) + } +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] +pub struct DeltaVersion(u64); +impl DeltaVersion { + pub const fn genesis() -> Self { + Self(0) + } + pub const fn __new(v: u64) -> Self { + Self(v) + } + fn step(&self) -> Self { + Self(self.0 + 1) + } + pub const fn value_u64(&self) -> u64 { + self.0 + } +} + +/* + schema delta +*/ + +#[derive(Debug)] +pub struct SchemaDeltaPart { + kind: SchemaDeltaKind, +} + +impl SchemaDeltaPart { + pub fn kind(&self) -> &SchemaDeltaKind { + &self.kind + } +} + +#[derive(Debug)] +pub enum SchemaDeltaKind { + FieldAdd(RawStr), + FieldRem(RawStr), +} + +impl SchemaDeltaPart { + fn new(kind: SchemaDeltaKind) -> Self { + Self { kind } + } + fn field_add(field_name: RawStr) -> Self { + Self::new(SchemaDeltaKind::FieldAdd(field_name)) + } + fn field_rem(field_name: RawStr) -> Self { + Self::new(SchemaDeltaKind::FieldRem(field_name)) + } +} + +/* + data delta +*/ + +#[derive(Debug, Clone)] +pub struct DataDelta { + data_version: DeltaVersion, + row: Row, + change: DataDeltaKind, +} + +impl DataDelta { + pub const fn new(data_version: DeltaVersion, row: Row, change: DataDeltaKind) -> Self { + Self { + data_version, + row, + change, + } + } + pub fn data_version(&self) -> DeltaVersion { + self.data_version + } + pub fn row(&self) -> &Row { + &self.row + } + pub fn change(&self) -> DataDeltaKind { + self.change + } +} + +#[derive(Debug, Clone, Copy, sky_macros::EnumMethods, PartialEq)] +#[repr(u8)] +pub enum DataDeltaKind { + Delete = 0, + Insert = 1, + Update = 2, +} diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs new file mode 100644 index 00000000..0f9b9d8d --- /dev/null +++ b/server/src/engine/core/model/mod.rs @@ -0,0 +1,761 @@ +/* + * Created on Mon Feb 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +pub(super) mod alt; +pub(in crate::engine) mod delta; + +#[cfg(test)] +use std::cell::RefCell; + +use { + super::index::PrimaryIndex, + crate::engine::{ + data::{ + cell::Datacell, + tag::{DataTag, FloatSpec, FullTag, SIntSpec, TagClass, TagSelector, UIntSpec}, + uuid::Uuid, + }, + error::{QueryError, QueryResult}, + fractal::{GenericTask, GlobalInstanceLike, Task}, + idx::{self, IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq}, + mem::{RawStr, VInline}, + ql::ddl::{ + crt::CreateModel, + drop::DropModel, + syn::{FieldSpec, LayerSpec}, + }, + txn::gns::{self as gnstxn, SpaceIDRef}, + }, + std::collections::hash_map::{Entry, HashMap}, +}; + +pub(in crate::engine::core) use self::delta::{DeltaState, DeltaVersion, SchemaDeltaKind}; + +use super::util::{EntityID, EntityIDRef}; +type Fields = IndexSTSeqCns; + +#[derive(Debug)] +pub struct Model { + uuid: Uuid, + p_key: RawStr, + p_tag: FullTag, + fields: Fields, + data: PrimaryIndex, + delta: DeltaState, + private: ModelPrivate, + decl: String, +} + +#[cfg(test)] +impl PartialEq for Model { + fn eq(&self, m: &Self) -> bool { + self.uuid == m.uuid + && self.p_key == m.p_key + && self.p_tag == m.p_tag + && self.fields == m.fields + } +} + +impl Model { + pub fn get_uuid(&self) -> Uuid { + self.uuid + } + pub fn p_key(&self) -> &str { + &self.p_key + } + pub fn p_tag(&self) -> FullTag { + self.p_tag + } + fn is_pk(&self, new: &str) -> bool { + self.p_key.as_bytes() == new.as_bytes() + } + fn not_pk(&self, new: &str) -> bool { + !self.is_pk(new) + } + fn guard_pk(&self, new: &str) -> QueryResult<()> { + if self.is_pk(new) { + Err(QueryError::QExecDdlModelAlterIllegal) + } else { + Ok(()) + } + } + pub fn primary_index(&self) -> &PrimaryIndex { + &self.data + } + pub fn delta_state(&self) -> &DeltaState { + &self.delta + } + pub fn fields(&self) -> &Fields { + &self.fields + } + pub fn model_mutator<'a>(&'a mut self) -> ModelMutator<'a> { + ModelMutator { model: self } + } + fn sync_decl(&mut self) { + self.decl = self.redescribe(); + } + pub fn describe(&self) -> &str { + &self.decl + } + fn redescribe(&self) -> String { + let mut ret = format!("{{"); + let mut it = self.fields().stseq_ord_kv().peekable(); + while let Some((field_name, field_decl)) = it.next() { + // legend: * -> primary, ! -> not null, ? -> null + if self.is_pk(&field_name) { + ret.push('*'); + } else if field_decl.is_nullable() { + ret.push('?'); + } else { + ret.push('!'); + } + ret.push_str(&field_name); + ret.push(':'); + // TODO(@ohsayan): it's all lists right now, so this is okay but fix it later + if field_decl.layers().len() == 1 { + ret.push_str(field_decl.layers()[0].tag().tag_selector().name_str()); + } else { + ret.push_str(&"[".repeat(field_decl.layers().len() - 1)); + ret.push_str( + field_decl.layers()[field_decl.layers().len() - 1] + .tag() + .tag_selector() + .name_str(), + ); + ret.push_str(&"]".repeat(field_decl.layers().len() - 1)) + } + if it.peek().is_some() { + ret.push(','); + } + } + ret.push('}'); + ret + } +} + +impl Model { + fn new_with_private( + uuid: Uuid, + p_key: RawStr, + p_tag: FullTag, + fields: Fields, + private: ModelPrivate, + ) -> Self { + let mut slf = Self { + uuid, + p_key, + p_tag, + fields, + data: PrimaryIndex::new_empty(), + delta: DeltaState::new_resolved(), + private, + decl: String::new(), + }; + slf.sync_decl(); + slf + } + pub fn new_restore( + uuid: Uuid, + p_key: Box, + p_tag: FullTag, + decl_fields: IndexSTSeqCns, Field>, + ) -> Self { + let mut private = ModelPrivate::empty(); + let p_key = unsafe { + // UNSAFE(@ohsayan): once again, all cool since we maintain the allocation + private.push_allocated(p_key) + }; + let mut fields = IndexSTSeqCns::idx_init(); + decl_fields + .stseq_owned_kv() + .map(|(field_key, field)| { + ( + unsafe { + // UNSAFE(@ohsayan): we ensure that priv is dropped iff model is dropped + private.push_allocated(field_key) + }, + field, + ) + }) + .for_each(|(field_key, field)| { + fields.st_insert(field_key, field); + }); + Self::new_with_private(uuid, p_key, p_tag, fields, private) + } + pub fn process_create( + CreateModel { + model_name: _, + fields, + props, + .. + }: CreateModel, + ) -> QueryResult { + let mut private = ModelPrivate::empty(); + let mut okay = props.is_empty() & !fields.is_empty(); + // validate fields + let mut field_spec = fields.into_iter(); + let mut fields = Fields::idx_init_cap(field_spec.len()); + let mut last_pk = None; + let mut pk_cnt = 0; + while (field_spec.len() != 0) & okay { + let FieldSpec { + field_name, + layers, + null, + primary, + } = field_spec.next().unwrap(); + let this_field_ptr = unsafe { + // UNSAFE(@ohsayan): this is going to go with our alloc, so we're good! if we fail too, the dtor for private will run + private.allocate_or_recycle(field_name.as_str()) + }; + if primary { + pk_cnt += 1usize; + last_pk = Some(unsafe { + // UNSAFE(@ohsayan): totally cool, it's all allocated + this_field_ptr.clone() + }); + okay &= !null; + } + let layer = Field::parse_layers(layers, null)?; + okay &= fields.st_insert(this_field_ptr, layer); + } + okay &= pk_cnt <= 1; + if okay { + let last_pk = last_pk.unwrap_or(unsafe { + // UNSAFE(@ohsayan): once again, all of this is allocated + fields.stseq_ord_key().next().unwrap().clone() + }); + let tag = fields.st_get(&last_pk).unwrap().layers()[0].tag; + if tag.tag_unique().is_unique() { + return Ok(Self::new_with_private( + Uuid::new(), + last_pk, + tag, + fields, + private, + )); + } + } + Err(QueryError::QExecDdlModelBadDefinition) + } +} + +impl Model { + pub fn transactional_exec_create( + global: &G, + stmt: CreateModel, + ) -> QueryResult> { + let (space_name, model_name) = (stmt.model_name.space(), stmt.model_name.entity()); + let if_nx = stmt.if_not_exists; + let model = Self::process_create(stmt)?; + global.namespace().ddl_with_space_mut(&space_name, |space| { + // TODO(@ohsayan): be extra cautious with post-transactional tasks (memck) + if space.models().contains(model_name) { + if if_nx { + return Ok(Some(false)); + } else { + return Err(QueryError::QExecDdlObjectAlreadyExists); + } + } + // since we've locked this down, no one else can parallely create another model in the same space (or remove) + if G::FS_IS_NON_NULL { + let mut txn_driver = global.namespace_txn_driver().lock(); + // prepare txn + let txn = gnstxn::CreateModelTxn::new( + SpaceIDRef::new(&space_name, &space), + &model_name, + &model, + ); + // attempt to initialize driver + global.initialize_model_driver( + &space_name, + space.get_uuid(), + &model_name, + model.get_uuid(), + )?; + // commit txn + match txn_driver.try_commit(txn) { + Ok(()) => {} + Err(e) => { + // failed to commit, request cleanup + global.taskmgr_post_standard_priority(Task::new( + GenericTask::delete_model_dir( + &space_name, + space.get_uuid(), + &model_name, + model.get_uuid(), + ), + )); + return Err(e.into()); + } + } + } + // update global state + let _ = space.models_mut().insert(model_name.into()); + let _ = global + .namespace() + .idx_models() + .write() + .insert(EntityID::new(&space_name, &model_name), model); + if if_nx { + Ok(Some(true)) + } else { + Ok(None) + } + }) + } + pub fn transactional_exec_drop( + global: &G, + stmt: DropModel, + ) -> QueryResult> { + let (space_name, model_name) = (stmt.entity.space(), stmt.entity.entity()); + global.namespace().ddl_with_space_mut(&space_name, |space| { + if !space.models().contains(model_name) { + if stmt.if_exists { + return Ok(Some(false)); + } else { + // the model isn't even present + return Err(QueryError::QExecObjectNotFound); + } + } + // get exclusive lock on models + let mut models_idx = global.namespace().idx_models().write(); + let model = models_idx + .get(&EntityIDRef::new(&space_name, &model_name)) + .unwrap(); + // the model must be empty for us to clean it up! (NB: consistent view + EX) + if (model.primary_index().count() != 0) & !(stmt.force) { + // nope, we can't drop this + return Err(QueryError::QExecDdlNotEmpty); + } + // okay this is looking good for us + if G::FS_IS_NON_NULL { + // prepare txn + let txn = gnstxn::DropModelTxn::new(gnstxn::ModelIDRef::new( + SpaceIDRef::new(&space_name, &space), + &model_name, + model.get_uuid(), + model.delta_state().schema_current_version().value_u64(), + )); + // commit txn + global.namespace_txn_driver().lock().try_commit(txn)?; + // request cleanup + global.purge_model_driver( + space_name, + space.get_uuid(), + model_name, + model.get_uuid(), + false, + ); + } + // update global state + let _ = models_idx.remove(&EntityIDRef::new(&space_name, &model_name)); + let _ = space.models_mut().remove(model_name); + if stmt.if_exists { + Ok(Some(true)) + } else { + Ok(None) + } + }) + } +} + +#[derive(Debug, PartialEq)] +struct ModelPrivate { + alloc: HashMap, bool, idx::meta::hash::HasherNativeFx>, +} + +impl ModelPrivate { + fn empty() -> Self { + Self { + alloc: HashMap::with_hasher(Default::default()), + } + } + pub(self) unsafe fn allocate_or_recycle(&mut self, new: &str) -> RawStr { + match self.alloc.get_key_value(new) { + Some((prev_alloc, _)) => { + // already allocated this + let ret = RawStr::new(prev_alloc.as_ptr(), prev_alloc.len()); + // set live! + *self.alloc.get_mut(ret.as_str()).unwrap() = false; + return ret; + } + None => { + // need to allocate + let alloc = new.to_owned().into_boxed_str(); + let ret = RawStr::new(alloc.as_ptr(), alloc.len()); + let _ = self.alloc.insert(alloc, false); + return ret; + } + } + } + pub(self) unsafe fn mark_pending_remove(&mut self, v: &str) -> RawStr { + let ret = self.alloc.get_key_value(v).unwrap().0; + let ret = RawStr::new(ret.as_ptr(), ret.len()); + *self.alloc.get_mut(v).unwrap() = true; + ret + } + pub(self) unsafe fn vacuum_marked(&mut self) { + self.alloc.retain(|_, dead| !*dead) + } + pub(self) unsafe fn push_allocated(&mut self, alloc: Box) -> RawStr { + match self.alloc.entry(alloc) { + Entry::Occupied(mut oe) => { + oe.insert(false); + RawStr::new(oe.key().as_ptr(), oe.key().len()) + } + Entry::Vacant(ve) => { + let ret = RawStr::new(ve.key().as_ptr(), ve.key().len()); + ve.insert(false); + return ret; + } + } + } +} + +pub struct ModelMutator<'a> { + model: &'a mut Model, +} + +impl<'a> ModelMutator<'a> { + pub unsafe fn vacuum_stashed(&mut self) { + self.model.private.vacuum_marked() + } + pub fn remove_field(&mut self, name: &str) -> bool { + // remove + let r = self.model.fields.st_delete(name); + // recycle + let ptr = unsafe { self.model.private.mark_pending_remove(name) }; + // publish delta + self.model.delta.unresolved_append_field_rem(ptr); + r + } + pub fn add_field(&mut self, name: Box, field: Field) -> bool { + unsafe { + // allocate + let fkeyptr = self.model.private.push_allocated(name); + // add + let r = self.model.fields.st_insert(fkeyptr.clone(), field); + // delta + self.model.delta.unresolved_append_field_add(fkeyptr); + r + } + } + pub fn update_field(&mut self, name: &str, field: Field) -> bool { + self.model.fields.st_update(name, field) + } +} + +impl<'a> Drop for ModelMutator<'a> { + fn drop(&mut self) { + self.model.sync_decl(); + } +} + +/* + Layer +*/ + +static G: [u8; 15] = [0, 13, 12, 5, 6, 4, 3, 6, 1, 10, 4, 5, 7, 5, 5]; +static S1: [u8; 7] = [13, 9, 4, 14, 2, 4, 7]; +static S2: [u8; 7] = [12, 8, 2, 6, 4, 9, 9]; + +static LUT: [(&str, FullTag); 14] = [ + ("bool", FullTag::BOOL), + ("uint8", FullTag::new_uint(TagSelector::UInt8)), + ("uint16", FullTag::new_uint(TagSelector::UInt16)), + ("uint32", FullTag::new_uint(TagSelector::UInt32)), + ("uint64", FullTag::new_uint(TagSelector::UInt64)), + ("sint8", FullTag::new_sint(TagSelector::SInt8)), + ("sint16", FullTag::new_sint(TagSelector::SInt16)), + ("sint32", FullTag::new_sint(TagSelector::SInt32)), + ("sint64", FullTag::new_sint(TagSelector::SInt64)), + ("float32", FullTag::new_float(TagSelector::Float32)), + ("float64", FullTag::new_float(TagSelector::Float64)), + ("binary", FullTag::BIN), + ("string", FullTag::STR), + ("list", FullTag::LIST), +]; + +#[cfg(test)] +pub static TY_BOOL: &str = LUT[0].0; +#[cfg(test)] +pub static TY_UINT: [&str; 4] = [LUT[1].0, LUT[2].0, LUT[3].0, LUT[4].0]; +#[cfg(test)] +pub static TY_SINT: [&str; 4] = [LUT[5].0, LUT[6].0, LUT[7].0, LUT[8].0]; +#[cfg(test)] +pub static TY_FLOAT: [&str; 2] = [LUT[9].0, LUT[10].0]; +#[cfg(test)] +pub static TY_BINARY: &str = LUT[11].0; +#[cfg(test)] +pub static TY_STRING: &str = LUT[12].0; +#[cfg(test)] +pub static TY_LIST: &str = LUT[13].0; + +#[derive(Debug, PartialEq, Clone)] +pub struct Field { + layers: VInline<1, Layer>, + nullable: bool, +} + +impl Field { + pub fn new(layers: VInline<1, Layer>, nullable: bool) -> Self { + Self { layers, nullable } + } + pub fn is_nullable(&self) -> bool { + self.nullable + } + pub fn layers(&self) -> &[Layer] { + &self.layers + } + pub fn parse_layers(spec: Vec, nullable: bool) -> QueryResult { + let mut layers = spec.into_iter().rev(); + let mut okay = true; + let mut fin = false; + let mut layerview = VInline::new(); + while (layers.len() != 0) & okay & !fin { + let LayerSpec { ty, props } = layers.next().unwrap(); + okay &= props.is_empty(); // FIXME(@ohsayan): you know what to do here + match Layer::get_layer(&ty) { + Some(l) => { + fin = l.tag.tag_selector() != TagSelector::List; + layerview.push(l); + } + None => okay = false, + } + } + okay &= fin & (layers.len() == 0); + if okay { + Ok(Self { + layers: layerview, + nullable, + }) + } else { + Err(QueryError::QExecDdlInvalidTypeDefinition) + } + } + #[inline(always)] + fn compute_index(&self, dc: &Datacell) -> usize { + if { + ((!self.is_nullable()) & dc.is_null()) + | ((self.layers[0].tag.tag_class() != dc.kind()) & !dc.is_null()) + } { + // illegal states: (1) bad null (2) tags don't match + 7 + } else { + dc.kind().value_word() + } + } + pub fn vt_data_fpath(&self, data: &mut Datacell) -> bool { + if (self.layers.len() == 1) | (data.is_null()) { + layertrace("fpath"); + unsafe { VTFN[self.compute_index(data)](self.layers()[0], data) } + } else { + Self::rvt_data(self.layers(), data) + } + } + fn rvt_data(layers: &[Layer], data: &mut Datacell) -> bool { + let layer = layers[0]; + let layers = &layers[1..]; + match (layer.tag().tag_class(), data.kind()) { + (TagClass::List, TagClass::List) => { + let mut okay = unsafe { + // UNSAFE(@ohsayan): +tagck + VTFN[TagClass::List.value_word()](layer, data) + }; + let list = unsafe { + // UNSAFE(@ohsayan): +tagck + data.read_list() + }; + let mut lread = list.write(); + let mut i = 0; + while (i < lread.len()) & okay { + okay &= Self::rvt_data(layers, &mut lread[i]); + i += 1; + } + okay + } + (tag_a, tag_b) if tag_a == tag_b => { + unsafe { + // UNSAFE(@ohsayan): same tags and lists have non-null elements + VTFN[tag_a.value_word()](layer, data) + } + } + _ => false, + } + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub struct Layer { + tag: FullTag, +} + +#[allow(unused)] +impl Layer { + pub const fn bool() -> Self { + Self::empty(FullTag::BOOL) + } + pub const fn uint8() -> Self { + Self::empty(FullTag::new_uint(TagSelector::UInt8)) + } + pub const fn uint16() -> Self { + Self::empty(FullTag::new_uint(TagSelector::UInt16)) + } + pub const fn uint32() -> Self { + Self::empty(FullTag::new_uint(TagSelector::UInt32)) + } + pub const fn uint64() -> Self { + Self::empty(FullTag::new_uint(TagSelector::UInt64)) + } + pub const fn sint8() -> Self { + Self::empty(FullTag::new_sint(TagSelector::SInt8)) + } + pub const fn sint16() -> Self { + Self::empty(FullTag::new_sint(TagSelector::SInt16)) + } + pub const fn sint32() -> Self { + Self::empty(FullTag::new_sint(TagSelector::SInt32)) + } + pub const fn sint64() -> Self { + Self::empty(FullTag::new_sint(TagSelector::SInt64)) + } + pub const fn float32() -> Self { + Self::empty(FullTag::new_float(TagSelector::Float32)) + } + pub const fn float64() -> Self { + Self::empty(FullTag::new_float(TagSelector::Float64)) + } + pub const fn bin() -> Self { + Self::empty(FullTag::BIN) + } + pub const fn str() -> Self { + Self::empty(FullTag::STR) + } + pub const fn list() -> Self { + Self::empty(FullTag::LIST) + } +} + +impl Layer { + pub fn tag(&self) -> FullTag { + self.tag + } + pub fn new_empty_props(tag: FullTag) -> Self { + Self::new(tag) + } + pub const fn new(tag: FullTag) -> Self { + Self { tag } + } + const fn empty(tag: FullTag) -> Self { + Self::new(tag) + } + fn hf(key: &[u8], v: [u8; 7]) -> u16 { + let mut tot = 0; + let mut i = 0; + while i < key.len() { + tot += v[i % v.len()] as u16 * key[i] as u16; + i += 1; + } + tot % 15 + } + fn pf(key: &[u8]) -> u16 { + (G[Self::hf(key, S1) as usize] as u16 + G[Self::hf(key, S2) as usize] as u16) % 15 + } + fn get_layer(ident: &str) -> Option { + let idx = Self::pf(ident.as_bytes()) as usize; + if idx < LUT.len() && LUT[idx].0 == ident { + Some(Self::empty(LUT[idx].1)) + } else { + None + } + } +} + +#[cfg(test)] +thread_local! { + static LAYER_TRACE: RefCell>> = RefCell::new(Vec::new()); +} + +#[inline(always)] +fn layertrace(_layer: impl ToString) { + #[cfg(test)] + { + LAYER_TRACE.with(|v| v.borrow_mut().push(_layer.to_string().into())); + } +} + +#[cfg(test)] +/// Obtain a layer trace and clear older traces +pub(super) fn layer_traces() -> Box<[Box]> { + LAYER_TRACE.with(|x| { + let ret = x.borrow().iter().cloned().collect(); + x.borrow_mut().clear(); + ret + }) +} + +static VTFN: [unsafe fn(Layer, &mut Datacell) -> bool; 8] = [ + vt_bool, + vt_uint, + vt_sint, + vt_float, + vt_bin, + vt_str, + vt_list, + |_, _| false, +]; +unsafe fn vt_bool(_: Layer, _: &mut Datacell) -> bool { + layertrace("bool"); + true +} +unsafe fn vt_uint(l: Layer, dc: &mut Datacell) -> bool { + layertrace("uint"); + dc.set_tag(l.tag()); + UIntSpec::from_full(l.tag()).check(dc.read_uint()) +} +unsafe fn vt_sint(l: Layer, dc: &mut Datacell) -> bool { + layertrace("sint"); + dc.set_tag(l.tag()); + SIntSpec::from_full(l.tag()).check(dc.read_sint()) +} +unsafe fn vt_float(l: Layer, dc: &mut Datacell) -> bool { + layertrace("float"); + dc.set_tag(l.tag()); + FloatSpec::from_full(l.tag()).check(dc.read_float()) +} +unsafe fn vt_bin(_: Layer, _: &mut Datacell) -> bool { + layertrace("binary"); + true +} +unsafe fn vt_str(_: Layer, _: &mut Datacell) -> bool { + layertrace("string"); + true +} +unsafe fn vt_list(_: Layer, _: &mut Datacell) -> bool { + layertrace("list"); + true +} diff --git a/server/src/actions/lists/macros.rs b/server/src/engine/core/query_meta.rs similarity index 71% rename from server/src/actions/lists/macros.rs rename to server/src/engine/core/query_meta.rs index fa428330..c85b1fe7 100644 --- a/server/src/actions/lists/macros.rs +++ b/server/src/engine/core/query_meta.rs @@ -1,5 +1,5 @@ /* - * Created on Wed Sep 15 2021 + * Created on Fri May 12 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2021, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,12 +24,12 @@ * */ -macro_rules! writelist { - ($con:expr, $listmap:expr, $items:expr) => {{ - $con.write_typed_non_null_array_header($items.len(), $listmap.get_value_tsymbol()) - .await?; - for item in $items { - $con.write_typed_non_null_array_element(&item).await?; - } - }}; +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)] +#[repr(u8)] +pub enum AssignmentOperator { + Assign = 0, + AddAssign = 1, + SubAssign = 2, + MulAssign = 3, + DivAssign = 4, } diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs new file mode 100644 index 00000000..5567d7ed --- /dev/null +++ b/server/src/engine/core/space.rs @@ -0,0 +1,319 @@ +/* + * Created on Mon Feb 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use super::EntityIDRef; + +use { + crate::engine::{ + data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric}, + error::{QueryError, QueryResult}, + fractal::{GenericTask, GlobalInstanceLike, Task}, + idx::STIndex, + ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace}, + storage::v1::{loader::SEInitState, RawFSInterface}, + txn::gns as gnstxn, + }, + std::collections::HashSet, +}; + +#[derive(Debug, PartialEq)] +pub struct Space { + uuid: Uuid, + models: HashSet>, + props: DictGeneric, +} + +#[derive(Debug, PartialEq)] +/// Procedure for `create space` +struct ProcedureCreate { + space_name: Box, + space: Space, + if_not_exists: bool, +} + +impl Space { + pub fn new(uuid: Uuid, models: HashSet>, props: DictGeneric) -> Self { + Self { + uuid, + models, + props, + } + } + #[cfg(test)] + pub fn new_auto_all() -> Self { + Self::new_auto(Default::default(), Default::default()) + } + pub fn get_uuid(&self) -> Uuid { + self.uuid + } + pub fn new_restore_empty(uuid: Uuid, props: DictGeneric) -> Self { + Self::new(uuid, Default::default(), props) + } + pub fn new_empty_auto(props: DictGeneric) -> Self { + Self::new_auto(Default::default(), props) + } + pub fn new_auto(models: HashSet>, props: DictGeneric) -> Self { + Self::new(Uuid::new(), models, props) + } + pub fn models(&self) -> &HashSet> { + &self.models + } + pub fn models_mut(&mut self) -> &mut HashSet> { + &mut self.models + } + pub fn props(&self) -> &DictGeneric { + &self.props + } + pub fn props_mut(&mut self) -> &mut DictGeneric { + &mut self.props + } + #[cfg(test)] + pub fn env(&self) -> &DictGeneric { + match self.props().get(Self::KEY_ENV).unwrap() { + DictEntryGeneric::Map(m) => m, + _ => panic!(), + } + } +} + +impl Space { + const KEY_ENV: &'static str = "env"; + #[inline] + /// Validate a `create` stmt + fn process_create( + CreateSpace { + space_name, + mut props, + if_not_exists, + }: CreateSpace, + ) -> QueryResult { + let space_name = space_name.to_string().into_boxed_str(); + // now let's check our props + match props.get(Self::KEY_ENV) { + Some(d) if props.len() == 1 => { + match d { + DictEntryGeneric::Data(d) if d.is_init() => { + // not the right type for a dict + return Err(QueryError::QExecDdlInvalidProperties); + } + DictEntryGeneric::Data(_) => { + // a null? make it empty + let _ = + props.insert(Self::KEY_ENV.into(), DictEntryGeneric::Map(into_dict!())); + } + DictEntryGeneric::Map(_) => {} + } + } + None if props.is_empty() => { + let _ = props.st_insert(Self::KEY_ENV.into(), DictEntryGeneric::Map(into_dict!())); + } + _ => { + // in all the other cases, we have illegal properties + // not the right type for a dict + return Err(QueryError::QExecDdlInvalidProperties); + } + } + Ok(ProcedureCreate { + space_name, + space: Space::new_empty_auto(dict::rflatten_metadata(props)), + if_not_exists, + }) + } +} + +impl Space { + pub fn transactional_exec_create( + global: &G, + space: CreateSpace, + ) -> QueryResult> { + // process create + let ProcedureCreate { + space_name, + space, + if_not_exists, + } = Self::process_create(space)?; + // lock the global namespace + global.namespace().ddl_with_spaces_write(|spaces| { + if spaces.st_contains(&space_name) { + if if_not_exists { + return Ok(Some(false)); + } else { + return Err(QueryError::QExecDdlObjectAlreadyExists); + } + } + // commit txn + if G::FS_IS_NON_NULL { + // prepare txn + let txn = gnstxn::CreateSpaceTxn::new(space.props(), &space_name, &space); + // try to create space for...the space + G::FileSystem::fs_create_dir_all(&SEInitState::space_dir( + &space_name, + space.get_uuid(), + ))?; + // commit txn + match global.namespace_txn_driver().lock().try_commit(txn) { + Ok(()) => {} + Err(e) => { + // tell fractal to clean it up sometime + global.taskmgr_post_standard_priority(Task::new( + GenericTask::delete_space_dir(&space_name, space.get_uuid()), + )); + return Err(e.into()); + } + } + } + // update global state + let _ = spaces.st_insert(space_name, space); + if if_not_exists { + Ok(Some(true)) + } else { + Ok(None) + } + }) + } + #[allow(unused)] + pub fn transactional_exec_alter( + global: &G, + AlterSpace { + space_name, + updated_props, + }: AlterSpace, + ) -> QueryResult<()> { + global.namespace().ddl_with_space_mut(&space_name, |space| { + match updated_props.get(Self::KEY_ENV) { + Some(DictEntryGeneric::Map(_)) if updated_props.len() == 1 => {} + Some(DictEntryGeneric::Data(l)) if updated_props.len() == 1 && l.is_null() => {} + None if updated_props.is_empty() => return Ok(()), + _ => return Err(QueryError::QExecDdlInvalidProperties), + } + // create patch + let patch = match dict::rprepare_metadata_patch(space.props(), updated_props) { + Some(patch) => patch, + None => return Err(QueryError::QExecDdlInvalidProperties), + }; + if G::FS_IS_NON_NULL { + // prepare txn + let txn = + gnstxn::AlterSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, space), &patch); + // commit + global.namespace_txn_driver().lock().try_commit(txn)?; + } + // merge + dict::rmerge_data_with_patch(space.props_mut(), patch); + // the `env` key may have been popped, so put it back (setting `env: null` removes the env key and we don't want to waste time enforcing this in the + // merge algorithm) + let _ = space + .props_mut() + .st_insert(Self::KEY_ENV.into(), DictEntryGeneric::Map(into_dict!())); + Ok(()) + }) + } + pub fn transactional_exec_drop( + global: &G, + DropSpace { + space: space_name, + force, + if_exists, + }: DropSpace, + ) -> QueryResult> { + if force { + global.namespace().ddl_with_all_mut(|spaces, models| { + let Some(space) = spaces.remove(space_name.as_str()) else { + if if_exists { + return Ok(Some(false)); + } else { + return Err(QueryError::QExecObjectNotFound); + } + }; + // commit drop + if G::FS_IS_NON_NULL { + // prepare txn + let txn = + gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, &space)); + // commit txn + global.namespace_txn_driver().lock().try_commit(txn)?; + // request cleanup + global.taskmgr_post_standard_priority(Task::new( + GenericTask::delete_space_dir(&space_name, space.get_uuid()), + )); + } + let space_uuid = space.get_uuid(); + for model in space.models.into_iter() { + let e: EntityIDRef<'static> = unsafe { + // UNSAFE(@ohsayan): I want to try what the borrow checker has been trying + core::mem::transmute(EntityIDRef::new(space_name.as_str(), &model)) + }; + let mdl = models.st_delete_return(&e).unwrap(); + global.purge_model_driver( + &space_name, + space_uuid, + &model, + mdl.get_uuid(), + true, + ); + } + let _ = spaces.st_delete(space_name.as_str()); + if if_exists { + Ok(Some(true)) + } else { + Ok(None) + } + }) + } else { + global.namespace().ddl_with_spaces_write(|spaces| { + let Some(space) = spaces.get(space_name.as_str()) else { + if if_exists { + return Ok(Some(false)); + } else { + return Err(QueryError::QExecObjectNotFound); + } + }; + if !space.models.is_empty() { + // nonempty, we can't do anything + return Err(QueryError::QExecDdlNotEmpty); + } + // okay, it's empty; good riddance + if G::FS_IS_NON_NULL { + // prepare txn + let txn = + gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, &space)); + // commit txn + global.namespace_txn_driver().lock().try_commit(txn)?; + // request cleanup + global.taskmgr_post_standard_priority(Task::new( + GenericTask::delete_space_dir(&space_name, space.get_uuid()), + )); + } + let _ = spaces.st_delete(space_name.as_str()); + if if_exists { + Ok(Some(true)) + } else { + Ok(None) + } + }) + } + } +} diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs new file mode 100644 index 00000000..d154cabc --- /dev/null +++ b/server/src/engine/core/tests/ddl_model/alt.rs @@ -0,0 +1,447 @@ +/* + * Created on Mon Mar 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::{ + model::{alt::AlterPlan, Model}, + tests::ddl_model::{create, exec_create}, + EntityIDRef, + }, + error::QueryResult, + fractal::GlobalInstanceLike, + ql::{ast::parse_ast_node_full, ddl::alt::AlterModel, tests::lex_insecure}, +}; + +fn with_plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) -> QueryResult<()> { + let model = create(model)?; + let tok = lex_insecure(plan.as_bytes()).unwrap(); + let alter = parse_ast_node_full(&tok[2..]).unwrap(); + let mv = AlterPlan::fdeltas(&model, alter)?; + Ok(f(mv)) +} +fn plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) { + with_plan(model, plan, f).unwrap() +} +fn exec_plan( + global: &impl GlobalInstanceLike, + new_space: bool, + model: &str, + plan: &str, + f: impl Fn(&Model), +) -> QueryResult<()> { + let mdl_name = exec_create(global, model, new_space)?; + let prev_uuid = { + global + .namespace() + .idx_models() + .read() + .get(&EntityIDRef::new("myspace", &mdl_name)) + .map(|mdl| mdl.get_uuid()) + .unwrap() + }; + let tok = lex_insecure(plan.as_bytes()).unwrap(); + let alter = parse_ast_node_full::(&tok[2..]).unwrap(); + Model::transactional_exec_alter(global, alter)?; + let models = global.namespace().idx_models().read(); + let model = models.get(&EntityIDRef::new("myspace", &mdl_name)).unwrap(); + assert_eq!(prev_uuid, model.get_uuid()); + f(model); + Ok(()) +} + +mod plan { + use crate::{ + engine::{ + core::model::{self, alt::AlterAction, Field, Layer}, + error::QueryError, + }, + vecfuse, + }; + /* + Simple + */ + #[test] + fn simple_add() { + super::plan( + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel add myfield { type: string, nullable: true }", + |plan| { + assert_eq!(plan.model.entity(), "mymodel"); + assert!(plan.no_lock); + assert_eq!( + plan.action, + AlterAction::Add( + into_dict! { "myfield" => Field::new([Layer::str()].into(), true) } + ) + ) + }, + ) + } + #[test] + fn simple_remove() { + super::plan( + "create model myspace.mymodel(username: string, password: binary, useless_field: uint8)", + "alter model myspace.mymodel remove useless_field", + |plan| { + assert_eq!(plan.model.entity(), "mymodel"); + assert!(plan.no_lock); + assert_eq!( + plan.action, + AlterAction::Remove(["useless_field".into()].into()) + ) + }, + ); + } + #[test] + fn simple_update() { + // FREEDOM! DAMN THE PASSWORD! + super::plan( + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel update password { nullable: true }", + |plan| { + assert_eq!(plan.model.entity(), "mymodel"); + assert!(plan.no_lock); + assert_eq!( + plan.action, + AlterAction::Update(into_dict! { + "password" => Field::new([Layer::bin()].into(), true) + }) + ); + }, + ); + } + #[test] + fn update_need_lock() { + // FIGHT THE NULL + super::plan( + "create model myspace.mymodel(username: string, null password: binary)", + "alter model myspace.mymodel update password { nullable: false }", + |plan| { + assert_eq!(plan.model.entity(), "mymodel"); + assert!(!plan.no_lock); + assert_eq!( + plan.action, + AlterAction::Update(into_dict! { + "password" => Field::new([Layer::bin()].into(), false) + }) + ); + }, + ); + } + /* + Illegal + */ + #[test] + fn illegal_remove_nx() { + assert_eq!( + super::with_plan( + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel remove password_e2e", + |_| {} + ) + .unwrap_err(), + QueryError::QExecUnknownField + ); + } + #[test] + fn illegal_remove_pk() { + assert_eq!( + super::with_plan( + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel remove username", + |_| {} + ) + .unwrap_err(), + QueryError::QExecDdlModelAlterIllegal + ); + } + #[test] + fn illegal_add_pk() { + assert_eq!( + super::with_plan( + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel add username { type: string }", + |_| {} + ) + .unwrap_err(), + QueryError::QExecDdlModelAlterIllegal + ); + } + #[test] + fn illegal_add_ex() { + assert_eq!( + super::with_plan( + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel add password { type: string }", + |_| {} + ) + .unwrap_err(), + QueryError::QExecDdlModelAlterIllegal + ); + } + #[test] + fn illegal_update_pk() { + assert_eq!( + super::with_plan( + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel update username { type: string }", + |_| {} + ) + .unwrap_err(), + QueryError::QExecDdlModelAlterIllegal + ); + } + #[test] + fn illegal_update_nx() { + assert_eq!( + super::with_plan( + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel update username_secret { type: string }", + |_| {} + ) + .unwrap_err(), + QueryError::QExecUnknownField + ); + } + fn bad_type_cast(orig_ty: &str, new_ty: &str) { + let create = + format!("create model myspace.mymodel(username: string, silly_field: {orig_ty})"); + let alter = format!("alter model myspace.mymodel update silly_field {{ type: {new_ty} }}"); + assert_eq!( + super::with_plan(&create, &alter, |_| {}).expect_err(&format!( + "found no error in transformation: {orig_ty} -> {new_ty}" + )), + QueryError::QExecDdlInvalidTypeDefinition, + "failed to match error in transformation: {orig_ty} -> {new_ty}", + ) + } + fn enumerated_bad_type_casts(orig_ty: O, new_ty: N) + where + O: IntoIterator, + N: IntoIterator + Clone, + { + for orig in orig_ty { + let new_ty = new_ty.clone(); + for new in new_ty { + bad_type_cast(orig, new); + } + } + } + #[test] + fn illegal_bool_direct_cast() { + enumerated_bad_type_casts( + ["bool"], + vecfuse![ + model::TY_UINT, + model::TY_SINT, + model::TY_BINARY, + model::TY_STRING, + model::TY_LIST + ], + ); + } + #[test] + fn illegal_uint_direct_cast() { + enumerated_bad_type_casts( + model::TY_UINT, + vecfuse![ + model::TY_BOOL, + model::TY_SINT, + model::TY_FLOAT, + model::TY_BINARY, + model::TY_STRING, + model::TY_LIST + ], + ); + } + #[test] + fn illegal_sint_direct_cast() { + enumerated_bad_type_casts( + model::TY_SINT, + vecfuse![ + model::TY_BOOL, + model::TY_UINT, + model::TY_FLOAT, + model::TY_BINARY, + model::TY_STRING, + model::TY_LIST + ], + ); + } + #[test] + fn illegal_float_direct_cast() { + enumerated_bad_type_casts( + model::TY_FLOAT, + vecfuse![ + model::TY_BOOL, + model::TY_UINT, + model::TY_SINT, + model::TY_BINARY, + model::TY_STRING, + model::TY_LIST + ], + ); + } + #[test] + fn illegal_binary_direct_cast() { + enumerated_bad_type_casts( + [model::TY_BINARY], + vecfuse![ + model::TY_BOOL, + model::TY_UINT, + model::TY_SINT, + model::TY_FLOAT, + model::TY_STRING, + model::TY_LIST + ], + ); + } + #[test] + fn illegal_string_direct_cast() { + enumerated_bad_type_casts( + [model::TY_STRING], + vecfuse![ + model::TY_BOOL, + model::TY_UINT, + model::TY_SINT, + model::TY_FLOAT, + model::TY_BINARY, + model::TY_LIST + ], + ); + } + #[test] + fn illegal_list_direct_cast() { + enumerated_bad_type_casts( + ["list { type: string }"], + vecfuse![ + model::TY_BOOL, + model::TY_UINT, + model::TY_SINT, + model::TY_FLOAT, + model::TY_BINARY, + model::TY_STRING + ], + ); + } +} + +mod exec { + use crate::engine::{ + core::model::{DeltaVersion, Field, Layer}, + error::QueryError, + fractal::test_utils::TestGlobal, + idx::{STIndex, STIndexSeq}, + }; + #[test] + fn simple_add() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_plan( + &global, + true, + "create model myspace.mymodel(username: string, col1: uint64)", + "alter model myspace.mymodel add (col2 { type: uint32, nullable: true }, col3 { type: uint16, nullable: true })", + |model| { + assert_eq!( + model + .fields() + .stseq_ord_kv() + .rev() + .take(2) + .map(|(id, f)| (id.as_str().to_owned(), f.clone())) + .collect::>(), + [ + ("col3".into(), Field::new([Layer::uint16()].into(), true)), + ("col2".into(), Field::new([Layer::uint32()].into(), true)) + ] + ); + assert_eq!( + model.delta_state().schema_current_version(), + DeltaVersion::__new(2) + ); + }, + ) + .unwrap(); + } + #[test] + fn simple_remove() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_plan( + &global, + true, + "create model myspace.mymodel(username: string, col1: uint64, col2: uint32, col3: uint16, col4: uint8)", + "alter model myspace.mymodel remove (col1, col2, col3, col4)", + |mdl| { + assert_eq!( + mdl + .fields() + .stseq_ord_kv() + .rev() + .map(|(a, b)| (a.as_str().to_owned(), b.clone())) + .collect::>(), + [("username".into(), Field::new([Layer::str()].into(), false))] + ); + assert_eq!( + mdl.delta_state().schema_current_version(), + DeltaVersion::__new(4) + ); + } + ).unwrap(); + } + #[test] + fn simple_update() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_plan( + &global, + true, + "create model myspace.mymodel(username: string, password: binary)", + "alter model myspace.mymodel update password { nullable: true }", + |model| { + assert!(model.fields().st_get("password").unwrap().is_nullable()); + assert_eq!( + model.delta_state().schema_current_version(), + DeltaVersion::genesis() + ); + }, + ) + .unwrap(); + } + #[test] + fn failing_alter_nullable_switch_need_lock() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_plan( + &global, + true, + "create model myspace.mymodel(username: string, null gh_handle: string)", + "alter model myspace.mymodel update gh_handle { nullable: false }", + |_| {}, + ) + .unwrap_err(), + QueryError::QExecNeedLock + ); + } +} diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs new file mode 100644 index 00000000..d519a365 --- /dev/null +++ b/server/src/engine/core/tests/ddl_model/crt.rs @@ -0,0 +1,182 @@ +/* + * Created on Sat Mar 04 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod validation { + use { + super::super::create, + crate::engine::{ + core::model::{DeltaVersion, Field, Layer}, + data::tag::{DataTag, FullTag}, + error::QueryError, + idx::STIndexSeq, + }, + }; + + #[test] + fn simple() { + let model = + create("create model myspace.mymodel(username: string, password: binary)").unwrap(); + assert_eq!(model.p_key(), "username"); + assert_eq!(model.p_tag(), FullTag::STR); + assert_eq!( + model + .fields() + .stseq_ord_value() + .cloned() + .collect::>(), + [ + Field::new([Layer::new_empty_props(FullTag::STR)].into(), false), + Field::new([Layer::new_empty_props(FullTag::BIN)].into(), false) + ] + ); + assert_eq!( + model.delta_state().schema_current_version(), + DeltaVersion::genesis() + ); + } + + #[test] + fn idiotic_order() { + let model = + create("create model myspace.mymodel(password: binary, primary username: string)") + .unwrap(); + assert_eq!(model.p_key(), "username"); + assert_eq!(model.p_tag(), FullTag::STR); + assert_eq!( + model + .fields() + .stseq_ord_value() + .cloned() + .collect::>(), + [ + Field::new([Layer::new_empty_props(FullTag::BIN)].into(), false), + Field::new([Layer::new_empty_props(FullTag::STR)].into(), false), + ] + ); + assert_eq!( + model.delta_state().schema_current_version(), + DeltaVersion::genesis() + ); + } + + #[test] + fn duplicate_primary_key() { + assert_eq!( + create( + "create model myspace.mymodel(primary username: string, primary contract_location: binary)" + ) + .unwrap_err(), + QueryError::QExecDdlModelBadDefinition + ); + } + + #[test] + fn duplicate_fields() { + assert_eq!( + create("create model myspace.mymodel(primary username: string, username: binary)") + .unwrap_err(), + QueryError::QExecDdlModelBadDefinition + ); + } + + #[test] + fn illegal_props() { + assert_eq!( + create("create model myspace.mymodel(primary username: string, password: binary) with { lol_prop: false }").unwrap_err(), + QueryError::QExecDdlModelBadDefinition + ); + } + + #[test] + fn illegal_pk() { + assert_eq!( + create( + "create model myspace.mymodel(primary username_bytes: list { type: uint8 }, password: binary)" + ) + .unwrap_err(), + QueryError::QExecDdlModelBadDefinition + ); + assert_eq!( + create("create model myspace.mymodel(primary username: float32, password: binary)") + .unwrap_err(), + QueryError::QExecDdlModelBadDefinition + ); + } +} + +/* + Exec +*/ + +mod exec { + use crate::engine::{ + core::{ + model::{DeltaVersion, Field, Layer}, + tests::ddl_model::{exec_create_new_space, with_model}, + }, + data::tag::{DataTag, FullTag}, + fractal::test_utils::TestGlobal, + idx::STIndexSeq, + }; + + const SPACE: &str = "myspace"; + + #[test] + fn simple() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + exec_create_new_space( + &global, + "create model myspace.mymodel(username: string, password: binary)", + ) + .unwrap(); + with_model(&global, SPACE, "mymodel", |model| { + let models: Vec<(String, Field)> = model + .fields() + .stseq_ord_kv() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect(); + assert_eq!(model.p_key(), "username"); + assert_eq!(model.p_tag(), FullTag::STR); + assert_eq!( + models, + [ + ( + "username".to_string(), + Field::new([Layer::str()].into(), false) + ), + ( + "password".to_string(), + Field::new([Layer::bin()].into(), false) + ) + ] + ); + assert_eq!( + model.delta_state().schema_current_version(), + DeltaVersion::genesis() + ); + }); + } +} diff --git a/server/src/engine/core/tests/ddl_model/layer.rs b/server/src/engine/core/tests/ddl_model/layer.rs new file mode 100644 index 00000000..5d95ea6f --- /dev/null +++ b/server/src/engine/core/tests/ddl_model/layer.rs @@ -0,0 +1,240 @@ +/* + * Created on Thu Mar 02 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::model::Field, + error::QueryResult, + ql::{ast::parse_ast_node_multiple_full, tests::lex_insecure}, +}; + +fn layerview_nullable(layer_def: &str, nullable: bool) -> QueryResult { + let tok = lex_insecure(layer_def.as_bytes()).unwrap(); + let spec = parse_ast_node_multiple_full(&tok).unwrap(); + Field::parse_layers(spec, nullable) +} +fn layerview(layer_def: &str) -> QueryResult { + layerview_nullable(layer_def, false) +} + +mod layer_spec_validation { + use { + super::layerview, + crate::engine::{core::model::Layer, error::QueryError}, + }; + + #[test] + fn string() { + assert_eq!(layerview("string").unwrap().layers(), [Layer::str()]); + } + + #[test] + fn nested_list() { + assert_eq!( + layerview("list { type: list { type: string } }") + .unwrap() + .layers(), + [Layer::list(), Layer::list(), Layer::str()] + ); + } + + #[test] + fn invalid_list() { + assert_eq!( + layerview("list").unwrap_err(), + QueryError::QExecDdlInvalidTypeDefinition + ); + } + + #[test] + fn invalid_flat() { + assert_eq!( + layerview("string { type: string }").unwrap_err(), + QueryError::QExecDdlInvalidTypeDefinition + ); + } +} + +mod layer_data_validation { + use { + super::{layerview, layerview_nullable}, + crate::engine::{core::model, data::cell::Datacell}, + }; + #[test] + fn bool() { + let mut dc = Datacell::new_bool(true); + let layer = layerview("bool").unwrap(); + assert!(layer.vt_data_fpath(&mut dc)); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]); + } + #[test] + fn uint() { + let targets = [ + ("uint8", u8::MAX as u64), + ("uint16", u16::MAX as _), + ("uint32", u32::MAX as _), + ("uint64", u64::MAX), + ]; + targets + .into_iter() + .enumerate() + .for_each(|(i, (layer, max))| { + let this_layer = layerview(layer).unwrap(); + let mut dc = Datacell::new_uint_default(max); + assert!(this_layer.vt_data_fpath(&mut dc), "{:?}", this_layer); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]); + for (lower, _) in targets[..i].iter() { + let layer = layerview(lower).unwrap(); + assert!(!layer.vt_data_fpath(&mut dc), "{:?}", layer); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]); + } + for (higher, _) in targets[i + 1..].iter() { + let layer = layerview(higher).unwrap(); + assert!(layer.vt_data_fpath(&mut dc), "{:?}", layer); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]); + } + }); + } + #[test] + fn sint() { + let targets = [ + ("sint8", (i8::MIN as i64, i8::MAX as i64)), + ("sint16", (i16::MIN as _, i16::MAX as _)), + ("sint32", (i32::MIN as _, i32::MAX as _)), + ("sint64", (i64::MIN, i64::MAX)), + ]; + targets + .into_iter() + .enumerate() + .for_each(|(i, (layer, (min, max)))| { + let this_layer = layerview(layer).unwrap(); + let mut dc_min = Datacell::new_sint_default(min); + let mut dc_max = Datacell::new_sint_default(max); + assert!(this_layer.vt_data_fpath(&mut dc_min), "{:?}", this_layer); + assert!(this_layer.vt_data_fpath(&mut dc_max), "{:?}", this_layer); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "sint", "fpath", "sint"]); + for (lower, _) in targets[..i].iter() { + let layer = layerview(lower).unwrap(); + assert!(!layer.vt_data_fpath(&mut dc_min), "{:?}", layer); + assert!(!layer.vt_data_fpath(&mut dc_max), "{:?}", layer); + assert_vecstreq_exact!( + model::layer_traces(), + ["fpath", "sint", "fpath", "sint"] + ); + } + for (higher, _) in targets[i + 1..].iter() { + let layer = layerview(higher).unwrap(); + assert!(layer.vt_data_fpath(&mut dc_min), "{:?}", layer); + assert!(layer.vt_data_fpath(&mut dc_max), "{:?}", layer); + assert_vecstreq_exact!( + model::layer_traces(), + ["fpath", "sint", "fpath", "sint"] + ); + } + }); + } + #[test] + fn float() { + // l + let f32_l = layerview("float32").unwrap(); + let f64_l = layerview("float64").unwrap(); + // dc + let f32_dc_min = Datacell::new_float_default(f32::MIN as _); + let f32_dc_max = Datacell::new_float_default(f32::MAX as _); + let f64_dc_min = Datacell::new_float_default(f64::MIN as _); + let f64_dc_max = Datacell::new_float_default(f64::MAX as _); + // check (32) + assert!(f32_l.vt_data_fpath(&mut f32_dc_min.clone())); + assert!(f32_l.vt_data_fpath(&mut f32_dc_max.clone())); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]); + assert!(f64_l.vt_data_fpath(&mut f32_dc_min.clone())); + assert!(f64_l.vt_data_fpath(&mut f32_dc_max.clone())); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]); + // check (64) + assert!(!f32_l.vt_data_fpath(&mut f64_dc_min.clone())); + assert!(!f32_l.vt_data_fpath(&mut f64_dc_max.clone())); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]); + assert!(f64_l.vt_data_fpath(&mut f64_dc_min.clone())); + assert!(f64_l.vt_data_fpath(&mut f64_dc_max.clone())); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]); + } + #[test] + fn bin() { + let layer = layerview("binary").unwrap(); + assert!(layer.vt_data_fpath(&mut Datacell::from("hello".as_bytes()))); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "binary"]); + } + #[test] + fn str() { + let layer = layerview("string").unwrap(); + assert!(layer.vt_data_fpath(&mut Datacell::from("hello"))); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "string"]); + } + #[test] + fn list_simple() { + let layer = layerview("list { type: string }").unwrap(); + let mut dc = Datacell::new_list(vec![ + Datacell::from("I"), + Datacell::from("love"), + Datacell::from("cats"), + ]); + assert!(layer.vt_data_fpath(&mut dc)); + assert_vecstreq_exact!( + model::layer_traces(), + ["list", "string", "string", "string"] + ); + } + #[test] + fn list_nested_l1() { + let layer = layerview("list { type: list { type: string } }").unwrap(); + let mut dc = Datacell::new_list(vec![ + Datacell::new_list(vec![Datacell::from("hello_11"), Datacell::from("hello_12")]), + Datacell::new_list(vec![Datacell::from("hello_21"), Datacell::from("hello_22")]), + Datacell::new_list(vec![Datacell::from("hello_31"), Datacell::from("hello_32")]), + ]); + assert!(layer.vt_data_fpath(&mut dc)); + assert_vecstreq_exact!( + model::layer_traces(), + [ + "list", // low + "list", "string", "string", // cs: 1 + "list", "string", "string", // cs: 2 + "list", "string", "string", // cs: 3 + ] + ); + } + #[test] + fn nullval_fpath() { + let layer = layerview_nullable("string", true).unwrap(); + assert!(layer.vt_data_fpath(&mut Datacell::null())); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]); + } + #[test] + fn nullval_nested_but_fpath() { + let layer = layerview_nullable("list { type: string }", true).unwrap(); + assert!(layer.vt_data_fpath(&mut Datacell::null())); + assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]); + } +} diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs new file mode 100644 index 00000000..b58cdc5e --- /dev/null +++ b/server/src/engine/core/tests/ddl_model/mod.rs @@ -0,0 +1,76 @@ +/* + * Created on Thu Mar 02 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod alt; +mod crt; +mod layer; + +use crate::engine::{ + core::{model::Model, EntityIDRef}, + error::QueryResult, + fractal::GlobalInstanceLike, + ql::{ast::parse_ast_node_full, ddl::crt::CreateModel, tests::lex_insecure}, +}; + +fn create(s: &str) -> QueryResult { + let tok = lex_insecure(s.as_bytes()).unwrap(); + let create_model = parse_ast_node_full(&tok[2..]).unwrap(); + Model::process_create(create_model) +} + +pub fn exec_create( + global: &impl GlobalInstanceLike, + create_stmt: &str, + create_new_space: bool, +) -> QueryResult { + let tok = lex_insecure(create_stmt.as_bytes()).unwrap(); + let create_model = parse_ast_node_full::(&tok[2..]).unwrap(); + let name = create_model.model_name.entity().to_owned(); + if create_new_space { + global + .namespace() + .create_empty_test_space(create_model.model_name.space()) + } + Model::transactional_exec_create(global, create_model).map(|_| name) +} + +pub fn exec_create_new_space( + global: &impl GlobalInstanceLike, + create_stmt: &str, +) -> QueryResult<()> { + exec_create(global, create_stmt, true).map(|_| ()) +} + +fn with_model( + global: &impl GlobalInstanceLike, + space_id: &str, + model_name: &str, + f: impl Fn(&Model), +) { + let models = global.namespace().idx_models().read(); + let model = models.get(&EntityIDRef::new(space_id, model_name)).unwrap(); + f(model) +} diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs new file mode 100644 index 00000000..8973c3ff --- /dev/null +++ b/server/src/engine/core/tests/ddl_space/alter.rs @@ -0,0 +1,152 @@ +/* + * Created on Thu Feb 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::space::Space, + data::{cell::Datacell, DictEntryGeneric}, + error::QueryError, + fractal::test_utils::TestGlobal, +}; + +#[test] +fn alter_add_prop_env_var() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_create_alter( + &global, + "create space myspace", + "alter space myspace with { env: { MY_NEW_PROP: 100 } }", + |space| { + assert_eq!( + space, + &Space::new_restore_empty( + space.get_uuid(), + into_dict!("env" => DictEntryGeneric::Map(into_dict!("MY_NEW_PROP" => Datacell::new_uint_default(100)))), + ) + ); + }, + ) + .unwrap(); +} + +#[test] +fn alter_update_prop_env_var() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + let uuid = super::exec_create( + &global, + "create space myspace with { env: { MY_NEW_PROP: 100 } }", + |space| { + assert_eq!( + space.env().get("MY_NEW_PROP").unwrap(), + &(Datacell::new_uint_default(100).into()) + ) + }, + ) + .unwrap(); + super::exec_alter( + &global, + "alter space myspace with { env: { MY_NEW_PROP: 200 } }", + |space| { + assert_eq!( + space, + &Space::new_restore_empty( + uuid, + into_dict! ("env" => DictEntryGeneric::Map(into_dict!("MY_NEW_PROP" => Datacell::new_uint_default(200)))), + ) + ) + }, + ) + .unwrap(); +} + +#[test] +fn alter_remove_prop_env_var() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + let uuid = super::exec_create( + &global, + "create space myspace with { env: { MY_NEW_PROP: 100 } }", + |space| { + assert_eq!( + space.env().get("MY_NEW_PROP").unwrap(), + &(Datacell::new_uint_default(100).into()) + ) + }, + ) + .unwrap(); + super::exec_alter( + &global, + "alter space myspace with { env: { MY_NEW_PROP: null } }", + |space| { + assert_eq!( + space, + &Space::new_restore_empty( + uuid, + into_dict!("env" => DictEntryGeneric::Map(into_dict!())) + ) + ) + }, + ) + .unwrap(); +} + +#[test] +fn alter_nx() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_alter( + &global, + "alter space myspace with { env: { MY_NEW_PROP: 100 } }", + |_| {}, + ) + .unwrap_err(), + QueryError::QExecObjectNotFound + ); +} + +#[test] +fn alter_remove_all_env() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + let uuid = super::exec_create( + &global, + "create space myspace with { env: { MY_NEW_PROP: 100 } }", + |space| { + assert_eq!( + space.env().get("MY_NEW_PROP").unwrap(), + &(Datacell::new_uint_default(100).into()) + ) + }, + ) + .unwrap(); + super::exec_alter(&global, "alter space myspace with { env: null }", |space| { + assert_eq!( + space, + &Space::new_restore_empty( + uuid, + into_dict!("env" => DictEntryGeneric::Map(into_dict!())) + ) + ) + }) + .unwrap(); +} diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs new file mode 100644 index 00000000..1010b5d4 --- /dev/null +++ b/server/src/engine/core/tests/ddl_space/create.rs @@ -0,0 +1,91 @@ +/* + * Created on Wed Feb 08 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::space::Space, + data::{cell::Datacell, DictEntryGeneric}, + error::QueryError, + fractal::test_utils::TestGlobal, +}; + +#[test] +fn exec_create_space_simple() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_create(&global, "create space myspace", |spc| { + assert!(spc.models().is_empty()) + }) + .unwrap(); +} + +#[test] +fn exec_create_space_with_env() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_create( + &global, + r#" + create space myspace with { + env: { + MAX_MODELS: 100 + } + } + "#, + |space| { + assert_eq!( + space, + &Space::new_restore_empty( + space.get_uuid(), + into_dict! { + "env" => DictEntryGeneric::Map(into_dict!("MAX_MODELS" => Datacell::new_uint_default(100))) + }, + ) + ); + }, + ) + .unwrap(); +} + +#[test] +fn exec_create_space_with_bad_env_type() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_create(&global, "create space myspace with { env: 100 }", |_| {}).unwrap_err(), + QueryError::QExecDdlInvalidProperties + ); +} + +#[test] +fn exec_create_space_with_random_property() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_create( + &global, + "create space myspace with { i_am_blue_da_ba_dee: 100 }", + |_| {} + ) + .unwrap_err(), + QueryError::QExecDdlInvalidProperties + ); +} diff --git a/server/src/engine/core/tests/ddl_space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs new file mode 100644 index 00000000..77940719 --- /dev/null +++ b/server/src/engine/core/tests/ddl_space/mod.rs @@ -0,0 +1,83 @@ +/* + * Created on Thu Feb 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod alter; +mod create; + +use crate::engine::{ + core::space::Space, + data::uuid::Uuid, + error::QueryResult, + fractal::GlobalInstanceLike, + ql::{ + ast::{self}, + tests::lex_insecure as lex, + }, +}; + +fn exec_create( + gns: &impl GlobalInstanceLike, + create: &str, + verify: impl Fn(&Space), +) -> QueryResult { + let tok = lex(create.as_bytes()).unwrap(); + let ast_node = + ast::parse_ast_node_full::(&tok[2..]).unwrap(); + let name = ast_node.space_name; + Space::transactional_exec_create(gns, ast_node)?; + gns.namespace().ddl_with_space_mut(&name, |space| { + verify(space); + Ok(space.get_uuid()) + }) +} + +fn exec_alter( + gns: &impl GlobalInstanceLike, + alter: &str, + verify: impl Fn(&Space), +) -> QueryResult { + let tok = lex(alter.as_bytes()).unwrap(); + let ast_node = + ast::parse_ast_node_full::(&tok[2..]).unwrap(); + let name = ast_node.space_name; + Space::transactional_exec_alter(gns, ast_node)?; + gns.namespace().ddl_with_space_mut(&name, |space| { + verify(space); + Ok(space.get_uuid()) + }) +} + +fn exec_create_alter( + gns: &impl GlobalInstanceLike, + crt: &str, + alt: &str, + verify_post_alt: impl Fn(&Space), +) -> QueryResult { + let uuid_crt = exec_create(gns, crt, |_| {})?; + let uuid_alt = exec_alter(gns, alt, verify_post_alt)?; + assert_eq!(uuid_crt, uuid_alt); + Ok(uuid_alt) +} diff --git a/server/native/flock-posix.c b/server/src/engine/core/tests/dml/delete.rs similarity index 50% rename from server/native/flock-posix.c rename to server/src/engine/core/tests/dml/delete.rs index f4e33a9c..017ba0ee 100644 --- a/server/native/flock-posix.c +++ b/server/src/engine/core/tests/dml/delete.rs @@ -1,5 +1,5 @@ /* - * Created on Fri Aug 07 2020 + * Created on Wed May 10 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2020, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -22,39 +22,35 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . * - */ +*/ -#include -#include +use crate::engine::{error::QueryError, fractal::test_utils::TestGlobal}; -/* Acquire an exclusive lock for a file with the given descriptor */ -int lock_exclusive(int descriptor) { - if (descriptor < 0) { - return EBADF; - } - if (flock(descriptor, LOCK_EX) == -1) { - return errno; - } - return 0; +#[test] +fn simple_delete() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_delete( + &global, + "create model myspace.mymodel(username: string, password: string)", + Some("insert into myspace.mymodel('sayan', 'pass123')"), + "delete from myspace.mymodel where username = 'sayan'", + "sayan", + ) + .unwrap(); } -int try_lock_exclusive(int descriptor) { - if (descriptor < 0) { - return EBADF; - } - if (flock(descriptor, LOCK_EX | LOCK_NB) == -1) { - return errno; - } - return 0; +#[test] +fn delete_nonexisting() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_delete( + &global, + "create model myspace.mymodel(username: string, password: string)", + None, + "delete from myspace.mymodel where username = 'sayan'", + "sayan", + ) + .unwrap_err(), + QueryError::QExecDmlRowNotFound + ); } - -/* Unlock a file with the given descriptor */ -int unlock(int descriptor) { - if (descriptor < 0) { - return EBADF; - } - if (flock(descriptor, LOCK_UN) == -1) { - return errno; - } - return 0; -} \ No newline at end of file diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs new file mode 100644 index 00000000..c1c99cbc --- /dev/null +++ b/server/src/engine/core/tests/dml/insert.rs @@ -0,0 +1,88 @@ +/* + * Created on Tue May 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{data::cell::Datacell, error::QueryError, fractal::test_utils::TestGlobal}; + +#[derive(sky_macros::Wrapper, Debug)] +struct Tuple(Vec<(Box, Datacell)>); + +#[test] +fn insert_simple() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_insert( + &global, + "create model myspace.mymodel(username: string, password: string)", + "insert into myspace.mymodel('sayan', 'pass123')", + "sayan", + |row| { + assert_veceq_transposed!(row.cloned_data(), Tuple(pairvec!(("password", "pass123")))); + }, + ) + .unwrap(); +} + +#[test] +fn insert_with_null() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_insert( + &global, + "create model myspace.mymodel(username: string, null useless_password: string, null useless_email: string, null useless_random_column: uint64)", + "insert into myspace.mymodel('sayan', null, null, null)", + "sayan", + |row| { + assert_veceq_transposed!( + row.cloned_data(), + Tuple( + pairvec!( + ("useless_password", Datacell::null()), + ("useless_email", Datacell::null()), + ("useless_random_column", Datacell::null()) + ) + ) + ) + } + ).unwrap(); +} + +#[test] +fn insert_duplicate() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + super::exec_insert( + &global, + "create model myspace.mymodel(username: string, password: string)", + "insert into myspace.mymodel('sayan', 'pass123')", + "sayan", + |row| { + assert_veceq_transposed!(row.cloned_data(), Tuple(pairvec!(("password", "pass123")))); + }, + ) + .unwrap(); + assert_eq!( + super::exec_insert_only(&global, "insert into myspace.mymodel('sayan', 'pass123')") + .unwrap_err(), + QueryError::QExecDmlDuplicate + ); +} diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs new file mode 100644 index 00000000..8385004c --- /dev/null +++ b/server/src/engine/core/tests/dml/mod.rs @@ -0,0 +1,202 @@ +/* + * Created on Tue May 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod delete; +mod insert; +mod select; +mod update; + +use crate::engine::{ + core::{dml, index::Row, model::Model, space::Space, EntityIDRef}, + data::{cell::Datacell, lit::Lit}, + error::QueryResult, + fractal::GlobalInstanceLike, + ql::{ + ast::parse_ast_node_full, + dml::{del::DeleteStatement, ins::InsertStatement}, + tests::lex_insecure, + }, + sync, +}; + +fn _exec_only_create_space_model(global: &impl GlobalInstanceLike, model: &str) -> QueryResult<()> { + let _ = global + .namespace() + .idx() + .write() + .insert("myspace".into(), Space::new_auto_all().into()); + let lex_create_model = lex_insecure(model.as_bytes()).unwrap(); + let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap(); + Model::transactional_exec_create(global, stmt_create_model).map(|_| ()) +} + +fn _exec_only_insert( + global: &impl GlobalInstanceLike, + insert: &str, + and_then: impl Fn(EntityIDRef) -> T, +) -> QueryResult { + let lex_insert = lex_insecure(insert.as_bytes()).unwrap(); + let stmt_insert = parse_ast_node_full::(&lex_insert[1..]).unwrap(); + let entity = stmt_insert.entity(); + dml::insert(global, stmt_insert)?; + let r = and_then(entity); + Ok(r) +} + +fn _exec_only_read_key_and_then( + global: &impl GlobalInstanceLike, + entity: EntityIDRef, + key_name: &str, + and_then: impl Fn(Row) -> T, +) -> QueryResult { + let guard = sync::atm::cpin(); + global.namespace().with_model(entity, |mdl| { + let row = mdl + .primary_index() + .select(Lit::from(key_name), &guard) + .unwrap() + .clone(); + drop(guard); + Ok(and_then(row)) + }) +} + +fn _exec_delete_only(global: &impl GlobalInstanceLike, delete: &str, key: &str) -> QueryResult<()> { + let lex_del = lex_insecure(delete.as_bytes()).unwrap(); + let delete = parse_ast_node_full::(&lex_del[1..]).unwrap(); + let entity = delete.entity(); + dml::delete(global, delete)?; + assert_eq!( + global.namespace().with_model(entity, |model| { + let g = sync::atm::cpin(); + Ok(model.primary_index().select(key.into(), &g).is_none()) + }), + Ok(true) + ); + Ok(()) +} + +fn _exec_only_select(global: &impl GlobalInstanceLike, select: &str) -> QueryResult> { + let lex_sel = lex_insecure(select.as_bytes()).unwrap(); + let select = parse_ast_node_full(&lex_sel[1..]).unwrap(); + let mut r = Vec::new(); + dml::select_custom(global, select, |cell| r.push(cell.clone()))?; + Ok(r) +} + +fn _exec_only_update(global: &impl GlobalInstanceLike, update: &str) -> QueryResult<()> { + let lex_upd = lex_insecure(update.as_bytes()).unwrap(); + let update = parse_ast_node_full(&lex_upd[1..]).unwrap(); + dml::update(global, update) +} + +pub(self) fn exec_insert( + global: &impl GlobalInstanceLike, + model: &str, + insert: &str, + key_name: &str, + f: impl Fn(Row) -> T, +) -> QueryResult { + _exec_only_create_space_model(global, model)?; + _exec_only_insert(global, insert, |entity| { + _exec_only_read_key_and_then(global, entity, key_name, |row| f(row)) + })? +} + +pub(self) fn exec_insert_only(global: &impl GlobalInstanceLike, insert: &str) -> QueryResult<()> { + _exec_only_insert(global, insert, |_| {}) +} + +pub(self) fn exec_delete( + global: &impl GlobalInstanceLike, + model: &str, + insert: Option<&str>, + delete: &str, + key: &str, +) -> QueryResult<()> { + _exec_only_create_space_model(global, model)?; + if let Some(insert) = insert { + _exec_only_insert(global, insert, |_| {})?; + } + _exec_delete_only(global, delete, key) +} + +pub(self) fn exec_select( + global: &impl GlobalInstanceLike, + model: &str, + insert: &str, + select: &str, +) -> QueryResult> { + _exec_only_create_space_model(global, model)?; + _exec_only_insert(global, insert, |_| {})?; + _exec_only_select(global, select) +} + +pub(self) fn exec_select_all( + global: &impl GlobalInstanceLike, + model: &str, + inserts: &[&str], + select: &str, +) -> QueryResult>> { + _exec_only_create_space_model(global, model)?; + for insert in inserts { + _exec_only_insert(global, insert, |_| {})?; + } + let lex_sel = lex_insecure(select.as_bytes()).unwrap(); + let select = parse_ast_node_full(&lex_sel[2..]).unwrap(); + let mut r: Vec> = Vec::new(); + dml::select_all( + global, + select, + &mut r, + |_, _, _| {}, + |rows, dc, col_cnt| match rows.last_mut() { + Some(row) if row.len() != col_cnt => row.push(dc.clone()), + _ => rows.push(vec![dc.clone()]), + }, + )?; + Ok(r) +} + +pub(self) fn exec_select_only( + global: &impl GlobalInstanceLike, + select: &str, +) -> QueryResult> { + _exec_only_select(global, select) +} + +pub(self) fn exec_update( + global: &impl GlobalInstanceLike, + model: &str, + insert: &str, + update: &str, + select: &str, +) -> QueryResult> { + _exec_only_create_space_model(global, model)?; + _exec_only_insert(global, insert, |_| {})?; + _exec_only_update(global, update)?; + _exec_only_select(global, select) +} diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs new file mode 100644 index 00000000..1e693f43 --- /dev/null +++ b/server/src/engine/core/tests/dml/select.rs @@ -0,0 +1,163 @@ +/* + * Created on Thu May 11 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::{data::cell::Datacell, error::QueryError, fractal::test_utils::TestGlobal}, + std::collections::HashMap, +}; + +#[test] +fn simple_select_wildcard() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_select( + &global, + "create model myspace.mymodel(username: string, password: string)", + "insert into myspace.mymodel('sayan', 'pass123')", + "select * from myspace.mymodel where username = 'sayan'", + ) + .unwrap(), + intovec!["sayan", "pass123"] + ); +} + +#[test] +fn simple_select_specified_same_order() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_select( + &global, + "create model myspace.mymodel(username: string, password: string)", + "insert into myspace.mymodel('sayan', 'pass123')", + "select username, password from myspace.mymodel where username = 'sayan'", + ) + .unwrap(), + intovec!["sayan", "pass123"] + ); +} + +#[test] +fn simple_select_specified_reversed_order() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_select( + &global, + "create model myspace.mymodel(username: string, password: string)", + "insert into myspace.mymodel('sayan', 'pass123')", + "select password, username from myspace.mymodel where username = 'sayan'", + ) + .unwrap(), + intovec!["pass123", "sayan"] + ); +} + +#[test] +fn select_null() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_select( + &global, + "create model myspace.mymodel(username: string, null password: string)", + "insert into myspace.mymodel('sayan', null)", + "select username, password from myspace.mymodel where username = 'sayan'", + ) + .unwrap(), + intovec!["sayan", Datacell::null()] + ); +} + +#[test] +fn select_nonexisting() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_select( + &global, + "create model myspace.mymodel(username: string, null password: string)", + "insert into myspace.mymodel('sayan', null)", + "select username, password from myspace.mymodel where username = 'notsayan'", + ) + .unwrap_err(), + QueryError::QExecDmlRowNotFound + ); +} + +/* + select all +*/ + +#[test] +fn select_all_wildcard() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + let ret = super::exec_select_all( + &global, + "create model myspace.mymodel(username: string, password: string)", + &[ + "insert into myspace.mymodel('sayan', 'password123')", + "insert into myspace.mymodel('robot', 'robot123')", + "insert into myspace.mymodel('douglas', 'galaxy123')", + "insert into myspace.mymodel('hgwells', 'timemachine')", + "insert into myspace.mymodel('orwell', '1984')", + ], + "select all * from myspace.mymodel LIMIT 100", + ) + .unwrap(); + let ret: HashMap> = ret + .into_iter() + .map(|mut d| (d.swap_remove(0).into_str().unwrap(), d)) + .collect(); + assert_eq!(ret.get("sayan").unwrap(), &intovec!["password123"]); + assert_eq!(ret.get("robot").unwrap(), &intovec!["robot123"]); + assert_eq!(ret.get("douglas").unwrap(), &intovec!["galaxy123"]); + assert_eq!(ret.get("hgwells").unwrap(), &intovec!["timemachine"]); + assert_eq!(ret.get("orwell").unwrap(), &intovec!["1984"]); +} + +#[test] +fn select_all_onefield() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + let ret = super::exec_select_all( + &global, + "create model myspace.mymodel(username: string, password: string)", + &[ + "insert into myspace.mymodel('sayan', 'password123')", + "insert into myspace.mymodel('robot', 'robot123')", + "insert into myspace.mymodel('douglas', 'galaxy123')", + "insert into myspace.mymodel('hgwells', 'timemachine')", + "insert into myspace.mymodel('orwell', '1984')", + ], + "select all username from myspace.mymodel LIMIT 100", + ) + .unwrap(); + let ret: HashMap> = ret + .into_iter() + .map(|mut d| (d.swap_remove(0).into_str().unwrap(), d)) + .collect(); + assert_eq!(ret.get("sayan").unwrap(), &intovec![]); + assert_eq!(ret.get("robot").unwrap(), &intovec![]); + assert_eq!(ret.get("douglas").unwrap(), &intovec![]); + assert_eq!(ret.get("hgwells").unwrap(), &intovec![]); + assert_eq!(ret.get("orwell").unwrap(), &intovec![]); +} diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs new file mode 100644 index 00000000..577cf465 --- /dev/null +++ b/server/src/engine/core/tests/dml/update.rs @@ -0,0 +1,160 @@ +/* + * Created on Sun May 14 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::dml, data::cell::Datacell, error::QueryError, fractal::test_utils::TestGlobal, +}; + +#[test] +fn simple() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_update( + &global, + "create model myspace.mymodel(username: string, email: string, followers: uint64, following: uint64)", + "insert into myspace.mymodel('sayan', 'sayan@example.com', 0, 100)", + "update myspace.mymodel set followers += 200000, following -= 15, email = 'sn@example.com' where username = 'sayan'", + "select * from myspace.mymodel where username = 'sayan'" + ).unwrap(), + intovec!["sayan", "sn@example.com", 200_000_u64, 85_u64], + ); + assert_eq!( + dml::update_flow_trace(), + ["sametag;nonnull", "sametag;nonnull", "sametag;nonnull"] + ); +} + +#[test] +fn with_null() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_update( + &global, + "create model myspace.mymodel(username: string, password: string, null email: string)", + "insert into myspace.mymodel('sayan', 'pass123', null)", + "update myspace.mymodel set email = 'sayan@example.com' where username = 'sayan'", + "select * from myspace.mymodel where username='sayan'" + ) + .unwrap(), + intovec!["sayan", "pass123", "sayan@example.com"] + ); + assert_eq!(dml::update_flow_trace(), ["sametag;orignull"]); +} + +#[test] +fn with_list() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_update( + &global, + "create model myspace.mymodel(link: string, click_ids: list { type: string })", + "insert into myspace.mymodel('example.com', [])", + "update myspace.mymodel set click_ids += 'ios_client_uuid' where link = 'example.com'", + "select * from myspace.mymodel where link = 'example.com'" + ) + .unwrap(), + intovec![ + "example.com", + Datacell::new_list(intovec!["ios_client_uuid"]) + ] + ); + assert_eq!(dml::update_flow_trace(), ["list;sametag"]); +} + +#[test] +fn fail_operation_on_null() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_update( + &global, + "create model myspace.mymodel(username: string, password: string, null email: string)", + "insert into myspace.mymodel('sayan', 'pass123', null)", + "update myspace.mymodel set email += '.com' where username = 'sayan'", + "select * from myspace.mymodel where username='sayan'" + ) + .unwrap_err(), + QueryError::QExecDmlValidationError + ); + assert_eq!( + dml::update_flow_trace(), + ["unknown_reason;exitmainloop", "rollback"] + ); +} + +#[test] +fn fail_unknown_fields() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_update( + &global, + "create model myspace.mymodel(username: string, password: string, null email: string)", + "insert into myspace.mymodel('sayan', 'pass123', null)", + "update myspace.mymodel set email2 = 'sayan@example.com', password += '4' where username = 'sayan'", + "select * from myspace.mymodel where username='sayan'" + ) + .unwrap_err(), + QueryError::QExecUnknownField + ); + assert_eq!(dml::update_flow_trace(), ["fieldnotfound", "rollback"]); + // verify integrity + assert_eq!( + super::exec_select_only( + &global, + "select * from myspace.mymodel where username='sayan'" + ) + .unwrap(), + intovec!["sayan", "pass123", Datacell::null()] + ); +} + +#[test] +fn fail_typedef_violation() { + let global = TestGlobal::new_with_tmp_nullfs_driver(); + assert_eq!( + super::exec_update( + &global, + "create model myspace.mymodel(username: string, password: string, rank: uint8)", + "insert into myspace.mymodel('sayan', 'pass123', 1)", + "update myspace.mymodel set password = 'pass1234', rank = 'one' where username = 'sayan'", + "select * from myspace.mymodel where username = 'sayan'" + ) + .unwrap_err(), + QueryError::QExecDmlValidationError + ); + assert_eq!( + dml::update_flow_trace(), + ["sametag;nonnull", "unknown_reason;exitmainloop", "rollback"] + ); + // verify integrity + assert_eq!( + super::exec_select_only( + &global, + "select * from myspace.mymodel where username = 'sayan'" + ) + .unwrap(), + intovec!["sayan", "pass123", 1u64] + ); +} diff --git a/server/src/admin/mod.rs b/server/src/engine/core/tests/mod.rs similarity index 85% rename from server/src/admin/mod.rs rename to server/src/engine/core/tests/mod.rs index 1791b211..a97da00f 100644 --- a/server/src/admin/mod.rs +++ b/server/src/engine/core/tests/mod.rs @@ -1,5 +1,5 @@ /* - * Created on Tue Nov 03 2020 + * Created on Wed Feb 08 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2020, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,7 +24,6 @@ * */ -//! Modules for administration of Skytable - -pub mod mksnap; -pub mod sys; +mod ddl_model; +mod ddl_space; +mod dml; diff --git a/server/src/engine/core/util.rs b/server/src/engine/core/util.rs new file mode 100644 index 00000000..b6c9bc84 --- /dev/null +++ b/server/src/engine/core/util.rs @@ -0,0 +1,157 @@ +/* + * Created on Thu Apr 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use std::{ + alloc::{dealloc, Layout}, + borrow::Borrow, + fmt, + hash::Hash, + marker::PhantomData, + mem::ManuallyDrop, + slice, str, +}; + +pub struct EntityID { + sp: *mut u8, + sl: usize, + ep: *mut u8, + el: usize, +} + +impl EntityID { + pub fn new(space: &str, entity: &str) -> Self { + let mut space = ManuallyDrop::new(space.to_owned().into_boxed_str().into_boxed_bytes()); + let mut entity = ManuallyDrop::new(entity.to_owned().into_boxed_str().into_boxed_bytes()); + Self { + sp: space.as_mut_ptr(), + sl: space.len(), + ep: entity.as_mut_ptr(), + el: entity.len(), + } + } + pub fn space(&self) -> &str { + unsafe { str::from_utf8_unchecked(slice::from_raw_parts(self.sp, self.sl)) } + } + pub fn entity(&self) -> &str { + unsafe { str::from_utf8_unchecked(slice::from_raw_parts(self.ep, self.el)) } + } +} + +impl Drop for EntityID { + fn drop(&mut self) { + unsafe { + dealloc(self.sp, Layout::array::(self.sl).unwrap_unchecked()); + dealloc(self.ep, Layout::array::(self.el).unwrap_unchecked()); + } + } +} + +impl PartialEq for EntityID { + fn eq(&self, other: &Self) -> bool { + self.space() == other.space() && self.entity() == other.entity() + } +} + +impl Eq for EntityID {} + +impl Hash for EntityID { + fn hash(&self, state: &mut H) { + self.space().hash(state); + self.entity().hash(state); + } +} + +impl fmt::Debug for EntityID { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EntityID") + .field("space", &self.space()) + .field("entity", &self.entity()) + .finish() + } +} + +#[derive(Clone, Copy)] +pub struct EntityIDRef<'a> { + sp: *const u8, + sl: usize, + ep: *const u8, + el: usize, + _lt: PhantomData<(&'a str, &'a str)>, +} + +impl<'a> EntityIDRef<'a> { + pub fn new(space: &'a str, entity: &'a str) -> Self { + Self { + sp: space.as_ptr(), + sl: space.len(), + ep: entity.as_ptr(), + el: entity.len(), + _lt: PhantomData, + } + } + pub fn space(&self) -> &'a str { + unsafe { str::from_utf8_unchecked(slice::from_raw_parts(self.sp, self.sl)) } + } + pub fn entity(&self) -> &'a str { + unsafe { str::from_utf8_unchecked(slice::from_raw_parts(self.ep, self.el)) } + } +} + +impl<'a> PartialEq for EntityIDRef<'a> { + fn eq(&self, other: &Self) -> bool { + self.space() == other.space() && self.entity() == other.entity() + } +} + +impl<'a> Eq for EntityIDRef<'a> {} + +impl<'a> Hash for EntityIDRef<'a> { + fn hash(&self, state: &mut H) { + self.space().hash(state); + self.entity().hash(state); + } +} + +impl<'a> fmt::Debug for EntityIDRef<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EntityIDRef") + .field("space", &self.space()) + .field("entity", &self.entity()) + .finish() + } +} + +impl<'a> Borrow> for EntityID { + fn borrow(&self) -> &EntityIDRef<'a> { + unsafe { core::mem::transmute(self) } + } +} + +impl<'a> From<(&'a str, &'a str)> for EntityIDRef<'a> { + fn from((s, e): (&'a str, &'a str)) -> Self { + Self::new(s, e) + } +} diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs new file mode 100644 index 00000000..c81a7c26 --- /dev/null +++ b/server/src/engine/data/cell.rs @@ -0,0 +1,562 @@ +/* + * Created on Tue Feb 28 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::core::index::PrimaryIndexKey; + +use super::tag::TagUnique; + +use { + crate::engine::{ + self, + data::{ + lit::Lit, + tag::{DataTag, FloatSpec, FullTag, SIntSpec, TagClass, UIntSpec}, + }, + mem::{DwordNN, DwordQN, NativeQword, SpecialPaddedWord, WordIO}, + }, + core::{ + fmt, + marker::PhantomData, + mem::{self, ManuallyDrop}, + ops::Deref, + slice, str, + }, + parking_lot::RwLock, +}; + +pub struct Datacell { + init: bool, + tag: FullTag, + data: DataRaw, +} + +impl Datacell { + // bool + pub fn new_bool(b: bool) -> Self { + unsafe { + // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag + Self::new( + FullTag::BOOL, + DataRaw::word(SpecialPaddedWord::store(b).dwordqn_promote()), + ) + } + } + pub unsafe fn read_bool(&self) -> bool { + self.load_word() + } + pub fn try_bool(&self) -> Option { + self.checked_tag(TagClass::Bool, || unsafe { + // UNSAFE(@ohsayan): correct because we just verified the tag + self.read_bool() + }) + } + pub fn bool(&self) -> bool { + self.try_bool().unwrap() + } + // uint + pub fn new_uint(u: u64, kind: UIntSpec) -> Self { + unsafe { + // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag + Self::new( + kind.into(), + DataRaw::word(SpecialPaddedWord::store(u).dwordqn_promote()), + ) + } + } + pub fn new_uint_default(u: u64) -> Self { + Self::new_uint(u, UIntSpec::DEFAULT) + } + pub unsafe fn read_uint(&self) -> u64 { + self.load_word() + } + pub fn try_uint(&self) -> Option { + self.checked_tag(TagClass::UnsignedInt, || unsafe { + // UNSAFE(@ohsayan): correct because we just verified the tag + self.read_uint() + }) + } + pub fn uint(&self) -> u64 { + self.try_uint().unwrap() + } + pub fn into_uint(self) -> Option { + if self.kind() != TagClass::UnsignedInt { + return None; + } + unsafe { + // UNSAFE(@ohsayan): +tagck + let md = ManuallyDrop::new(self); + Some(md.data.word.dwordnn_load_qw()) + } + } + // sint + pub fn new_sint(i: i64, kind: SIntSpec) -> Self { + unsafe { + // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag + Self::new( + kind.into(), + DataRaw::word(SpecialPaddedWord::store(i).dwordqn_promote()), + ) + } + } + pub fn new_sint_default(s: i64) -> Self { + Self::new_sint(s, SIntSpec::DEFAULT) + } + pub unsafe fn read_sint(&self) -> i64 { + self.load_word() + } + pub fn try_sint(&self) -> Option { + self.checked_tag(TagClass::SignedInt, || unsafe { + // UNSAFE(@ohsayan): Correct because we just verified the tag + self.read_sint() + }) + } + pub fn sint(&self) -> i64 { + self.try_sint().unwrap() + } + // float + pub fn new_float(f: f64, spec: FloatSpec) -> Self { + unsafe { + // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag + Self::new( + spec.into(), + DataRaw::word(SpecialPaddedWord::store(f).dwordqn_promote()), + ) + } + } + pub fn new_float_default(f: f64) -> Self { + Self::new_float(f, FloatSpec::DEFAULT) + } + pub unsafe fn read_float(&self) -> f64 { + self.load_word() + } + pub fn try_float(&self) -> Option { + self.checked_tag(TagClass::Float, || unsafe { + // UNSAFE(@ohsayan): Correcrt because we just verified the tag + self.read_float() + }) + } + pub fn float(&self) -> f64 { + self.try_float().unwrap() + } + // bin + pub fn new_bin(s: Box<[u8]>) -> Self { + let mut md = ManuallyDrop::new(s); + unsafe { + // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag + Self::new( + FullTag::BIN, + DataRaw::word(WordIO::store((md.len(), md.as_mut_ptr()))), + ) + } + } + pub unsafe fn read_bin(&self) -> &[u8] { + let (l, p) = self.load_word(); + slice::from_raw_parts::(p, l) + } + pub fn try_bin(&self) -> Option<&[u8]> { + self.checked_tag(TagClass::Bin, || unsafe { + // UNSAFE(@ohsayan): Correct because we just verified the tag + self.read_bin() + }) + } + pub fn bin(&self) -> &[u8] { + self.try_bin().unwrap() + } + pub fn into_bin(self) -> Option> { + if self.kind() != TagClass::Bin { + return None; + } + unsafe { + // UNSAFE(@ohsayan): no double free + tagck + let md = ManuallyDrop::new(self); + let (a, b) = md.data.word.dwordqn_load_qw_nw(); + Some(Vec::from_raw_parts( + b as *const u8 as *mut u8, + a as usize, + a as usize, + )) + } + } + // str + pub fn new_str(s: Box) -> Self { + let mut md = ManuallyDrop::new(s.into_boxed_bytes()); + unsafe { + // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag + Self::new( + FullTag::STR, + DataRaw::word(WordIO::store((md.len(), md.as_mut_ptr()))), + ) + } + } + pub unsafe fn read_str(&self) -> &str { + let (l, p) = self.load_word(); + str::from_utf8_unchecked(slice::from_raw_parts(p, l)) + } + pub fn try_str(&self) -> Option<&str> { + self.checked_tag(TagClass::Str, || unsafe { + // UNSAFE(@ohsayan): Correct because we just verified the tag + self.read_str() + }) + } + pub fn str(&self) -> &str { + self.try_str().unwrap() + } + pub fn into_str(self) -> Option { + if self.kind() != TagClass::Str { + return None; + } + unsafe { + // UNSAFE(@ohsayan): no double free + tagck + let md = ManuallyDrop::new(self); + let (a, b) = md.data.word.dwordqn_load_qw_nw(); + Some(String::from_raw_parts( + b as *const u8 as *mut u8, + a as usize, + a as usize, + )) + } + } + // list + pub fn new_list(l: Vec) -> Self { + unsafe { + // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag + Self::new(FullTag::LIST, DataRaw::rwl(RwLock::new(l))) + } + } + pub unsafe fn read_list(&self) -> &RwLock> { + &self.data.rwl + } + pub fn try_list(&self) -> Option<&RwLock>> { + self.checked_tag(TagClass::List, || unsafe { + // UNSAFE(@ohsayan): Correct because we just verified the tag + self.read_list() + }) + } + pub fn list(&self) -> &RwLock> { + self.try_list().unwrap() + } + pub fn into_list(self) -> Option> { + if self.kind() != TagClass::List { + return None; + } + unsafe { + // UNSAFE(@ohsayan): +tagck +avoid double free + let md = ManuallyDrop::new(self); + let rwl = core::ptr::read(&md.data.rwl); + Some(ManuallyDrop::into_inner(rwl).into_inner()) + } + } + pub unsafe fn new_qw(qw: u64, tag: FullTag) -> Datacell { + Self::new( + tag, + DataRaw::word(SpecialPaddedWord::store(qw).dwordqn_promote()), + ) + } + pub unsafe fn set_tag(&mut self, tag: FullTag) { + self.tag = tag; + } +} + +direct_from! { + Datacell => { + bool as new_bool, + u64 as new_uint_default, + i64 as new_sint_default, + f64 as new_float_default, + Vec as new_bin, + Box<[u8]> as new_bin, + &'static [u8] as new_bin, + String as new_str, + Box as new_str, + &'static str as new_str, + Vec as new_list, + Box<[Self]> as new_list, + } +} + +impl<'a> From> for Datacell { + fn from(l: Lit<'a>) -> Self { + match l.kind().tag_class() { + tag if tag < TagClass::Bin => unsafe { + // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type doesn't need any advanced construction + Datacell::new( + l.kind(), + // DO NOT RELY ON the payload's bit pattern; it's padded + DataRaw::word(l.data().dwordqn_promote()), + ) + }, + TagClass::Bin | TagClass::Str => unsafe { + // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type requires a new heap for construction + let mut bin = ManuallyDrop::new(l.bin().to_owned().into_boxed_slice()); + Datacell::new( + l.kind(), + DataRaw::word(DwordQN::dwordqn_store_qw_nw( + bin.len() as u64, + bin.as_mut_ptr() as usize, + )), + ) + }, + _ => unsafe { + // UNSAFE(@ohsayan): a Lit will never be higher than a string + impossible!() + }, + } + } +} + +#[cfg(test)] +impl From for Datacell { + fn from(i: i32) -> Self { + if i.is_negative() { + Self::new_sint_default(i as _) + } else { + Self::new_uint_default(i as _) + } + } +} + +impl From<[Datacell; N]> for Datacell { + fn from(l: [Datacell; N]) -> Self { + Self::new_list(l.into()) + } +} + +impl Datacell { + pub const fn tag(&self) -> FullTag { + self.tag + } + pub fn kind(&self) -> TagClass { + self.tag.tag_class() + } + pub fn null() -> Self { + unsafe { + // UNSAFE(@ohsayan): This is a hack. It's safe because we set init to false + Self::_new( + FullTag::BOOL, + DataRaw::word(NativeQword::dwordnn_store_qw(0)), + false, + ) + } + } + pub fn is_null(&self) -> bool { + !self.init + } + pub fn is_init(&self) -> bool { + self.init + } + unsafe fn load_word<'a, T>(&'a self) -> T + where + NativeQword: WordIO, + { + self.data.word.load() + } + unsafe fn _new(tag: FullTag, data: DataRaw, init: bool) -> Self { + Self { init, tag, data } + } + unsafe fn new(tag: FullTag, data: DataRaw) -> Self { + Self::_new(tag, data, true) + } + fn checked_tag(&self, tag: TagClass, f: impl FnOnce() -> T) -> Option { + ((self.kind() == tag) & (self.is_init())).then(f) + } + pub unsafe fn as_raw(&self) -> NativeQword { + mem::transmute_copy(&self.data.word) + } +} + +impl fmt::Debug for Datacell { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut f = f.debug_struct("Datacell"); + f.field("tag", &self.tag); + macro_rules! fmtdbg { + ($($match:ident => $ret:expr),* $(,)?) => { + match self.kind() { + $(TagClass::$match if self.is_init() => { f.field("data", &Some($ret));},)* + TagClass::Bool if self.is_null() => {f.field("data", &Option::::None);}, + _ => unreachable!("incorrect state"), + } + } + } + fmtdbg!( + Bool => self.bool(), + UnsignedInt => self.uint(), + SignedInt => self.sint(), + Float => self.float(), + Bin => self.bin(), + Str => self.str(), + List => self.list(), + ); + f.finish() + } +} + +impl PartialEq for Datacell { + fn eq(&self, other: &Datacell) -> bool { + if self.is_null() { + return other.is_null(); + } + match (self.kind(), other.kind()) { + (TagClass::Bool, TagClass::Bool) => self.bool() == other.bool(), + (TagClass::UnsignedInt, TagClass::UnsignedInt) => self.uint() == other.uint(), + (TagClass::SignedInt, TagClass::SignedInt) => self.sint() == other.sint(), + (TagClass::Float, TagClass::Float) => self.float() == other.float(), + (TagClass::Bin, TagClass::Bin) => self.bin() == other.bin(), + (TagClass::Str, TagClass::Str) => self.str() == other.str(), + (TagClass::List, TagClass::List) => { + let l1_l = self.list().read(); + let l2_l = other.list().read(); + let l1: &[Self] = l1_l.as_ref(); + let l2: &[Self] = l2_l.as_ref(); + l1 == l2 + } + _ => false, + } + } +} + +impl Eq for Datacell {} + +union! { + union DataRaw { + !word: NativeQword, + !rwl: RwLock>, + } +} + +impl DataRaw { + fn word(word: NativeQword) -> Self { + Self { + word: ManuallyDrop::new(word), + } + } + fn rwl(rwl: RwLock>) -> Self { + Self { + rwl: ManuallyDrop::new(rwl), + } + } +} + +impl Drop for Datacell { + fn drop(&mut self) { + match self.kind() { + TagClass::Str | TagClass::Bin => unsafe { + // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class + let (l, p) = self.load_word(); + engine::mem::dealloc_array::(p, l) + }, + TagClass::List => unsafe { + // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class + ManuallyDrop::drop(&mut self.data.rwl) + }, + _ => {} + } + } +} + +#[cfg(test)] +impl Clone for Datacell { + fn clone(&self) -> Self { + let data = match self.kind() { + TagClass::Str | TagClass::Bin => unsafe { + // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class + let mut block = ManuallyDrop::new(self.read_bin().to_owned().into_boxed_slice()); + DataRaw::word(DwordQN::dwordqn_store_qw_nw( + block.len() as u64, + block.as_mut_ptr() as usize, + )) + }, + TagClass::List => unsafe { + // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class + let data = self.read_list().read().iter().cloned().collect(); + DataRaw::rwl(RwLock::new(data)) + }, + _ => unsafe { + // UNSAFE(@ohsayan): we have checked that the cell is a stack class + DataRaw::word(mem::transmute_copy(&self.data.word)) + }, + }; + unsafe { + // UNSAFE(@ohsayan): same tag, we correctly init data and also return the same init state + Self::_new(self.tag, data, self.init) + } + } +} + +#[derive(Debug)] +pub struct VirtualDatacell<'a> { + dc: ManuallyDrop, + _lt: PhantomData>, +} + +impl<'a> VirtualDatacell<'a> { + pub fn new(lit: Lit<'a>, tag: TagUnique) -> Self { + debug_assert_eq!(lit.kind().tag_unique(), tag); + Self { + dc: ManuallyDrop::new(unsafe { + // UNSAFE(@ohsayan): this is a "reference" to a "virtual" aka fake DC. this just works because of memory layouts + Datacell::new(lit.kind(), DataRaw::word(lit.data().dwordqn_promote())) + }), + _lt: PhantomData, + } + } + pub fn new_pk(pk: &'a PrimaryIndexKey, tag: FullTag) -> Self { + debug_assert_eq!(pk.tag(), tag.tag_unique()); + Self { + dc: ManuallyDrop::new(unsafe { + Datacell::new(tag, DataRaw::word(pk.data().dwordqn_promote())) + }), + _lt: PhantomData, + } + } +} + +impl<'a> Deref for VirtualDatacell<'a> { + type Target = Datacell; + fn deref(&self) -> &Self::Target { + &self.dc + } +} + +impl<'a> PartialEq for VirtualDatacell<'a> { + fn eq(&self, other: &Datacell) -> bool { + self.dc.deref() == other + } +} + +impl<'a> Clone for VirtualDatacell<'a> { + fn clone(&self) -> Self { + unsafe { core::mem::transmute_copy(self) } + } +} + +#[test] +fn virtual_dc_damn() { + let dc = Lit::new_str("hello, world"); + assert_eq!( + VirtualDatacell::new(dc, TagUnique::Str), + Datacell::from("hello, world") + ); +} diff --git a/server/src/engine/data/dict.rs b/server/src/engine/data/dict.rs new file mode 100644 index 00000000..bdce358a --- /dev/null +++ b/server/src/engine/data/dict.rs @@ -0,0 +1,213 @@ +/* + * Created on Thu Feb 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::{ + data::{cell::Datacell, lit::Lit}, + idx::STIndex, + }, + std::collections::HashMap, +}; + +/// A generic dictionary built from scratch from syntactical elements +pub type DictGeneric = HashMap, DictEntryGeneric>; + +#[derive(Debug, PartialEq)] +#[cfg_attr(test, derive(Clone))] +/// A generic dict entry: either a literal or a recursive dictionary +pub enum DictEntryGeneric { + /// A literal + Data(Datacell), + /// A map + Map(DictGeneric), +} + +impl DictEntryGeneric { + pub fn as_dict_mut(&mut self) -> Option<&mut DictGeneric> { + match self { + Self::Map(m) => Some(m), + _ => None, + } + } + pub fn into_dict(self) -> Option { + match self { + Self::Map(m) => Some(m), + _ => None, + } + } + pub fn into_data(self) -> Option { + match self { + Self::Data(d) => Some(d), + _ => None, + } + } +} + +/* + patchsets +*/ + +/// Accepts a dict with possible null values, and removes those null values +pub fn rflatten_metadata(mut new: DictGeneric) -> DictGeneric { + _rflatten_metadata(&mut new); + new +} + +fn _rflatten_metadata(new: &mut DictGeneric) { + new.retain(|_, v| match v { + DictEntryGeneric::Data(d) => d.is_init(), + DictEntryGeneric::Map(m) => { + _rflatten_metadata(m); + true + } + }); +} + +/// Recursively merge a [`DictGeneric`] into a [`DictGeneric`] with the use of an intermediary +/// patchset to avoid inconsistent states +pub fn rmerge_metadata(current: &mut DictGeneric, new: DictGeneric) -> bool { + match rprepare_metadata_patch(current as &_, new) { + Some(patch) => { + rmerge_data_with_patch(current, patch); + true + } + None => false, + } +} + +pub fn rprepare_metadata_patch(current: &DictGeneric, new: DictGeneric) -> Option { + let mut patch = Default::default(); + if rmerge_metadata_prepare_patch(current, new, &mut patch) { + Some(patch) + } else { + None + } +} + +pub fn rmerge_data_with_patch(current: &mut DictGeneric, patch: DictGeneric) { + for (key, patch) in patch { + match patch { + DictEntryGeneric::Data(d) if d.is_init() => { + current.st_upsert(key, DictEntryGeneric::Data(d)); + } + DictEntryGeneric::Data(_) => { + // null + let _ = current.remove(&key); + } + DictEntryGeneric::Map(m) => match current.get_mut(&key) { + Some(current_recursive) => match current_recursive { + DictEntryGeneric::Map(current_m) => { + rmerge_data_with_patch(current_m, m); + } + _ => { + // can never reach here since the patch is always correct + unreachable!() + } + }, + None => { + let mut new = DictGeneric::new(); + rmerge_data_with_patch(&mut new, m); + } + }, + } + } +} + +fn rmerge_metadata_prepare_patch( + current: &DictGeneric, + new: DictGeneric, + patch: &mut DictGeneric, +) -> bool { + let mut new = new.into_iter(); + let mut okay = true; + while new.len() != 0 && okay { + let (key, new_entry) = new.next().unwrap(); + match (current.get(&key), new_entry) { + // non-null -> non-null: merge flatten update + (Some(DictEntryGeneric::Data(this_data)), DictEntryGeneric::Data(new_data)) + if new_data.is_init() => + { + if this_data.kind() == new_data.kind() { + patch.insert(key, DictEntryGeneric::Data(new_data)); + } else { + okay = false; + } + } + (Some(DictEntryGeneric::Data(_)), DictEntryGeneric::Map(_)) => { + okay = false; + } + ( + Some(DictEntryGeneric::Map(this_recursive_data)), + DictEntryGeneric::Map(new_recursive_data), + ) => { + let mut this_patch = DictGeneric::new(); + let _okay = rmerge_metadata_prepare_patch( + this_recursive_data, + new_recursive_data, + &mut this_patch, + ); + patch.insert(key, DictEntryGeneric::Map(this_patch)); + okay &= _okay; + } + // null -> non-null: flatten insert + (None, DictEntryGeneric::Data(l)) if l.is_init() => { + let _ = patch.insert(key, DictEntryGeneric::Data(l)); + } + (None, DictEntryGeneric::Map(m)) => { + let mut this_patch = DictGeneric::new(); + okay &= rmerge_metadata_prepare_patch(&into_dict!(), m, &mut this_patch); + let _ = patch.insert(key, DictEntryGeneric::Map(this_patch)); + } + // non-null -> null: remove + (Some(_), DictEntryGeneric::Data(l)) => { + debug_assert!(l.is_null()); + patch.insert(key, DictEntryGeneric::Data(Datacell::null())); + } + (None, DictEntryGeneric::Data(l)) => { + debug_assert!(l.is_null()); + // ignore + } + } + } + okay +} + +/* + impls +*/ + +impl<'a> From> for DictEntryGeneric { + fn from(l: Lit<'a>) -> Self { + Self::Data(Datacell::from(l)) + } +} + +direct_from! { + DictEntryGeneric => { + Datacell as Data, + DictGeneric as Map, + } +} diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs new file mode 100644 index 00000000..ed9cb223 --- /dev/null +++ b/server/src/engine/data/lit.rs @@ -0,0 +1,416 @@ +/* + * Created on Wed Sep 20 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::{ + data::tag::{DataTag, FullTag, TagClass, TagUnique}, + mem::{DwordQN, SpecialPaddedWord}, + }, + core::{ + fmt, + hash::{Hash, Hasher}, + marker::PhantomData, + mem::ManuallyDrop, + slice, str, + }, +}; + +/* + NOTE(@ohsayan): Heinous hackery that should not ever be repeated. Just don't touch anything here. +*/ + +/// A literal representation +pub struct Lit<'a> { + tag: FullTag, + dtc: u8, + word: SpecialPaddedWord, + _lt: PhantomData<&'a [u8]>, +} + +impl<'a> Lit<'a> { + /// Create a new bool literal + pub fn new_bool(b: bool) -> Self { + Self::_quad(b as _, FullTag::BOOL) + } + /// Create a new unsigned integer + pub fn new_uint(u: u64) -> Self { + Self::_quad(u, FullTag::UINT) + } + /// Create a new signed integer + pub fn new_sint(s: i64) -> Self { + Self::_quad(s as _, FullTag::SINT) + } + /// Create a new float64 + pub fn new_float(f: f64) -> Self { + Self::_quad(f.to_bits(), FullTag::FLOAT) + } + /// Returns a "shallow clone" + /// + /// This function will fall apart if lifetimes aren't handled correctly (aka will segfault) + pub fn as_ir(&'a self) -> Lit<'a> { + unsafe { + // UNSAFE(@ohsayan): this is a dirty, uncanny and wild hack that everyone should be forbidden from doing + let mut slf: Lit<'a> = core::mem::transmute_copy(self); + slf.dtc = Self::DTC_NONE; + slf + } + } +} + +#[allow(unused)] +impl<'a> Lit<'a> { + /// Attempt to read a bool + pub fn try_bool(&self) -> Option { + (self.tag.tag_class() == TagClass::Bool).then_some(unsafe { + // UNSAFE(@ohsayan): +tagck + self.bool() + }) + } + /// Attempt to read an unsigned integer + pub fn try_uint(&self) -> Option { + (self.tag.tag_class() == TagClass::UnsignedInt).then_some(unsafe { + // UNSAFE(@ohsayan): +tagck + self.uint() + }) + } + /// Attempt to read a signed integer + pub fn try_sint(&self) -> Option { + (self.tag.tag_class() == TagClass::SignedInt).then_some(unsafe { + // UNSAFE(@ohsayan): +tagck + self.sint() + }) + } + /// Attempt to read a float + pub fn try_float(&self) -> Option { + (self.tag.tag_class() == TagClass::Float).then_some(unsafe { + // UNSAFE(@ohsayan): +tagck + self.float() + }) + } + /// Read a bool directly. This function isn't exactly unsafe, but we want to provide a type preserving API + pub unsafe fn bool(&self) -> bool { + self.uint() == 1 + } + /// Read an unsigned integer directly. This function isn't exactly unsafe, but we want to provide a type + /// preserving API + pub unsafe fn uint(&self) -> u64 { + self.word.dwordqn_load_qw_nw().0 + } + /// Read a signed integer directly. This function isn't exactly unsafe, but we want to provide a type + /// preserving API + pub unsafe fn sint(&self) -> i64 { + self.uint() as _ + } + /// Read a floating point number directly. This function isn't exactly unsafe, but we want to provide a type + /// preserving API + pub unsafe fn float(&self) -> f64 { + f64::from_bits(self.uint()) + } +} + +#[allow(unused)] +impl<'a> Lit<'a> { + /// Attempt to read a binary value + pub fn try_bin(&self) -> Option<&'a [u8]> { + (self.tag.tag_class() == TagClass::Bin).then(|| unsafe { + // UNSAFE(@ohsayan): +tagck + self.bin() + }) + } + /// Attempt to read a string value + pub fn try_str(&self) -> Option<&'a str> { + (self.tag.tag_class() == TagClass::Str).then(|| unsafe { + // UNSAFE(@ohsayan): +tagck + self.str() + }) + } + /// Read a string value directly + /// + /// ## Safety + /// The underlying repr MUST be a string. Otherwise you'll segfault or cause other library functions to misbehave + pub unsafe fn str(&self) -> &'a str { + str::from_utf8_unchecked(self.bin()) + } + /// Read a binary value directly + /// + /// ## Safety + /// The underlying repr MUST be a string. Otherwise you'll segfault + pub unsafe fn bin(&self) -> &'a [u8] { + let (q, n) = self.word.dwordqn_load_qw_nw(); + slice::from_raw_parts(n as *const u8 as *mut u8, q as _) + } +} + +impl<'a> Lit<'a> { + /// Create a new string (referenced) + pub fn new_str(s: &'a str) -> Self { + unsafe { + /* + UNSAFE(@ohsayan): the mut cast is just for typesake so it doesn't matter while we also set DTC + to none so it shouldn't matter anyway + */ + Self::_str(s.as_ptr() as *mut u8, s.len(), Self::DTC_NONE) + } + } + /// Create a new boxed string + pub fn new_boxed_str(s: Box) -> Self { + let mut md = ManuallyDrop::new(s); // mut -> aliasing! + unsafe { + // UNSAFE(@ohsayan): correct aliasing, and DTC to destroy heap + Self::_str(md.as_mut_ptr(), md.len(), Self::DTC_HSTR) + } + } + /// Create a new string + pub fn new_string(s: String) -> Self { + Self::new_boxed_str(s.into_boxed_str()) + } + /// Create a new binary (referenced) + pub fn new_bin(b: &'a [u8]) -> Self { + unsafe { + // UNSAFE(@ohsayan): mut cast is once again just a typesake change + Self::_wide_word(b.as_ptr() as *mut _, b.len(), Self::DTC_NONE, FullTag::BIN) + } + } +} + +impl<'a> Lit<'a> { + /// Returns the type of this literal + pub fn kind(&self) -> FullTag { + self.tag + } + /// Returns the internal representation of this type + pub unsafe fn data(&self) -> &SpecialPaddedWord { + &self.word + } + pub fn __vdata(&self) -> &'a [u8] { + let (vlen, data) = self.word.dwordqn_load_qw_nw(); + let len = vlen as usize * (self.kind().tag_unique() >= TagUnique::Bin) as usize; + unsafe { + // UNSAFE(@ohsayan): either because of static or lt + slice::from_raw_parts(data as *const u8, len) + } + } +} + +impl<'a> Lit<'a> { + const DTC_NONE: u8 = 0; + const DTC_HSTR: u8 = 1; + unsafe fn _new(tag: FullTag, dtc: u8, word: SpecialPaddedWord) -> Self { + Self { + tag, + dtc, + word, + _lt: PhantomData, + } + } + fn _quad(quad: u64, tag: FullTag) -> Self { + unsafe { + // UNSAFE(@ohsayan): we initialize the correct bit pattern + Self::_new(tag, Self::DTC_NONE, SpecialPaddedWord::new_quad(quad)) + } + } + unsafe fn _wide_word(ptr: *mut u8, len: usize, dtc: u8, tag: FullTag) -> Self { + Self::_new(tag, dtc, SpecialPaddedWord::new(len as _, ptr as _)) + } + unsafe fn _str(ptr: *mut u8, len: usize, dtc: u8) -> Self { + Self::_wide_word(ptr, len, dtc, FullTag::STR) + } + unsafe fn _drop_zero(_: SpecialPaddedWord) {} + unsafe fn _drop_hstr(word: SpecialPaddedWord) { + let (a, b) = word.dwordqn_load_qw_nw(); + drop(Vec::from_raw_parts( + b as *const u8 as *mut u8, + a as _, + a as _, + )); + } +} + +impl<'a> Drop for Lit<'a> { + fn drop(&mut self) { + static DFN: [unsafe fn(SpecialPaddedWord); 2] = [Lit::_drop_zero, Lit::_drop_hstr]; + unsafe { DFN[self.dtc as usize](core::mem::transmute_copy(&self.word)) } + } +} + +impl<'a> Clone for Lit<'a> { + fn clone(&self) -> Lit<'a> { + static CFN: [unsafe fn(SpecialPaddedWord) -> SpecialPaddedWord; 2] = unsafe { + [ + |stack| core::mem::transmute(stack), + |hstr| { + let (q, n) = hstr.dwordqn_load_qw_nw(); + let mut md = ManuallyDrop::new( + slice::from_raw_parts(n as *const u8, q as usize).to_owned(), + ); + md.shrink_to_fit(); + SpecialPaddedWord::new(q, md.as_mut_ptr() as _) + }, + ] + }; + unsafe { + Self::_new( + self.tag, + self.dtc, + CFN[self.dtc as usize](core::mem::transmute_copy(&self.word)), + ) + } + } +} + +impl<'a> fmt::Debug for Lit<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut field = f.debug_struct("Lit"); + field.field("tag", &self.tag); + unsafe { + macro_rules! d { + ($expr:expr) => {{ + field.field("data", &$expr); + }}; + } + match self.tag.tag_class() { + TagClass::Bool => d!(self.bool()), + TagClass::UnsignedInt => d!(self.uint()), + TagClass::SignedInt => d!(self.sint()), + TagClass::Float => d!(self.float()), + TagClass::Bin => d!(self.bin()), + TagClass::Str => d!(self.str()), + TagClass::List => panic!("found 2D in 1D"), + } + } + field.finish() + } +} + +impl<'a> Hash for Lit<'a> { + fn hash(&self, state: &mut H) { + self.tag.tag_unique().hash(state); + self.__vdata().hash(state); + } +} + +impl<'a> PartialEq for Lit<'a> { + fn eq(&self, other: &Self) -> bool { + unsafe { + // UNSAFE(@ohsayan): +tagck + match (self.tag.tag_class(), other.tag.tag_class()) { + (TagClass::Bool, TagClass::Bool) => self.bool() == other.bool(), + (TagClass::UnsignedInt, TagClass::UnsignedInt) => self.uint() == other.uint(), + (TagClass::SignedInt, TagClass::SignedInt) => self.sint() == other.sint(), + (TagClass::Float, TagClass::Float) => self.float() == other.float(), + (TagClass::Bin, TagClass::Bin) => self.bin() == other.bin(), + (TagClass::Str, TagClass::Str) => self.str() == other.str(), + _ => false, + } + } + } +} + +direct_from! { + Lit<'a> => { + bool as new_bool, + u64 as new_uint, + i64 as new_sint, + f64 as new_float, + &'a str as new_str, + String as new_string, + Box as new_boxed_str, + &'a [u8] as new_bin, + } +} + +impl<'a> ToString for Lit<'a> { + fn to_string(&self) -> String { + unsafe { + match self.kind().tag_class() { + TagClass::Bool => self.bool().to_string(), + TagClass::UnsignedInt => self.uint().to_string(), + TagClass::SignedInt => self.sint().to_string(), + TagClass::Float => self.float().to_string(), + TagClass::Bin => format!("{:?}", self.bin()), + TagClass::Str => format!("{:?}", self.str()), + TagClass::List => panic!("found 2D in 1D"), + } + } + } +} + +#[test] +fn stk_variants() { + let stk1 = [ + Lit::new_bool(true), + Lit::new_uint(u64::MAX), + Lit::new_sint(i64::MIN), + Lit::new_float(f64::MIN), + Lit::new_str("hello"), + Lit::new_bin(b"world"), + ]; + let stk2 = stk1.clone(); + assert_eq!(stk1, stk2); +} + +#[test] +fn hp_variants() { + let hp1 = [ + Lit::new_string("hello".into()), + Lit::new_string("world".into()), + ]; + let hp2 = hp1.clone(); + assert_eq!(hp1, hp2); +} + +#[test] +fn lt_link() { + let l = Lit::new_string("hello".into()); + let l_ir = l.as_ir(); + assert_eq!(l, l_ir); +} + +#[test] +fn token_array_lt_test() { + let tokens = vec![Lit::new_string("hello".to_string()), Lit::new_str("hi")]; + #[derive(Debug)] + pub struct SelectStatement<'a> { + primary_key: Lit<'a>, + shorthand: Lit<'a>, + } + let select_stmt = SelectStatement { + primary_key: tokens[0].as_ir(), + shorthand: tokens[1].as_ir(), + }; + { + { + let SelectStatement { + primary_key, + shorthand, + } = &select_stmt; + let _ = primary_key.as_ir(); + let _ = shorthand.as_ir(); + } + } + drop(select_stmt); + drop(tokens); +} diff --git a/server/src/dbnet/macros.rs b/server/src/engine/data/mod.rs similarity index 80% rename from server/src/dbnet/macros.rs rename to server/src/engine/data/mod.rs index 64faab26..be15bd35 100644 --- a/server/src/dbnet/macros.rs +++ b/server/src/engine/data/mod.rs @@ -1,5 +1,5 @@ /* - * Created on Thu Aug 05 2021 + * Created on Sat Feb 04 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2021, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,11 +24,13 @@ * */ -macro_rules! skip_loop_err { - ($expr:expr) => { - match $expr { - Ok(ret) => ret, - Err(_) => continue, - } - }; -} +pub mod cell; +pub mod dict; +pub mod lit; +pub mod tag; +pub mod uuid; +// test +#[cfg(test)] +mod tests; + +pub use dict::{DictEntryGeneric, DictGeneric}; diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs new file mode 100644 index 00000000..62e6f73a --- /dev/null +++ b/server/src/engine/data/tag.rs @@ -0,0 +1,268 @@ +/* + * Created on Mon Feb 27 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +macro_rules! strid { + ($(#[$attr:meta])*$vis:vis enum $enum:ident {$($(#[$var_attr:meta])* $variant:ident $(= $dscr:expr)?),* $(,)?}) => { + $(#[$attr])* $vis enum $enum { $($(#[$var_attr])* $variant $(= $dscr)?),*} + impl $enum { + pub const fn name_str(&self) -> &'static str { match self { $(Self::$variant => stringify!($variant),)* } } + } + } +} + +#[repr(u8)] +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)] +pub enum TagClass { + Bool = 0, + UnsignedInt = 1, + SignedInt = 2, + Float = 3, + Bin = 4, + Str = 5, + List = 6, +} + +strid! { + #[repr(u8)] + #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)] + pub enum TagSelector { + Bool = 0, + UInt8 = 1, + UInt16 = 2, + UInt32 = 3, + UInt64 = 4, + SInt8 = 5, + SInt16 = 6, + SInt32 = 7, + SInt64 = 8, + Float32 = 9, + Float64 = 10, + Binary = 11, + String = 12, + List = 13, + } +} + +impl TagSelector { + pub const fn into_full(self) -> FullTag { + FullTag::new(self.tag_class(), self, self.tag_unique()) + } + pub const unsafe fn from_raw(v: u8) -> Self { + core::mem::transmute(v) + } + pub const fn tag_unique(&self) -> TagUnique { + [ + TagUnique::Illegal, + TagUnique::UnsignedInt, + TagUnique::UnsignedInt, + TagUnique::UnsignedInt, + TagUnique::UnsignedInt, + TagUnique::SignedInt, + TagUnique::SignedInt, + TagUnique::SignedInt, + TagUnique::SignedInt, + TagUnique::Illegal, + TagUnique::Illegal, + TagUnique::Bin, + TagUnique::Str, + TagUnique::Illegal, + ][self.value_word()] + } + pub const fn tag_class(&self) -> TagClass { + [ + TagClass::Bool, + TagClass::UnsignedInt, + TagClass::UnsignedInt, + TagClass::UnsignedInt, + TagClass::UnsignedInt, + TagClass::SignedInt, + TagClass::SignedInt, + TagClass::SignedInt, + TagClass::SignedInt, + TagClass::Float, + TagClass::Float, + TagClass::Bin, + TagClass::Str, + TagClass::List, + ][self.value_word()] + } +} + +#[repr(u8)] +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)] +pub enum TagUnique { + UnsignedInt = 0, + SignedInt = 1, + Bin = 2, + Str = 3, + Illegal = 0xFF, +} + +impl TagUnique { + pub const fn is_unique(&self) -> bool { + self.value_u8() != Self::Illegal.value_u8() + } + pub const fn try_from_raw(raw: u8) -> Option { + if raw > 3 { + return None; + } + Some(unsafe { core::mem::transmute(raw) }) + } +} + +pub trait DataTag { + const BOOL: Self; + const UINT: Self; + const SINT: Self; + const FLOAT: Self; + const BIN: Self; + const STR: Self; + const LIST: Self; + fn tag_class(&self) -> TagClass; + fn tag_selector(&self) -> TagSelector; + fn tag_unique(&self) -> TagUnique; +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub struct FullTag { + class: TagClass, + selector: TagSelector, + unique: TagUnique, +} + +impl FullTag { + const fn new(class: TagClass, selector: TagSelector, unique: TagUnique) -> Self { + Self { + class, + selector, + unique, + } + } + pub const fn new_uint(selector: TagSelector) -> Self { + Self::new(TagClass::UnsignedInt, selector, TagUnique::UnsignedInt) + } + pub const fn new_sint(selector: TagSelector) -> Self { + Self::new(TagClass::SignedInt, selector, TagUnique::SignedInt) + } + pub const fn new_float(selector: TagSelector) -> Self { + Self::new(TagClass::Float, selector, TagUnique::Illegal) + } +} + +macro_rules! fulltag { + ($class:ident, $selector:ident, $unique:ident) => { + FullTag::new(TagClass::$class, TagSelector::$selector, TagUnique::$unique) + }; + ($class:ident, $selector:ident) => { + fulltag!($class, $selector, Illegal) + }; +} + +impl DataTag for FullTag { + const BOOL: Self = fulltag!(Bool, Bool); + const UINT: Self = fulltag!(UnsignedInt, UInt64, UnsignedInt); + const SINT: Self = fulltag!(SignedInt, SInt64, SignedInt); + const FLOAT: Self = fulltag!(Float, Float64); + const BIN: Self = fulltag!(Bin, Binary, Bin); + const STR: Self = fulltag!(Str, String, Str); + const LIST: Self = fulltag!(List, List); + fn tag_class(&self) -> TagClass { + self.class + } + fn tag_selector(&self) -> TagSelector { + self.selector + } + fn tag_unique(&self) -> TagUnique { + self.unique + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +#[repr(transparent)] +pub struct UIntSpec(FullTag); +impl UIntSpec { + pub const LIM_MAX: [u64; 4] = as_array![u8::MAX, u16::MAX, u32::MAX, u64::MAX]; + pub const DEFAULT: Self = Self::UINT64; + pub const UINT64: Self = Self(FullTag::new_uint(TagSelector::UInt64)); + pub const unsafe fn from_full(f: FullTag) -> Self { + Self(f) + } + pub fn check(&self, v: u64) -> bool { + v <= Self::LIM_MAX[self.0.tag_selector().value_word() - 1] + } +} + +impl From for FullTag { + fn from(value: UIntSpec) -> Self { + value.0 + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +#[repr(transparent)] +pub struct SIntSpec(FullTag); +impl SIntSpec { + pub const LIM_MIN: [i64; 4] = as_array![i8::MIN, i16::MIN, i32::MIN, i64::MIN]; + pub const LIM_MAX: [i64; 4] = as_array![i8::MAX, i16::MAX, i32::MAX, i64::MAX]; + pub const DEFAULT: Self = Self::SINT64; + pub const SINT64: Self = Self(FullTag::new_sint(TagSelector::SInt64)); + pub const unsafe fn from_full(f: FullTag) -> Self { + Self(f) + } + pub fn check(&self, i: i64) -> bool { + let tag = self.0.tag_selector().value_word() - 5; + (i >= Self::LIM_MIN[tag]) & (i <= Self::LIM_MAX[tag]) + } +} + +impl From for FullTag { + fn from(value: SIntSpec) -> Self { + value.0 + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +#[repr(transparent)] +pub struct FloatSpec(FullTag); +impl FloatSpec { + pub const LIM_MIN: [f64; 2] = as_array![f32::MIN, f64::MIN]; + pub const LIM_MAX: [f64; 2] = as_array![f32::MAX, f64::MAX]; + pub const DEFAULT: Self = Self::F64; + pub const F64: Self = Self(FullTag::new_float(TagSelector::Float64)); + pub const unsafe fn from_full(f: FullTag) -> Self { + Self(f) + } + pub fn check(&self, f: f64) -> bool { + let tag = self.0.tag_selector().value_word() - 9; + (f >= Self::LIM_MIN[tag]) & (f <= Self::LIM_MAX[tag]) + } +} + +impl From for FullTag { + fn from(value: FloatSpec) -> Self { + value.0 + } +} diff --git a/server/src/engine/data/tests/md_dict_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs new file mode 100644 index 00000000..700c7a5c --- /dev/null +++ b/server/src/engine/data/tests/md_dict_tests.rs @@ -0,0 +1,103 @@ +/* + * Created on Thu Feb 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::data::{ + cell::Datacell, + dict::{self, DictEntryGeneric, DictGeneric}, +}; + +#[test] +fn t_simple_flatten() { + let generic_dict: DictGeneric = into_dict! { + "a_valid_key" => DictEntryGeneric::Data(100u64.into()), + "a_null_key" => Datacell::null(), + }; + let expected: DictGeneric = into_dict!( + "a_valid_key" => Datacell::new_uint_default(100) + ); + let ret = dict::rflatten_metadata(generic_dict); + assert_eq!(ret, expected); +} + +#[test] +fn t_simple_patch() { + let mut current: DictGeneric = into_dict! { + "a" => Datacell::new_uint_default(2), + "b" => Datacell::new_uint_default(3), + "z" => Datacell::new_sint_default(-100), + }; + let new: DictGeneric = into_dict! { + "a" => Datacell::new_uint_default(1), + "b" => Datacell::new_uint_default(2), + "z" => Datacell::null(), + }; + let expected: DictGeneric = into_dict! { + "a" => Datacell::new_uint_default(1), + "b" => Datacell::new_uint_default(2), + }; + assert!(dict::rmerge_metadata(&mut current, new)); + assert_eq!(current, expected); +} + +#[test] +fn t_bad_patch() { + let mut current: DictGeneric = into_dict! { + "a" => Datacell::new_uint_default(2), + "b" => Datacell::new_uint_default(3), + "z" => Datacell::new_sint_default(-100), + }; + let backup = current.clone(); + let new: DictGeneric = into_dict! { + "a" => Datacell::new_uint_default(1), + "b" => Datacell::new_uint_default(2), + "z" => Datacell::new_str("omg".into()), + }; + assert!(!dict::rmerge_metadata(&mut current, new)); + assert_eq!(current, backup); +} + +#[test] +fn patch_null_out_dict() { + let mut current: DictGeneric = into_dict! { + "a" => Datacell::new_uint_default(2), + "b" => Datacell::new_uint_default(3), + "z" => DictEntryGeneric::Map(into_dict!( + "c" => Datacell::new_uint_default(1), + "d" => Datacell::new_uint_default(2) + )), + }; + let expected: DictGeneric = into_dict! { + "a" => Datacell::new_uint_default(2), + "b" => Datacell::new_uint_default(3), + }; + let new: DictGeneric = into_dict! { + "a" => Datacell::new_uint_default(2), + "b" => Datacell::new_uint_default(3), + "z" => Datacell::null(), + }; + assert!(dict::rmerge_metadata(&mut current, new)); + assert_eq!(current, expected); +} diff --git a/server/src/engine/data/tests/mod.rs b/server/src/engine/data/tests/mod.rs new file mode 100644 index 00000000..1586b8e4 --- /dev/null +++ b/server/src/engine/data/tests/mod.rs @@ -0,0 +1,35 @@ +/* + * Created on Thu Feb 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod md_dict_tests; +use super::lit::Lit; + +#[test] +fn t_largest_int_lit() { + let x = Lit::new_uint(u64::MAX); + let y = Lit::new_uint(u64::MAX); + assert_eq!(x, y); +} diff --git a/server/src/engine/data/uuid.rs b/server/src/engine/data/uuid.rs new file mode 100644 index 00000000..18207b1a --- /dev/null +++ b/server/src/engine/data/uuid.rs @@ -0,0 +1,54 @@ +/* + * Created on Mon Aug 14 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use core::fmt; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Uuid { + data: uuid::Uuid, +} + +impl Uuid { + pub fn new() -> Self { + Self { + data: uuid::Uuid::new_v4(), + } + } + pub fn from_bytes(b: [u8; 16]) -> Self { + Self { + data: uuid::Uuid::from_u128_le(u128::from_le_bytes(b)), + } + } + pub fn to_le_bytes(self) -> [u8; 16] { + self.data.to_u128_le().to_le_bytes() + } +} + +impl fmt::Display for Uuid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.data.fmt(f) + } +} diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs new file mode 100644 index 00000000..9c31013e --- /dev/null +++ b/server/src/engine/error.rs @@ -0,0 +1,221 @@ +/* + * Created on Sat Feb 04 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use {super::config::ConfigError, crate::util::os::SysIOError, std::fmt}; + +pub type RuntimeResult = Result; +pub type QueryResult = Result; + +/// an enumeration of 'flat' errors that the server actually responds to the client with, since we do not want to send specific information +/// about anything (as that will be a security hole). The variants correspond with their actual response codes +#[derive(Debug, Clone, Copy, PartialEq, sky_macros::EnumMethods)] +#[repr(u8)] +pub enum QueryError { + // system + /// I/O error + SysServerError = 0, + /// out of memory + SysOutOfMemory = 1, + /// unknown server error + SysUnknownError = 2, + /// system auth error + SysAuthError = 3, + /// transactional error + SysTransactionalError = 4, + /// insufficient permissions error + SysPermissionDenied = 5, + SysNetworkSystemIllegalClientPacket = 6, + // QL + /// something like an integer that randomly has a character to attached to it like `1234q` + LexInvalidInput = 25, + /// unexpected byte + LexUnexpectedByte = 26, + /// expected a longer statement + QLUnexpectedEndOfStatement = 27, + /// incorrect syntax for "something" + QLInvalidSyntax = 28, + /// invalid collection definition definition + QLInvalidCollectionSyntax = 29, + /// invalid type definition syntax + QLInvalidTypeDefinitionSyntax = 30, + /// expected a full entity definition + QLExpectedEntity = 31, + /// expected a statement, found something else + QLExpectedStatement = 32, + /// unknown statement + QLUnknownStatement = 33, + // exec + /// the object to be used as the "query container" is missing (for example, insert when the model was missing) + QExecObjectNotFound = 100, + /// an unknown field was attempted to be accessed/modified/... + QExecUnknownField = 101, + /// invalid property for an object + QExecDdlInvalidProperties = 102, + /// create space/model, but the object already exists + QExecDdlObjectAlreadyExists = 103, + /// an object that was attempted to be removed is non-empty, and for this object, removals require it to be empty + QExecDdlNotEmpty = 104, + /// invalid type definition + QExecDdlInvalidTypeDefinition = 105, + /// bad model definition + QExecDdlModelBadDefinition = 106, + /// illegal alter model query + QExecDdlModelAlterIllegal = 107, + // exec DML + /// violated the uniqueness property + QExecDmlDuplicate = 108, + /// the data could not be validated for being accepted into a field/function/etc. + QExecDmlValidationError = 109, + /// the where expression has an unindexed column essentially implying that we can't run this query because of perf concerns + QExecDmlWhereHasUnindexedColumn = 110, + /// the row matching the given match expression was not found + QExecDmlRowNotFound = 111, + /// this query needs a lock for execution, but that wasn't explicitly allowed anywhere + QExecNeedLock = 112, +} + +impl From for QueryError { + fn from(e: super::fractal::error::Error) -> Self { + match e.kind() { + ErrorKind::IoError(_) | ErrorKind::Storage(_) => QueryError::SysServerError, + ErrorKind::Txn(_) => QueryError::SysTransactionalError, + ErrorKind::Other(_) => QueryError::SysUnknownError, + ErrorKind::Config(_) => unreachable!("config error cannot propagate here"), + } + } +} + +macro_rules! enumerate_err { + ($(#[$attr:meta])* $vis:vis enum $errname:ident { $($(#[$varattr:meta])* $variant:ident = $errstring:expr),* $(,)? }) => { + $(#[$attr])* + $vis enum $errname { $($(#[$varattr])* $variant),* } + impl core::fmt::Display for $errname { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self {$( Self::$variant => write!(f, "{}", $errstring),)*} + } + } + impl std::error::Error for $errname {} + } +} + +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +/// A "master" error kind enumeration for all kinds of runtime errors +pub enum ErrorKind { + /// An I/O error + IoError(SysIOError), + /// An SDSS error + Storage(StorageError), + /// A transactional error + Txn(TransactionError), + /// other errors + Other(String), + /// configuration errors + Config(ConfigError), +} + +impl fmt::Display for ErrorKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::IoError(io) => write!(f, "io error: {io}"), + Self::Storage(se) => write!(f, "storage error: {se}"), + Self::Txn(txe) => write!(f, "txn error: {txe}"), + Self::Other(oe) => write!(f, "error: {oe}"), + Self::Config(cfg) => write!(f, "config error: {cfg}"), + } + } +} + +impl std::error::Error for ErrorKind {} + +direct_from! { + ErrorKind => { + std::io::Error as IoError, + SysIOError as IoError, + } +} + +enumerate_err! { + #[derive(Debug, PartialEq)] + /// Errors that occur when restoring transactional data + pub enum TransactionError { + /// corrupted txn payload. has more bytes than expected + DecodeCorruptedPayloadMoreBytes = "txn-payload-unexpected-content", + /// transaction payload is corrupted. has lesser bytes than expected + DecodedUnexpectedEof = "txn-payload-unexpected-eof", + /// unknown transaction operation. usually indicates a corrupted payload + DecodeUnknownTxnOp = "txn-payload-unknown-payload", + /// While restoring a certain item, a non-resolvable conflict was encountered in the global state, because the item was + /// already present (when it was expected to not be present) + OnRestoreDataConflictAlreadyExists = "txn-payload-conflict-already-exists", + /// On restore, a certain item that was expected to be present was missing in the global state + OnRestoreDataMissing = "txn-payload-conflict-missing", + /// On restore, a certain item that was expected to match a certain value, has a different value + OnRestoreDataConflictMismatch = "txn-payload-conflict-mismatch", + } +} + +enumerate_err! { + #[derive(Debug, PartialEq)] + /// SDSS based storage engine errors + pub enum StorageError { + // header + /// version mismatch + HeaderDecodeVersionMismatch = "header-version-mismatch", + /// The entire header is corrupted + HeaderDecodeCorruptedHeader = "header-corrupted", + /// Expected header values were not matched with the current header + HeaderDecodeDataMismatch = "header-data-mismatch", + // journal + /// While attempting to handle a basic failure (such as adding a journal entry), the recovery engine ran into an exceptional + /// situation where it failed to make a necessary repair the log + JournalWRecoveryStageOneFailCritical = "journal-recovery-failure", + /// An entry in the journal is corrupted + JournalLogEntryCorrupted = "journal-entry-corrupted", + /// The structure of the journal is corrupted + JournalCorrupted = "journal-corrupted", + // internal file structures + /// While attempting to decode a structure in an internal segment of a file, the storage engine ran into a possibly irrecoverable error + InternalDecodeStructureCorrupted = "structure-decode-corrupted", + /// the payload (non-static) part of a structure in an internal segment of a file is corrupted + InternalDecodeStructureCorruptedPayload = "structure-decode-corrupted-payload", + /// the data for an internal structure was decoded but is logically invalid + InternalDecodeStructureIllegalData = "structure-decode-illegal-data", + /// when attempting to flush a data batch, the batch journal crashed and a recovery event was triggered. But even then, + /// the data batch journal could not be fixed + DataBatchRecoveryFailStageOne = "batch-recovery-failure", + /// when attempting to restore a data batch from disk, the batch journal crashed and had a corruption, but it is irrecoverable + DataBatchRestoreCorruptedBatch = "batch-corrupted-batch", + /// when attempting to restore a data batch from disk, the driver encountered a corrupted entry + DataBatchRestoreCorruptedEntry = "batch-corrupted-entry", + /// we failed to close the data batch + DataBatchCloseError = "batch-persist-close-failed", + /// the data batch file is corrupted + DataBatchRestoreCorruptedBatchFile = "batch-corrupted-file", + /// the system database is corrupted + SysDBCorrupted = "sysdb-corrupted", + } +} diff --git a/server/src/engine/fractal/context.rs b/server/src/engine/fractal/context.rs new file mode 100644 index 00000000..5b46c4f8 --- /dev/null +++ b/server/src/engine/fractal/context.rs @@ -0,0 +1,197 @@ +/* + * Created on Sun Oct 01 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#![allow(unused)] + +use core::fmt; +use std::cell::RefCell; + +/// The current engine context +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum Subsystem { + Init, // the init system + Storage, // the storage engine + Database, // the database engine + Network, // the network layer +} + +impl Subsystem { + pub const fn as_str(self) -> &'static str { + match self { + Self::Init => "init system", + Self::Storage => "storage error", + Self::Database => "engine error", + Self::Network => "network error", + } + } +} + +/* + diagnostics +*/ + +#[derive(Clone)] +/// A dmsg +pub enum Dmsg { + A(Box), + B(&'static str), +} + +impl PartialEq for Dmsg { + fn eq(&self, other: &Self) -> bool { + self.as_ref() == other.as_ref() + } +} + +impl AsRef for Dmsg { + fn as_ref(&self) -> &str { + match self { + Self::A(a) => a, + Self::B(b) => b, + } + } +} + +direct_from! { + Dmsg => { + String as A, + Box as A, + &'static str as B, + } +} + +impl fmt::Display for Dmsg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(self.as_ref(), f) + } +} + +impl fmt::Debug for Dmsg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(self.as_ref(), f) + } +} + +/* + context +*/ + +macro_rules! exported { + ($($vis:vis impl $ty:ty { $($(#[$attr:meta])* $fnvis:vis fn $fn:ident($($fnarg:ident: $fnarg_ty:ty),*) $(-> $fnret:ty)? $fnblock:block)*})*) => { + $(impl $ty { $( $(#[$attr])* $fnvis fn $fn($($fnarg: $fnarg_ty),*) $( -> $fnret)? $fnblock )*} + $($(#[$attr])* $vis fn $fn($($fnarg: $fnarg_ty),*) $( -> $fnret)? { <$ty>::$fn($($fnarg),*) })*)* + } +} + +struct LocalContext { + origin: Option, + dmsg: Option, +} + +fn if_test(f: impl FnOnce()) { + if cfg!(test) { + f() + } +} + +/// A copy of the local context (might be either popped or cloned) +#[derive(Debug, PartialEq, Clone)] +pub struct LocalCtxInstance { + origin: Option, + dmsg: Option, +} + +impl LocalCtxInstance { + fn new(origin: Option, dmsg: Option) -> Self { + Self { origin, dmsg } + } + pub fn origin(&self) -> Option { + self.origin + } + pub fn dmsg(&self) -> Option<&Dmsg> { + self.dmsg.as_ref() + } +} + +impl From for LocalCtxInstance { + fn from(LocalContext { origin, dmsg }: LocalContext) -> Self { + Self { origin, dmsg } + } +} + +exported! { + pub impl LocalContext { + // all + fn set(origin: Subsystem, dmsg: impl Into) { Self::_ctx(|ctx| { ctx.origin = Some(origin); ctx.dmsg = Some(dmsg.into()) }) } + fn test_set(origin: Subsystem, dmsg: impl Into) { if_test(|| Self::set(origin, dmsg)) } + // dmsg + /// set a local dmsg + fn set_dmsg(dmsg: impl Into) { Self::_ctx(|ctx| ctx.dmsg = Some(dmsg.into())) } + /// (only in test) set a local dmsg + fn test_set_dmsg(dmsg: impl Into) { if_test(|| Self::set_dmsg(dmsg)) } + /// Set a local dmsg iff not already set + fn set_dmsg_if_unset(dmsg: impl Into) { Self::_ctx(|ctx| { ctx.dmsg.get_or_insert(dmsg.into()); }) } + /// (only in test) set a local dmsg iff not already set + fn test_set_dmsg_if_unset(dmsg: impl Into) { if_test(|| Self::set_dmsg_if_unset(dmsg)) } + // origin + /// set a local origin + fn set_origin(origin: Subsystem) { Self::_ctx(|ctx| ctx.origin = Some(origin)) } + /// (only in test) set a local origin + fn test_set_origin(origin: Subsystem) { if_test(|| Self::set_origin(origin)) } + /// set origin iff unset + fn set_origin_if_unset(origin: Subsystem) { Self::_ctx(|ctx| { ctx.origin.get_or_insert(origin); }) } + /// (only in test) set a local origin iff not already set + fn test_set_origin_if_unset(origin: Subsystem) { if_test(|| Self::set_origin_if_unset(origin)) } + } + pub(super) impl LocalContext { + // alter context + /// pop the origin from the local context + fn pop_origin() -> Option { Self::_ctx(|ctx| ctx.origin.take()) } + /// pop the dmsg from the local context + fn pop_dmsg() -> Option { Self::_ctx(|ctx| ctx.dmsg.take()) } + /// pop the entire context + fn pop() -> LocalCtxInstance { Self::_ctx(|ctx| core::mem::replace(ctx, LocalContext::null()).into()) } + /// get the origin + fn get_origin() -> Option { Self::_ctx(|ctx| ctx.origin.clone()) } + /// get the dmsg + fn get_dmsg() -> Option { Self::_ctx(|ctx| ctx.dmsg.clone()) } + /// get a clone of the local context + fn cloned() -> LocalCtxInstance { Self::_ctx(|ctx| LocalCtxInstance::new(ctx.origin.clone(), ctx.dmsg.clone())) } + } +} + +impl LocalContext { + fn _new(origin: Option, dmsg: Option) -> Self { + Self { origin, dmsg } + } + fn null() -> Self { + Self::_new(None, None) + } + fn _ctx(f: impl FnOnce(&mut Self) -> T) -> T { + thread_local! { static CTX: RefCell = RefCell::new(LocalContext::null()) } + CTX.with(|lctx| f(&mut lctx.borrow_mut())) + } +} diff --git a/server/src/engine/fractal/drivers.rs b/server/src/engine/fractal/drivers.rs new file mode 100644 index 00000000..6a633660 --- /dev/null +++ b/server/src/engine/fractal/drivers.rs @@ -0,0 +1,89 @@ +/* + * Created on Sun Sep 10 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::util, + crate::engine::{ + error::RuntimeResult, + storage::v1::{data_batch::DataBatchPersistDriver, RawFSInterface}, + txn::gns::GNSTransactionDriverAnyFS, + }, + parking_lot::Mutex, + std::sync::Arc, +}; + +/// GNS driver +pub(super) struct FractalGNSDriver { + #[allow(unused)] + status: util::Status, + pub(super) txn_driver: Mutex>, +} + +impl FractalGNSDriver { + pub(super) fn new(txn_driver: GNSTransactionDriverAnyFS) -> Self { + Self { + status: util::Status::new_okay(), + txn_driver: Mutex::new(txn_driver), + } + } + pub fn txn_driver(&self) -> &Mutex> { + &self.txn_driver + } +} + +/// Model driver +pub struct FractalModelDriver { + #[allow(unused)] + hooks: Arc, + batch_driver: Mutex>, +} + +impl FractalModelDriver { + /// Initialize a model driver with default settings + pub fn init(batch_driver: DataBatchPersistDriver) -> Self { + Self { + hooks: Arc::new(FractalModelHooks::new()), + batch_driver: Mutex::new(batch_driver), + } + } + /// Returns a reference to the batch persist driver + pub fn batch_driver(&self) -> &Mutex> { + &self.batch_driver + } + pub fn close(self) -> RuntimeResult<()> { + self.batch_driver.into_inner().close() + } +} + +/// Model hooks +#[derive(Debug)] +pub struct FractalModelHooks; + +impl FractalModelHooks { + fn new() -> Self { + Self + } +} diff --git a/server/src/engine/fractal/error.rs b/server/src/engine/fractal/error.rs new file mode 100644 index 00000000..0998e146 --- /dev/null +++ b/server/src/engine/fractal/error.rs @@ -0,0 +1,328 @@ +/* + * Created on Mon Oct 02 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::context::{self, Dmsg, Subsystem}, + crate::engine::{ + config::ConfigError, + error::{ErrorKind, StorageError, TransactionError}, + }, + core::fmt, +}; + +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +/// An error implementation with context tracing and propagation +/// +/// - All errors that are classified in [`ErrorKind`] will automatically inherit all local context, unless explicitly orphaned, +/// or manually constructed (see [`IntoError::err_noinherit`]) +/// - All other errors will generally take the context from parent +/// +/// Error propagation and tracing relies on the fact that the first error that occurs will end the routine in question, entering +/// a new local context; if otherwise, it will fail. To manage such custom conditions, look at [`ErrorContext`] or manually +/// constructing [`Error`]s. +pub struct Error { + kind: ErrorKind, + origin: Option, + dmsg: Option, +} + +impl Error { + /// Returns the error kind + pub fn kind(&self) -> &ErrorKind { + &self.kind + } + /// Replace the origin in self + pub fn add_origin(self, origin: Subsystem) -> Self { + Self::_new(self.kind, Some(origin), self.dmsg) + } + /// Replace the dmsg in self + pub fn add_dmsg(self, dmsg: impl Into) -> Self { + Self::_new(self.kind, self.origin, Some(dmsg.into())) + } +} + +impl Error { + /// ctor + fn _new(kind: ErrorKind, origin: Option, dmsg: Option) -> Self { + Self { kind, origin, dmsg } + } + /// new full error + pub fn new(kind: ErrorKind, origin: Subsystem, dmsg: impl Into) -> Self { + Self::_new(kind, Some(origin), Some(dmsg.into())) + } + /// new error with kind and no ctx + pub fn with_kind(kind: ErrorKind) -> Self { + Self::_new(kind, None, None) + } + /// new error with kind and origin + fn with_origin(kind: ErrorKind, origin: Subsystem) -> Self { + Self::_new(kind, Some(origin), None) + } + /// remove the dmsg from self + fn remove_dmsg(self) -> Self { + Self::_new(self.kind, self.origin, None) + } + /// remove the origin from self + fn remove_origin(self) -> Self { + Self::_new(self.kind, None, self.dmsg) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.origin { + Some(orig) => write!(f, "{} error: ", orig.as_str()), + None => write!(f, "runtime error: "), + }?; + match self.dmsg.as_ref() { + Some(dmsg) => write!(f, "{dmsg}; ")?, + None => {} + } + write!(f, "{}", self.kind) + } +} + +impl std::error::Error for Error {} + +/* + generic error casts +*/ + +// for all other direct error casts, always inherit context +impl> From for Error { + fn from(e: E) -> Self { + Self::_new(e.into(), context::get_origin(), context::get_dmsg()) + } +} + +/* + error casts used during result private context mutation +*/ + +// only used when you're modifying context +pub trait IntoError { + fn err_noinherit(self) -> Error; + fn err_inherit_parent(self) -> Error; +} + +// error kinds do not carry any context +impl> IntoError for E { + fn err_noinherit(self) -> Error { + Error::with_kind(self.into()) + } + fn err_inherit_parent(self) -> Error { + Self::err_noinherit(self) + } +} + +impl IntoError for Error { + fn err_noinherit(self) -> Error { + Error::with_kind(self.kind) + } + fn err_inherit_parent(self) -> Error { + self + } +} + +/* + error context and tracing +*/ + +pub trait ErrorContext { + // no inherit + /// set the origin (do not inherit parent or local) + fn set_origin(self, origin: Subsystem) -> Result; + /// set the dmsg (do not inherit parent or local) + fn set_dmsg(self, dmsg: impl Into) -> Result; + /// set the origin and dmsg (do not inherit) + fn set_ctx(self, origin: Subsystem, dmsg: impl Into) -> Result; + // inherit parent + /// set the origin (inherit rest from parent) + fn ip_set_origin(self, origin: Subsystem) -> Result; + /// set the dmsg (inherit rest from origin) + fn ip_set_dmsg(self, dmsg: impl Into) -> Result; + // inherit local + /// set the origin (inherit rest from local) + fn il_set_origin(self, origin: Subsystem) -> Result; + /// set the dmsg (inherit rest from local) + fn il_set_dmsg(self, dmsg: impl Into) -> Result; + /// inherit everything from local (assuming this has no context) + fn inherit_local(self) -> Result; + // inherit any + /// set the origin (inherit rest from either parent, then local) + fn inherit_set_origin(self, origin: Subsystem) -> Result; + /// set the dmsg (inherit rest from either parent, then local) + fn inherit_set_dmsg(self, dmsg: impl Into) -> Result; + // orphan + /// orphan the entire context (if any) + fn orphan(self) -> Result; + /// orphan the origin (if any) + fn orphan_origin(self) -> Result; + /// orphan the dmsg (if any) + fn orphan_dmsg(self) -> Result; +} + +impl ErrorContext for Result +where + E: IntoError, +{ + // no inherit + fn set_origin(self, origin: Subsystem) -> Result { + self.map_err(|e| e.err_noinherit().add_origin(origin)) + } + fn set_dmsg(self, dmsg: impl Into) -> Result { + self.map_err(|e| e.err_noinherit().add_dmsg(dmsg)) + } + fn set_ctx(self, origin: Subsystem, dmsg: impl Into) -> Result { + self.map_err(|e| Error::new(e.err_noinherit().kind, origin, dmsg)) + } + // inherit local + fn il_set_origin(self, origin: Subsystem) -> Result { + self.map_err(|e| Error::_new(e.err_noinherit().kind, Some(origin), context::pop_dmsg())) + } + fn il_set_dmsg(self, dmsg: impl Into) -> Result { + self.map_err(|e| { + Error::_new( + e.err_noinherit().kind, + context::pop_origin(), + Some(dmsg.into()), + ) + }) + } + fn inherit_local(self) -> Result { + self.map_err(|e| { + Error::_new( + e.err_noinherit().kind, + context::get_origin(), + context::get_dmsg(), + ) + }) + } + // inherit parent + fn ip_set_origin(self, origin: Subsystem) -> Result { + self.map_err(|e| e.err_inherit_parent().add_origin(origin)) + } + fn ip_set_dmsg(self, dmsg: impl Into) -> Result { + self.map_err(|e| e.err_inherit_parent().add_dmsg(dmsg)) + } + // inherit any + fn inherit_set_dmsg(self, dmsg: impl Into) -> Result { + self.map_err(|e| { + // inherit from parent + let mut e = e.err_inherit_parent(); + // inherit from local if parent has no ctx + e.origin = e.origin.or_else(|| context::pop_origin()); + e.add_dmsg(dmsg) + }) + } + fn inherit_set_origin(self, origin: Subsystem) -> Result { + self.map_err(|e| { + // inherit from parent + let mut e = e.err_inherit_parent(); + // inherit form local if parent has no ctx + e.dmsg = e.dmsg.or_else(|| context::pop_dmsg()); + e.add_origin(origin) + }) + } + fn orphan(self) -> Result { + self.map_err(|e| e.err_noinherit()) + } + fn orphan_dmsg(self) -> Result { + self.map_err(|e| e.err_inherit_parent().remove_dmsg()) + } + fn orphan_origin(self) -> Result { + self.map_err(|e| e.err_inherit_parent().remove_origin()) + } +} + +/* + foreign type casts +*/ + +macro_rules! impl_other_err_tostring { + ($($ty:ty => $origin:ident),* $(,)?) => { + $( + impl From<$ty> for Error { + fn from(e: $ty) -> Self { Self::_new(ErrorKind::Other(e.to_string()), Some(Subsystem::$origin), context::pop_dmsg()) } + } + impl IntoError for $ty { + fn err_noinherit(self) -> Error { Error::with_kind(ErrorKind::Other(self.to_string())) } + fn err_inherit_parent(self) -> Error { Self::err_noinherit(self) } + } + )* + } +} + +impl_other_err_tostring! { + openssl::ssl::Error => Network, + openssl::error::Error => Network, + openssl::error::ErrorStack => Network, +} + +impl From for Error { + fn from(value: StorageError) -> Self { + Self::_new( + ErrorKind::Storage(value), + context::pop_origin(), + context::pop_dmsg(), + ) + } +} + +impl From for Error { + fn from(value: TransactionError) -> Self { + Self::_new( + ErrorKind::Txn(value), + context::pop_origin(), + context::pop_dmsg(), + ) + } +} + +impl From for Error { + fn from(e: ConfigError) -> Self { + Self::with_origin(ErrorKind::Config(e), Subsystem::Init) + } +} + +impl IntoError for StorageError { + fn err_noinherit(self) -> Error { + Error::with_kind(ErrorKind::Storage(self)) + } + fn err_inherit_parent(self) -> Error { + self.into() + } +} + +impl IntoError for TransactionError { + fn err_noinherit(self) -> Error { + Error::with_kind(ErrorKind::Txn(self)) + } + fn err_inherit_parent(self) -> Error { + self.into() + } +} diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs new file mode 100644 index 00000000..ea9332ab --- /dev/null +++ b/server/src/engine/fractal/mgr.rs @@ -0,0 +1,442 @@ +/* + * Created on Sat Sep 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::ModelUniqueID, + crate::{ + engine::{ + core::{ + model::{delta::DataDelta, Model}, + EntityIDRef, + }, + data::uuid::Uuid, + storage::v1::LocalFS, + }, + util::os, + }, + std::path::PathBuf, + tokio::{ + fs, + sync::{ + broadcast, + mpsc::{UnboundedReceiver, UnboundedSender}, + }, + task::JoinHandle, + }, +}; + +pub const GENERAL_EXECUTOR_WINDOW: u64 = 5 * 60; + +/// A task for the [`FractalMgr`] to perform +pub struct Task { + threshold: usize, + task: T, +} + +impl Task { + const THRESHOLD: usize = 10; + /// Create a new task with the default threshold + pub fn new(task: T) -> Self { + Self::with_threshold(task, Self::THRESHOLD) + } + /// Create a task with the given threshold + fn with_threshold(task: T, threshold: usize) -> Self { + Self { threshold, task } + } +} + +/// A general task +pub enum GenericTask { + #[allow(unused)] + /// Delete a single file + DeleteFile(PathBuf), + /// Delete a directory (and all its children) + DeleteDirAll(PathBuf), +} + +impl GenericTask { + pub fn delete_model_dir( + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + ) -> Self { + Self::DeleteDirAll( + crate::engine::storage::v1::loader::SEInitState::model_dir( + space_name, space_uuid, model_name, model_uuid, + ) + .into(), + ) + } + pub fn delete_space_dir(space_name: &str, space_uuid: Uuid) -> Self { + Self::DeleteDirAll( + crate::engine::storage::v1::loader::SEInitState::space_dir(space_name, space_uuid) + .into(), + ) + } +} + +/// A critical task +pub enum CriticalTask { + /// Write a new data batch + WriteBatch(ModelUniqueID, usize), +} + +/// The task manager +pub(super) struct FractalMgr { + hp_dispatcher: UnboundedSender>, + general_dispatcher: UnboundedSender>, + runtime_stats: FractalRTStat, +} + +pub(super) struct FractalRTStat { + mem_free_bytes: u64, + per_mdl_delta_max_size: usize, +} + +impl FractalRTStat { + fn init(model_cnt: usize) -> Self { + let mem_free_bytes = os::free_memory_in_bytes(); + let allowed_delta_limit = mem_free_bytes as f64 * 0.02; + let per_model_limit = allowed_delta_limit / model_cnt.max(1) as f64; + Self { + mem_free_bytes, + per_mdl_delta_max_size: per_model_limit as usize / sizeof!(DataDelta), + } + } + #[allow(unused)] + pub(super) fn mem_free_bytes(&self) -> u64 { + self.mem_free_bytes + } + pub(super) fn per_mdl_delta_max_size(&self) -> usize { + self.per_mdl_delta_max_size + } +} + +impl FractalMgr { + pub(super) fn new( + hp_dispatcher: UnboundedSender>, + general_dispatcher: UnboundedSender>, + model_count: usize, + ) -> Self { + Self { + hp_dispatcher, + general_dispatcher, + runtime_stats: FractalRTStat::init(model_count), + } + } + pub fn get_rt_stat(&self) -> &FractalRTStat { + &self.runtime_stats + } + /// Add a high priority task to the queue + /// + /// ## Panics + /// + /// This will panic if the high priority executor has crashed or exited + pub fn post_high_priority(&self, task: Task) { + self.hp_dispatcher.send(task).unwrap() + } + /// Add a low priority task to the queue + /// + /// ## Panics + /// + /// This will panic if the low priority executor has crashed or exited + pub fn post_low_priority(&self, task: Task) { + self.general_dispatcher.send(task).unwrap() + } +} + +/// Handles to all the services that fractal needs. These are spawned on the default runtime +pub struct FractalHandle { + pub hp_handle: JoinHandle<()>, + pub lp_handle: JoinHandle<()>, +} + +#[must_use = "fractal engine won't boot unless you call boot"] +pub struct FractalBoot { + global: super::Global, + lp_recv: UnboundedReceiver>, + hp_recv: UnboundedReceiver>, +} + +impl FractalBoot { + pub(super) fn prepare( + global: super::Global, + lp_recv: UnboundedReceiver>, + hp_recv: UnboundedReceiver>, + ) -> Self { + Self { + global, + lp_recv, + hp_recv, + } + } + pub fn boot(self, sigterm: &broadcast::Sender<()>, rs_window: u64) -> FractalHandle { + let Self { + global, + lp_recv: lp_receiver, + hp_recv: hp_receiver, + } = self; + FractalMgr::start_all(global, sigterm, lp_receiver, hp_receiver, rs_window) + } +} + +impl FractalMgr { + /// Start all background services, and return their handles + pub(super) fn start_all( + global: super::Global, + sigterm: &broadcast::Sender<()>, + lp_receiver: UnboundedReceiver>, + hp_receiver: UnboundedReceiver>, + rs_window: u64, + ) -> FractalHandle { + let fractal_mgr = global.get_state().fractal_mgr(); + let global_1 = global.clone(); + let global_2 = global.clone(); + let sigterm_rx = sigterm.subscribe(); + let hp_handle = tokio::spawn(async move { + FractalMgr::hp_executor_svc(fractal_mgr, global_1, hp_receiver, sigterm_rx).await + }); + let sigterm_rx = sigterm.subscribe(); + let lp_handle = tokio::spawn(async move { + FractalMgr::general_executor_svc( + fractal_mgr, + global_2, + lp_receiver, + sigterm_rx, + rs_window, + ) + .await + }); + FractalHandle { + hp_handle, + lp_handle, + } + } +} + +// services +impl FractalMgr { + /// The high priority executor service runs in the background to take care of high priority tasks and take any + /// appropriate action. It will exclusively own the high priority queue since it is the only broker that is + /// allowed to perform HP tasks + pub async fn hp_executor_svc( + &'static self, + global: super::Global, + mut receiver: UnboundedReceiver>, + mut sigterm: broadcast::Receiver<()>, + ) { + loop { + let task = tokio::select! { + task = receiver.recv() => { + match task { + Some(t) => t, + None => { + info!("fhp: exiting executor service because all tasks closed"); + break; + } + } + } + _ = sigterm.recv() => { + info!("fhp: finishing pending tasks"); + while let Ok(task) = receiver.try_recv() { + let global = global.clone(); + tokio::task::spawn_blocking(move || self.hp_executor(global, task)).await.unwrap() + } + info!("fhp: exited executor service"); + break; + } + }; + let global = global.clone(); + tokio::task::spawn_blocking(move || self.hp_executor(global, task)) + .await + .unwrap() + } + } + fn hp_executor( + &'static self, + global: super::Global, + Task { threshold, task }: Task, + ) { + // TODO(@ohsayan): check threshold and update hooks + match task { + CriticalTask::WriteBatch(model_id, observed_size) => { + info!("fhp: {model_id} has reached cache capacity. writing to disk"); + let mdl_drivers = global.get_state().get_mdl_drivers().read(); + let Some(mdl_driver) = mdl_drivers.get(&model_id) else { + // because we maximize throughput, the model driver may have been already removed but this task + // was way behind in the queue + return; + }; + let res = global._namespace().with_model( + EntityIDRef::new(model_id.space().into(), model_id.model().into()), + |model| { + if model.get_uuid() != model_id.uuid() { + // once again, throughput maximization will lead to, in extremely rare cases, this + // branch returning. but it is okay + return Ok(()); + } + Self::try_write_model_data_batch(model, observed_size, mdl_driver) + }, + ); + match res { + Ok(()) => { + if observed_size != 0 { + info!("fhp: completed maintenance task for {model_id}, synced={observed_size}") + } + } + Err(_) => { + error!( + "fhp: error writing data batch for model {}. retrying ...", + model_id.uuid() + ); + // enqueue again for retrying + self.hp_dispatcher + .send(Task::with_threshold( + CriticalTask::WriteBatch(model_id, observed_size), + threshold - 1, + )) + .unwrap(); + } + } + } + } + } + /// The general priority task or simply the general queue takes of care of low priority and other standard priority + /// tasks (such as those running on a schedule). A low priority task can be promoted to a high priority task, and the + /// discretion of the GP executor. Similarly, the executor owns the general purpose task queue since it is the sole broker + /// for such tasks + pub async fn general_executor_svc( + &'static self, + global: super::Global, + mut lpq: UnboundedReceiver>, + mut sigterm: broadcast::Receiver<()>, + rs_window: u64, + ) { + let dur = std::time::Duration::from_secs(rs_window); + loop { + tokio::select! { + _ = sigterm.recv() => { + info!("flp: finishing any pending maintenance tasks"); + let global = global.clone(); + tokio::task::spawn_blocking(|| self.general_executor(global)).await.unwrap(); + info!("flp: exited executor service"); + break; + }, + _ = tokio::time::sleep(dur) => { + let global = global.clone(); + tokio::task::spawn_blocking(|| self.general_executor(global)).await.unwrap() + } + task = lpq.recv() => { + let Task { threshold, task } = match task { + Some(t) => t, + None => { + info!("flp: exiting executor service because all tasks closed"); + break; + } + }; + // TODO(@ohsayan): threshold + match task { + GenericTask::DeleteFile(f) => { + if let Err(_) = fs::remove_file(&f).await { + self.general_dispatcher.send( + Task::with_threshold(GenericTask::DeleteFile(f), threshold - 1) + ).unwrap(); + } + } + GenericTask::DeleteDirAll(dir) => { + if let Err(_) = fs::remove_dir_all(&dir).await { + self.general_dispatcher.send( + Task::with_threshold(GenericTask::DeleteDirAll(dir), threshold - 1) + ).unwrap(); + } + } + } + } + } + } + } + fn general_executor(&'static self, global: super::Global) { + let mdl_drivers = global.get_state().get_mdl_drivers().read(); + for (model_id, driver) in mdl_drivers.iter() { + let mut observed_len = 0; + let res = global._namespace().with_model( + EntityIDRef::new(model_id.space().into(), model_id.model().into()), + |model| { + if model.get_uuid() != model_id.uuid() { + // once again, throughput maximization will lead to, in extremely rare cases, this + // branch returning. but it is okay + return Ok(()); + } + // mark that we're taking these deltas + observed_len = model + .delta_state() + .__fractal_take_full_from_data_delta(super::FractalToken::new()); + Self::try_write_model_data_batch(model, observed_len, driver) + }, + ); + match res { + Ok(()) => { + if observed_len != 0 { + info!( + "flp: completed maintenance task for {model_id}, synced={observed_len}" + ) + } + } + Err(_) => { + // this failure is *not* good, so we want to promote this to a critical task + self.hp_dispatcher + .send(Task::new(CriticalTask::WriteBatch( + model_id.clone(), + observed_len, + ))) + .unwrap() + } + } + } + } +} + +// util +impl FractalMgr { + /// Attempt to write a model data batch with the observed size. + /// + /// The zero check is essential + fn try_write_model_data_batch( + model: &Model, + observed_size: usize, + mdl_driver: &super::FractalModelDriver, + ) -> crate::engine::error::QueryResult<()> { + if observed_size == 0 { + // no changes, all good + return Ok(()); + } + // try flushing the batch + let mut batch_driver = mdl_driver.batch_driver().lock(); + batch_driver.write_new_batch(model, observed_size)?; + Ok(()) + } +} diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs new file mode 100644 index 00000000..cc8321b5 --- /dev/null +++ b/server/src/engine/fractal/mod.rs @@ -0,0 +1,363 @@ +/* + * Created on Sat Sep 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + self::sys_store::SystemStore, + super::{ + core::{dml::QueryExecMeta, model::Model, GlobalNS}, + data::uuid::Uuid, + storage::{ + self, + v1::{LocalFS, RawFSInterface}, + }, + txn::gns::GNSTransactionDriverAnyFS, + }, + crate::engine::error::RuntimeResult, + parking_lot::{Mutex, RwLock}, + std::{collections::HashMap, fmt, mem::MaybeUninit}, + tokio::sync::mpsc::unbounded_channel, +}; + +pub mod context; +mod drivers; +pub mod error; +mod mgr; +pub mod sys_store; +#[cfg(test)] +pub mod test_utils; +mod util; +pub use { + drivers::FractalModelDriver, + mgr::{CriticalTask, GenericTask, Task, GENERAL_EXECUTOR_WINDOW}, + util::FractalToken, +}; + +pub type ModelDrivers = HashMap>; + +/* + global state init +*/ + +/// Returned by [`enable_and_start_all`]. This contains a [`Global`] handle that can be used to easily access global +/// data +pub struct GlobalStateStart { + pub global: Global, + pub boot: mgr::FractalBoot, +} + +/// Enable all drivers and start all engines (or others that you must start) +/// +/// ## Safety +/// +/// Must be called iff this is the only thread calling it +pub unsafe fn load_and_enable_all( + gns: GlobalNS, + config: SystemStore, + gns_driver: GNSTransactionDriverAnyFS, + model_drivers: ModelDrivers, +) -> GlobalStateStart { + let model_cnt_on_boot = model_drivers.len(); + let gns_driver = drivers::FractalGNSDriver::new(gns_driver); + let mdl_driver = RwLock::new(model_drivers); + let (hp_sender, hp_recv) = unbounded_channel(); + let (lp_sender, lp_recv) = unbounded_channel(); + let global_state = GlobalState::new( + gns, + gns_driver, + mdl_driver, + mgr::FractalMgr::new(hp_sender, lp_sender, model_cnt_on_boot), + config, + ); + *Global::__gref_raw() = MaybeUninit::new(global_state); + let token = Global::new(); + GlobalStateStart { + global: token.clone(), + boot: mgr::FractalBoot::prepare(token.clone(), lp_recv, hp_recv), + } +} + +/* + global access +*/ + +/// Something that represents the global state +pub trait GlobalInstanceLike { + type FileSystem: RawFSInterface; + const FS_IS_NON_NULL: bool = Self::FileSystem::NOT_NULL; + // stat + fn get_max_delta_size(&self) -> usize; + // global namespace + fn namespace(&self) -> &GlobalNS; + fn namespace_txn_driver(&self) -> &Mutex>; + // model drivers + fn initialize_model_driver( + &self, + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + ) -> RuntimeResult<()>; + fn purge_model_driver( + &self, + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + skip_delete: bool, + ); + // taskmgr + fn taskmgr_post_high_priority(&self, task: Task); + fn taskmgr_post_standard_priority(&self, task: Task); + // default impls + fn request_batch_resolve_if_cache_full( + &self, + space_name: &str, + model_name: &str, + model: &Model, + hint: QueryExecMeta, + ) { + let current_delta_size = hint.delta_hint(); + let index_size = model.primary_index().count(); + let five = (index_size as f64 * 0.05) as usize; + let max_delta = five.max(self.get_max_delta_size()); + if current_delta_size >= max_delta { + let obtained_delta_size = model + .delta_state() + .__fractal_take_full_from_data_delta(FractalToken::new()); + self.taskmgr_post_high_priority(Task::new(CriticalTask::WriteBatch( + ModelUniqueID::new(space_name, model_name, model.get_uuid()), + obtained_delta_size, + ))); + } + } + // config handle + fn sys_store(&self) -> &SystemStore; +} + +impl GlobalInstanceLike for Global { + type FileSystem = LocalFS; + // ns + fn namespace(&self) -> &GlobalNS { + self._namespace() + } + fn namespace_txn_driver(&self) -> &Mutex> { + self.get_state().gns_driver.txn_driver() + } + // taskmgr + fn taskmgr_post_high_priority(&self, task: Task) { + self._post_high_priority_task(task) + } + fn taskmgr_post_standard_priority(&self, task: Task) { + self._post_standard_priority_task(task) + } + // stat + fn get_max_delta_size(&self) -> usize { + self._get_max_delta_size() + } + // sys + fn sys_store(&self) -> &SystemStore { + &self.get_state().config + } + // model + fn purge_model_driver( + &self, + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + skip_delete: bool, + ) { + let id = ModelUniqueID::new(space_name, model_name, model_uuid); + self.get_state() + .mdl_driver + .write() + .remove(&id) + .expect("tried to remove non existent driver"); + if !skip_delete { + self.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir( + space_name, space_uuid, model_name, model_uuid, + ))); + } + } + fn initialize_model_driver( + &self, + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + ) -> RuntimeResult<()> { + // create dir + LocalFS::fs_create_dir(&storage::v1::loader::SEInitState::model_dir( + space_name, space_uuid, model_name, model_uuid, + ))?; + // init driver + let driver = + storage::v1::data_batch::create(&storage::v1::loader::SEInitState::model_path( + space_name, space_uuid, model_name, model_uuid, + ))?; + self.get_state().mdl_driver.write().insert( + ModelUniqueID::new(space_name, model_name, model_uuid), + drivers::FractalModelDriver::init(driver), + ); + Ok(()) + } +} + +#[derive(Debug, Clone)] +/// A handle to the global state +pub struct Global(()); + +impl Global { + unsafe fn new() -> Self { + Self(()) + } + fn get_state(&self) -> &'static GlobalState { + unsafe { self.__gref() } + } + /// Returns a handle to the [`GlobalNS`] + fn _namespace(&self) -> &'static GlobalNS { + &unsafe { self.__gref() }.gns + } + /// Post an urgent task + fn _post_high_priority_task(&self, task: Task) { + self.get_state().fractal_mgr().post_high_priority(task) + } + /// Post a task with normal priority + /// + /// NB: It is not guaranteed that the task will remain as a low priority task because the scheduler can choose + /// to promote the task to a high priority task, if it deems necessary. + fn _post_standard_priority_task(&self, task: Task) { + self.get_state().fractal_mgr().post_low_priority(task) + } + /// Returns the maximum size a model's delta size can hit before it should immediately issue a batch write request + /// to avoid memory pressure + fn _get_max_delta_size(&self) -> usize { + self.get_state() + .fractal_mgr() + .get_rt_stat() + .per_mdl_delta_max_size() + } + unsafe fn __gref_raw() -> &'static mut MaybeUninit { + static mut G: MaybeUninit = MaybeUninit::uninit(); + &mut G + } + unsafe fn __gref(&self) -> &'static GlobalState { + Self::__gref_raw().assume_init_ref() + } + pub unsafe fn unload_all(self) { + // TODO(@ohsayan): handle errors + let GlobalState { + gns_driver, + mdl_driver, + .. + } = Self::__gref_raw().assume_init_read(); + let gns_driver = gns_driver.txn_driver.into_inner().into_inner(); + let mdl_drivers = mdl_driver.into_inner(); + gns_driver.close().unwrap(); + for (_, driver) in mdl_drivers { + driver.close().unwrap(); + } + } +} + +/* + global state +*/ + +/// The global state +struct GlobalState { + gns: GlobalNS, + gns_driver: drivers::FractalGNSDriver, + mdl_driver: RwLock>, + task_mgr: mgr::FractalMgr, + config: SystemStore, +} + +impl GlobalState { + fn new( + gns: GlobalNS, + gns_driver: drivers::FractalGNSDriver, + mdl_driver: RwLock>, + task_mgr: mgr::FractalMgr, + config: SystemStore, + ) -> Self { + Self { + gns, + gns_driver, + mdl_driver, + task_mgr, + config, + } + } + pub(self) fn get_mdl_drivers(&self) -> &RwLock> { + &self.mdl_driver + } + pub(self) fn fractal_mgr(&self) -> &mgr::FractalMgr { + &self.task_mgr + } +} + +// these impls are completely fine +unsafe impl Send for GlobalState {} +unsafe impl Sync for GlobalState {} + +/// An unique signature that identifies a model, and only that model (guaranteed by the OS's random source) +// NB(@ohsayan): if there are collisions, which I absolutely do not expect any instances of, pool in the space's UUID +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct ModelUniqueID { + space: Box, + model: Box, + uuid: Uuid, +} + +impl fmt::Display for ModelUniqueID { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "model-{}@{}", self.model(), self.space()) + } +} + +impl ModelUniqueID { + /// Create a new unique model ID + pub fn new(space: &str, model: &str, uuid: Uuid) -> Self { + Self { + space: space.into(), + model: model.into(), + uuid, + } + } + /// Returns the space name + pub fn space(&self) -> &str { + self.space.as_ref() + } + /// Returns the model name + pub fn model(&self) -> &str { + self.model.as_ref() + } + /// Returns the uuid + pub fn uuid(&self) -> Uuid { + self.uuid + } +} diff --git a/server/src/engine/fractal/sys_store.rs b/server/src/engine/fractal/sys_store.rs new file mode 100644 index 00000000..f20820fa --- /dev/null +++ b/server/src/engine/fractal/sys_store.rs @@ -0,0 +1,274 @@ +/* + * Created on Sun Sep 10 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::{ + config::{ConfigAuth, ConfigMode}, + error::{QueryError, QueryResult}, + storage::v1::RawFSInterface, + }, + parking_lot::RwLock, + std::{ + collections::{hash_map::Entry, HashMap}, + marker::PhantomData, + }, +}; + +#[derive(Debug)] +pub struct SystemStore { + syscfg: SysConfig, + _fs: PhantomData, +} + +impl SystemStore { + pub fn system_store(&self) -> &SysConfig { + &self.syscfg + } +} + +#[derive(Debug)] +/// The global system configuration +pub struct SysConfig { + auth_data: RwLock, + host_data: SysHostData, + run_mode: ConfigMode, +} + +impl PartialEq for SysConfig { + fn eq(&self, other: &Self) -> bool { + self.run_mode == other.run_mode + && self.host_data == other.host_data + && self.auth_data.read().eq(&other.auth_data.read()) + } +} + +impl SysConfig { + /// Initialize a new system config + pub fn new(auth_data: RwLock, host_data: SysHostData, run_mode: ConfigMode) -> Self { + Self { + auth_data, + host_data, + run_mode, + } + } + pub fn new_full(new_auth: ConfigAuth, host_data: SysHostData, run_mode: ConfigMode) -> Self { + Self::new( + RwLock::new(SysAuth::new( + into_dict!(SysAuthUser::USER_ROOT => SysAuthUser::new( + rcrypt::hash(new_auth.root_key.as_str(), rcrypt::DEFAULT_COST) + .unwrap() + .into_boxed_slice())), + )), + host_data, + run_mode, + ) + } + pub fn new_auth(new_auth: ConfigAuth, run_mode: ConfigMode) -> Self { + Self::new_full(new_auth, SysHostData::new(0, 0), run_mode) + } + #[cfg(test)] + /// A test-mode default setting with the root password set to `password12345678` + pub(super) fn test_default() -> Self { + Self { + auth_data: RwLock::new(SysAuth::new( + into_dict!(SysAuthUser::USER_ROOT => SysAuthUser::new( + rcrypt::hash("password12345678", rcrypt::DEFAULT_COST) + .unwrap() + .into_boxed_slice())), + )), + host_data: SysHostData::new(0, 0), + run_mode: ConfigMode::Dev, + } + } + /// Returns a handle to the authentication data + pub fn auth_data(&self) -> &RwLock { + &self.auth_data + } + /// Returns a reference to host data + pub fn host_data(&self) -> &SysHostData { + &self.host_data + } +} + +#[derive(Debug, PartialEq)] +/// The host data section (system.host) +pub struct SysHostData { + startup_counter: u64, + settings_version: u32, +} + +impl SysHostData { + /// New [`SysHostData`] + pub fn new(startup_counter: u64, settings_version: u32) -> Self { + Self { + startup_counter, + settings_version, + } + } + /// Returns the startup counter + /// + /// Note: + /// - If this is `0` -> this is the first boot + /// - If this is `1` -> this is the second boot (... and so on) + pub fn startup_counter(&self) -> u64 { + self.startup_counter + } + /// Returns the settings version + /// + /// Note: + /// - If this is `0` -> this is the initial setting (first boot) + /// + /// If it stays at 0, this means that the settings were never changed + pub fn settings_version(&self) -> u32 { + self.settings_version + } +} + +impl SystemStore { + pub fn _new(syscfg: SysConfig) -> Self { + Self { + syscfg, + _fs: PhantomData, + } + } + fn _try_sync_or(&self, auth: &mut SysAuth, rb: impl FnOnce(&mut SysAuth)) -> QueryResult<()> { + match self.sync_db(auth) { + Ok(()) => Ok(()), + Err(e) => { + error!("failed to sync system store: {e}"); + rb(auth); + Err(e.into()) + } + } + } + /// Create a new user with the given details + pub fn create_new_user(&self, username: String, password: String) -> QueryResult<()> { + // TODO(@ohsayan): we want to be very careful with this + let _username = username.clone(); + let mut auth = self.system_store().auth_data().write(); + match auth.users.entry(username.into()) { + Entry::Vacant(ve) => { + ve.insert(SysAuthUser::new( + rcrypt::hash(password, rcrypt::DEFAULT_COST) + .unwrap() + .into_boxed_slice(), + )); + self._try_sync_or(&mut auth, |auth| { + auth.users.remove(_username.as_str()); + }) + } + Entry::Occupied(_) => Err(QueryError::SysAuthError), + } + } + pub fn alter_user(&self, username: String, password: String) -> QueryResult<()> { + let mut auth = self.system_store().auth_data().write(); + match auth.users.get_mut(username.as_str()) { + Some(user) => { + let last_pass_hash = core::mem::replace( + &mut user.key, + rcrypt::hash(password, rcrypt::DEFAULT_COST) + .unwrap() + .into_boxed_slice(), + ); + self._try_sync_or(&mut auth, |auth| { + auth.users.get_mut(username.as_str()).unwrap().key = last_pass_hash; + }) + } + None => Err(QueryError::SysAuthError), + } + } + pub fn drop_user(&self, username: &str) -> QueryResult<()> { + let mut auth = self.system_store().auth_data().write(); + if username == SysAuthUser::USER_ROOT { + // you can't remove root! + return Err(QueryError::SysAuthError); + } + match auth.users.remove_entry(username) { + Some((username, user)) => self._try_sync_or(&mut auth, |auth| { + let _ = auth.users.insert(username, user); + }), + None => Err(QueryError::SysAuthError), + } + } +} + +/* + auth +*/ + +#[derive(Debug, PartialEq)] +/// The auth data section (system.auth) +pub struct SysAuth { + users: HashMap, SysAuthUser>, +} + +impl SysAuth { + /// New [`SysAuth`] with the given settings + pub fn new(users: HashMap, SysAuthUser>) -> Self { + Self { users } + } + pub fn verify_user_check_root + ?Sized>( + &self, + username: &str, + password: &T, + ) -> QueryResult { + match self.users.get(username) { + Some(user) if rcrypt::verify(password, user.key()).unwrap() => { + Ok(username == SysAuthUser::USER_ROOT) + } + Some(_) | None => Err(QueryError::SysAuthError), + } + } + /// Verify the user with the given details + pub fn verify_user + ?Sized>( + &self, + username: &str, + password: &T, + ) -> QueryResult<()> { + self.verify_user_check_root(username, password).map(|_| ()) + } + pub fn users(&self) -> &HashMap, SysAuthUser> { + &self.users + } +} + +#[derive(Debug, PartialEq)] +/// The auth user +pub struct SysAuthUser { + key: Box<[u8]>, +} + +impl SysAuthUser { + pub const USER_ROOT: &'static str = "root"; + /// Create a new [`SysAuthUser`] + pub fn new(key: Box<[u8]>) -> Self { + Self { key } + } + /// Get the key + pub fn key(&self) -> &[u8] { + self.key.as_ref() + } +} diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs new file mode 100644 index 00000000..ed555cd1 --- /dev/null +++ b/server/src/engine/fractal/test_utils.rs @@ -0,0 +1,170 @@ +/* + * Created on Wed Sep 13 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + sys_store::{SysConfig, SystemStore}, + CriticalTask, FractalModelDriver, GenericTask, GlobalInstanceLike, ModelUniqueID, Task, + }, + crate::engine::{ + core::GlobalNS, + data::uuid::Uuid, + storage::{ + self, + v1::{ + memfs::{NullFS, VirtualFS}, + RawFSInterface, + }, + }, + txn::gns::GNSTransactionDriverAnyFS, + }, + parking_lot::{Mutex, RwLock}, + std::collections::HashMap, +}; + +/// A `test` mode global implementation +pub struct TestGlobal { + gns: GlobalNS, + hp_queue: RwLock>>, + lp_queue: RwLock>>, + #[allow(unused)] + max_delta_size: usize, + txn_driver: Mutex>, + model_drivers: RwLock>>, + sys_cfg: SystemStore, +} + +impl TestGlobal { + fn new( + gns: GlobalNS, + max_delta_size: usize, + txn_driver: GNSTransactionDriverAnyFS, + ) -> Self { + Self { + gns, + hp_queue: RwLock::default(), + lp_queue: RwLock::default(), + max_delta_size, + txn_driver: Mutex::new(txn_driver), + model_drivers: RwLock::default(), + sys_cfg: SystemStore::_new(SysConfig::test_default()), + } + } +} + +impl TestGlobal { + pub fn new_with_driver_id(log_name: &str) -> Self { + let gns = GlobalNS::empty(); + let driver = storage::v1::loader::open_gns_driver(log_name, &gns) + .unwrap() + .into_inner(); + Self::new(gns, 0, GNSTransactionDriverAnyFS::new(driver)) + } +} + +impl TestGlobal { + pub fn new_with_vfs_driver(log_name: &str) -> Self { + Self::new_with_driver_id(log_name) + } +} + +impl TestGlobal { + pub fn new_with_nullfs_driver(log_name: &str) -> Self { + Self::new_with_driver_id(log_name) + } + pub fn new_with_tmp_nullfs_driver() -> Self { + Self::new_with_nullfs_driver("") + } +} + +impl GlobalInstanceLike for TestGlobal { + type FileSystem = Fs; + fn namespace(&self) -> &GlobalNS { + &self.gns + } + fn namespace_txn_driver(&self) -> &Mutex> { + &self.txn_driver + } + fn taskmgr_post_high_priority(&self, task: Task) { + self.hp_queue.write().push(task) + } + fn taskmgr_post_standard_priority(&self, task: Task) { + self.lp_queue.write().push(task) + } + fn get_max_delta_size(&self) -> usize { + 100 + } + fn sys_store(&self) -> &SystemStore { + &self.sys_cfg + } + fn purge_model_driver( + &self, + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + skip_delete: bool, + ) { + let id = ModelUniqueID::new(space_name, model_name, model_uuid); + self.model_drivers + .write() + .remove(&id) + .expect("tried to remove non-existent model"); + if !skip_delete { + self.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir( + space_name, space_uuid, model_name, model_uuid, + ))); + } + } + fn initialize_model_driver( + &self, + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + ) -> crate::engine::error::RuntimeResult<()> { + // create model dir + Fs::fs_create_dir(&storage::v1::loader::SEInitState::model_dir( + space_name, space_uuid, model_name, model_uuid, + ))?; + let driver = + storage::v1::data_batch::create(&storage::v1::loader::SEInitState::model_path( + space_name, space_uuid, model_name, model_uuid, + ))?; + self.model_drivers.write().insert( + ModelUniqueID::new(space_name, model_name, model_uuid), + FractalModelDriver::init(driver), + ); + Ok(()) + } +} + +impl Drop for TestGlobal { + fn drop(&mut self) { + let mut txn_driver = self.txn_driver.lock(); + txn_driver.__journal_mut().__close_mut().unwrap(); + } +} diff --git a/server/src/engine/fractal/util.rs b/server/src/engine/fractal/util.rs new file mode 100644 index 00000000..323e79d5 --- /dev/null +++ b/server/src/engine/fractal/util.rs @@ -0,0 +1,80 @@ +/* + * Created on Sat Sep 09 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#![allow(unused)] + +use std::sync::atomic::{AtomicBool, Ordering}; + +#[derive(Debug)] +pub struct Status { + okay: AtomicBool, +} + +impl Status { + pub const fn new_okay() -> Self { + Self::new(true) + } + pub const fn new_iffy() -> Self { + Self::new(false) + } + const fn new(v: bool) -> Self { + Self { + okay: AtomicBool::new(v), + } + } +} + +impl Status { + pub fn is_iffy(&self) -> bool { + !self._get() + } + pub fn is_healthy(&self) -> bool { + self._get() + } + fn _get(&self) -> bool { + self.okay.load(Ordering::Acquire) + } +} + +impl Status { + pub(super) fn set_okay(&self) { + self._set(true) + } + pub(super) fn set_iffy(&self) { + self._set(false) + } + fn _set(&self, v: bool) { + self.okay.store(v, Ordering::Release) + } +} + +/// A special token for fractal calls +pub struct FractalToken(()); +impl FractalToken { + pub(super) fn new() -> Self { + Self(()) + } +} diff --git a/server/src/engine/idx/meta/hash.rs b/server/src/engine/idx/meta/hash.rs new file mode 100644 index 00000000..9e884750 --- /dev/null +++ b/server/src/engine/idx/meta/hash.rs @@ -0,0 +1,239 @@ +/* + * Created on Sat Apr 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use std::hash::{BuildHasher, Hasher}; + +pub type HasherNativeFx = HasherRawFx; + +const ROTATE: u32 = 5; +const PRIME32: u32 = 0x9E3779B9; // golden +const PRIME64: u64 = 0x517CC1B727220A95; // archimedes (obtained from rustc) + +pub trait WriteNumeric { + fn self_u32(self) -> u32; +} + +macro_rules! impl_numeric_writes { + ($($ty:ty),*) => { + $(impl WriteNumeric for $ty { fn self_u32(self) -> u32 { self as u32 } })* + }; +} + +impl_numeric_writes!(u8, i8, u16, i16, u32, i32); + +pub trait HashWord: Sized { + const STATE: Self; + fn fin(&self) -> u64; + fn h_bytes(&mut self, bytes: &[u8]); + fn h_quad(&mut self, quad: u64); + fn h_word(&mut self, v: impl WriteNumeric); +} + +impl HashWord for u32 { + const STATE: Self = 0; + fn fin(&self) -> u64 { + (*self) as _ + } + fn h_bytes(&mut self, mut bytes: &[u8]) { + let mut state = *self; + while bytes.len() >= 4 { + // no need for ptr am; let opt with loop invariant + state = self::hash32( + state, + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + ); + bytes = &bytes[4..]; + } + + if bytes.len() >= 2 { + state = self::hash32(state, u16::from_ne_bytes([bytes[0], bytes[1]]) as u32); + bytes = &bytes[2..]; + } + + if !bytes.is_empty() { + state = self::hash32(state, bytes[0] as u32); + } + + *self = state; + } + fn h_quad(&mut self, quad: u64) { + let mut state = *self; + let [x, y]: [u32; 2] = unsafe { core::mem::transmute(quad.to_ne_bytes()) }; + state = self::hash32(state, x); + state = self::hash32(state, y); + *self = state; + } + fn h_word(&mut self, v: impl WriteNumeric) { + *self = self::hash32(*self, v.self_u32()); + } +} + +impl HashWord for u64 { + const STATE: Self = 0; + fn fin(&self) -> u64 { + (*self) as _ + } + fn h_bytes(&mut self, mut bytes: &[u8]) { + let mut state = *self; + while bytes.len() >= 8 { + state = self::hash64( + state, + u64::from_ne_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]), + ); + bytes = &bytes[8..]; + } + + if bytes.len() >= 4 { + state = self::hash64( + state, + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64, + ); + bytes = &bytes[4..]; + } + + if bytes.len() >= 2 { + state = self::hash64(state, u16::from_ne_bytes([bytes[0], bytes[1]]) as u64); + bytes = &bytes[2..]; + } + + if !bytes.is_empty() { + state = self::hash64(state, bytes[0] as u64); + } + + *self = state; + } + fn h_quad(&mut self, quad: u64) { + *self = self::hash64(*self, quad); + } + fn h_word(&mut self, v: impl WriteNumeric) { + *self = self::hash64(*self, v.self_u32() as _) + } +} + +impl HashWord for usize { + const STATE: Self = 0; + fn fin(&self) -> u64 { + (*self) as _ + } + fn h_bytes(&mut self, bytes: &[u8]) { + if cfg!(target_pointer_width = "32") { + let mut slf = *self as u32; + ::h_bytes(&mut slf, bytes); + *self = slf as usize; + } else { + let mut slf = *self as u64; + ::h_bytes(&mut slf, bytes); + *self = slf as usize; + } + } + fn h_quad(&mut self, quad: u64) { + if cfg!(target_pointer_width = "32") { + let mut slf = *self as u32; + ::h_quad(&mut slf, quad); + *self = slf as usize; + } else { + let mut slf = *self as u64; + ::h_quad(&mut slf, quad); + *self = slf as usize; + } + } + fn h_word(&mut self, v: impl WriteNumeric) { + if cfg!(target_pointer_width = "32") { + let mut slf = *self as u32; + ::h_word(&mut slf, v); + *self = slf as usize; + } else { + let mut slf = *self as u64; + ::h_word(&mut slf, v); + *self = slf as usize; + } + } +} + +fn hash32(state: u32, word: u32) -> u32 { + (state.rotate_left(ROTATE) ^ word).wrapping_mul(PRIME32) +} +fn hash64(state: u64, word: u64) -> u64 { + (state.rotate_left(ROTATE) ^ word).wrapping_mul(PRIME64) +} + +#[derive(Debug)] +pub struct HasherRawFx(T); + +impl HasherRawFx { + pub const fn new() -> Self { + Self(T::STATE) + } +} + +impl Hasher for HasherRawFx { + fn finish(&self) -> u64 { + self.0.fin() + } + fn write(&mut self, bytes: &[u8]) { + T::h_bytes(&mut self.0, bytes) + } + fn write_u8(&mut self, i: u8) { + T::h_word(&mut self.0, i) + } + fn write_u16(&mut self, i: u16) { + T::h_word(&mut self.0, i) + } + fn write_u32(&mut self, i: u32) { + T::h_word(&mut self.0, i) + } + fn write_u64(&mut self, i: u64) { + T::h_quad(&mut self.0, i) + } + fn write_u128(&mut self, i: u128) { + let [a, b]: [u64; 2] = unsafe { core::mem::transmute(i) }; + T::h_quad(&mut self.0, a); + T::h_quad(&mut self.0, b); + } + fn write_usize(&mut self, i: usize) { + if cfg!(target_pointer_width = "32") { + T::h_word(&mut self.0, i as u32); + } else { + T::h_quad(&mut self.0, i as u64); + } + } +} + +impl BuildHasher for HasherRawFx { + type Hasher = Self; + + fn build_hasher(&self) -> Self::Hasher { + Self::new() + } +} + +impl Default for HasherRawFx { + fn default() -> Self { + Self::new() + } +} diff --git a/server/src/engine/idx/meta/mod.rs b/server/src/engine/idx/meta/mod.rs new file mode 100644 index 00000000..e3576450 --- /dev/null +++ b/server/src/engine/idx/meta/mod.rs @@ -0,0 +1,65 @@ +/* + * Created on Sun Jan 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +pub mod hash; + +use core::{ + borrow::Borrow, + hash::{BuildHasher, Hash}, +}; + +pub trait AsHasher: BuildHasher + Default {} +impl AsHasher for T where T: BuildHasher + Default {} + +/// The [`Comparable`] trait is like [`PartialEq`], but is different due to its expectations, and escapes its scandalous relations with [`Eq`] and the consequential +/// implications across the [`std`]. +/// +/// ☢️ WARNING ☢️: In some cases implementations of the [`Comparable`] set of traits COMPLETELY VIOLATES [`Eq`]'s invariants. BE VERY CAREFUL WHEN USING IN EXPRESSIONS +/* + FIXME(@ohsayan): The gradual idea is to completely move to Comparable, but that means we'll have to go ahead as much as replacing the impls for some items in the + standard library. We don't have the time to do that right now, but I hope we can do it soon +*/ +pub trait Comparable: Hash { + fn cmp_eq(&self, key: &K) -> bool; +} + +pub trait ComparableUpgradeable: Comparable { + fn upgrade(&self) -> K; +} + +impl, T: Eq + Hash + ?Sized> Comparable for T { + fn cmp_eq(&self, key: &K) -> bool { + self == key.borrow() + } +} + +impl + Hash + Comparable + ?Sized> ComparableUpgradeable + for T +{ + fn upgrade(&self) -> K { + self.to_owned() + } +} diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs new file mode 100644 index 00000000..a7678741 --- /dev/null +++ b/server/src/engine/idx/mod.rs @@ -0,0 +1,358 @@ +/* + * Created on Thu Jan 19 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#![deny(unreachable_patterns)] + +pub mod meta; +pub mod mtchm; +mod stdhm; +mod stord; +#[cfg(test)] +mod tests; + +use { + self::meta::Comparable, + crate::engine::sync::atm::Guard, + core::{borrow::Borrow, hash::Hash}, +}; + +pub mod stdord_iter { + pub use super::stord::iter::IndexSTSeqDllIterOrdKV; +} + +// re-exports +pub type IndexSTSeqCns = stord::IndexSTSeqDll>; +#[cfg(test)] +pub type IndexSTSeqLib = stord::IndexSTSeqDll>; +pub type IndexMTRaw = mtchm::imp::Raw; +pub type IndexST = + std::collections::hash_map::HashMap; + +/// Any type implementing this trait can be used as a key inside memory engine structures +pub trait AsKey: Hash + Eq + 'static { + /// Read the key + fn read_key(&self) -> &Self; +} + +impl AsKey for T { + fn read_key(&self) -> &Self { + self + } +} + +/// If your T can be cloned/copied and implements [`AsKey`], then this trait will automatically be implemented +pub trait AsKeyClone: AsKey + Clone { + /// Read the key and return a clone + fn read_key_clone(&self) -> Self; +} + +impl AsKeyClone for T { + #[inline(always)] + fn read_key_clone(&self) -> Self { + Clone::clone(self) + } +} + +pub trait AsValue: 'static { + fn read_value(&self) -> &Self; +} +impl AsValue for T { + fn read_value(&self) -> &Self { + self + } +} + +/// Any type implementing this trait can be used as a value inside memory engine structures +pub trait AsValueClone: AsValue + Clone { + /// Read the value and return a clone + fn read_value_clone(&self) -> Self; +} + +impl AsValueClone for T { + #[inline(always)] + fn read_value_clone(&self) -> Self { + Clone::clone(self) + } +} + +#[cfg(debug_assertions)] +/// A dummy metrics object +pub struct DummyMetrics; + +/// The base spec for any index. Iterators have meaningless order, and that is intentional and oftentimes +/// consequential. For more specialized impls, use the [`STIndex`], [`MTIndex`] or [`STIndexSeq`] traits +pub trait IndexBaseSpec: Sized { + /// Index supports prealloc? + const PREALLOC: bool; + #[cfg(debug_assertions)] + /// A type representing debug metrics + type Metrics; + // init + /// Initialize an empty instance of the index + fn idx_init() -> Self; + /// Initialize a pre-loaded instance of the index + fn idx_init_with(s: Self) -> Self; + /// Init the idx with the given cap + /// + /// By default doesn't attempt to allocate + fn idx_init_cap(_: usize) -> Self { + if Self::PREALLOC { + panic!("expected prealloc"); + } + Self::idx_init() + } + #[cfg(debug_assertions)] + /// Returns a reference to the index metrics + fn idx_metrics(&self) -> &Self::Metrics; +} + +/// An unordered MTIndex +pub trait MTIndex: IndexBaseSpec { + type IterKV<'t, 'g, 'v>: Iterator + where + 'g: 't + 'v, + 't: 'v, + K: 'v, + V: 'v, + Self: 't; + type IterKey<'t, 'g, 'v>: Iterator + where + 'g: 't + 'v, + 't: 'v, + K: 'v, + Self: 't; + type IterVal<'t, 'g, 'v>: Iterator + where + 'g: 't + 'v, + 't: 'v, + V: 'v, + Self: 't; + fn mt_iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterKV<'t, 'g, 'v>; + fn mt_iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterKey<'t, 'g, 'v>; + fn mt_iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterVal<'t, 'g, 'v>; + /// Returns the length of the index + fn mt_len(&self) -> usize; + /// Attempts to compact the backing storage + fn mt_compact(&self) {} + /// Clears all the entries in the MTIndex + fn mt_clear(&self, g: &Guard); + // write + /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is + /// violated + fn mt_insert(&self, e: E, g: &Guard) -> bool + where + V: AsValue; + /// Updates or inserts the given value + fn mt_upsert(&self, e: E, g: &Guard) + where + V: AsValue; + // read + fn mt_contains(&self, key: &Q, g: &Guard) -> bool + where + Q: ?Sized + Comparable; + /// Returns a reference to the value corresponding to the key, if it exists + fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V> + where + Q: ?Sized + Comparable, + 't: 'v, + 'g: 't + 'v; + fn mt_get_element<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E> + where + Q: ?Sized + Comparable, + 't: 'v, + 'g: 't + 'v; + /// Returns a clone of the value corresponding to the key, if it exists + fn mt_get_cloned(&self, key: &Q, g: &Guard) -> Option + where + Q: ?Sized + Comparable, + V: AsValueClone; + // update + /// Returns true if the entry is updated + fn mt_update(&self, e: E, g: &Guard) -> bool + where + K: AsKeyClone, + V: AsValue; + /// Updates the entry and returns the old value, if it exists + fn mt_update_return<'t, 'g, 'v>(&'t self, e: E, g: &'g Guard) -> Option<&'v V> + where + K: AsKeyClone, + V: AsValue, + 't: 'v, + 'g: 't + 'v; + // delete + /// Returns true if the entry was deleted + fn mt_delete(&self, key: &Q, g: &Guard) -> bool + where + Q: ?Sized + Comparable; + /// Removes the entry and returns it, if it exists + fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V> + where + Q: ?Sized + Comparable, + 't: 'v, + 'g: 't + 'v; + fn mt_delete_return_entry<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E> + where + Q: ?Sized + Comparable, + 't: 'v, + 'g: 't + 'v; +} + +pub trait MTIndexExt: MTIndex { + type IterEntry<'t, 'g, 'v>: Iterator + where + 'g: 't + 'v, + 't: 'v, + K: 'v, + V: 'v, + E: 'v, + Self: 't; + fn mt_iter_entry<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterEntry<'t, 'g, 'v>; +} + +/// An unordered STIndex +pub trait STIndex: IndexBaseSpec { + /// An iterator over the keys and values + type IterKV<'a>: Iterator + where + Self: 'a, + K: 'a, + V: 'a; + /// An iterator over the keys + type IterKey<'a>: Iterator + where + Self: 'a, + K: 'a; + /// An iterator over the values + type IterValue<'a>: Iterator + where + Self: 'a, + V: 'a; + /// returns the length of the idx + fn st_len(&self) -> usize; + /// Attempts to compact the backing storage + fn st_compact(&mut self) {} + /// Clears all the entries in the STIndex + fn st_clear(&mut self); + // write + /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is + /// violated + fn st_insert(&mut self, key: K, val: V) -> bool + where + K: AsKey, + V: AsValue; + /// Updates or inserts the given value + fn st_upsert(&mut self, key: K, val: V) + where + K: AsKey, + V: AsValue; + // read + fn st_contains(&self, key: &Q) -> bool + where + K: Borrow + AsKey, + Q: ?Sized + AsKey; + /// Returns a reference to the value corresponding to the key, if it exists + fn st_get(&self, key: &Q) -> Option<&V> + where + K: AsKey + Borrow, + Q: ?Sized + AsKey; + /// Returns a clone of the value corresponding to the key, if it exists + fn st_get_cloned(&self, key: &Q) -> Option + where + K: AsKey + Borrow, + Q: ?Sized + AsKey, + V: AsValueClone; + fn st_get_mut(&mut self, key: &Q) -> Option<&mut V> + where + K: AsKey + Borrow, + Q: ?Sized + AsKey; + // update + /// Returns true if the entry is updated + fn st_update(&mut self, key: &Q, val: V) -> bool + where + K: AsKey + Borrow, + V: AsValue, + Q: ?Sized + AsKey; + /// Updates the entry and returns the old value, if it exists + fn st_update_return(&mut self, key: &Q, val: V) -> Option + where + K: AsKey + Borrow, + V: AsValue, + Q: ?Sized + AsKey; + // delete + /// Returns true if the entry was deleted + fn st_delete(&mut self, key: &Q) -> bool + where + K: AsKey + Borrow, + Q: ?Sized + AsKey; + /// Removes the entry and returns it, if it exists + fn st_delete_return(&mut self, key: &Q) -> Option + where + K: AsKey + Borrow, + Q: ?Sized + AsKey; + fn st_delete_if(&mut self, key: &Q, iff: impl Fn(&V) -> bool) -> Option + where + K: AsKey + Borrow, + Q: ?Sized + AsKey; + // iter + /// Returns an iterator over a tuple of keys and values + fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a>; + /// Returns an iterator over the keys + fn st_iter_key<'a>(&'a self) -> Self::IterKey<'a>; + /// Returns an iterator over the values + fn st_iter_value<'a>(&'a self) -> Self::IterValue<'a>; +} + +pub trait STIndexSeq: STIndex { + /// An ordered iterator over the keys and values + type IterOrdKV<'a>: Iterator + DoubleEndedIterator + where + Self: 'a, + K: 'a, + V: 'a; + /// An ordered iterator over the keys + type IterOrdKey<'a>: Iterator + DoubleEndedIterator + where + Self: 'a, + K: 'a; + /// An ordered iterator over the values + type IterOrdValue<'a>: Iterator + DoubleEndedIterator + where + Self: 'a, + V: 'a; + type OwnedIterKV: Iterator + DoubleEndedIterator; + type OwnedIterKeys: Iterator + DoubleEndedIterator; + type OwnedIterValues: Iterator + DoubleEndedIterator; + /// Returns an ordered iterator over the KV pairs + fn stseq_ord_kv<'a>(&'a self) -> Self::IterOrdKV<'a>; + /// Returns an ordered iterator over the keys + fn stseq_ord_key<'a>(&'a self) -> Self::IterOrdKey<'a>; + /// Returns an ordered iterator over the values + fn stseq_ord_value<'a>(&'a self) -> Self::IterOrdValue<'a>; + // owned + fn stseq_owned_kv(self) -> Self::OwnedIterKV; + fn stseq_owned_keys(self) -> Self::OwnedIterKeys; + fn stseq_owned_values(self) -> Self::OwnedIterValues; +} diff --git a/server/src/engine/idx/mtchm/access.rs b/server/src/engine/idx/mtchm/access.rs new file mode 100644 index 00000000..e762f64c --- /dev/null +++ b/server/src/engine/idx/mtchm/access.rs @@ -0,0 +1,125 @@ +/* + * Created on Fri Jan 27 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::meta::TreeElement, + crate::engine::idx::meta::Comparable, + core::{hash::Hash, marker::PhantomData}, +}; + +pub trait ReadMode { + type Ret<'a>; + type Target: Comparable + ?Sized + Hash; + fn target(&self) -> &Self::Target; + fn ex<'a>(v: &'a T) -> Self::Ret<'a>; + fn nx<'a>() -> Self::Ret<'a>; +} + +pub struct RModeExists<'a, T, U: ?Sized> { + target: &'a U, + _d: PhantomData, +} + +impl<'a, T, U: ?Sized> RModeExists<'a, T, U> { + pub fn new(target: &'a U) -> Self { + Self { + target, + _d: PhantomData, + } + } +} + +impl<'re, T: TreeElement, U: Comparable + ?Sized> ReadMode for RModeExists<'re, T, U> { + type Ret<'a> = bool; + type Target = U; + fn target(&self) -> &Self::Target { + self.target + } + fn ex(_: &T) -> Self::Ret<'_> { + true + } + fn nx<'a>() -> Self::Ret<'a> { + false + } +} + +pub struct RModeRef<'a, T, U: ?Sized> { + target: &'a U, + _d: PhantomData, +} + +impl<'a, T, U: ?Sized> RModeRef<'a, T, U> { + pub fn new(target: &'a U) -> Self { + Self { + target, + _d: PhantomData, + } + } +} + +impl<'re, T: TreeElement, U: Comparable + ?Sized> ReadMode for RModeRef<'re, T, U> { + type Ret<'a> = Option<&'a T::Value>; + type Target = U; + fn target(&self) -> &Self::Target { + self.target + } + fn ex(c: &T) -> Self::Ret<'_> { + Some(c.val()) + } + fn nx<'a>() -> Self::Ret<'a> { + None + } +} + +pub struct RModeElementRef<'a, T, U: ?Sized> { + target: &'a U, + _d: PhantomData, +} + +impl<'a, T, U: ?Sized> RModeElementRef<'a, T, U> { + pub fn new(target: &'a U) -> Self { + Self { + target, + _d: PhantomData, + } + } +} + +impl<'re, T: TreeElement, U: Comparable + ?Sized> ReadMode + for RModeElementRef<'re, T, U> +{ + type Ret<'a> = Option<&'a T>; + type Target = U; + fn target(&self) -> &Self::Target { + self.target + } + fn ex(c: &T) -> Self::Ret<'_> { + Some(c) + } + fn nx<'a>() -> Self::Ret<'a> { + None + } +} diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs new file mode 100644 index 00000000..6497b6e7 --- /dev/null +++ b/server/src/engine/idx/mtchm/imp.rs @@ -0,0 +1,213 @@ +/* + * Created on Sat Jan 28 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::idx::MTIndexExt; + +#[cfg(debug_assertions)] +use super::CHTRuntimeLog; +use { + super::{ + iter::{IterKV, IterKey, IterVal}, + meta::{Config, TreeElement}, + patch::{DeleteRetEntry, VanillaInsert, VanillaUpdate, VanillaUpdateRet, VanillaUpsert}, + RawTree, + }, + crate::engine::{ + idx::{meta::Comparable, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, MTIndex}, + sync::atm::Guard, + }, +}; + +pub type Raw = RawTree; +#[cfg(test)] +pub type ChmCopy = Raw<(K, V), C>; + +impl IndexBaseSpec for Raw { + const PREALLOC: bool = false; + + #[cfg(debug_assertions)] + type Metrics = CHTRuntimeLog; + + fn idx_init() -> Self { + Self::new() + } + + fn idx_init_with(s: Self) -> Self { + s + } + + #[cfg(debug_assertions)] + fn idx_metrics(&self) -> &Self::Metrics { + &self.m + } +} + +impl MTIndexExt for Raw { + type IterEntry<'t, 'g, 'v> = super::iter::IterEntry<'t, 'g, 'v, E, C> + where + 'g: 't + 'v, + 't: 'v, + E::Key: 'v, + E::Value: 'v, + E: 'v, + Self: 't; + fn mt_iter_entry<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterEntry<'t, 'g, 'v> { + super::iter::IterEntry::new(self, g) + } +} + +impl MTIndex for Raw { + type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, E, C> + where + 'g: 't + 'v, + 't: 'v, + E::Key: 'v, + E::Value: 'v, + Self: 't; + + type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, E, C> + where + 'g: 't + 'v, + 't: 'v, + E::Key: 'v, + Self: 't; + + type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, E, C> + where + 'g: 't + 'v, + 't: 'v, + E::Value: 'v, + Self: 't; + + fn mt_iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterKV<'t, 'g, 'v> { + super::iter::IterKV::new(self, g) + } + + fn mt_iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterKey<'t, 'g, 'v> { + super::iter::IterKey::new(self, g) + } + + fn mt_iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterVal<'t, 'g, 'v> { + super::iter::IterVal::new(self, g) + } + + fn mt_len(&self) -> usize { + self.len() + } + fn mt_clear(&self, g: &Guard) { + self.transactional_clear(g) + } + + fn mt_insert(&self, e: E, g: &Guard) -> bool + where + E::Value: AsValue, + { + self.patch(VanillaInsert(e), g) + } + + fn mt_upsert(&self, e: E, g: &Guard) + where + E::Value: AsValue, + { + self.patch(VanillaUpsert(e), g) + } + + fn mt_contains(&self, key: &Q, g: &Guard) -> bool + where + Q: ?Sized + Comparable, + { + self.contains_key(key, g) + } + + fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E::Value> + where + Q: ?Sized + Comparable, + 't: 'v, + 'g: 't + 'v, + { + self.get(key, g) + } + + fn mt_get_element<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E> + where + Q: ?Sized + Comparable, + 't: 'v, + 'g: 't + 'v, + { + self.get_full(key, g) + } + + fn mt_get_cloned(&self, key: &Q, g: &Guard) -> Option + where + Q: ?Sized + Comparable, + E::Value: AsValueClone, + { + self.get(key, g).cloned() + } + + fn mt_update(&self, e: E, g: &Guard) -> bool + where + E::Key: AsKeyClone, + E::Value: AsValue, + { + self.patch(VanillaUpdate(e), g) + } + + fn mt_update_return<'t, 'g, 'v>(&'t self, e: E, g: &'g Guard) -> Option<&'v E::Value> + where + E::Key: AsKeyClone, + E::Value: AsValue, + 't: 'v, + 'g: 't + 'v, + { + self.patch(VanillaUpdateRet(e), g) + } + + fn mt_delete(&self, key: &Q, g: &Guard) -> bool + where + Q: ?Sized + Comparable, + { + self.remove(key, g) + } + + fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E::Value> + where + Q: ?Sized + Comparable, + 't: 'v, + 'g: 't + 'v, + { + self.remove_return(key, g) + } + + fn mt_delete_return_entry<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E> + where + Q: ?Sized + Comparable, + 't: 'v, + 'g: 't + 'v, + { + self._remove(DeleteRetEntry::new(key), g) + } +} diff --git a/server/src/engine/idx/mtchm/iter.rs b/server/src/engine/idx/mtchm/iter.rs new file mode 100644 index 00000000..10b008f6 --- /dev/null +++ b/server/src/engine/idx/mtchm/iter.rs @@ -0,0 +1,317 @@ +/* + * Created on Fri Jan 27 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + meta::{Config, DefConfig, NodeFlag, TreeElement}, + Node, RawTree, + }, + crate::engine::{ + mem::UArray, + sync::atm::{Guard, Shared}, + }, + std::marker::PhantomData, +}; + +pub struct IterKV<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + i: RawIter<'t, 'g, 'v, T, C, CfgIterKV>, +} + +impl<'t, 'g, 'v, T, C> IterKV<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + pub fn new(t: &'t RawTree, g: &'g Guard) -> Self { + Self { + i: RawIter::new(t, g), + } + } +} + +impl<'t, 'g, 'v, T, C> Iterator for IterKV<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + type Item = (&'v T::Key, &'v T::Value); + + fn next(&mut self) -> Option { + self.i.next() + } +} + +pub struct IterEntry<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + i: RawIter<'t, 'g, 'v, T, C, CfgIterEntry>, +} + +impl<'t, 'g, 'v, T, C> IterEntry<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + pub fn new(t: &'t RawTree, g: &'g Guard) -> Self { + Self { + i: RawIter::new(t, g), + } + } +} + +impl<'t, 'g, 'v, T, C> Iterator for IterEntry<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + type Item = &'v T; + fn next(&mut self) -> Option { + self.i.next() + } +} + +pub struct IterKey<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + i: RawIter<'t, 'g, 'v, T, C, CfgIterKey>, +} + +impl<'t, 'g, 'v, T, C> IterKey<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + pub fn new(t: &'t RawTree, g: &'g Guard) -> Self { + Self { + i: RawIter::new(t, g), + } + } +} + +impl<'t, 'g, 'v, T, C> Iterator for IterKey<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + type Item = &'v T::Key; + + fn next(&mut self) -> Option { + self.i.next() + } +} + +pub struct IterVal<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + i: RawIter<'t, 'g, 'v, T, C, CfgIterVal>, +} + +impl<'t, 'g, 'v, T, C> IterVal<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + pub fn new(t: &'t RawTree, g: &'g Guard) -> Self { + Self { + i: RawIter::new(t, g), + } + } +} + +impl<'t, 'g, 'v, T, C> Iterator for IterVal<'t, 'g, 'v, T, C> +where + 't: 'v, + 'g: 'v + 't, + C: Config, + T: TreeElement, +{ + type Item = &'v T::Value; + + fn next(&mut self) -> Option { + self.i.next() + } +} + +trait IterConfig { + type Ret<'a> + where + T: 'a; + fn some<'a>(v: &'a T) -> Option>; +} + +struct CfgIterEntry; +impl IterConfig for CfgIterEntry { + type Ret<'a> = &'a T where T: 'a; + fn some<'a>(v: &'a T) -> Option> { + Some(v) + } +} + +struct CfgIterKV; +impl IterConfig for CfgIterKV { + type Ret<'a> = (&'a T::Key, &'a T::Value) where T: 'a; + fn some<'a>(v: &'a T) -> Option> { + Some((v.key(), v.val())) + } +} + +struct CfgIterKey; +impl IterConfig for CfgIterKey { + type Ret<'a> = &'a T::Key where T::Key: 'a; + fn some<'a>(v: &'a T) -> Option> { + Some(v.key()) + } +} + +struct CfgIterVal; +impl IterConfig for CfgIterVal { + type Ret<'a> = &'a T::Value where T::Value: 'a; + fn some<'a>(v: &'a T) -> Option> { + Some(v.val()) + } +} + +struct DFSCNodeCtx<'g, C: Config> { + sptr: Shared<'g, Node>, + idx: usize, +} + +struct RawIter<'t, 'g, 'v, T, C, I> +where + 't: 'v, + 'g: 'v + 't, + I: IterConfig, + C: Config, +{ + g: &'g Guard, + stack: UArray<{ ::BRANCH_MX + 1 }, DFSCNodeCtx<'g, C>>, + _m: PhantomData<(&'v T, C, &'t RawTree, I)>, +} + +impl<'t, 'g, 'v, T, C, I> RawIter<'t, 'g, 'v, T, C, I> +where + 't: 'v, + 'g: 'v + 't, + I: IterConfig, + C: Config, +{ + pub(super) fn new(tree: &'t RawTree, g: &'g Guard) -> Self { + let mut stack = UArray::new(); + let sptr = tree.root.ld_acq(g); + stack.push(DFSCNodeCtx { sptr, idx: 0 }); + Self { + g, + stack, + _m: PhantomData, + } + } + /// depth-first search the tree + fn _next(&mut self) -> Option> { + while !self.stack.is_empty() { + let l = self.stack.len() - 1; + let ref mut current = self.stack[l]; + let ref node = current.sptr; + let flag = super::ldfl(¤t.sptr); + match flag { + _ if node.is_null() => { + self.stack.pop(); + } + flag if super::hf(flag, NodeFlag::DATA) => { + let data = unsafe { + // UNSAFE(@ohsayan): flagck + RawTree::::read_data(current.sptr) + }; + if current.idx < data.len() { + let ref ret = data[current.idx]; + current.idx += 1; + return I::some(ret); + } else { + self.stack.pop(); + } + } + _ if current.idx < C::MAX_TREE_HEIGHT => { + let this_node = unsafe { + // UNSAFE(@ohsayan): guard + node.deref() + }; + let sptr = this_node.branch[current.idx].ld_acq(&self.g); + current.idx += 1; + self.stack.push(DFSCNodeCtx { sptr, idx: 0 }); + } + _ => { + self.stack.pop(); + } + } + } + None + } +} + +impl<'t, 'g, 'v, T, C, I> Iterator for RawIter<'t, 'g, 'v, T, C, I> +where + 't: 'v, + 'g: 'v + 't, + I: IterConfig, + C: Config, +{ + type Item = I::Ret<'v>; + + fn next(&mut self) -> Option { + self._next() + } +} diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs new file mode 100644 index 00000000..d916bcb2 --- /dev/null +++ b/server/src/engine/idx/mtchm/meta.rs @@ -0,0 +1,139 @@ +/* + * Created on Thu Jan 26 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::{ + idx::{meta::AsHasher, AsKey, AsKeyClone, AsValue, AsValueClone}, + mem::VInline, + }, + std::{collections::hash_map::RandomState, sync::Arc}, +}; + +const LNODE_STACK: usize = 1; +pub type DefConfig = Config2B; +pub type LNode = VInline; + +pub trait PreConfig: Sized + 'static { + type HState: AsHasher; + const BITS: u32; +} + +pub trait Config: PreConfig { + const BRANCH_MX: usize = ::BITS as _; + const BRANCH_LG: usize = { + let mut index = ::BRANCH_MX; + let mut log = 0usize; + while { + index >>= 1; + index != 0 + } { + log += 1; + } + log + }; + const MASK: u64 = (::BITS - 1) as _; + const MAX_TREE_HEIGHT_UB: usize = 0x40; + const MAX_TREE_HEIGHT: usize = + ::MAX_TREE_HEIGHT_UB / ::BRANCH_LG; + const LEVEL_ZERO: usize = 0; +} + +impl Config for T {} + +pub struct Config2B(T); +impl PreConfig for Config2B { + const BITS: u32 = u16::BITS; + type HState = T; +} + +pub trait TreeElement: Clone + 'static { + type Key: AsKey; + type IKey; + type Value: AsValue; + type IValue; + type VEx1; + type VEx2; + fn key(&self) -> &Self::Key; + fn val(&self) -> &Self::Value; + fn new(k: Self::IKey, v: Self::IValue, vex1: Self::VEx1, vex2: Self::VEx2) -> Self; +} + +impl TreeElement for (K, V) { + type IKey = K; + type Key = K; + type IValue = V; + type Value = V; + type VEx1 = (); + type VEx2 = (); + #[inline(always)] + fn key(&self) -> &K { + &self.0 + } + #[inline(always)] + fn val(&self) -> &V { + &self.1 + } + fn new(k: Self::Key, v: Self::Value, _: (), _: ()) -> Self { + (k, v) + } +} + +impl TreeElement for Arc<(K, V)> { + type IKey = K; + type Key = K; + type IValue = V; + type Value = V; + type VEx1 = (); + type VEx2 = (); + #[inline(always)] + fn key(&self) -> &K { + &self.0 + } + #[inline(always)] + fn val(&self) -> &V { + &self.1 + } + fn new(k: Self::Key, v: Self::Value, _: (), _: ()) -> Self { + Arc::new((k, v)) + } +} + +flags! { + pub struct NodeFlag: usize { + PENDING_DELETE = 0b01, + DATA = 0b10, + } +} + +flags! { + #[derive(PartialEq, Eq)] + pub struct CompressState: u8 { + NULL = 0b00, + SNODE = 0b01, + CASFAIL = 0b10, + RESTORED = 0b11, + } +} diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs new file mode 100644 index 00000000..68023225 --- /dev/null +++ b/server/src/engine/idx/mtchm/mod.rs @@ -0,0 +1,742 @@ +/* + * Created on Thu Jan 26 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod access; +pub mod imp; +pub(super) mod iter; +pub mod meta; +mod patch; +#[cfg(test)] +mod tests; + +#[cfg(debug_assertions)] +use crate::engine::sync::atm::ORD_ACQ; +use { + self::{ + iter::{IterKV, IterKey, IterVal}, + meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement}, + }, + crate::engine::{ + idx::meta::Comparable, + mem::UArray, + sync::atm::{self, cpin, upin, Atomic, Guard, Owned, Shared, ORD_ACR, ORD_RLX}, + }, + crossbeam_epoch::CompareExchangeError, + std::{ + fmt, + hash::Hash, + hash::{BuildHasher, Hasher}, + marker::PhantomData, + mem, + sync::atomic::AtomicUsize, + }, +}; + +/* + HACK(@ohsayan): Until https://github.com/rust-lang/rust/issues/76560 is stabilized which is likely to take a while, + we need to settle for trait objects. +*/ + +#[cfg(debug_assertions)] +struct CHTMetricsData { + split: AtomicUsize, + hln: AtomicUsize, +} + +pub struct CHTRuntimeLog { + #[cfg(debug_assertions)] + data: CHTMetricsData, + #[cfg(not(debug_assertions))] + data: (), +} + +impl CHTRuntimeLog { + #[cfg(debug_assertions)] + const ZERO: AtomicUsize = AtomicUsize::new(0); + #[cfg(not(debug_assertions))] + const NEW: Self = Self { data: () }; + #[cfg(debug_assertions)] + const NEW: Self = Self { + data: CHTMetricsData { + split: Self::ZERO, + hln: Self::ZERO, + }, + }; + const fn new() -> Self { + Self::NEW + } + dbgfn! { + fn hsplit(self: &Self) { + self.data.split.fetch_add(1, ORD_ACQ); + } else { + () + } + fn hlnode(self: &Self) { + self.data.hln.fetch_add(1, ORD_ACQ); + } else { + () + } + #[cfg(test)] + fn replnode(self: &Self) -> usize { + self.data.hln.load(ORD_RLX) + } else { + 0 + } + } +} + +impl Drop for CHTRuntimeLog { + fn drop(&mut self) { + let _ = self.data; + } +} + +pub struct Node { + branch: [Atomic; ::BRANCH_MX], +} + +impl Node { + const NULL: Atomic = Atomic::null(); + const NULL_BRANCH: [Atomic; ::BRANCH_MX] = + [Self::NULL; ::BRANCH_MX]; + const _SZ: usize = mem::size_of::() / mem::size_of::>(); + const _ALIGN: usize = C::BRANCH_MX / Self::_SZ; + const _EQ: () = assert!(Self::_ALIGN == 1); + #[inline(always)] + const fn null() -> Self { + let _ = Self::_EQ; + Self { + branch: Self::NULL_BRANCH, + } + } +} + +#[inline(always)] +fn gc(g: &Guard) { + g.flush(); +} + +#[inline(always)] +fn ldfl(c: &Shared>) -> usize { + c.tag() +} + +#[inline(always)] +const fn hf(c: usize, f: NodeFlag) -> bool { + (c & f.d()) == f.d() +} + +#[inline(always)] +const fn cf(c: usize, r: NodeFlag) -> usize { + c & !r.d() +} + +trait CTFlagAlign { + const FL_A: bool; + const FL_B: bool; + const FLCK_A: () = assert!(Self::FL_A & Self::FL_B); + const FLCK: () = Self::FLCK_A; +} + +impl CTFlagAlign for RawTree { + const FL_A: bool = atm::ensure_flag_align::, { NodeFlag::bits() }>(); + const FL_B: bool = atm::ensure_flag_align::, { NodeFlag::bits() }>(); +} + +impl Default for RawTree { + fn default() -> Self { + Self::_new(C::HState::default()) + } +} + +pub struct RawTree { + root: Atomic>, + h: C::HState, + l: AtomicUsize, + m: CHTRuntimeLog, + _m: PhantomData, +} + +impl RawTree { + #[inline(always)] + const fn _new(h: C::HState) -> Self { + let _ = Self::FLCK; + Self { + root: Atomic::null(), + h, + l: AtomicUsize::new(0), + _m: PhantomData, + m: CHTRuntimeLog::new(), + } + } + #[inline(always)] + fn len(&self) -> usize { + self.l.load(ORD_RLX) + } + #[inline(always)] + #[cfg(test)] + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl RawTree { + #[inline(always)] + fn new() -> Self { + Self::_new(C::HState::default()) + } +} + +impl RawTree { + fn hash(&self, k: &Q) -> u64 + where + Q: ?Sized + Hash, + { + let mut state = self.h.build_hasher(); + k.hash(&mut state); + state.finish() + } +} + +// iter +impl RawTree { + fn iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKV<'t, 'g, 'v, T, C> { + IterKV::new(self, g) + } + fn iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKey<'t, 'g, 'v, T, C> { + IterKey::new(self, g) + } + #[allow(unused)] + fn iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterVal<'t, 'g, 'v, T, C> { + IterVal::new(self, g) + } +} + +impl RawTree { + fn transactional_clear(&self, g: &Guard) { + self.iter_key(g).for_each(|k| { + let _ = self.remove(k, g); + }); + } + fn patch<'g, P: patch::PatchWrite>(&'g self, mut patch: P, g: &'g Guard) -> P::Ret<'g> { + let hash = self.hash(patch.target()); + let mut level = C::LEVEL_ZERO; + let mut current = &self.root; + let mut parent = None; + let mut child = None; + loop { + let node = current.ld_acq(g); + match ldfl(&node) { + flag if hf(flag, NodeFlag::PENDING_DELETE) => { + /* + FIXME(@ohsayan): + this node is about to be deleted (well, maybe) so we'll attempt a cleanup as well. we might not exactly + need to do this. also this is a potentially expensive thing since we're going all the way back to the root, + we might be able to optimize this with a fixed-size queue. + */ + unsafe { + // UNSAFE(@ohsayan): we know that isn't the root and def doesn't have data (that's how the algorithm works) + Self::compress(parent.unwrap(), child.unwrap(), g); + } + level = C::LEVEL_ZERO; + current = &self.root; + parent = None; + child = None; + } + _ if node.is_null() => { + // this is an empty slot + if P::WMODE == patch::WRITEMODE_REFRESH { + // I call that a job well done + return P::nx_ret(); + } + if (P::WMODE == patch::WRITEMODE_ANY) | (P::WMODE == patch::WRITEMODE_FRESH) { + let new = Self::new_data(patch.nx_new()); + match current.cx_rel(node, new, g) { + Ok(_) => { + // we're done here + self.incr_len(); + return P::nx_ret(); + } + Err(CompareExchangeError { new, .. }) => unsafe { + /* + UNSAFE(@ohsayan): so we attempted to CAS it but the CAS failed. in that case, destroy the + lnode we created. We never published the value so no other thread has watched, making this + safe + */ + Self::ldrop(new.into_shared(g)); + }, + } + } + } + flag if hf(flag, NodeFlag::DATA) => { + // so we have an lnode. well maybe an snode + let data = unsafe { + // UNSAFE(@ohsayan): flagck + Self::read_data(node) + }; + debug_assert!(!data.is_empty(), "logic,empty node not compressed"); + if !patch.target().cmp_eq(data[0].key()) && level < C::MAX_TREE_HEIGHT_UB { + /* + so this is a collision and since we haven't reached the max height, we should always + create a new branch so let's do that + */ + self.m.hsplit(); + debug_assert_eq!(data.len(), 1, "logic,lnode before height ub"); + if P::WMODE == patch::WRITEMODE_REFRESH { + // another job well done; an snode with the wrong key; so basically it's missing + return P::nx_ret(); + } + let next_chunk = (self.hash(data[0].key()) >> level) & C::MASK; + let mut new_branch = Node::null(); + // stick this one in + new_branch.branch[next_chunk as usize] = Atomic::from(node); + // we don't care about what happens + let _ = current.cx_rel(node, Owned::new(new_branch), g); + } else { + /* + in this case we either have the same key or we found an lnode. resolve any conflicts and attempt + to update + */ + let p = data.iter().position(|e| patch.target().cmp_eq(e.key())); + match p { + Some(v) if P::WMODE == patch::WRITEMODE_FRESH => { + return P::ex_ret(&data[v]) + } + Some(i) + if P::WMODE == patch::WRITEMODE_REFRESH + || P::WMODE == patch::WRITEMODE_ANY => + { + // update the entry and create a new node + let mut new_ln = LNode::new(); + new_ln.extend(data[..i].iter().cloned()); + new_ln.extend(data[i + 1..].iter().cloned()); + new_ln.push(patch.ex_apply(&data[i])); + match current.cx_rel(node, Self::new_lnode(new_ln), g) { + Ok(new) => { + if cfg!(debug_assertions) + && unsafe { Self::read_data(new) }.len() > 1 + { + self.m.hlnode(); + } + unsafe { + /* + UNSAFE(@ohsayan): swapped out, and we'll be the last thread to see this once the epoch proceeds + sufficiently + */ + g.defer_destroy(Shared::>::from( + node.as_raw() as *const LNode<_> + )) + } + return P::ex_ret(&data[i]); + } + Err(CompareExchangeError { new, .. }) => { + // failed to swap it in + unsafe { + Self::ldrop(new.into_shared(g)); + } + } + } + } + None if P::WMODE == patch::WRITEMODE_ANY + || P::WMODE == patch::WRITEMODE_FRESH => + { + // no funk here + let mut new_node = data.clone(); + new_node.push(patch.nx_new()); + match current.cx_rel(node, Self::new_lnode(new_node), g) { + Ok(new) => { + if cfg!(debug_assertions) + && unsafe { Self::read_data(new) }.len() > 1 + { + self.m.hlnode(); + } + // swapped out + unsafe { + // UNSAFE(@ohsayan): last thread to see this (well, sorta) + g.defer_destroy(Shared::>::from( + node.as_raw() as *const LNode<_>, + )); + } + self.incr_len(); + return P::nx_ret(); + } + Err(CompareExchangeError { new, .. }) => { + // failed to swap it + unsafe { + // UNSAFE(@ohsayan): never published this, so we're the last one + Self::ldrop(new.into_shared(g)) + } + } + } + } + None if P::WMODE == patch::WRITEMODE_REFRESH => return P::nx_ret(), + _ => { + unreachable!("logic, WMODE mismatch: `{}`", P::WMODE); + } + } + } + } + _ => { + // branch + let nxidx = (hash >> level) & C::MASK; + level += C::BRANCH_LG; + parent = Some(current); + child = Some(node); + current = &unsafe { node.deref() }.branch[nxidx as usize]; + } + } + } + } + + fn contains_key<'g, Q: ?Sized + Comparable>(&'g self, k: &Q, g: &'g Guard) -> bool { + self._lookup(access::RModeExists::new(k), g) + } + fn get<'g, Q: ?Sized + Comparable>( + &'g self, + k: &Q, + g: &'g Guard, + ) -> Option<&'g T::Value> { + self._lookup(access::RModeRef::new(k), g) + } + fn get_full<'g, Q: ?Sized + Comparable>( + &'g self, + k: &Q, + g: &'g Guard, + ) -> Option<&'g T> { + self._lookup(access::RModeElementRef::new(k), g) + } + fn _lookup<'g, R: access::ReadMode>(&'g self, read_spec: R, g: &'g Guard) -> R::Ret<'g> { + let mut hash = self.hash(read_spec.target()); + let mut current = &self.root; + loop { + let node = current.ld_acq(g); + match ldfl(&node) { + _ if node.is_null() => { + // honestly, if this ran on the root I'm going to die laughing (@ohsayan) + return R::nx(); + } + flag if hf(flag, NodeFlag::DATA) => { + let mut ret = R::nx(); + return unsafe { + // UNSAFE(@ohsayan): checked flag + nullck + Self::read_data(node).iter().find_map(|e_current| { + read_spec.target().cmp_eq(e_current.key()).then(|| { + ret = R::ex(e_current); + }) + }); + ret + }; + } + _ => { + // branch + current = &unsafe { node.deref() }.branch[(hash & C::MASK) as usize]; + hash >>= C::BRANCH_LG; + } + } + } + } + fn remove<'g, Q: Comparable + ?Sized>(&'g self, k: &Q, g: &'g Guard) -> bool { + self._remove(patch::Delete::new(k), g) + } + fn remove_return<'g, Q: Comparable + ?Sized>( + &'g self, + k: &Q, + g: &'g Guard, + ) -> Option<&'g T::Value> { + self._remove(patch::DeleteRet::new(k), g) + } + fn _remove<'g, P: patch::PatchDelete>(&'g self, patch: P, g: &'g Guard) -> P::Ret<'g> { + let hash = self.hash(patch.target()); + let mut current = &self.root; + let mut level = C::LEVEL_ZERO; + let mut levels = UArray::<{ ::BRANCH_MX }, _>::new(); + 'retry: loop { + let node = current.ld_acq(g); + match ldfl(&node) { + _ if node.is_null() => { + // lol + return P::nx(); + } + flag if hf(flag, NodeFlag::PENDING_DELETE) => { + let (p, c) = levels.pop().unwrap(); + unsafe { + /* + we hit a node that might be deleted, we aren't allowed to change it, so we'll attempt a + compression as well. same thing here as the other routines....can we do anything to avoid + the expensive root traversal? + */ + Self::compress(p, c, g); + } + levels.clear(); + level = C::LEVEL_ZERO; + current = &self.root; + } + flag if hf(flag, NodeFlag::DATA) => { + let data = unsafe { + // UNSAFE(@ohsayan): flagck + Self::read_data(node) + }; + let mut ret = P::nx(); + let mut rem = false; + // this node shouldn't be empty + debug_assert!(!data.is_empty(), "logic,empty node not collected"); + // build new lnode + let r: LNode = data + .iter() + .filter_map(|this_elem| { + if patch.target().cmp_eq(this_elem.key()) { + ret = P::ex(this_elem); + rem = true; + None + } else { + Some(this_elem.clone()) + } + }) + .collect(); + let replace = if r.is_empty() { + // don't create dead nodes + Shared::null() + } else { + Self::new_lnode(r).into_shared(g) + }; + match current.cx_rel(node, replace, g) { + Ok(_) => { + // swapped it out + unsafe { + // UNSAFE(@ohsayan): flagck + g.defer_destroy(Shared::>::from( + node.as_raw() as *const LNode<_> + )); + } + } + Err(CompareExchangeError { new, .. }) if !new.is_null() => { + // failed to swap it in, and it had some data + unsafe { + // UNSAFE(@ohsayan): Never published it, all ours + g.defer_destroy(Shared::>::from( + new.as_raw() as *const LNode<_> + )); + } + continue 'retry; + } + Err(_) => continue 'retry, + } + // attempt compressions + for (p, c) in levels.into_iter().rev() { + let live_nodes = unsafe { + // UNSAFE(@ohsayan): guard + c.deref() + } + .branch + .iter() + .filter(|n| !n.ld_rlx(g).is_null()) + .count(); + if live_nodes > 1 { + break; + } + if unsafe { + // UNSAFE(@ohsayan): we know for a fact that we only have sensible levels + Self::compress(p, c, g) + } == CompressState::RESTORED + { + // simply restored the earlier state, so let's stop + break; + } + } + self.decr_len_by(rem as _); + gc(g); + return ret; + } + _ => { + // branch + levels.push((current, node)); + let nxidx = (hash >> level) & C::MASK; + level += C::BRANCH_LG; + current = &unsafe { node.deref() }.branch[nxidx as usize]; + } + } + } + } +} + +// low-level methods +impl RawTree { + fn decr_len_by(&self, by: usize) { + self.l.fetch_sub(by, ORD_RLX); + } + fn incr_len(&self) { + self.l.fetch_add(1, ORD_RLX); + } + #[inline(always)] + fn new_lnode(node: LNode) -> Owned> { + unsafe { + Owned::>::from_raw(Box::into_raw(Box::new(node)) as *mut Node<_>) + .with_tag(NodeFlag::DATA.d()) + } + } + /// Returns a new inner node, in the form of a data probe leaf + /// ☢ WARNING ☢: Do not drop this naively for god's sake + #[inline(always)] + fn new_data(data: T) -> Owned> { + let mut d = LNode::new(); + unsafe { + // UNSAFE(@ohsayan): empty arr + d.push_unchecked(data) + }; + Self::new_lnode(d) + } + unsafe fn read_data<'g>(d: Shared<'g, Node>) -> &'g LNode { + debug_assert!(hf(ldfl(&d), NodeFlag::DATA)); + (d.as_raw() as *const LNode<_>) + .as_ref() + .expect("logic,nullptr in lnode") + } + /// SAFETY: Ensure you have some actual data and not random garbage + #[inline(always)] + unsafe fn ldrop(leaf: Shared>) { + debug_assert!(hf(ldfl(&leaf), NodeFlag::DATA)); + drop(Owned::>::from_raw(leaf.as_raw() as *mut _)) + } + unsafe fn _rdrop(node: Shared>) { + match ldfl(&node) { + _ if node.is_null() => {} + flag if hf(flag, NodeFlag::DATA) => Self::ldrop(node), + _ => { + // a branch + let this_branch = node.into_owned(); + for child in &this_branch.branch { + Self::rdrop(child) + } + drop(this_branch); + } + } + } + unsafe fn rdrop(n: &Atomic>) { + let g = upin(); + let node = n.ld_acq(g); + Self::_rdrop(node); + } + unsafe fn compress<'g>( + parent: &Atomic>, + child: Shared<'g, Node>, + g: &'g Guard, + ) -> CompressState { + /* + We look at the child's children and determine whether we can clean the child up. Although the amount of + memory we can save is not something very signficant but it becomes important with larger cardinalities + */ + debug_assert!(!hf(ldfl(&child), NodeFlag::DATA), "logic,compress lnode"); + debug_assert_eq!(ldfl(&child), 0, "logic,compress pending delete node"); + let branch = child.deref(); + let mut continue_compress = true; + let mut last_leaf = None; + let mut new_child = Node::null(); + let mut cnt = 0_usize; + + let mut i = 0; + while i < C::BRANCH_MX { + let child_ref = &branch.branch[i]; + let this_child = child_ref.fetch_or(NodeFlag::PENDING_DELETE.d(), ORD_ACR, g); + let this_child = this_child.with_tag(cf(ldfl(&this_child), NodeFlag::PENDING_DELETE)); + match ldfl(&this_child) { + // lol, dangling child + _ if this_child.is_null() => {} + // some data in here + flag if hf(flag, NodeFlag::DATA) => { + last_leaf = Some(this_child); + cnt += Self::read_data(this_child).len(); + } + // branch + _ => { + continue_compress = false; + cnt += 1; + } + } + new_child.branch[i] = Atomic::from(this_child); + i += 1; + } + + let insert; + let ret; + let mut drop = None; + + match last_leaf { + Some(node) if continue_compress && cnt == 1 => { + // snode + insert = node; + ret = CompressState::SNODE; + } + None if cnt == 0 => { + // a dangling branch + insert = Shared::null(); + ret = CompressState::NULL; + } + _ => { + // we can't compress this since we have a lot of children + let new = Owned::new(new_child).into_shared(g); + insert = new; + drop = Some(new); + ret = CompressState::RESTORED; + } + } + + // all logic done; let's see what fate the CAS brings us + match parent.cx_rel(child, insert, g) { + Ok(_) => { + unsafe { + // UNSAFE(@ohsayan): We're the thread in the last epoch who's seeing this; so, we're good + g.defer_destroy(child); + } + ret + } + Err(_) => { + mem::drop(drop.map(|n| Shared::into_owned(n))); + CompressState::CASFAIL + } + } + } +} + +impl Drop for RawTree { + fn drop(&mut self) { + unsafe { + // UNSAFE(@ohsayan): sole live owner + Self::rdrop(&self.root); + } + gc(&cpin()) + } +} + +impl fmt::Debug for RawTree +where + T::Key: fmt::Debug, + T::Value: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let g = cpin(); + f.debug_map().entries(self.iter_kv(&g)).finish() + } +} diff --git a/server/src/engine/idx/mtchm/patch.rs b/server/src/engine/idx/mtchm/patch.rs new file mode 100644 index 00000000..a92436c1 --- /dev/null +++ b/server/src/engine/idx/mtchm/patch.rs @@ -0,0 +1,275 @@ +/* + * Created on Sun Feb 19 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::meta::TreeElement, + crate::engine::idx::meta::Comparable, + std::{hash::Hash, marker::PhantomData}, +}; + +/// write mode flag +pub type WriteFlag = u8; +/// fresh +pub const WRITEMODE_FRESH: WriteFlag = 0b01; +/// refresh +pub const WRITEMODE_REFRESH: WriteFlag = 0b10; +/// any +pub const WRITEMODE_ANY: WriteFlag = 0b11; + +/// A [`Patch`] is intended to atomically update the state of the tree, which means that all your deltas should be atomic +/// +/// Make sure you override the [`unreachable!`] behavior! +pub trait PatchWrite { + const WMODE: WriteFlag; + type Ret<'a>; + type Target: Hash + Comparable; + fn target<'a>(&'a self) -> &Self::Target; + fn nx_new(&mut self) -> E { + unreachable!() + } + fn nx_ret<'a>() -> Self::Ret<'a>; + fn ex_apply(&mut self, _: &E) -> E { + unreachable!() + } + fn ex_ret<'a>(current: &'a E) -> Self::Ret<'a>; +} + +/* + vanilla +*/ + +pub struct VanillaInsert(pub T); +impl PatchWrite for VanillaInsert { + const WMODE: WriteFlag = WRITEMODE_FRESH; + type Ret<'a> = bool; + type Target = T::Key; + fn target<'a>(&'a self) -> &Self::Target { + self.0.key() + } + // nx + fn nx_new(&mut self) -> T { + self.0.clone() + } + fn nx_ret<'a>() -> Self::Ret<'a> { + true + } + // ex + fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> { + false + } +} + +pub struct VanillaUpsert(pub T); +impl PatchWrite for VanillaUpsert { + const WMODE: WriteFlag = WRITEMODE_ANY; + type Ret<'a> = (); + type Target = T::Key; + fn target<'a>(&'a self) -> &Self::Target { + self.0.key() + } + // nx + fn nx_new(&mut self) -> T { + self.0.clone() + } + fn nx_ret<'a>() -> Self::Ret<'a> {} + // ex + fn ex_apply(&mut self, _: &T) -> T { + self.0.clone() + } + fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> {} +} + +pub struct VanillaUpsertRet(pub T); +impl PatchWrite for VanillaUpsertRet { + const WMODE: WriteFlag = WRITEMODE_ANY; + type Ret<'a> = Option<&'a T::Value>; + type Target = T::Key; + fn target<'a>(&'a self) -> &Self::Target { + self.0.key() + } + // nx + fn nx_new(&mut self) -> T { + self.0.clone() + } + fn nx_ret<'a>() -> Self::Ret<'a> { + None + } + // ex + fn ex_apply(&mut self, _: &T) -> T { + self.0.clone() + } + fn ex_ret<'a>(c: &'a T) -> Self::Ret<'a> { + Some(c.val()) + } +} + +pub struct VanillaUpdate(pub T); +impl PatchWrite for VanillaUpdate { + const WMODE: WriteFlag = WRITEMODE_REFRESH; + type Ret<'a> = bool; + type Target = T::Key; + fn target<'a>(&'a self) -> &Self::Target { + self.0.key() + } + // nx + fn nx_ret<'a>() -> Self::Ret<'a> { + false + } + // ex + fn ex_apply(&mut self, _: &T) -> T { + self.0.clone() + } + fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> { + true + } +} + +pub struct VanillaUpdateRet(pub T); +impl PatchWrite for VanillaUpdateRet { + const WMODE: WriteFlag = WRITEMODE_REFRESH; + type Ret<'a> = Option<&'a T::Value>; + type Target = T::Key; + fn target<'a>(&'a self) -> &Self::Target { + self.0.key() + } + // nx + fn nx_ret<'a>() -> Self::Ret<'a> { + None + } + // ex + fn ex_apply(&mut self, _: &T) -> T { + self.0.clone() + } + fn ex_ret<'a>(c: &'a T) -> Self::Ret<'a> { + Some(c.val()) + } +} + +/* + delete +*/ + +pub trait PatchDelete { + type Ret<'a>; + type Target: Comparable + ?Sized + Hash; + fn target(&self) -> &Self::Target; + fn ex<'a>(v: &'a T) -> Self::Ret<'a>; + fn nx<'a>() -> Self::Ret<'a>; +} + +pub struct Delete<'a, T: TreeElement, U: ?Sized> { + target: &'a U, + _m: PhantomData, +} + +impl<'a, T: TreeElement, U: ?Sized> Delete<'a, T, U> { + pub fn new(target: &'a U) -> Self { + Self { + target, + _m: PhantomData, + } + } +} + +impl<'d, T: TreeElement, U: Comparable + ?Sized> PatchDelete for Delete<'d, T, U> { + type Ret<'a> = bool; + type Target = U; + fn target(&self) -> &Self::Target { + &self.target + } + #[inline(always)] + fn ex<'a>(_: &'a T) -> Self::Ret<'a> { + true + } + #[inline(always)] + fn nx<'a>() -> Self::Ret<'a> { + false + } +} + +pub struct DeleteRetEntry<'a, T: TreeElement, U: ?Sized> { + target: &'a U, + _m: PhantomData, +} + +impl<'a, T: TreeElement, U: ?Sized> DeleteRetEntry<'a, T, U> { + pub fn new(target: &'a U) -> Self { + Self { + target, + _m: PhantomData, + } + } +} + +impl<'dr, T: TreeElement, U: Comparable + ?Sized> PatchDelete + for DeleteRetEntry<'dr, T, U> +{ + type Ret<'a> = Option<&'a T>; + + type Target = U; + + fn target(&self) -> &Self::Target { + self.target + } + + fn ex<'a>(v: &'a T) -> Self::Ret<'a> { + Some(v) + } + + fn nx<'a>() -> Self::Ret<'a> { + None + } +} + +pub struct DeleteRet<'a, T: TreeElement, U: ?Sized> { + target: &'a U, + _m: PhantomData, +} + +impl<'a, T: TreeElement, U: ?Sized> DeleteRet<'a, T, U> { + pub fn new(target: &'a U) -> Self { + Self { + target, + _m: PhantomData, + } + } +} + +impl<'dr, T: TreeElement, U: Comparable + ?Sized> PatchDelete for DeleteRet<'dr, T, U> { + type Ret<'a> = Option<&'a T::Value>; + type Target = U; + fn target(&self) -> &Self::Target { + &self.target + } + #[inline(always)] + fn ex<'a>(c: &'a T) -> Self::Ret<'a> { + Some(c.val()) + } + #[inline(always)] + fn nx<'a>() -> Self::Ret<'a> { + None + } +} diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs new file mode 100644 index 00000000..412818c9 --- /dev/null +++ b/server/src/engine/idx/mtchm/tests.rs @@ -0,0 +1,343 @@ +/* + * Created on Sun Jan 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + imp::ChmCopy as _ChmCopy, + meta::{Config, DefConfig}, + }, + crate::engine::{ + idx::{IndexBaseSpec, MTIndex}, + sync::atm::{cpin, Guard}, + }, + std::{ + hash::{BuildHasher, Hasher}, + sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, + thread::{self, JoinHandle}, + time::Duration, + }, +}; + +type Chm = ChmCopy; +type ChmCopy = _ChmCopy; + +struct LolHash { + seed: usize, +} + +impl LolHash { + const fn with_seed(seed: usize) -> Self { + Self { seed } + } + const fn init_default_seed() -> Self { + Self::with_seed(0) + } +} + +impl Default for LolHash { + fn default() -> Self { + Self::init_default_seed() + } +} + +impl Hasher for LolHash { + fn finish(&self) -> u64 { + self.seed as _ + } + fn write(&mut self, _: &[u8]) {} +} + +struct LolState { + seed: usize, +} + +impl BuildHasher for LolState { + type Hasher = LolHash; + + fn build_hasher(&self) -> Self::Hasher { + LolHash::with_seed(self.seed) + } +} + +impl Default for LolState { + fn default() -> Self { + Self { seed: 0 } + } +} + +type ChmU8 = Chm; + +// empty +#[test] +fn drop_empty() { + let idx = ChmU8::idx_init(); + drop(idx); +} + +#[test] +fn get_empty() { + let idx = ChmU8::idx_init(); + assert!(idx.mt_get(&10, &cpin()).is_none()); +} + +#[test] +fn update_empty() { + let idx = ChmU8::idx_init(); + assert!(!idx.mt_update((10, 20), &cpin())); +} + +const SPAM_QCOUNT: usize = if crate::util::IS_ON_CI { + 1_024 +} else if cfg!(miri) { + 32 +} else { + 16_384 +}; +const SPAM_TENANTS: usize = if cfg!(miri) { 2 } else { 32 }; + +#[derive(Clone, Debug)] +struct ControlToken(Arc>); +impl ControlToken { + fn new() -> Self { + Self(Arc::new(RwLock::new(()))) + } + fn acquire_hold(&self) -> RwLockWriteGuard<'_, ()> { + self.0.write().unwrap() + } + fn acquire_permit(&self) -> RwLockReadGuard<'_, ()> { + self.0.read().unwrap() + } +} + +const TUP_INCR: fn(usize) -> (usize, usize) = |x| (x, x + 1); + +fn prepare_distr_data(source_buf: &[StringTup], distr_buf: &mut Vec>) { + distr_buf.try_reserve(SPAM_TENANTS).unwrap(); + distr_buf.extend(source_buf.chunks(SPAM_QCOUNT / SPAM_TENANTS).map(|chunk| { + chunk + .iter() + .map(|(k, v)| (Arc::clone(k), Arc::clone(v))) + .collect() + })); +} + +fn prepare_data(source_buf: &mut Vec, f: F) +where + F: Fn(usize) -> (X, Y), + X: ToString, + Y: ToString, +{ + source_buf.try_reserve(SPAM_QCOUNT).unwrap(); + source_buf.extend( + (0..SPAM_QCOUNT) + .into_iter() + .map(f) + .map(|(k, v)| (Arc::new(k.to_string()), Arc::new(v.to_string()))), + ); +} + +type StringTup = (Arc, Arc); + +fn tdistribute_jobs( + token: &ControlToken, + tree: &Arc<_ChmCopy>, + distr_data: Vec>, + f: F, +) -> Vec> +where + F: FnOnce(ControlToken, Arc<_ChmCopy>, Vec) + Send + 'static + Copy, + K: Send + Sync + 'static, + V: Send + Sync + 'static, + C::HState: Send + Sync, +{ + let r = distr_data + .into_iter() + .enumerate() + .map(|(tid, this_data)| { + let this_token = token.clone(); + let this_idx = tree.clone(); + thread::Builder::new() + .name(tid.to_string()) + .spawn(move || f(this_token, this_idx, this_data)) + .unwrap() + }) + .collect(); + thread::sleep(Duration::from_millis(1 * SPAM_TENANTS as u64)); + r +} + +fn tjoin_all(handles: Vec>) -> Box<[T]> { + handles + .into_iter() + .map(JoinHandle::join) + .map(|h| match h { + Ok(v) => v, + Err(e) => { + panic!("thread died with: {:?}", e.downcast_ref::<&str>()) + } + }) + .collect() +} + +fn modify_and_verify_integrity( + token: &ControlToken, + tree: &Arc<_ChmCopy>, + source_buf: &[StringTup], + action: fn(token: ControlToken, tree: Arc<_ChmCopy>, thread_chunk: Vec), + verify: fn(g: &Guard, tree: &_ChmCopy, src: &[StringTup]), +) where + K: Send + Sync + 'static, + V: Send + Sync + 'static, + C::HState: Send + Sync, +{ + let mut distr_data = Vec::new(); + prepare_distr_data(source_buf, &mut distr_data); + let hold = token.acquire_hold(); + let threads = tdistribute_jobs(token, tree, distr_data, action); + // BLOW THAT INTERCORE TRAFFIC + drop(hold); + let _x: Box<[()]> = tjoin_all(threads); + let pin = cpin(); + verify(&pin, tree, source_buf); +} + +fn _action_put( + token: ControlToken, + idx: Arc<_ChmCopy, Arc, C>>, + data: Vec, +) { + let _token = token.acquire_permit(); + let g = cpin(); + data.into_iter().for_each(|(k, v)| { + assert!(idx.mt_insert((k, v), &g)); + }); +} +fn _verify_eq( + pin: &Guard, + idx: &_ChmCopy, Arc, C>, + source: &[(Arc, Arc)], +) { + assert_eq!(idx.len(), SPAM_QCOUNT); + source.into_iter().for_each(|(k, v)| { + assert_eq!( + idx.mt_get(k, &pin) + .expect(&format!("failed to find key: {}", k)) + .as_str(), + v.as_str() + ); + }); +} + +#[test] +fn multispam_insert() { + let idx = Arc::new(ChmCopy::default()); + let token = ControlToken::new(); + let mut data = Vec::new(); + prepare_data(&mut data, TUP_INCR); + modify_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq); +} + +#[test] +fn multispam_update() { + let idx = Arc::new(ChmCopy::default()); + let token = ControlToken::new(); + let mut data = Vec::new(); + prepare_data(&mut data, TUP_INCR); + modify_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq); + // update our data set + data.iter_mut().enumerate().for_each(|(i, (_, v))| { + *v = Arc::new((i + 2).to_string()); + }); + // store and verify integrity + modify_and_verify_integrity( + &token, + &idx, + &data, + |tok, idx, chunk| { + let g = cpin(); + let _permit = tok.acquire_permit(); + chunk.into_iter().for_each(|(k, v)| { + let ret = idx + .mt_update_return((k.clone(), v), &g) + .expect(&format!("couldn't find key: {}", k)); + assert_eq!( + ret.as_str().parse::().unwrap(), + (k.parse::().unwrap() + 1) + ); + }); + // hmm + }, + |pin, idx, source| { + assert_eq!(idx.len(), SPAM_QCOUNT); + source.into_iter().for_each(|(k, v)| { + let ret = idx + .mt_get(k, &pin) + .expect(&format!("couldn't find key: {}", k)); + assert_eq!(ret.as_str(), v.as_str()); + }); + }, + ); +} + +#[test] +fn multispam_delete() { + let idx = Arc::new(ChmCopy::default()); + let token = ControlToken::new(); + let mut data = Vec::new(); + prepare_data(&mut data, TUP_INCR); + modify_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq); + // now expunge + modify_and_verify_integrity( + &token, + &idx, + &data, + |tok, idx, chunk| { + let g = cpin(); + let _permit = tok.acquire_permit(); + chunk.into_iter().for_each(|(k, v)| { + assert_eq!(idx.mt_delete_return(&k, &g).unwrap().as_str(), v.as_str()); + }); + }, + |g, idx, orig| { + assert!(orig.into_iter().all(|(k, _)| idx.mt_get(k, &g).is_none())); + assert!( + idx.is_empty(), + "expected empty, found {} elements instead", + idx.len() + ); + }, + ); +} + +#[test] +fn multispam_lol() { + let idx = Arc::new(super::RawTree::>::new()); + let token = ControlToken::new(); + let mut data = Vec::new(); + prepare_data(&mut data, TUP_INCR); + modify_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq); + assert_eq!(idx.idx_metrics().replnode(), SPAM_QCOUNT - 1); +} diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs new file mode 100644 index 00000000..0820ada7 --- /dev/null +++ b/server/src/engine/idx/stdhm.rs @@ -0,0 +1,224 @@ +/* + * Created on Mon Jan 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(debug_assertions)] +use super::DummyMetrics; +use { + super::{AsKey, AsValue, AsValueClone, IndexBaseSpec, STIndex}, + crate::engine::mem::StatelessLen, + std::{ + borrow::Borrow, + collections::{ + hash_map::{ + Entry, Iter as StdMapIterKV, Keys as StdMapIterKey, Values as StdMapIterVal, + }, + HashMap as StdMap, + }, + hash::BuildHasher, + mem, + }, +}; + +impl IndexBaseSpec for StdMap +where + S: BuildHasher + Default, +{ + const PREALLOC: bool = true; + + #[cfg(debug_assertions)] + type Metrics = DummyMetrics; + + fn idx_init() -> Self { + StdMap::with_hasher(S::default()) + } + + fn idx_init_with(s: Self) -> Self { + s + } + + fn idx_init_cap(cap: usize) -> Self { + Self::with_capacity_and_hasher(cap, S::default()) + } + + #[cfg(debug_assertions)] + fn idx_metrics(&self) -> &Self::Metrics { + &DummyMetrics + } +} + +impl STIndex for StdMap +where + K: AsKey, + V: AsValue, + S: BuildHasher + Default, +{ + type IterKV<'a> = StdMapIterKV<'a, K, V> + where + Self: 'a, + K: 'a, + V: 'a; + + type IterKey<'a> = StdMapIterKey<'a, K, V> + where + Self: 'a, + K: 'a; + + type IterValue<'a> = StdMapIterVal<'a, K, V> + where + Self: 'a, + V: 'a; + + fn st_compact(&mut self) { + self.shrink_to_fit() + } + + fn st_len(&self) -> usize { + self.len() + } + + fn st_clear(&mut self) { + self.clear() + } + + fn st_insert(&mut self, key: K, val: V) -> bool { + match self.entry(key) { + Entry::Vacant(ve) => { + ve.insert(val); + true + } + _ => false, + } + } + + fn st_upsert(&mut self, key: K, val: V) { + let _ = self.insert(key, val); + } + + fn st_contains(&self, k: &Q) -> bool + where + K: Borrow, + Q: ?Sized + AsKey, + { + self.contains_key(k) + } + + fn st_get(&self, key: &Q) -> Option<&V> + where + K: Borrow, + Q: ?Sized + AsKey, + { + self.get(key) + } + + fn st_get_cloned(&self, key: &Q) -> Option + where + K: Borrow, + Q: ?Sized + AsKey, + V: AsValueClone, + { + self.get(key).cloned() + } + + fn st_get_mut(&mut self, key: &Q) -> Option<&mut V> + where + K: AsKey + Borrow, + Q: ?Sized + AsKey, + { + self.get_mut(key) + } + + fn st_update(&mut self, key: &Q, val: V) -> bool + where + K: Borrow, + Q: ?Sized + AsKey, + { + self.get_mut(key).map(move |e| *e = val).is_some() + } + + fn st_update_return(&mut self, key: &Q, val: V) -> Option + where + K: Borrow, + Q: ?Sized + AsKey, + { + self.get_mut(key).map(move |e| { + let mut new = val; + mem::swap(&mut new, e); + new + }) + } + + fn st_delete(&mut self, key: &Q) -> bool + where + K: Borrow, + Q: ?Sized + AsKey, + { + self.remove(key).is_some() + } + + fn st_delete_return(&mut self, key: &Q) -> Option + where + K: Borrow, + Q: ?Sized + AsKey, + { + self.remove(key) + } + + fn st_delete_if(&mut self, key: &Q, iff: impl Fn(&V) -> bool) -> Option + where + K: AsKey + Borrow, + Q: ?Sized + AsKey, + { + match self.get(key) { + Some(v) => { + if iff(v) { + self.remove(key); + Some(true) + } else { + Some(false) + } + } + None => None, + } + } + + fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a> { + self.iter() + } + + fn st_iter_key<'a>(&'a self) -> Self::IterKey<'a> { + self.keys() + } + + fn st_iter_value<'a>(&'a self) -> Self::IterValue<'a> { + self.values() + } +} + +impl StatelessLen for StdMap { + fn stateless_len(&self) -> usize { + self.len() + } +} diff --git a/server/src/engine/idx/stord/config.rs b/server/src/engine/idx/stord/config.rs new file mode 100644 index 00000000..757f33e8 --- /dev/null +++ b/server/src/engine/idx/stord/config.rs @@ -0,0 +1,132 @@ +/* + * Created on Sun Jan 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{IndexSTSeqDllNode, IndexSTSeqDllNodePtr}, + crate::engine::idx::meta::AsHasher, + std::{collections::hash_map::RandomState, marker::PhantomData, ptr}, +}; + +#[derive(Debug)] +pub struct LiberalStrategy { + f: *mut IndexSTSeqDllNode, +} + +impl AllocStrategy for LiberalStrategy { + const NEW: Self = Self { f: ptr::null_mut() }; + const METRIC_REFRESH: bool = true; + #[inline(always)] + unsafe fn free(&mut self, n: *mut IndexSTSeqDllNode) { + (*n).n = self.f; + self.f = n; + } + #[inline(always)] + fn alloc( + &mut self, + node: IndexSTSeqDllNode, + refresh_metric: &mut bool, + ) -> IndexSTSeqDllNodePtr { + if self.f.is_null() { + IndexSTSeqDllNode::alloc_box(node) + } else { + *refresh_metric = true; + unsafe { + // UNSAFE(@ohsayan): Safe because we already did a nullptr check + let f = self.f; + self.f = (*self.f).n; + ptr::write(f, node); + IndexSTSeqDllNodePtr::new_unchecked(f) + } + } + } + #[inline(always)] + fn cleanup(&mut self) { + unsafe { + // UNSAFE(@ohsayan): All nullck + let mut c = self.f; + while !c.is_null() { + let nx = (*c).n; + IndexSTSeqDllNode::dealloc_headless(c); + c = nx; + } + } + self.f = ptr::null_mut(); + } +} + +#[derive(Debug)] +pub struct ConservativeStrategy { + _d: PhantomData>, +} + +impl AllocStrategy for ConservativeStrategy { + const NEW: Self = Self { _d: PhantomData }; + const METRIC_REFRESH: bool = false; + #[inline(always)] + unsafe fn free(&mut self, n: *mut IndexSTSeqDllNode) { + IndexSTSeqDllNode::dealloc_headless(n) + } + #[inline(always)] + fn alloc(&mut self, node: IndexSTSeqDllNode, _: &mut bool) -> IndexSTSeqDllNodePtr { + IndexSTSeqDllNode::alloc_box(node) + } + #[inline(always)] + fn cleanup(&mut self) {} +} + +pub trait AllocStrategy: Sized { + // HACK(@ohsayan): I trust the optimizer, but not so much + const METRIC_REFRESH: bool; + const NEW: Self; + fn alloc( + &mut self, + node: IndexSTSeqDllNode, + refresh_metric: &mut bool, + ) -> IndexSTSeqDllNodePtr; + unsafe fn free(&mut self, f: *mut IndexSTSeqDllNode); + fn cleanup(&mut self); +} + +pub trait Config { + type Hasher: AsHasher; + type AllocStrategy: AllocStrategy; +} + +#[derive(Debug, Default)] +pub struct ConservativeConfig(PhantomData>); + +impl Config for ConservativeConfig { + type Hasher = RandomState; + type AllocStrategy = ConservativeStrategy; +} + +#[derive(Debug, Default)] +pub struct LiberalConfig(PhantomData>); + +impl Config for LiberalConfig { + type Hasher = RandomState; + type AllocStrategy = LiberalStrategy; +} diff --git a/server/src/engine/idx/stord/iter.rs b/server/src/engine/idx/stord/iter.rs new file mode 100644 index 00000000..c70d5850 --- /dev/null +++ b/server/src/engine/idx/stord/iter.rs @@ -0,0 +1,607 @@ +/* + * Created on Sun Jan 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + config::Config, IndexSTSeqDll, IndexSTSeqDllKeyptr, IndexSTSeqDllNode, IndexSTSeqDllNodePtr, + }, + crate::engine::idx::{AsKey, AsValue}, + std::{ + collections::{ + hash_map::{Iter as StdMapIter, Keys as StdMapIterKey, Values as StdMapIterVal}, + HashMap as StdMap, + }, + fmt::{self, Debug}, + iter::FusedIterator, + marker::PhantomData, + mem::ManuallyDrop, + ptr::{self, NonNull}, + }, +}; + +macro_rules! unsafe_marker_impl { + (unsafe impl for $ty:ty) => { + unsafe impl<'a, K: Send, V: Send> Send for $ty {} + unsafe impl<'a, K: Sync, V: Sync> Sync for $ty {} + }; +} + +pub struct IndexSTSeqDllIterUnordKV<'a, K: 'a, V: 'a> { + i: StdMapIter<'a, IndexSTSeqDllKeyptr, IndexSTSeqDllNodePtr>, +} + +// UNSAFE(@ohsayan): aliasing guarantees correctness +unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordKV<'a, K, V>); + +impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKV<'a, K, V> { + #[inline(always)] + pub(super) fn new( + m: &'a StdMap, NonNull>, S>, + ) -> Self { + Self { i: m.iter() } + } +} + +impl<'a, K, V> Clone for IndexSTSeqDllIterUnordKV<'a, K, V> { + fn clone(&self) -> Self { + Self { i: self.i.clone() } + } +} + +impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKV<'a, K, V> { + type Item = (&'a K, &'a V); + + fn next(&mut self) -> Option { + self.i.next().map(|(_, n)| { + let n = n.as_ptr(); + unsafe { + // UNSAFE(@ohsayan): nullck + (&(*n).k, &(*n).v) + } + }) + } + fn size_hint(&self) -> (usize, Option) { + <_ as Iterator>::size_hint(&self.i) + } +} + +impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKV<'a, K, V> { + fn len(&self) -> usize { + self.i.len() + } +} + +impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {} + +impl<'a, K: 'a + Debug, V: 'a + Debug> Debug for IndexSTSeqDllIterUnordKV<'a, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.clone()).finish() + } +} + +pub struct IndexSTSeqDllIterUnordKey<'a, K: 'a, V: 'a> { + k: StdMapIterKey<'a, IndexSTSeqDllKeyptr, IndexSTSeqDllNodePtr>, +} + +// UNSAFE(@ohsayan): aliasing guarantees correctness +unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordKey<'a, K, V>); + +impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKey<'a, K, V> { + #[inline(always)] + pub(super) fn new( + m: &'a StdMap, NonNull>, S>, + ) -> Self { + Self { k: m.keys() } + } +} + +impl<'a, K, V> Clone for IndexSTSeqDllIterUnordKey<'a, K, V> { + fn clone(&self) -> Self { + Self { k: self.k.clone() } + } +} + +impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKey<'a, K, V> { + type Item = &'a K; + fn next(&mut self) -> Option { + self.k.next().map(|k| { + unsafe { + // UNSAFE(@ohsayan): nullck + &*k.p + } + }) + } + fn size_hint(&self) -> (usize, Option) { + <_ as Iterator>::size_hint(&self.k) + } +} + +impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKey<'a, K, V> { + fn len(&self) -> usize { + self.k.len() + } +} + +impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKey<'a, K, V> {} + +impl<'a, K: Debug, V> Debug for IndexSTSeqDllIterUnordKey<'a, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.clone()).finish() + } +} + +pub struct IndexSTSeqDllIterUnordValue<'a, K: 'a, V: 'a> { + v: StdMapIterVal<'a, IndexSTSeqDllKeyptr, IndexSTSeqDllNodePtr>, +} + +// UNSAFE(@ohsayan): aliasing guarantees correctness +unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordValue<'a, K, V>); + +impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordValue<'a, K, V> { + #[inline(always)] + pub(super) fn new( + m: &'a StdMap, NonNull>, S>, + ) -> Self { + Self { v: m.values() } + } +} + +impl<'a, K, V> Clone for IndexSTSeqDllIterUnordValue<'a, K, V> { + fn clone(&self) -> Self { + Self { v: self.v.clone() } + } +} + +impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordValue<'a, K, V> { + type Item = &'a V; + fn next(&mut self) -> Option { + self.v.next().map(|n| { + unsafe { + // UNSAFE(@ohsayan): nullck + &(*n.as_ptr()).v + } + }) + } + fn size_hint(&self) -> (usize, Option) { + <_ as Iterator>::size_hint(&self.v) + } +} + +impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordValue<'a, K, V> { + fn len(&self) -> usize { + self.v.len() + } +} + +impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordValue<'a, K, V> {} + +impl<'a, K, V: Debug> Debug for IndexSTSeqDllIterUnordValue<'a, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.clone()).finish() + } +} + +trait IndexSTSeqDllIterOrdConfig { + type Ret<'a> + where + K: 'a, + V: 'a; + /// ## Safety + /// Ptr must be non-null + unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode) -> Option> + where + K: 'a, + V: 'a; +} + +struct IndexSTSeqDllIterOrdConfigFull; + +impl IndexSTSeqDllIterOrdConfig for IndexSTSeqDllIterOrdConfigFull { + type Ret<'a> = (&'a K, &'a V) where K: 'a, V: 'a; + #[inline(always)] + unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode) -> Option> + where + K: 'a, + V: 'a, + { + Some((&(*ptr).k, &(*ptr).v)) + } +} + +struct IndexSTSeqDllIterOrdConfigKey; + +impl IndexSTSeqDllIterOrdConfig for IndexSTSeqDllIterOrdConfigKey { + type Ret<'a> = &'a K + where + K: 'a, + V: 'a; + #[inline(always)] + unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode) -> Option<&'a K> + where + K: 'a, + V: 'a, + { + Some(&(*ptr).k) + } +} + +struct IndexSTSeqDllIterOrdConfigValue; + +impl IndexSTSeqDllIterOrdConfig for IndexSTSeqDllIterOrdConfigValue { + type Ret<'a> = &'a V + where + K: 'a, + V: 'a; + #[inline(always)] + unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode) -> Option<&'a V> + where + K: 'a, + V: 'a, + { + Some(&(*ptr).v) + } +} + +pub(super) struct OrderedOwnedIteratorRaw { + h: *mut IndexSTSeqDllNode, + t: *mut IndexSTSeqDllNode, + r: usize, +} + +impl OrderedOwnedIteratorRaw { + pub(super) fn new>(mut idx: IndexSTSeqDll) -> Self { + // clean up if needed + idx.vacuum_full(); + let mut idx = ManuallyDrop::new(idx); + // chuck the map + drop(unsafe { ptr::read((&mut idx.m) as *mut _) }); + // we own everything now + unsafe { + Self { + h: if idx.h.is_null() { + ptr::null_mut() + } else { + (*idx.h).p + }, + t: idx.h, + r: idx.len(), + } + } + } +} + +impl OrderedOwnedIteratorRaw { + #[inline(always)] + fn _next(&mut self) -> Option<(K, V)> { + if self.h == self.t { + None + } else { + self.r -= 1; + unsafe { + // UNSAFE(@ohsayan): +nullck + let this = ptr::read(self.h); + // destroy this node + IndexSTSeqDllNode::dealloc_headless(self.h); + self.h = (*self.h).p; + Some((this.k, this.v)) + } + } + } + #[inline(always)] + fn _next_back(&mut self) -> Option<(K, V)> { + if self.h == self.t { + None + } else { + self.r -= 1; + unsafe { + // UNSAFE(@ohsayan): +nullck + self.t = (*self.t).n; + let this = ptr::read(self.t); + IndexSTSeqDllNode::dealloc_headless(self.t); + Some((this.k, this.v)) + } + } + } +} + +impl Drop for OrderedOwnedIteratorRaw { + fn drop(&mut self) { + // clean up what's left + while let Some(_) = self._next() {} + } +} + +pub struct OrderedOwnedIteratorKV(pub(super) OrderedOwnedIteratorRaw); + +impl Iterator for OrderedOwnedIteratorKV { + type Item = (K, V); + fn next(&mut self) -> Option { + self.0._next() + } +} + +impl DoubleEndedIterator for OrderedOwnedIteratorKV { + fn next_back(&mut self) -> Option { + self.0._next_back() + } +} + +pub struct OrderedOwnedIteratorKey(pub(super) OrderedOwnedIteratorRaw); + +impl Iterator for OrderedOwnedIteratorKey { + type Item = K; + fn next(&mut self) -> Option { + self.0._next().map(|(k, _)| k) + } +} +impl DoubleEndedIterator for OrderedOwnedIteratorKey { + fn next_back(&mut self) -> Option { + self.0._next_back().map(|(k, _)| k) + } +} + +pub struct OrderedOwnedIteratorValue(pub(super) OrderedOwnedIteratorRaw); + +impl Iterator for OrderedOwnedIteratorValue { + type Item = V; + fn next(&mut self) -> Option { + self.0._next().map(|(_, v)| v) + } +} +impl DoubleEndedIterator for OrderedOwnedIteratorValue { + fn next_back(&mut self) -> Option { + self.0._next_back().map(|(_, v)| v) + } +} + +struct IndexSTSeqDllIterOrdBase<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig> { + h: *const IndexSTSeqDllNode, + t: *const IndexSTSeqDllNode, + r: usize, + _l: PhantomData<(&'a K, &'a V, C)>, +} + +impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig> IndexSTSeqDllIterOrdBase<'a, K, V, C> { + #[inline(always)] + fn new>(idx: &'a IndexSTSeqDll) -> Self { + Self { + h: if idx.h.is_null() { + ptr::null_mut() + } else { + unsafe { + // UNSAFE(@ohsayan): nullck + (*idx.h).p + } + }, + t: idx.h, + r: idx.len(), + _l: PhantomData, + } + } + #[inline(always)] + fn _next(&mut self) -> Option> { + if self.h == self.t { + None + } else { + self.r -= 1; + unsafe { + // UNSAFE(@ohsayan): Assuming we had a legal init, this should be fine + let this = C::read_ret(self.h); + self.h = (*self.h).p; + this + } + } + } + #[inline(always)] + fn _next_back(&mut self) -> Option> { + if self.h == self.t { + None + } else { + self.r -= 1; + unsafe { + // UNSAFE(@ohsayan): legal init, then ok + self.t = (*self.t).n; + // UNSAFE(@ohsayan): non-null (sentinel) + C::read_ret(self.t) + } + } + } +} + +impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig> Debug + for IndexSTSeqDllIterOrdBase<'a, K, V, C> +where + C::Ret<'a>: Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.clone()).finish() + } +} + +impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig> Iterator + for IndexSTSeqDllIterOrdBase<'a, K, V, C> +{ + type Item = C::Ret<'a>; + fn next(&mut self) -> Option { + self._next() + } + fn size_hint(&self) -> (usize, Option) { + (self.r, Some(self.r)) + } +} + +impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig> DoubleEndedIterator + for IndexSTSeqDllIterOrdBase<'a, K, V, C> +{ + fn next_back(&mut self) -> Option { + self._next_back() + } +} + +impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig> ExactSizeIterator + for IndexSTSeqDllIterOrdBase<'a, K, V, C> +{ + fn len(&self) -> usize { + self.r + } +} + +impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig> Clone + for IndexSTSeqDllIterOrdBase<'a, K, V, C> +{ + fn clone(&self) -> Self { + Self { ..*self } + } +} + +#[derive(Debug)] +pub struct IndexSTSeqDllIterOrdKV<'a, K: 'a, V: 'a> { + i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigFull>, +} +impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterOrdKV<'a, K, V> { + pub(super) fn new>(arg: &'a IndexSTSeqDll) -> Self { + Self { + i: IndexSTSeqDllIterOrdBase::new(arg), + } + } +} + +// UNSAFE(@ohsayan): aliasing guarantees correctness +unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdKV<'a, K, V>); + +impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKV<'a, K, V> { + type Item = (&'a K, &'a V); + fn next(&mut self) -> Option { + self.i.next() + } + fn size_hint(&self) -> (usize, Option) { + self.i.size_hint() + } +} + +impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdKV<'a, K, V> { + fn next_back(&mut self) -> Option { + self.i.next_back() + } +} + +impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKV<'a, K, V> { + fn len(&self) -> usize { + self.i.len() + } +} + +impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKV<'a, K, V> { + fn clone(&self) -> Self { + Self { i: self.i.clone() } + } +} + +#[derive(Debug)] +pub struct IndexSTSeqDllIterOrdKey<'a, K: 'a, V: 'a> { + i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigKey>, +} +impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterOrdKey<'a, K, V> { + pub(super) fn new>(arg: &'a IndexSTSeqDll) -> Self { + Self { + i: IndexSTSeqDllIterOrdBase::new(arg), + } + } +} + +// UNSAFE(@ohsayan): aliasing guarantees correctness +unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdKey<'a, K, V>); + +impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKey<'a, K, V> { + type Item = &'a K; + fn next(&mut self) -> Option { + self.i.next() + } + fn size_hint(&self) -> (usize, Option) { + self.i.size_hint() + } +} + +impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdKey<'a, K, V> { + fn next_back(&mut self) -> Option { + self.i.next_back() + } +} + +impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKey<'a, K, V> { + fn len(&self) -> usize { + self.i.len() + } +} + +impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKey<'a, K, V> { + fn clone(&self) -> Self { + Self { i: self.i.clone() } + } +} + +#[derive(Debug)] +pub struct IndexSTSeqDllIterOrdValue<'a, K: 'a, V: 'a> { + i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigValue>, +} +impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterOrdValue<'a, K, V> { + pub(super) fn new>(arg: &'a IndexSTSeqDll) -> Self { + Self { + i: IndexSTSeqDllIterOrdBase::new(arg), + } + } +} + +// UNSAFE(@ohsayan): aliasing guarantees correctness +unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdValue<'a, K, V>); + +impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdValue<'a, K, V> { + type Item = &'a V; + fn next(&mut self) -> Option { + self.i.next() + } + fn size_hint(&self) -> (usize, Option) { + self.i.size_hint() + } +} + +impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdValue<'a, K, V> { + fn next_back(&mut self) -> Option { + self.i.next_back() + } +} + +impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdValue<'a, K, V> { + fn len(&self) -> usize { + self.i.len() + } +} + +impl<'a, K, V> Clone for IndexSTSeqDllIterOrdValue<'a, K, V> { + fn clone(&self) -> Self { + Self { i: self.i.clone() } + } +} diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs new file mode 100644 index 00000000..6357cd26 --- /dev/null +++ b/server/src/engine/idx/stord/mod.rs @@ -0,0 +1,783 @@ +/* + * Created on Mon Jan 16 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +pub(super) mod config; +pub(super) mod iter; + +use { + self::{ + config::{AllocStrategy, Config}, + iter::{ + IndexSTSeqDllIterOrdKV, IndexSTSeqDllIterOrdKey, IndexSTSeqDllIterOrdValue, + IndexSTSeqDllIterUnordKV, IndexSTSeqDllIterUnordKey, IndexSTSeqDllIterUnordValue, + }, + }, + super::{AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, STIndex, STIndexSeq}, + crate::engine::mem::StatelessLen, + std::{ + alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout}, + borrow::Borrow, + collections::HashMap as StdMap, + fmt::{self, Debug}, + hash::{Hash, Hasher}, + mem, + ptr::{self, NonNull}, + }, +}; + +/* + For the ordered index impl, we resort to some crazy unsafe code, especially because there's no other way to + deal with non-primitive Ks. That's why we'll ENTIRELY AVOID exporting any structures; if we end up using a node + or a ptr struct anywhere inappropriate, it'll most likely SEGFAULT. So yeah, better be careful with this one. + Second note, I'm not a big fan of the DLL and will most likely try a different approach in the future; this one + is the most convenient option for now. + + -- Sayan (@ohsayan) // Jan. 16 '23 +*/ + +#[repr(transparent)] +/// # WARNING: Segfault/UAF alert +/// +/// Yeah, this type is going to segfault if you decide to use it in random places. Literally, don't use it if +/// you're unsure about it's validity. For example, if you simply `==` this or attempt to use it an a hashmap, +/// you can segfault. IFF, the ptr is valid will it not segfault +struct IndexSTSeqDllKeyptr { + p: *const K, +} + +impl IndexSTSeqDllKeyptr { + #[inline(always)] + fn new(r: &K) -> Self { + Self { p: r as *const _ } + } +} + +impl Hash for IndexSTSeqDllKeyptr { + #[inline(always)] + fn hash(&self, state: &mut H) + where + H: Hasher, + { + unsafe { + /* + UNSAFE(@ohsayan): BAD. THIS IS NOT SAFE, but dang it, it's the only way we can do this without + dynamic rule checking. I wish there was a `'self` lifetime + */ + (*self.p).hash(state) + } + } +} + +impl PartialEq for IndexSTSeqDllKeyptr { + #[inline(always)] + fn eq(&self, other: &Self) -> bool { + unsafe { + /* + UNSAFE(@ohsayan): BAD. THIS IS NOT SAFE, but dang it, it's the only way we can do this without + dynamic rule checking. I wish there was a `'self` lifetime + */ + (*self.p).eq(&*other.p) + } + } +} + +impl Eq for IndexSTSeqDllKeyptr {} + +// stupid type for trait impl conflict riddance +#[derive(Debug, Hash, PartialEq, Eq)] +#[repr(transparent)] +struct IndexSTSeqDllQref(Q); + +impl IndexSTSeqDllQref { + #[inline(always)] + unsafe fn from_ref(r: &Q) -> &Self { + mem::transmute(r) + } +} + +impl Borrow> for IndexSTSeqDllKeyptr +where + K: Borrow, + Q: ?Sized, +{ + #[inline(always)] + fn borrow(&self) -> &IndexSTSeqDllQref { + unsafe { + /* + UNSAFE(@ohsayan): BAD. This deref ain't safe either. ref is good though + */ + IndexSTSeqDllQref::from_ref((*self.p).borrow()) + } + } +} + +#[derive(Debug)] +pub struct IndexSTSeqDllNode { + k: K, + v: V, + n: *mut Self, + p: *mut Self, +} + +impl IndexSTSeqDllNode { + const LAYOUT: Layout = Layout::new::(); + #[inline(always)] + fn new(k: K, v: V, n: *mut Self, p: *mut Self) -> Self { + Self { k, v, n, p } + } + #[inline(always)] + fn new_null(k: K, v: V) -> Self { + Self::new(k, v, ptr::null_mut(), ptr::null_mut()) + } + #[inline(always)] + fn _alloc_with_garbage() -> *mut Self { + unsafe { + // UNSAFE(@ohsayan): aight shut up, it's a malloc + let ptr = std_alloc(Self::LAYOUT) as *mut Self; + assert!(!ptr.is_null(), "damn the allocator failed"); + ptr + } + } + #[inline(always)] + fn _alloc(Self { k, v, p, n }: Self) -> *mut Self { + unsafe { + // UNSAFE(@ohsayan): grow up, we're writing to a fresh block + let ptr = Self::_alloc_with_garbage(); + (*ptr).k = k; + (*ptr).v = v; + if WPTR_N { + (*ptr).n = n; + } + if WPTR_P { + (*ptr).p = p; + } + ptr + } + } + #[inline(always)] + unsafe fn _drop(slf: *mut Self) { + let _ = Box::from_raw(slf); + } + #[inline(always)] + /// LEAK: K, V + unsafe fn dealloc_headless(slf: *mut Self) { + std_dealloc(slf as *mut u8, Self::LAYOUT) + } + #[inline(always)] + unsafe fn unlink(node: *mut Self) { + (*((*node).p)).n = (*node).n; + (*((*node).n)).p = (*node).p; + } + #[inline(always)] + unsafe fn link(from: *mut Self, to: *mut Self) { + (*to).n = (*from).n; + (*to).p = from; + (*from).n = to; + (*(*to).n).p = to; + } + #[inline(always)] + fn alloc_box(node: IndexSTSeqDllNode) -> NonNull> { + unsafe { + // UNSAFE(@ohsayan): Safe because of box alloc + NonNull::new_unchecked(Box::into_raw(Box::new(node))) + } + } +} + +pub type IndexSTSeqDllNodePtr = NonNull>; + +#[cfg(debug_assertions)] +pub struct IndexSTSeqDllMetrics { + stat_f: usize, +} + +#[cfg(debug_assertions)] +impl IndexSTSeqDllMetrics { + #[cfg(test)] + pub const fn raw_f(&self) -> usize { + self.stat_f + } + const fn new() -> IndexSTSeqDllMetrics { + Self { stat_f: 0 } + } +} + +/// An ST-index with ordering. Inefficient ordered scanning since not in block +pub struct IndexSTSeqDll> { + m: StdMap, IndexSTSeqDllNodePtr, C::Hasher>, + h: *mut IndexSTSeqDllNode, + a: C::AllocStrategy, + #[cfg(debug_assertions)] + metrics: IndexSTSeqDllMetrics, +} + +impl> IndexSTSeqDll { + const DEF_CAP: usize = 0; + #[inline(always)] + const fn _new( + m: StdMap, IndexSTSeqDllNodePtr, C::Hasher>, + h: *mut IndexSTSeqDllNode, + ) -> IndexSTSeqDll { + Self { + m, + h, + a: C::AllocStrategy::NEW, + #[cfg(debug_assertions)] + metrics: IndexSTSeqDllMetrics::new(), + } + } + #[inline(always)] + fn _new_map(m: StdMap, IndexSTSeqDllNodePtr, C::Hasher>) -> Self { + Self::_new(m, ptr::null_mut()) + } + #[inline(always)] + pub fn with_hasher(hasher: C::Hasher) -> Self { + Self::with_capacity_and_hasher(Self::DEF_CAP, hasher) + } + #[inline(always)] + pub fn with_capacity_and_hasher(cap: usize, hasher: C::Hasher) -> Self { + Self::_new_map(StdMap::with_capacity_and_hasher(cap, hasher)) + } + fn metrics_update_f_empty(&mut self) { + #[cfg(debug_assertions)] + { + self.metrics.stat_f = 0; + } + } +} + +impl> IndexSTSeqDll { + pub fn with_capacity(cap: usize) -> Self { + Self::with_capacity_and_hasher(cap, C::Hasher::default()) + } +} + +impl + Default> Default for IndexSTSeqDll { + fn default() -> Self { + Self::with_hasher(C::Hasher::default()) + } +} + +impl> IndexSTSeqDll { + #[inline(always)] + fn metrics_update_f_decr(&mut self) { + #[cfg(debug_assertions)] + { + self.metrics.stat_f -= 1; + } + } + #[inline(always)] + fn metrics_update_f_incr(&mut self) { + #[cfg(debug_assertions)] + { + self.metrics.stat_f += 1; + } + } + #[inline(always)] + fn ensure_sentinel(&mut self) { + if self.h.is_null() { + let ptr = IndexSTSeqDllNode::_alloc_with_garbage(); + unsafe { + // UNSAFE(@ohsayan): Fresh alloc + self.h = ptr; + (*ptr).p = ptr; + (*ptr).n = ptr; + } + } + } + #[inline(always)] + /// ## Safety + /// + /// Head must not be null + unsafe fn drop_nodes_full(&mut self) { + // don't drop sentinenl + let mut c = (*self.h).n; + while c != self.h { + let nx = (*c).n; + IndexSTSeqDllNode::_drop(c); + c = nx; + } + } + #[inline(always)] + /// NOTE: `&mut Self` for aliasing + /// ## Safety + /// Ensure head is non null + unsafe fn link(&mut self, node: IndexSTSeqDllNodePtr) { + IndexSTSeqDllNode::link(self.h, node.as_ptr()) + } + pub fn len(&self) -> usize { + self.m.len() + } +} + +impl> IndexSTSeqDll { + #[inline(always)] + fn _iter_unord_kv<'a>(&'a self) -> IndexSTSeqDllIterUnordKV<'a, K, V> { + IndexSTSeqDllIterUnordKV::new(&self.m) + } + #[inline(always)] + fn _iter_unord_k<'a>(&'a self) -> IndexSTSeqDllIterUnordKey<'a, K, V> { + IndexSTSeqDllIterUnordKey::new(&self.m) + } + #[inline(always)] + fn _iter_unord_v<'a>(&'a self) -> IndexSTSeqDllIterUnordValue<'a, K, V> { + IndexSTSeqDllIterUnordValue::new(&self.m) + } + #[inline(always)] + fn _iter_ord_kv<'a>(&'a self) -> IndexSTSeqDllIterOrdKV<'a, K, V> { + IndexSTSeqDllIterOrdKV::new(self) + } + #[inline(always)] + fn _iter_ord_k<'a>(&'a self) -> IndexSTSeqDllIterOrdKey<'a, K, V> { + IndexSTSeqDllIterOrdKey::new(self) + } + #[inline(always)] + fn _iter_ord_v<'a>(&'a self) -> IndexSTSeqDllIterOrdValue<'a, K, V> { + IndexSTSeqDllIterOrdValue::new(self) + } +} + +impl> IndexSTSeqDll { + #[inline(always)] + /// Clean up unused and cached memory + fn vacuum_full(&mut self) { + self.m.shrink_to_fit(); + self.a.cleanup(); + if C::AllocStrategy::METRIC_REFRESH { + self.metrics_update_f_empty(); + } + } +} + +impl> IndexSTSeqDll { + #[inline(always)] + fn _insert(&mut self, k: K, v: V) -> bool { + if self.m.contains_key(&IndexSTSeqDllKeyptr::new(&k)) { + return false; + } + self.__insert(k, v) + } + fn _get(&self, k: &Q) -> Option<&V> + where + K: Borrow, + Q: AsKey, + { + self.m + .get(unsafe { + // UNSAFE(@ohsayan): Ref with correct bounds + IndexSTSeqDllQref::from_ref(k) + }) + .map(|e| unsafe { + // UNSAFE(@ohsayan): ref is non-null and ensures aliasing reqs + &(e.as_ref()).read_value().v + }) + } + fn _get_mut(&mut self, k: &Q) -> Option<&mut V> + where + K: Borrow, + Q: AsKey, + { + self.m + .get_mut(unsafe { IndexSTSeqDllQref::from_ref(k) }) + .map(|e| unsafe { &mut e.as_mut().v }) + } + #[inline(always)] + fn _update(&mut self, k: &Q, v: V) -> Option + where + K: Borrow, + Q: AsKey, + { + match self.m.get(unsafe { + // UNSAFE(@ohsayan): Just got a ref with the right bounds + IndexSTSeqDllQref::from_ref(k) + }) { + Some(e) => unsafe { + // UNSAFE(@ohsayan): Impl guarantees that entry presence == nullck head + self.__update(*e, v) + }, + None => None, + } + } + #[inline(always)] + fn _upsert(&mut self, k: K, v: V) -> Option { + match self.m.get(&IndexSTSeqDllKeyptr::new(&k)) { + Some(e) => unsafe { + // UNSAFE(@ohsayan): Impl guarantees that entry presence == nullck head + self.__update(*e, v) + }, + None => { + let _ = self.__insert(k, v); + None + } + } + } + #[inline(always)] + fn _remove(&mut self, k: &Q) -> Option + where + K: Borrow, + Q: AsKey + ?Sized, + { + self.m + .remove(unsafe { + // UNSAFE(@ohsayan): good trait bounds and type + IndexSTSeqDllQref::from_ref(k) + }) + .map(|n| unsafe { + let n = n.as_ptr(); + // UNSAFE(@ohsayan): Correct init and aligned to K + drop(ptr::read(&(*n).k)); + // UNSAFE(@ohsayan): Correct init and aligned to V + let v = ptr::read(&(*n).v); + // UNSAFE(@ohsayan): non-null guaranteed by as_ptr + IndexSTSeqDllNode::unlink(n); + self.a.free(n); + if C::AllocStrategy::METRIC_REFRESH { + self.metrics_update_f_incr(); + } + v + }) + } + #[inline(always)] + fn __insert(&mut self, k: K, v: V) -> bool { + self.ensure_sentinel(); + let mut refresh = false; + let node = self + .a + .alloc(IndexSTSeqDllNode::new_null(k, v), &mut refresh); + if C::AllocStrategy::METRIC_REFRESH & cfg!(debug_assertions) & refresh { + self.metrics_update_f_decr(); + } + let kptr = unsafe { + // UNSAFE(@ohsayan): All g, we allocated it rn + IndexSTSeqDllKeyptr::new(&node.as_ref().k) + }; + let _ = self.m.insert(kptr, node); + unsafe { + // UNSAFE(@ohsayan): sentinel check done + self.link(node); + } + true + } + #[inline(always)] + /// ## Safety + /// + /// Has sentinel + unsafe fn __update(&mut self, e: NonNull>, v: V) -> Option { + let old = unsafe { + // UNSAFE(@ohsayan): Same type layout, alignments and non-null + ptr::replace(&mut (*e.as_ptr()).v, v) + }; + self._refresh(e); + Some(old) + } + #[inline(always)] + /// ## Safety + /// + /// Has sentinel + unsafe fn _refresh(&mut self, e: NonNull>) { + // UNSAFE(@ohsayan): Since it's in the collection, it is a valid ptr + IndexSTSeqDllNode::unlink(e.as_ptr()); + // UNSAFE(@ohsayan): As we found a node, our impl guarantees that the head is not-null + self.link(e); + } + #[inline(always)] + fn _clear(&mut self) { + self.m.clear(); + if !self.h.is_null() { + unsafe { + // UNSAFE(@ohsayan): nullck + self.drop_nodes_full(); + // UNSAFE(@ohsayan): Drop won't kill sentinel; link back to self + (*self.h).p = self.h; + (*self.h).n = self.h; + } + } + } +} + +impl> Drop for IndexSTSeqDll { + fn drop(&mut self) { + if !self.h.is_null() { + unsafe { + // UNSAFE(@ohsayan): nullck + self.drop_nodes_full(); + // UNSAFE(@ohsayan): nullck: drop doesn't clear sentinel + IndexSTSeqDllNode::dealloc_headless(self.h); + } + } + self.a.cleanup(); + } +} + +impl> FromIterator<(K, V)> for IndexSTSeqDll { + fn from_iter>(iter: T) -> Self { + let mut slf = Self::with_hasher(C::Hasher::default()); + iter.into_iter() + .for_each(|(k, v)| assert!(slf._insert(k, v))); + slf + } +} + +impl> IndexBaseSpec for IndexSTSeqDll { + const PREALLOC: bool = true; + + #[cfg(debug_assertions)] + type Metrics = IndexSTSeqDllMetrics; + + fn idx_init() -> Self { + Self::with_hasher(C::Hasher::default()) + } + + fn idx_init_with(s: Self) -> Self { + s + } + + fn idx_init_cap(cap: usize) -> Self { + Self::with_capacity_and_hasher(cap, C::Hasher::default()) + } + + #[cfg(debug_assertions)] + fn idx_metrics(&self) -> &Self::Metrics { + &self.metrics + } +} + +impl> STIndex for IndexSTSeqDll +where + K: AsKey, + V: AsValue, +{ + type IterKV<'a> = IndexSTSeqDllIterUnordKV<'a, K, V> + where + Self: 'a, + K: 'a, + V: 'a; + + type IterKey<'a> = IndexSTSeqDllIterUnordKey<'a, K, V> + where + Self: 'a, + K: 'a; + + type IterValue<'a> = IndexSTSeqDllIterUnordValue<'a, K, V> + where + Self: 'a, + V: 'a; + + fn st_compact(&mut self) { + self.vacuum_full(); + } + + fn st_len(&self) -> usize { + self.len() + } + + fn st_clear(&mut self) { + self._clear() + } + + fn st_insert(&mut self, key: K, val: V) -> bool { + self._insert(key, val) + } + + fn st_upsert(&mut self, key: K, val: V) { + let _ = self._upsert(key, val); + } + + fn st_contains(&self, key: &Q) -> bool + where + K: Borrow + AsKey, + Q: ?Sized + AsKey, + { + self.m.contains_key(unsafe { + // UNSAFE(@ohsayan): Valid ref with correct bounds + IndexSTSeqDllQref::from_ref(key) + }) + } + + fn st_get(&self, key: &Q) -> Option<&V> + where + K: Borrow, + Q: ?Sized + AsKey, + { + self._get(key) + } + + fn st_get_cloned(&self, key: &Q) -> Option + where + K: Borrow, + Q: ?Sized + AsKey, + V: AsValueClone, + { + self._get(key).cloned() + } + + fn st_get_mut(&mut self, k: &Q) -> Option<&mut V> + where + K: AsKey + Borrow, + Q: ?Sized + AsKey, + { + self._get_mut(k) + } + + fn st_update(&mut self, key: &Q, val: V) -> bool + where + K: Borrow, + Q: ?Sized + AsKey, + { + self._update(key, val).is_some() + } + + fn st_update_return(&mut self, key: &Q, val: V) -> Option + where + K: Borrow, + Q: ?Sized + AsKey, + { + self._update(key, val) + } + + fn st_delete(&mut self, key: &Q) -> bool + where + K: Borrow, + Q: ?Sized + AsKey, + { + self._remove(key).is_some() + } + + fn st_delete_return(&mut self, key: &Q) -> Option + where + K: Borrow, + Q: ?Sized + AsKey, + { + self._remove(key) + } + + fn st_delete_if(&mut self, key: &Q, iff: impl Fn(&V) -> bool) -> Option + where + K: AsKey + Borrow, + Q: ?Sized + AsKey, + { + match self._get(key) { + Some(v) if iff(v) => { + self._remove(key); + Some(true) + } + Some(_) => Some(false), + None => None, + } + } + + fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a> { + self._iter_unord_kv() + } + + fn st_iter_key<'a>(&'a self) -> Self::IterKey<'a> { + self._iter_unord_k() + } + + fn st_iter_value<'a>(&'a self) -> Self::IterValue<'a> { + self._iter_unord_v() + } +} + +impl STIndexSeq for IndexSTSeqDll +where + K: AsKey, + V: AsValue, + C: Config, +{ + type IterOrdKV<'a> = IndexSTSeqDllIterOrdKV<'a, K, V> + where + Self: 'a, + K: 'a, + V: 'a; + type IterOrdKey<'a> = IndexSTSeqDllIterOrdKey<'a, K, V> + where + Self: 'a, + K: 'a; + type IterOrdValue<'a> = IndexSTSeqDllIterOrdValue<'a, K, V> + where + Self: 'a, + V: 'a; + type OwnedIterKV = iter::OrderedOwnedIteratorKV; + type OwnedIterKeys = iter::OrderedOwnedIteratorKey; + type OwnedIterValues = iter::OrderedOwnedIteratorValue; + fn stseq_ord_kv<'a>(&'a self) -> Self::IterOrdKV<'a> { + self._iter_ord_kv() + } + fn stseq_ord_key<'a>(&'a self) -> Self::IterOrdKey<'a> { + self._iter_ord_k() + } + fn stseq_ord_value<'a>(&'a self) -> Self::IterOrdValue<'a> { + self._iter_ord_v() + } + fn stseq_owned_keys(self) -> Self::OwnedIterKeys { + iter::OrderedOwnedIteratorKey(iter::OrderedOwnedIteratorRaw::new(self)) + } + fn stseq_owned_values(self) -> Self::OwnedIterValues { + iter::OrderedOwnedIteratorValue(iter::OrderedOwnedIteratorRaw::new(self)) + } + fn stseq_owned_kv(self) -> Self::OwnedIterKV { + iter::OrderedOwnedIteratorKV(iter::OrderedOwnedIteratorRaw::new(self)) + } +} + +impl> Clone for IndexSTSeqDll { + fn clone(&self) -> Self { + let mut slf = Self::with_capacity_and_hasher(self.len(), C::Hasher::default()); + self._iter_ord_kv() + .map(|(k, v)| (k.clone(), v.clone())) + .for_each(|(k, v)| { + slf._insert(k, v); + }); + slf + } +} + +unsafe impl + Send> Send for IndexSTSeqDll {} +unsafe impl + Sync> Sync for IndexSTSeqDll {} + +impl + fmt::Debug> fmt::Debug + for IndexSTSeqDll +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_map().entries(self._iter_ord_kv()).finish() + } +} + +impl> PartialEq for IndexSTSeqDll { + fn eq(&self, other: &Self) -> bool { + self.len() == other.len() + && self + ._iter_ord_kv() + .all(|(k, v)| other._get(k).unwrap().eq(v)) + } +} + +impl> StatelessLen for IndexSTSeqDll { + fn stateless_len(&self) -> usize { + self.len() + } +} diff --git a/server/src/engine/idx/tests.rs b/server/src/engine/idx/tests.rs new file mode 100644 index 00000000..b552e5f4 --- /dev/null +++ b/server/src/engine/idx/tests.rs @@ -0,0 +1,187 @@ +/* + * Created on Wed Jan 18 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use super::*; + +mod idx_st_seq_dll { + use super::{IndexBaseSpec, IndexSTSeqLib, STIndex, STIndexSeq}; + + #[cfg(not(miri))] + const SPAM_CNT: usize = 131_072; + #[cfg(miri)] + const SPAM_CNT: usize = 128; + + type Index = IndexSTSeqLib; + + /// Returns an index with: `i -> "{i+1}"` starting from 0 upto the value of [`SPAM_CNT`] + fn mkidx() -> IndexSTSeqLib { + let mut idx = IndexSTSeqLib::idx_init(); + for int in 0..SPAM_CNT { + assert!(idx.st_insert(int, (int + 1).to_string())); + } + // verify data + for int in 0..SPAM_CNT { + assert_eq!(idx.st_get(&int).unwrap().as_str(), (int + 1).to_string()); + } + assert_eq!(idx.idx_metrics().raw_f(), 0); + idx + } + + #[inline(always)] + fn s(s: &str) -> String { + s.to_owned() + } + #[test] + fn empty_drop() { + let idx = Index::idx_init(); + assert_eq!(idx.idx_metrics().raw_f(), 0); + drop(idx); + } + #[test] + fn spam_read_nx() { + let idx = IndexSTSeqLib::::idx_init(); + for int in SPAM_CNT..SPAM_CNT * 2 { + assert!(idx.st_get(&int).is_none()); + } + } + #[test] + fn spam_insert_ex() { + let mut idx = mkidx(); + for int in 0..SPAM_CNT { + assert!(!idx.st_insert(int, (int + 2).to_string())); + } + } + #[test] + fn spam_update_nx() { + let mut idx = IndexSTSeqLib::::idx_init(); + for int in 0..SPAM_CNT { + assert!(!idx.st_update(&int, (int + 2).to_string())); + } + } + #[test] + fn spam_delete_nx() { + let mut idx = IndexSTSeqLib::::idx_init(); + for int in 0..SPAM_CNT { + assert!(!idx.st_delete(&int)); + } + } + #[test] + fn simple_crud() { + let mut idx = Index::idx_init(); + assert!(idx.st_insert(s("hello"), s("world"))); + assert_eq!(idx.st_get("hello").as_deref().unwrap(), "world"); + assert!(idx.st_update("hello", s("world2"))); + assert_eq!(idx.st_get("hello").as_deref().unwrap(), "world2"); + assert_eq!(idx.st_delete_return("hello").unwrap(), "world2"); + assert_eq!(idx.idx_metrics().raw_f(), 1); + } + #[test] + fn spam_crud() { + let mut idx = IndexSTSeqLib::idx_init(); + for int in 0..SPAM_CNT { + assert!(idx.st_insert(int, int + 1)); + assert_eq!(*idx.st_get(&int).unwrap(), int + 1); + assert!(idx.st_update(&int, int + 2)); + assert_eq!(*idx.st_get(&int).unwrap(), int + 2); + assert_eq!(idx.st_delete_return(&int).unwrap(), int + 2); + } + assert_eq!(idx.idx_metrics().raw_f(), 1); + } + #[test] + fn spam_read() { + let mut idx = IndexSTSeqLib::idx_init(); + for int in 0..SPAM_CNT { + let v = (int + 1).to_string(); + assert!(idx.st_insert(int, v.clone())); + assert_eq!(idx.st_get(&int).as_deref().unwrap(), &v); + } + assert_eq!(idx.idx_metrics().raw_f(), 0); + } + #[test] + fn spam_update() { + let mut idx = mkidx(); + for int in 0..SPAM_CNT { + assert_eq!( + idx.st_update_return(&int, (int + 2).to_string()).unwrap(), + (int + 1).to_string() + ); + } + assert_eq!(idx.idx_metrics().raw_f(), 0); + } + #[test] + fn spam_delete() { + let mut idx = mkidx(); + for int in 0..SPAM_CNT { + assert_eq!(idx.st_delete_return(&int).unwrap(), (int + 1).to_string()); + assert_eq!(idx.idx_metrics().raw_f(), int + 1); + } + assert_eq!(idx.idx_metrics().raw_f(), SPAM_CNT); + } + #[test] + fn compact() { + let mut idx = mkidx(); + assert_eq!(idx.idx_metrics().raw_f(), 0); + for int in 0..SPAM_CNT { + let _ = idx.st_delete(&int); + } + assert_eq!(idx.idx_metrics().raw_f(), SPAM_CNT); + idx.st_clear(); + assert_eq!(idx.idx_metrics().raw_f(), SPAM_CNT); + idx.st_compact(); + assert_eq!(idx.idx_metrics().raw_f(), 0); + } + // pointless testing random iterators + #[test] + fn iter_ord() { + let idx1 = mkidx(); + let idx2: Vec<(usize, String)> = + idx1.stseq_ord_kv().map(|(k, v)| (*k, v.clone())).collect(); + (0..SPAM_CNT) + .into_iter() + .zip(idx2.into_iter()) + .for_each(|(i, (k, v))| { + assert_eq!(i, k); + assert_eq!((i + 1).to_string(), v); + }); + } + #[test] + fn iter_ord_rev() { + let idx1 = mkidx(); + let idx2: Vec<(usize, String)> = idx1 + .stseq_ord_kv() + .rev() + .map(|(k, v)| (*k, v.clone())) + .collect(); + (0..SPAM_CNT) + .rev() + .into_iter() + .zip(idx2.into_iter()) + .for_each(|(i, (k, v))| { + assert_eq!(i, k); + assert_eq!((i + 1).to_string(), v); + }); + } +} diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs new file mode 100644 index 00000000..6bb0fc1a --- /dev/null +++ b/server/src/engine/macros.rs @@ -0,0 +1,397 @@ +/* + * Created on Wed Oct 12 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software fation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(test)] +macro_rules! into_array { + ($($e:expr),* $(,)?) => { [$($e.into()),*] }; +} + +macro_rules! as_array { + ($($e:expr),* $(,)?) => { [$($e as _),*] }; +} + +macro_rules! extract { + ($src:expr, $what:pat => $ret:expr) => { + if let $what = $src { + $ret + } else { + $crate::impossible!() + } + }; +} + +#[cfg(test)] +macro_rules! multi_assert_eq { + ($($lhs:expr),* => $rhs:expr) => { + $(assert_eq!($lhs, $rhs);)* + }; +} + +macro_rules! direct_from { + ($for:ident<$lt:lifetime> => {$($other:ty as $me:ident),*$(,)?}) => { + $(impl<$lt> ::core::convert::From<$other> for $for<$lt> {fn from(v: $other) -> Self {Self::$me(v.into())}})* + }; + ($for:ty => {$($other:ty as $me:ident),*$(,)?}) => { + $(impl ::core::convert::From<$other> for $for {fn from(v: $other) -> Self {Self::$me(v.into())}})* + }; + ($for:ty[_] => {$($other:ty as $me:ident),*$(,)?}) => { + $(impl ::core::convert::From<$other> for $for {fn from(_: $other) -> Self {Self::$me}})* + }; +} + +macro_rules! flags { + ($(#[$attr:meta])* $vis:vis struct $group:ident: $ty:ty { $($const:ident = $expr:expr),+ $(,)?}) => ( + $(#[$attr])* #[repr(transparent)] $vis struct $group {r#const: $ty} + #[allow(unused)] + impl $group { + $(pub const $const: Self = Self { r#const: $expr };)* + #[inline(always)] pub const fn d(&self) -> $ty { self.r#const } + const _BASE_HB: $ty = 1 << (<$ty>::BITS - 1); + #[inline(always)] pub const fn name(&self) -> &'static str { + match self.r#const {$(capture if capture == $expr => ::core::stringify!($const),)* _ => ::core::unreachable!()} + } + const LEN: usize = { let mut i = 0; $(let _ = $expr; i += 1;)+{i} }; + const A: [$ty; $group::LEN] = [$($expr,)+]; + const SANITY: () = { + let a = &Self::A; let l = a.len(); let mut i = 0; + while i < l { let mut j = i + 1; while j < l { if a[i] == a[j] { panic!("conflict"); } j += 1; } i += 1; } + }; + const ALL: $ty = { let mut r: $ty = 0; $( r |= $expr;)+ r }; + pub const fn has_flags_in(v: $ty) -> bool { Self::ALL & v != 0 } + pub const fn bits() -> usize { let r: $ty = ($($expr+)+0); r.count_ones() as _ } + } + impl ::core::fmt::Debug for $group { + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + const _V : () = $group::SANITY; + ::core::write!(f, "{}::{}", ::core::stringify!($group), Self::name(self)) + } + } + ); +} + +macro_rules! __kw_misc { + ($ident:ident) => { + $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::Misc( + $crate::engine::ql::lex::KeywordMisc::$ident, + )) + }; +} + +macro_rules! __kw_stmt { + ($ident:ident) => { + $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::Statement( + $crate::engine::ql::lex::KeywordStmt::$ident, + )) + }; +} + +/* + Frankly, this is just for lazy people like me. Do not judge + -- Sayan (@ohsayan) +*/ +macro_rules! Token { + // misc symbol + (@) => { + __sym_token!(SymAt) + }; + (#) => { + __sym_token!(SymHash) + }; + ($) => { + __sym_token!(SymDollar) + }; + (%) => { + __sym_token!(SymPercent) + }; + (.) => { + __sym_token!(SymPeriod) + }; + (,) => { + __sym_token!(SymComma) + }; + (_) => { + __sym_token!(SymUnderscore) + }; + (?) => { + __sym_token!(SymQuestion) + }; + (:) => { + __sym_token!(SymColon) + }; + (;) => { + __sym_token!(SymSemicolon) + }; + (~) => { + __sym_token!(SymTilde) + }; + // logical + (!) => { + __sym_token!(OpLogicalNot) + }; + (^) => { + __sym_token!(OpLogicalXor) + }; + (&) => { + __sym_token!(OpLogicalAnd) + }; + (|) => { + __sym_token!(OpLogicalOr) + }; + // operator misc. + (=) => { + __sym_token!(OpAssign) + }; + // arithmetic + (+) => { + __sym_token!(OpArithmeticAdd) + }; + (-) => { + __sym_token!(OpArithmeticSub) + }; + (*) => { + __sym_token!(OpArithmeticMul) + }; + (/) => { + __sym_token!(OpArithmeticDiv) + }; + // relational + (>) => { + __sym_token!(OpComparatorGt) + }; + (<) => { + __sym_token!(OpComparatorLt) + }; + // ddl keywords + (use) => { + __kw_stmt!(Use) + }; + (create) => { + __kw_stmt!(Create) + }; + (alter) => { + __kw_stmt!(Alter) + }; + (drop) => { + __kw_stmt!(Drop) + }; + (model) => { + __kw_misc!(Model) + }; + (space) => { + __kw_misc!(Space) + }; + (primary) => { + __kw_misc!(Primary) + }; + // ddl misc + (with) => { + __kw_misc!(With) + }; + (add) => { + __kw_misc!(Add) + }; + (remove) => { + __kw_misc!(Remove) + }; + (sort) => { + __kw_misc!(Sort) + }; + (type) => { + __kw_misc!(Type) + }; + // dml + (insert) => { + __kw_stmt!(Insert) + }; + (select) => { + __kw_stmt!(Select) + }; + (update) => { + __kw_stmt!(Update) + }; + (delete) => { + __kw_stmt!(Delete) + }; + // dml misc + (set) => { + __kw_misc!(Set) + }; + (limit) => { + __kw_misc!(Limit) + }; + (from) => { + __kw_misc!(From) + }; + (into) => { + __kw_misc!(Into) + }; + (where) => { + __kw_misc!(Where) + }; + (if) => { + __kw_misc!(If) + }; + (and) => { + __kw_misc!(And) + }; + (as) => { + __kw_misc!(As) + }; + (by) => { + __kw_misc!(By) + }; + (asc) => { + __kw_misc!(Asc) + }; + (desc) => { + __kw_misc!(Desc) + }; + // types + (string) => { + __kw_misc!(String) + }; + (binary) => { + __kw_misc!(Binary) + }; + (list) => { + __kw_misc!(List) + }; + (map) => { + __kw_misc!(Map) + }; + (bool) => { + __kw_misc!(Bool) + }; + (int) => { + __kw_misc!(Int) + }; + (double) => { + __kw_misc!(Double) + }; + (float) => { + __kw_misc!(Float) + }; + // tt + (open {}) => { + __sym_token!(TtOpenBrace) + }; + (close {}) => { + __sym_token!(TtCloseBrace) + }; + (() open) => { + __sym_token!(TtOpenParen) + }; + (() close) => { + __sym_token!(TtCloseParen) + }; + (open []) => { + __sym_token!(TtOpenSqBracket) + }; + (close []) => { + __sym_token!(TtCloseSqBracket) + }; + // misc + (null) => { + __kw_misc!(Null) + }; + (not) => { + __kw_misc!(Not) + }; + (return) => { + __kw_misc!(Return) + }; + (allow) => { + __kw_misc!(Allow) + }; + (all) => { + __kw_misc!(All) + }; + (exists) => { + __kw_stmt!(Exists) + }; +} + +macro_rules! union { + ($(#[$attr:meta])* $vis:vis union $name:ident $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name] [] $tail);); + ($(#[$attr:meta])* $vis:vis union $name:ident<$($lt:lifetime),*> $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name<$($lt),*>] [] $tail);); + (@parse $decl:tt [$($head:tt)*] {}) => (union!(@defeat0 $decl [$($head)*]);); + (@parse $decl:tt [$($head:tt)*] {$(#[$attr:meta])* $vis:vis !$ident:ident:$ty:ty,$($tail:tt)*}) => ( + union!(@parse $decl [$($head)* $(#[$attr])* $vis $ident: ::core::mem::ManuallyDrop::<$ty>,] {$($tail)*}); + ); + (@parse $decl:tt [$($head:tt)*] {$(#[$attr:meta])* $vis:vis $ident:ident:$ty:ty,$($tail:tt)*}) => ( + union!(@parse $decl [$($head)* $(#[$attr])* $vis $ident: $ty, ] { $($tail)* }); + ); + (@defeat0 [$($decls:tt)*] [$($head:tt)*]) => (union!(@defeat1 $($decls)* { $($head)* });); + (@defeat1 $i:item) => ($i); +} + +macro_rules! dbgfn { + ($($(#[$attr:meta])* $vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block:block)*) => { + $(dbgfn!(@int $(#[$attr])* $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block {panic!("called dbg symbol in non-dbg build")});)* + }; + ($($(#[$attr:meta])* $vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block:block else $block_b:block)*) => { + $(dbgfn!(@int $(#[$attr])* $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block $block_b);)* + }; + (@int $(#[$attr:meta])* $vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block_a:block $block_b:block) => { + #[cfg(debug_assertions)] + $(#[$attr])* $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block_a + #[cfg(not(debug_assertions))] + $(#[$attr])* $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block_b + } +} + +/// Convert all the KV pairs into an iterator and then turn it into an appropriate collection +/// (inferred). +/// +/// **Warning: This is going to be potentially slow due to the iterator creation** +macro_rules! into_dict { + () => { ::core::default::Default::default() }; + ($($key:expr => $value:expr),+ $(,)?) => {{ + [$(($key.into(), $value.into())),+] + .map(|(k, v)| (k, v)) + .into_iter() + .collect() + }}; +} + +#[cfg(test)] +macro_rules! pairvec { + ($($x:expr),*) => {{ let mut v = Vec::new(); $( let (a, b) = $x; v.push((a.into(), b.into())); )* v }}; +} + +#[cfg(test)] +macro_rules! intovec { + ($($x:expr),* $(,)?) => { vec![$(core::convert::From::from($x),)*] }; +} + +macro_rules! sizeof { + ($ty:ty) => { + ::core::mem::size_of::<$ty>() + }; + ($ty:ty, $by:literal) => { + ::core::mem::size_of::<$ty>() * $by + }; +} diff --git a/server/src/engine/mem/astr.rs b/server/src/engine/mem/astr.rs new file mode 100644 index 00000000..6b4052b7 --- /dev/null +++ b/server/src/engine/mem/astr.rs @@ -0,0 +1,174 @@ +/* + * Created on Sat Feb 25 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::UArray, + crate::engine::ql::lex::Ident, + std::{ + borrow::Borrow, + fmt, mem, + ops::{Deref, DerefMut}, + }, +}; + +#[derive(PartialEq, Eq, Hash, Clone)] +#[repr(transparent)] +pub struct AStr { + base: UArray, +} +impl AStr { + #[inline(always)] + pub fn check(v: &str) -> bool { + v.len() <= N + } + #[inline(always)] + pub fn try_new(s: &str) -> Option { + if Self::check(s) { + Some(unsafe { + // UNSAFE(@ohsayan): verified len + Self::from_len_unchecked(s) + }) + } else { + None + } + } + #[inline(always)] + pub fn new(s: &str) -> Self { + Self::try_new(s).expect("length overflow") + } + #[inline(always)] + pub unsafe fn from_len_unchecked_ident(i: Ident<'_>) -> Self { + Self::from_len_unchecked(i.as_str()) + } + #[inline(always)] + pub unsafe fn from_len_unchecked(s: &str) -> Self { + Self { + base: UArray::from_slice(s.as_bytes()), + } + } + #[inline(always)] + pub unsafe fn from_len_unchecked_bytes(b: &[u8]) -> Self { + Self::from_len_unchecked(mem::transmute(b)) + } + #[inline(always)] + pub fn _as_str(&self) -> &str { + unsafe { + // UNSAFE(@ohsayan): same layout + mem::transmute(self._as_bytes()) + } + } + #[inline(always)] + pub fn _as_mut_str(&mut self) -> &mut str { + unsafe { + // UNSAFE(@ohsayan): same layout + mem::transmute(self._as_bytes_mut()) + } + } + pub fn _as_bytes(&self) -> &[u8] { + self.base.as_slice() + } + pub fn _as_bytes_mut(&mut self) -> &mut [u8] { + self.base.as_slice_mut() + } +} +impl fmt::Debug for AStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self._as_str()) + } +} +impl Deref for AStr { + type Target = str; + #[inline(always)] + fn deref(&self) -> &Self::Target { + self._as_str() + } +} +impl DerefMut for AStr { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + self._as_mut_str() + } +} +impl<'a, const N: usize> From> for AStr { + #[inline(always)] + fn from(value: Ident<'a>) -> Self { + Self::new(value.as_str()) + } +} +impl<'a, const N: usize> From<&'a str> for AStr { + #[inline(always)] + fn from(s: &str) -> Self { + Self::new(s) + } +} +impl PartialEq for AStr { + #[inline(always)] + fn eq(&self, other: &str) -> bool { + self._as_bytes() == other.as_bytes() + } +} +impl PartialEq> for str { + #[inline(always)] + fn eq(&self, other: &AStr) -> bool { + other._as_bytes() == self.as_bytes() + } +} +impl PartialEq<[u8]> for AStr { + #[inline(always)] + fn eq(&self, other: &[u8]) -> bool { + self._as_bytes() == other + } +} +impl PartialEq> for [u8] { + #[inline(always)] + fn eq(&self, other: &AStr) -> bool { + self == other.as_bytes() + } +} +impl AsRef<[u8]> for AStr { + #[inline(always)] + fn as_ref(&self) -> &[u8] { + self._as_bytes() + } +} +impl AsRef for AStr { + #[inline(always)] + fn as_ref(&self) -> &str { + self._as_str() + } +} +impl Default for AStr { + #[inline(always)] + fn default() -> Self { + Self::new("") + } +} +impl Borrow<[u8]> for AStr { + #[inline(always)] + fn borrow(&self) -> &[u8] { + self._as_bytes() + } +} diff --git a/server/src/engine/mem/ll.rs b/server/src/engine/mem/ll.rs new file mode 100644 index 00000000..b8ed9866 --- /dev/null +++ b/server/src/engine/mem/ll.rs @@ -0,0 +1,103 @@ +/* + * Created on Fri Sep 01 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use core::ops::{Deref, DerefMut}; + +#[derive(Debug, Clone, Copy, Default, Hash, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr(target_arch = "s390x", repr(align(256)))] +#[cfg_attr( + any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "x86_64", + ), + repr(align(128)) +)] +#[cfg_attr( + any( + target_arch = "arm", + target_arch = "hexagon", + target_arch = "mips", + target_arch = "mips64", + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "sparc" + ), + repr(align(32)) +)] +#[cfg_attr( + not(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "hexagon", + target_arch = "m68k", + target_arch = "mips", + target_arch = "mips64", + target_arch = "powerpc64", + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "sparc", + target_arch = "x86_64", + )), + repr(align(64)) +)] +#[cfg_attr(target_arch = "m68k", repr(align(16)))] +/** + cache line padding (to avoid unintended cache line invalidation) + - 256-bit (on a side note, good lord): + -> s390x: https://community.ibm.com/community/user/ibmz-and-linuxone/viewdocument/microprocessor-optimization-primer + - 128-bit: + -> aarch64: ARM64's big.LITTLE (it's a funny situation because there's a silly situation where one set of cores have one cache line + size while the other ones have a different size; see this excellent article: https://www.mono-project.com/news/2016/09/12/arm64-icache/) + -> powerpc64: https://reviews.llvm.org/D33656 + -> x86_64: Intel's Sandy Bridge+ (https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf) + - 64-bit: default for all non-specific targets + - 32-bit: arm, hexagon, mips, mips64, riscv64, and sparc have 32-byte cache line size + - 16-bit: m68k (not very useful for us, but yeah) +*/ +pub struct CachePadded { + data: T, +} + +impl CachePadded { + pub const fn new(data: T) -> Self { + Self { data } + } +} + +impl Deref for CachePadded { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.data + } +} + +impl DerefMut for CachePadded { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.data + } +} diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs new file mode 100644 index 00000000..46ce25c2 --- /dev/null +++ b/server/src/engine/mem/mod.rs @@ -0,0 +1,136 @@ +/* + * Created on Sun Jan 22 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod astr; +mod ll; +mod numbuf; +mod rawslice; +pub mod scanner; +mod stackop; +mod uarray; +mod vinline; +mod word; +// test +#[cfg(test)] +mod tests; +// re-exports +pub use { + astr::AStr, + ll::CachePadded, + numbuf::IntegerRepr, + rawslice::RawStr, + scanner::BufferedScanner, + uarray::UArray, + vinline::VInline, + word::{DwordNN, DwordQN, WordIO, ZERO_BLOCK}, +}; +// imports +use std::alloc::{self, Layout}; + +pub unsafe fn dealloc_array(ptr: *mut T, l: usize) { + alloc::dealloc(ptr as *mut u8, Layout::array::(l).unwrap_unchecked()) +} + +/// Native double pointer width (note, native != arch native, but host native) +pub struct NativeDword([usize; 2]); +/// Native triple pointer width (note, native != arch native, but host native) +pub struct NativeTword([usize; 3]); +/// Native quad pointer width (note, native != arch native, but host native) +pub struct NativeQword([usize; 4]); +/// A special word with a special bit pattern padded (with a quad) +/// +/// **WARNING**: DO NOT EXPECT this to have the same bit pattern as that of native word sizes. It's called "special" FOR A REASON +pub struct SpecialPaddedWord { + a: u64, + b: usize, +} + +impl SpecialPaddedWord { + pub const unsafe fn new(a: u64, b: usize) -> Self { + Self { a, b } + } + pub fn new_quad(a: u64) -> Self { + Self { + a, + b: ZERO_BLOCK.as_ptr() as usize, + } + } +} + +pub trait StatelessLen { + fn stateless_len(&self) -> usize; + fn stateless_empty(&self) -> bool { + self.stateless_len() == 0 + } +} + +impl StatelessLen for Vec { + fn stateless_len(&self) -> usize { + self.len() + } +} + +impl StatelessLen for Box<[T]> { + fn stateless_len(&self) -> usize { + self.len() + } +} + +impl StatelessLen for String { + fn stateless_len(&self) -> usize { + self.len() + } +} + +impl StatelessLen for str { + fn stateless_len(&self) -> usize { + self.len() + } +} + +impl StatelessLen for [T] { + fn stateless_len(&self) -> usize { + self.len() + } +} + +impl StatelessLen for VInline { + fn stateless_len(&self) -> usize { + self.len() + } +} + +impl StatelessLen for AStr { + fn stateless_len(&self) -> usize { + self.len() + } +} + +impl StatelessLen for UArray { + fn stateless_len(&self) -> usize { + self.len() + } +} diff --git a/server/src/engine/mem/numbuf.rs b/server/src/engine/mem/numbuf.rs new file mode 100644 index 00000000..7f0dc7e3 --- /dev/null +++ b/server/src/engine/mem/numbuf.rs @@ -0,0 +1,174 @@ +/* + * Created on Thu Nov 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +/* + derived from the implementation in libcore +*/ + +use core::{mem, ptr, slice}; + +pub trait Int { + type Buffer: Default; + fn init_buf() -> Self::Buffer { + Self::Buffer::default() + } + fn init(self, buf: &mut Self::Buffer) -> &[u8]; +} + +pub struct IntegerRepr { + b: I::Buffer, +} + +impl IntegerRepr { + pub fn new() -> Self { + Self { b: I::init_buf() } + } + pub fn as_bytes(&mut self, i: I) -> &[u8] { + i.init(&mut self.b) + } + pub fn scoped(i: I, mut f: impl FnMut(&[u8]) -> T) -> T { + let mut slf = Self::new(); + f(slf.as_bytes(i)) + } + #[cfg(test)] + pub fn as_str(&mut self, i: I) -> &str { + unsafe { core::mem::transmute(self.as_bytes(i)) } + } +} + +const DEC_DIGITS_LUT: &[u8] = b"\ + 0001020304050607080910111213141516171819\ + 2021222324252627282930313233343536373839\ + 4041424344454647484950515253545556575859\ + 6061626364656667686970717273747576777879\ + 8081828384858687888990919293949596979899"; + +macro_rules! impl_int { + ($($($int:ty => $max:literal),* as $cast:ty),*) => { + $($(impl Int for $int { + type Buffer = [u8; $max]; + fn init(self, buf: &mut Self::Buffer) -> &[u8] { + #[allow(unused_comparisons)] + let negative = self < 0; + let mut n = if negative { + // two's complement (invert, add 1) + ((!(self as $cast)).wrapping_add(1)) + } else { + self as $cast + }; + let mut curr_idx = buf.len() as isize; + let buf_ptr = buf.as_mut_ptr(); + let lut_ptr = DEC_DIGITS_LUT.as_ptr(); + unsafe { + if mem::size_of::() >= 2 { + while n >= 10_000 { + let rem = (n % 10_000) as isize; + n /= 10_000; + let d1 = (rem / 100) << 1; + let d2 = (rem % 100) << 1; + curr_idx -= 4; + ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr_idx), 2); + ptr::copy_nonoverlapping(lut_ptr.offset(d2), buf_ptr.offset(curr_idx + 2), 2); + } + } + // 4 chars left + let mut n = n as isize; + // 2 chars + if n >= 100 { + let d1 = (n % 100) << 1; + n /= 100; + curr_idx -= 2; + ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr_idx), 2); + } + // 1 or 2 left + if n < 10 { + curr_idx -= 1; + *buf_ptr.offset(curr_idx) = (n as u8) + b'0'; + } else { + let d1 = n << 1; + curr_idx -= 2; + ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr_idx), 2); + } + if negative { + curr_idx -= 1; + *buf_ptr.offset(curr_idx) = b'-'; + } + slice::from_raw_parts(buf_ptr.offset(curr_idx), buf.len() - curr_idx as usize) + } + } + })*)* + }; +} + +impl_int!(u8 => 3, i8 => 4, u16 => 5, i16 => 6, u32 => 10, i32 => 11 as u32, u64 => 20, i64 => 20 as u64); + +#[cfg(test)] +mod tests { + fn ibufeq(v: I) { + let mut buf = super::IntegerRepr::new(); + assert_eq!(buf.as_str(v), v.to_string()); + } + #[test] + fn u8() { + ibufeq(u8::MIN); + ibufeq(u8::MAX); + } + #[test] + fn i8() { + ibufeq(i8::MIN); + ibufeq(i8::MAX); + } + #[test] + fn u16() { + ibufeq(u16::MIN); + ibufeq(u16::MAX); + } + #[test] + fn i16() { + ibufeq(i16::MIN); + ibufeq(i16::MAX); + } + #[test] + fn u32() { + ibufeq(u32::MIN); + ibufeq(u32::MAX); + } + #[test] + fn i32() { + ibufeq(i32::MIN); + ibufeq(i32::MAX); + } + #[test] + fn u64() { + ibufeq(u64::MIN); + ibufeq(u64::MAX); + } + #[test] + fn i64() { + ibufeq(i64::MIN); + ibufeq(i64::MAX); + } +} diff --git a/server/src/engine/mem/rawslice.rs b/server/src/engine/mem/rawslice.rs new file mode 100644 index 00000000..f0faf014 --- /dev/null +++ b/server/src/engine/mem/rawslice.rs @@ -0,0 +1,158 @@ +/* + * Created on Thu Nov 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use core::{ + borrow::Borrow, + fmt, + hash::{Hash, Hasher}, + ops::Deref, + slice, str, +}; + +#[derive(PartialEq, Eq, PartialOrd, Ord)] +pub struct RawStr { + base: RawSlice, +} + +impl RawStr { + pub unsafe fn new(p: *const u8, l: usize) -> Self { + Self { + base: RawSlice::new(p, l), + } + } + pub unsafe fn clone(&self) -> Self { + Self { + base: self.base.clone(), + } + } + pub fn as_str(&self) -> &str { + unsafe { + // UNSAFE(@ohsayan): up to caller to ensure proper pointers + str::from_utf8_unchecked(self.base.as_slice()) + } + } +} + +impl From<&'static str> for RawStr { + fn from(s: &'static str) -> Self { + unsafe { Self::new(s.as_ptr(), s.len()) } + } +} + +impl Borrow for RawStr { + fn borrow(&self) -> &str { + unsafe { core::mem::transmute(self.clone()) } + } +} + +impl Deref for RawStr { + type Target = str; + fn deref(&self) -> &Self::Target { + self.as_str() + } +} + +impl fmt::Debug for RawStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(self.as_str(), f) + } +} + +impl fmt::Display for RawStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(self.as_str(), f) + } +} + +impl Hash for RawStr { + fn hash(&self, state: &mut H) { + self.as_str().hash(state) + } +} + +pub struct RawSlice { + t: *const T, + l: usize, +} + +unsafe impl Send for RawSlice {} +unsafe impl Sync for RawSlice {} + +impl RawSlice { + #[inline(always)] + pub unsafe fn new(t: *const T, l: usize) -> Self { + Self { t, l } + } + pub fn as_slice(&self) -> &[T] { + unsafe { + // UNSAFE(@ohsayan): the caller MUST guarantee that this remains valid throughout the usage of the slice + slice::from_raw_parts(self.t, self.l) + } + } + pub unsafe fn clone(&self) -> Self { + Self { ..*self } + } +} + +impl Hash for RawSlice { + fn hash(&self, state: &mut H) { + self.as_slice().hash(state) + } +} + +impl PartialEq for RawSlice { + fn eq(&self, other: &Self) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl Eq for RawSlice {} + +impl PartialOrd for RawSlice { + fn partial_cmp(&self, other: &Self) -> Option { + self.as_slice().partial_cmp(other.as_slice()) + } +} + +impl Ord for RawSlice { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.as_slice().cmp(other.as_slice()) + } +} + +impl fmt::Debug for RawSlice { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.as_slice()).finish() + } +} + +impl Deref for RawSlice { + type Target = [T]; + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} diff --git a/server/src/engine/mem/scanner.rs b/server/src/engine/mem/scanner.rs new file mode 100644 index 00000000..5c3b925c --- /dev/null +++ b/server/src/engine/mem/scanner.rs @@ -0,0 +1,419 @@ +/* + * Created on Fri Sep 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use core::{ptr, slice}; + +pub type BufferedScanner<'a> = Scanner<'a, u8>; + +#[derive(Debug, PartialEq)] +/// A scanner over a slice buffer `[T]` +pub struct Scanner<'a, T> { + d: &'a [T], + __cursor: usize, +} + +impl<'a, T> Scanner<'a, T> { + /// Create a new scanner, starting at position 0 + pub const fn new(d: &'a [T]) -> Self { + unsafe { + // UNSAFE(@ohsayan): starting with 0 is always correct + Self::new_with_cursor(d, 0) + } + } + /// Create a new scanner, starting with the given position + /// + /// ## Safety + /// + /// `i` must be a valid index into the given slice + pub const unsafe fn new_with_cursor(d: &'a [T], i: usize) -> Self { + Self { d, __cursor: i } + } +} + +impl<'a, T> Scanner<'a, T> { + pub const fn buffer_len(&self) -> usize { + self.d.len() + } + /// Returns the remaining number of **items** + pub const fn remaining(&self) -> usize { + self.buffer_len() - self.__cursor + } + /// Returns the current cursor position + pub const fn cursor(&self) -> usize { + self.__cursor + } + /// Returns the buffer from the current position + pub fn current_buffer(&self) -> &[T] { + &self.d[self.__cursor..] + } + /// Returns the ptr to the cursor + /// + /// WARNING: The pointer might be invalid! + pub const fn cursor_ptr(&self) -> *const T { + unsafe { + // UNSAFE(@ohsayan): assuming that the cursor is correctly initialized, this is always fine + self.d.as_ptr().add(self.__cursor) + } + } + /// Returns true if the scanner has reached eof + pub fn eof(&self) -> bool { + self.remaining() == 0 + } + /// Returns true if the scanner has atleast `sizeof` bytes remaining + pub fn has_left(&self, sizeof: usize) -> bool { + self.remaining() >= sizeof + } + /// Returns true if the rounded cursor matches the predicate + pub fn rounded_cursor_matches(&self, f: impl Fn(&T) -> bool) -> bool { + f(&self.d[self.rounded_cursor()]) + } + /// Same as `rounded_cursor_matches`, but with the added guarantee that no rounding was done + pub fn rounded_cursor_not_eof_matches(&self, f: impl Fn(&T) -> bool) -> bool { + self.rounded_cursor_matches(f) & !self.eof() + } + /// A shorthand for equality in `rounded_cursor_not_eof_matches` + pub fn rounded_cursor_not_eof_equals(&self, v_t: T) -> bool + where + T: PartialEq, + { + self.rounded_cursor_matches(|v| v_t.eq(v)) & !self.eof() + } +} + +impl<'a, T> Scanner<'a, T> { + pub fn inner_buffer(&self) -> &'a [T] { + &self.d + } + /// Manually set the cursor position + /// + /// ## Safety + /// The index must be valid + pub unsafe fn set_cursor(&mut self, i: usize) { + self.__cursor = i; + } + /// Increment the cursor + /// + /// ## Safety + /// The buffer must not have reached EOF + pub unsafe fn incr_cursor(&mut self) { + self.incr_cursor_by(1) + } + /// Increment the cursor by the given amount + /// + /// ## Safety + /// The buffer must have atleast `by` remaining + pub unsafe fn incr_cursor_by(&mut self, by: usize) { + self.__cursor += by; + } + /// Increment the cursor if the given the condition is satisfied + /// + /// ## Safety + /// Custom logic should ensure only legal cursor increments + pub unsafe fn incr_cursor_if(&mut self, iff: bool) { + self.incr_cursor_by(iff as _) + } + /// Decrement the cursor + /// + /// ## Safety + /// The cursor must **not be at 0** + pub unsafe fn decr_cursor(&mut self) { + self.decr_cursor_by(1) + } + /// Decrement the cursor by the given amount + /// + /// ## Safety + /// Should not overflow (overflow safety is ... nevermind) + pub unsafe fn decr_cursor_by(&mut self, by: usize) { + self.__cursor -= by; + } + /// Returns the current cursor + /// + /// ## Safety + /// Buffer should NOT be at EOF + pub unsafe fn deref_cursor(&self) -> T + where + T: Copy, + { + *self.cursor_ptr() + } + /// Returns the rounded cursor + pub fn rounded_cursor(&self) -> usize { + (self.buffer_len() - 1).min(self.__cursor) + } + /// Returns the current cursor value with rounding + pub fn rounded_cursor_value(&self) -> T + where + T: Copy, + { + self.d[self.rounded_cursor()] + } +} + +impl<'a> Scanner<'a, u8> { + #[cfg(test)] + /// Attempt to parse the next byte + pub fn try_next_byte(&mut self) -> Option { + if self.eof() { + None + } else { + Some(unsafe { + // UNSAFE(@ohsayan): +remaining check + self.next_byte() + }) + } + } + /// Attempt to parse the next block (variable) + pub fn try_next_variable_block(&mut self, len: usize) -> Option<&'a [u8]> { + if self.has_left(len) { + Some(unsafe { + // UNSAFE(@ohsayan): +remaining check + self.next_chunk_variable(len) + }) + } else { + None + } + } +} + +/// Incomplete buffered reads +#[derive(Debug, PartialEq)] +pub enum ScannerDecodeResult { + /// The value was decoded + Value(T), + /// We need more data to determine if we have the correct value + NeedMore, + /// Found an error while decoding a value + Error, +} + +impl<'a> Scanner<'a, u8> { + /// Keep moving the cursor ahead while the predicate returns true + pub fn trim_ahead(&mut self, f: impl Fn(u8) -> bool) { + while self.rounded_cursor_not_eof_matches(|b| f(*b)) { + unsafe { + // UNSAFE(@ohsayan): not eof + self.incr_cursor() + } + } + } + /// Attempt to parse a `\n` terminated integer (we move past the LF, so you can't see it) + /// + /// If we were unable to read in the integer, then the cursor will be restored to its starting position + // TODO(@ohsayan): optimize + pub fn try_next_ascii_u64_lf_separated_with_result_or_restore_cursor( + &mut self, + ) -> ScannerDecodeResult { + self.try_next_ascii_u64_lf_separated_with_result_or::() + } + pub fn try_next_ascii_u64_lf_separated_with_result_or( + &mut self, + ) -> ScannerDecodeResult { + let mut okay = true; + let start = self.cursor(); + let ret = self.try_next_ascii_u64_stop_at_lf(&mut okay); + let payload_ok = okay; + let lf = self.rounded_cursor_not_eof_matches(|b| *b == b'\n'); + okay &= lf; + unsafe { + // UNSAFE(@ohsayan): not eof + // skip LF + self.incr_cursor_if(okay) + }; + if okay { + ScannerDecodeResult::Value(ret) + } else { + if RESTORE_CURSOR { + unsafe { + // UNSAFE(@ohsayan): we correctly restore the cursor + self.set_cursor(start) + } + } + if payload_ok { + // payload was ok, but we missed a null + ScannerDecodeResult::NeedMore + } else { + // payload was NOT ok + ScannerDecodeResult::Error + } + } + } + /// Attempt to parse a LF terminated integer (we move past the LF) + /// If we were unable to read in the integer, then the cursor will be restored to its starting position + pub fn try_next_ascii_u64_lf_separated_or_restore_cursor(&mut self) -> Option { + self.try_next_ascii_u64_lf_separated_or::() + } + pub fn try_next_ascii_u64_lf_separated_or( + &mut self, + ) -> Option { + let start = self.cursor(); + let mut okay = true; + let ret = self.try_next_ascii_u64_stop_at_lf(&mut okay); + let lf = self.rounded_cursor_not_eof_matches(|b| *b == b'\n'); + unsafe { + // UNSAFE(@ohsayan): not eof + self.incr_cursor_if(lf & okay) + } + if okay & lf { + Some(ret) + } else { + if RESTORE_CURSOR { + unsafe { + // UNSAFE(@ohsayan): we correctly restore the cursor + self.set_cursor(start) + } + } + None + } + } + /// Extracts whatever integer is possible using the current bytestream, stopping at a LF (but **not** skipping it) + pub fn try_next_ascii_u64_stop_at_lf(&mut self, g_okay: &mut bool) -> u64 { + self.try_next_ascii_u64_stop_at::(g_okay, |byte| byte != b'\n') + } + /// Extracts whatever integer is possible using the current bytestream, stopping only when either an overflow occurs or when + /// the closure returns false + pub fn try_next_ascii_u64_stop_at( + &mut self, + g_okay: &mut bool, + keep_going_if: impl Fn(u8) -> bool, + ) -> u64 { + let mut ret = 0u64; + let mut okay = true; + while self.rounded_cursor_not_eof_matches(|b| keep_going_if(*b)) & okay { + let b = self.d[self.cursor()]; + if ASCII_CHECK { + okay &= b.is_ascii_digit(); + } + ret = match ret.checked_mul(10) { + Some(r) => r, + None => { + okay = false; + break; + } + }; + ret = match ret.checked_add((b & 0x0F) as u64) { + Some(r) => r, + None => { + okay = false; + break; + } + }; + unsafe { + // UNSAFE(@ohsayan): loop invariant + self.incr_cursor_by(1) + } + } + *g_okay &= okay; + ret + } +} + +impl<'a> Scanner<'a, u8> { + /// Attempt to parse the next [`i64`] value, stopping and skipping the STOP_BYTE + /// + /// WARNING: The cursor is NOT reversed + pub fn try_next_ascii_i64_separated_by(&mut self) -> (bool, i64) { + let (okay, int) = self.try_next_ascii_i64_stop_at(|b| b == STOP_BYTE); + let lf = self.rounded_cursor_not_eof_equals(STOP_BYTE); + unsafe { + // UNSAFE(@ohsayan): not eof + self.incr_cursor_if(lf & okay) + } + (lf & okay, int) + } + /// Attempt to parse the next [`i64`] value, stopping at the stop condition or stopping if an error occurred + /// + /// WARNING: It is NOT guaranteed that the stop condition was met + pub fn try_next_ascii_i64_stop_at(&mut self, stop_if: impl Fn(u8) -> bool) -> (bool, i64) { + let mut ret = 0i64; + // check if we have a direction + let current = self.rounded_cursor_value(); + let direction_negative = current == b'-'; + // skip negative + unsafe { + // UNSAFE(@ohsayan): not eof + self.incr_cursor_if(direction_negative) + } + let mut okay = direction_negative | current.is_ascii_digit() & !self.eof(); + while self.rounded_cursor_not_eof_matches(|b| !stop_if(*b)) & okay { + let byte = unsafe { + // UNSAFE(@ohsayan): loop invariant + self.next_byte() + }; + okay &= byte.is_ascii_digit(); + ret = match ret.checked_mul(10) { + Some(r) => r, + None => { + okay = false; + break; + } + }; + if direction_negative { + ret = match ret.checked_sub((byte & 0x0f) as i64) { + Some(r) => r, + None => { + okay = false; + break; + } + }; + } else { + ret = match ret.checked_add((byte & 0x0f) as i64) { + Some(r) => r, + None => { + okay = false; + break; + } + } + } + } + (okay, ret) + } +} + +impl<'a> Scanner<'a, u8> { + /// Load the next [`u64`] LE + pub unsafe fn next_u64_le(&mut self) -> u64 { + u64::from_le_bytes(self.next_chunk()) + } + /// Load the next block + pub unsafe fn next_chunk(&mut self) -> [u8; N] { + let mut b = [0u8; N]; + ptr::copy_nonoverlapping(self.cursor_ptr(), b.as_mut_ptr(), N); + self.incr_cursor_by(N); + b + } + /// Load the next variable-sized block + pub unsafe fn next_chunk_variable(&mut self, size: usize) -> &'a [u8] { + let r = slice::from_raw_parts(self.cursor_ptr(), size); + self.incr_cursor_by(size); + r + } + /// Load the next byte + pub unsafe fn next_byte(&mut self) -> u8 { + let r = *self.cursor_ptr(); + self.incr_cursor_by(1); + r + } +} diff --git a/server/src/engine/mem/stackop.rs b/server/src/engine/mem/stackop.rs new file mode 100644 index 00000000..280d52d6 --- /dev/null +++ b/server/src/engine/mem/stackop.rs @@ -0,0 +1,85 @@ +/* + * Created on Tue May 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[derive(Debug, PartialEq, Clone, Copy)] +pub struct ByteStack { + array: [u8; N], +} + +#[allow(dead_code)] +impl ByteStack { + #[inline(always)] + pub const fn data_copy(&self) -> [u8; N] { + self.array + } + #[inline(always)] + pub const fn new(array: [u8; N]) -> Self { + Self { array } + } + #[inline(always)] + pub const fn zeroed() -> Self { + Self::new([0u8; N]) + } + #[inline(always)] + pub const fn slice(&self) -> &[u8] { + &self.array + } + #[inline(always)] + pub const fn read_byte(&self, position: usize) -> u8 { + self.array[position] + } + #[inline(always)] + pub const fn read_word(&self, position: usize) -> u16 { + unsafe { core::mem::transmute([self.read_byte(position), self.read_byte(position + 1)]) } + } + #[inline(always)] + pub const fn read_dword(&self, position: usize) -> u32 { + unsafe { + core::mem::transmute([ + self.read_word(position), + self.read_word(position + sizeof!(u16)), + ]) + } + } + #[inline(always)] + pub const fn read_qword(&self, position: usize) -> u64 { + unsafe { + core::mem::transmute([ + self.read_dword(position), + self.read_dword(position + sizeof!(u32)), + ]) + } + } + #[inline(always)] + pub const fn read_xmmword(&self, position: usize) -> u128 { + unsafe { + core::mem::transmute([ + self.read_qword(position), + self.read_qword(position + sizeof!(u64)), + ]) + } + } +} diff --git a/server/src/engine/mem/tests/mod.rs b/server/src/engine/mem/tests/mod.rs new file mode 100644 index 00000000..b689c053 --- /dev/null +++ b/server/src/engine/mem/tests/mod.rs @@ -0,0 +1,319 @@ +/* + * Created on Sun Jan 22 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use super::*; +mod scanner; +mod word; + +mod vinline { + use super::VInline; + const CAP: usize = 8; + #[test] + fn drop_empty() { + let vi = VInline::::new(); + drop(vi); + } + /// This will: + /// - returns an array [0..upto] + /// - verify length + /// - verify payload + /// - verify capacity (if upto <= CAP) + /// - verify stack/heap logic + fn cmkvi(upto: usize, map: F) -> VInline + where + F: Clone + FnMut(usize) -> T, + { + let map2 = map.clone(); + let r: VInline = (0..upto).map(map).collect(); + assert_eq!(r.len(), upto); + if upto <= CAP { + assert_eq!(r.capacity(), CAP); + assert!(r.on_stack()); + } else { + assert!(r.on_heap()); + } + assert!((0..upto).map(map2).zip(r.iter()).all(|(x, y)| { x == *y })); + r + } + fn mkvi(upto: usize) -> VInline { + cmkvi(upto, |v| v) + } + fn mkvi_str(upto: usize) -> VInline { + cmkvi(upto, |v| v.to_string()) + } + #[test] + fn push_on_stack() { + let vi = mkvi(CAP); + assert!(vi.on_stack()); + } + #[test] + fn push_on_heap() { + let vi = mkvi(CAP + 1); + assert_eq!(vi.capacity(), CAP * 2); + } + #[test] + fn remove_on_stack() { + let mut vi = mkvi(CAP); + assert_eq!(vi.remove(6), 6); + assert_eq!(vi.len(), CAP - 1); + assert_eq!(vi.capacity(), CAP); + assert_eq!(vi.as_ref(), [0, 1, 2, 3, 4, 5, 7]); + } + #[test] + fn remove_on_heap() { + let mut vi = mkvi(CAP + 1); + assert_eq!(vi.remove(6), 6); + assert_eq!(vi.len(), CAP); + assert_eq!(vi.capacity(), CAP * 2); + assert_eq!(vi.as_ref(), [0, 1, 2, 3, 4, 5, 7, 8]); + } + #[test] + fn optimize_capacity_none_on_stack() { + let mut vi = mkvi(CAP); + vi.optimize_capacity(); + assert_eq!(vi.capacity(), CAP); + assert!(vi.on_stack()); + } + #[test] + fn optimize_capacity_none_on_heap() { + let mut vi = mkvi(CAP + 1); + assert_eq!(vi.capacity(), CAP * 2); + vi.extend(CAP + 1..CAP * 2); + assert_eq!(vi.capacity(), CAP * 2); + vi.optimize_capacity(); + assert_eq!(vi.capacity(), CAP * 2); + } + #[test] + fn optimize_capacity_on_heap() { + let mut vi = mkvi(CAP + 1); + assert_eq!(vi.capacity(), CAP * 2); + vi.optimize_capacity(); + assert_eq!(vi.capacity(), CAP + 1); + } + #[test] + fn optimize_capacity_mv_stack() { + let mut vi = mkvi(CAP + 1); + assert_eq!(vi.capacity(), CAP * 2); + let _ = vi.remove_compact(0); + assert_eq!(vi.len(), CAP); + assert_eq!(vi.capacity(), CAP); + assert!(vi.on_stack()); + } + #[test] + fn clear_stack() { + let mut vi = mkvi(CAP); + vi.clear(); + assert_eq!(vi.capacity(), CAP); + assert_eq!(vi.len(), 0); + } + #[test] + fn clear_heap() { + let mut vi = mkvi(CAP + 1); + vi.clear(); + assert_eq!(vi.capacity(), CAP * 2); + assert_eq!(vi.len(), 0); + } + #[test] + fn clone_stack() { + let v1 = mkvi(CAP); + let v2 = v1.clone(); + assert_eq!(v1, v2); + } + #[test] + fn clone_heap() { + let v1 = mkvi(CAP + 1); + let v2 = v1.clone(); + assert_eq!(v1, v2); + } + #[test] + fn into_iter_stack() { + let v1 = mkvi_str(CAP); + let v: Vec = v1.into_iter().collect(); + (0..CAP) + .zip(v) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn into_iter_stack_partial() { + let v1 = mkvi_str(CAP); + let v: Vec = v1.into_iter().take(CAP / 2).collect(); + (0..CAP / 2) + .zip(v) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn into_iter_heap() { + let v1 = mkvi_str(CAP + 2); + let v: Vec = v1.into_iter().collect(); + (0..CAP) + .zip(v) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn into_iter_heap_partial() { + let v1 = mkvi_str(CAP + 2); + let v: Vec = v1.into_iter().take(CAP / 2).collect(); + (0..CAP / 2) + .zip(v) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn into_iter_rev_stack() { + let v1 = mkvi_str(CAP); + let v: Vec = v1.into_iter().rev().collect(); + (0..CAP) + .rev() + .zip(v) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn into_iter_rev_stack_partial() { + let v1 = mkvi_str(CAP); + let v: Vec = v1.into_iter().rev().take(CAP / 2).collect(); + (CAP / 2..CAP) + .rev() + .zip(v.into_iter()) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn into_iter_rev_heap() { + let v1 = mkvi_str(CAP + 2); + let v: Vec = v1.into_iter().rev().collect(); + (0..CAP + 2) + .rev() + .zip(v) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn into_iter_rev_heap_partial() { + let v1 = mkvi_str(CAP + 2); + let v: Vec = v1.into_iter().rev().take(CAP / 2).collect(); + (0..CAP + 2).rev().zip(v).for_each(|(x, y)| { + assert_eq!(x.to_string(), y); + }) + } +} + +mod uarray { + use super::UArray; + const CAP: usize = 8; + #[test] + fn empty() { + let a = UArray::::new(); + drop(a); + } + #[test] + fn push_okay() { + let mut a = UArray::::new(); + a.push(1); + a.push(2); + a.push(3); + a.push(4); + } + #[test] + #[should_panic(expected = "stack,capof")] + fn push_panic() { + let mut a = UArray::::new(); + a.push(1); + a.push(2); + a.push(3); + a.push(4); + a.push(5); + a.push(6); + a.push(7); + a.push(8); + a.push(9); + } + #[test] + fn slice() { + let a: UArray = (1u8..=8).collect(); + assert_eq!(a.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8]); + } + #[test] + fn slice_mut() { + let mut a: UArray = (0u8..8).collect(); + a.iter_mut().for_each(|v| *v += 1); + assert_eq!(a.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8]) + } + #[test] + fn into_iter_empty() { + let a: UArray = UArray::new(); + let r: Vec = a.into_iter().collect(); + assert!(r.is_empty()); + } + #[test] + fn into_iter() { + let a: UArray = (0u8..8).collect(); + let r: Vec = a.into_iter().collect(); + (0..8) + .zip(r.into_iter()) + .for_each(|(x, y)| assert_eq!(x, y)); + } + #[test] + fn into_iter_partial() { + let a: UArray = (0u8..8).map(|v| ToString::to_string(&v)).collect(); + let r: Vec = a.into_iter().take(4).collect(); + (0..4) + .zip(r.into_iter()) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn clone() { + let a: UArray = (0u8..CAP as _).collect(); + let b = a.clone(); + assert_eq!(a, b); + } + #[test] + fn into_iter_rev() { + let a: UArray = (0u8..8).map(|v| v.to_string()).collect(); + let r: Vec = a.into_iter().rev().collect(); + (0..8) + .rev() + .zip(r.into_iter()) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn into_iter_rev_partial() { + let a: UArray = (0u8..8).map(|v| v.to_string()).collect(); + let r: Vec = a.into_iter().rev().take(4).collect(); + (4..8) + .rev() + .zip(r.into_iter()) + .for_each(|(x, y)| assert_eq!(x.to_string(), y)); + } + #[test] + fn pop_array() { + let mut a: UArray = (0u8..8).map(|v| v.to_string()).collect(); + assert_eq!(a.pop().unwrap(), "7"); + assert_eq!(a.len(), CAP - 1); + } + #[test] + fn clear_array() { + let mut a: UArray = (0u8..8).map(|v| v.to_string()).collect(); + a.clear(); + assert!(a.is_empty()); + } +} diff --git a/server/src/engine/mem/tests/scanner.rs b/server/src/engine/mem/tests/scanner.rs new file mode 100644 index 00000000..e1b2ee94 --- /dev/null +++ b/server/src/engine/mem/tests/scanner.rs @@ -0,0 +1,249 @@ +/* + * Created on Wed Sep 20 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::mem::scanner::{BufferedScanner, ScannerDecodeResult}; + +fn s(b: &[u8]) -> BufferedScanner { + BufferedScanner::new(b) +} + +/* + lf separated +*/ + +#[test] +fn read_u64_lf_separated() { + let mut s = s(b"18446744073709551615\n"); + assert_eq!( + s.try_next_ascii_u64_lf_separated_or_restore_cursor() + .unwrap(), + u64::MAX + ); + assert_eq!(s.cursor(), s.buffer_len()); +} + +#[test] +fn read_u64_lf_separated_missing() { + let mut s = s(b"18446744073709551615"); + assert!(s + .try_next_ascii_u64_lf_separated_or_restore_cursor() + .is_none()); + assert_eq!(s.cursor(), 0); +} + +#[test] +fn read_u64_lf_separated_invalid() { + let mut scn = s(b"1844674407370955161A\n"); + assert!(scn + .try_next_ascii_u64_lf_separated_or_restore_cursor() + .is_none()); + assert_eq!(scn.cursor(), 0); + let mut scn = s(b"?1844674407370955161A\n"); + assert!(scn + .try_next_ascii_u64_lf_separated_or_restore_cursor() + .is_none()); + assert_eq!(scn.cursor(), 0); +} + +#[test] +fn read_u64_lf_separated_zero() { + let mut s = s(b"0\n"); + assert_eq!( + s.try_next_ascii_u64_lf_separated_or_restore_cursor() + .unwrap(), + 0 + ); + assert_eq!(s.cursor(), s.buffer_len()); +} + +#[test] +fn read_u64_lf_overflow() { + let mut s = s(b"184467440737095516155\n"); + assert!(s + .try_next_ascii_u64_lf_separated_or_restore_cursor() + .is_none()); + assert_eq!(s.cursor(), 0); +} + +/* + lf separated allow unbuffered +*/ + +#[test] +fn incomplete_read_u64_okay() { + let mut scn = s(b"18446744073709551615\n"); + assert_eq!( + scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(), + ScannerDecodeResult::Value(u64::MAX) + ); + assert_eq!(scn.cursor(), scn.buffer_len()); +} + +#[test] +fn incomplete_read_u64_missing_lf() { + let mut scn = s(b"18446744073709551615"); + assert_eq!( + scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(), + ScannerDecodeResult::NeedMore + ); + assert_eq!(scn.cursor(), 0); +} + +#[test] +fn incomplete_read_u64_lf_error() { + let mut scn = s(b"1844674407370955161A\n"); + assert_eq!( + scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(), + ScannerDecodeResult::Error + ); + assert_eq!(scn.cursor(), 0); + let mut scn = s(b"?1844674407370955161A\n"); + assert_eq!( + scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(), + ScannerDecodeResult::Error + ); + assert_eq!(scn.cursor(), 0); +} + +#[test] +fn incomplete_read_u64_lf_zero() { + let mut scn = s(b"0\n"); + assert_eq!( + scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(), + ScannerDecodeResult::Value(0) + ) +} + +#[test] +fn incomplete_read_u64_lf_overflow() { + let mut s = s(b"184467440737095516155\n"); + assert_eq!( + s.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(), + ScannerDecodeResult::Error + ); + assert_eq!(s.cursor(), 0); +} + +/* + lf separated i64 +*/ + +fn concat(a: impl ToString, b: impl ToString) -> Vec { + let (a, b) = (a.to_string(), b.to_string()); + let mut s = String::with_capacity(a.len() + b.len()); + s.push_str(a.as_str()); + s.push_str(b.as_str()); + s.into_bytes() +} + +#[test] +fn read_i64_lf_separated_okay() { + let buf = concat(i64::MAX, "\n"); + let mut scn = s(&buf); + assert_eq!( + scn.try_next_ascii_i64_separated_by::(), + (true, i64::MAX) + ); + assert_eq!(scn.cursor(), scn.buffer_len()); + let buf = concat(i64::MIN, "\n"); + let mut scn = s(&buf); + assert_eq!( + scn.try_next_ascii_i64_separated_by::(), + (true, i64::MIN) + ); + assert_eq!(scn.cursor(), scn.buffer_len()); +} + +#[test] +fn read_i64_lf_separated_missing() { + let buf = concat(i64::MAX, ""); + let mut scn = s(&buf); + assert_eq!( + scn.try_next_ascii_i64_separated_by::(), + (false, i64::MAX) + ); + assert_eq!(scn.cursor(), scn.buffer_len()); + let buf = concat(i64::MIN, ""); + let mut scn = s(&buf); + assert_eq!( + scn.try_next_ascii_i64_separated_by::(), + (false, i64::MIN) + ); + assert_eq!(scn.cursor(), scn.buffer_len()); +} + +#[test] +fn read_i64_lf_separated_invalid() { + let buf = concat(i64::MAX, "A\n"); + let mut scn = s(&buf); + assert_eq!( + scn.try_next_ascii_i64_separated_by::(), + (false, i64::MAX) + ); + assert_eq!(scn.cursor(), scn.buffer_len() - 1); + let buf = concat("A", format!("{}\n", i64::MIN)); + let mut scn = s(&buf); + assert_eq!(scn.try_next_ascii_i64_separated_by::(), (false, 0)); + assert_eq!(scn.cursor(), 0); +} + +#[test] +fn read_i64_lf_overflow() { + let buf = concat(u64::MAX, "\n"); + let mut scn = s(&buf); + assert_eq!( + scn.try_next_ascii_i64_separated_by::(), + (false, 1844674407370955161) + ); + assert_eq!(scn.cursor(), scn.buffer_len() - 1); +} + +#[test] +fn read_i64_lf_underflow() { + let buf = concat(i64::MIN, "1\n"); + let mut scn = s(&buf); + assert_eq!( + scn.try_next_ascii_i64_separated_by::(), + (false, -9223372036854775808) + ); + assert_eq!(scn.cursor(), scn.buffer_len() - 1); +} + +#[test] +fn rounding() { + let mut scanner = s(b"123"); + for i in 1..=u8::MAX { + match i { + 1..=3 => { + assert_eq!(scanner.try_next_byte().unwrap(), (i + b'0')); + } + _ => { + assert_eq!(scanner.rounded_cursor_value(), b'3'); + } + } + } + assert_eq!(scanner.cursor(), scanner.buffer_len()); +} diff --git a/server/src/engine/mem/tests/word.rs b/server/src/engine/mem/tests/word.rs new file mode 100644 index 00000000..50a41a41 --- /dev/null +++ b/server/src/engine/mem/tests/word.rs @@ -0,0 +1,170 @@ +/* + * Created on Mon Apr 24 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::mem::{ + word::{DwordQN, QwordNNNN, TwordNNN, WordIO, ZERO_BLOCK}, + NativeDword, NativeQword, NativeTword, SpecialPaddedWord, + }, + core::{slice, str}, +}; + +fn wordld>(w: &W, x: T) -> (T, T) { + (w.load(), x) +} + +macro_rules! assert_wordeq { + ($a:expr, $b:expr) => {{ + let (a, b) = wordld(&$a, $b); + assert_eq!(a, b); + }}; +} + +macro_rules! assert_wordeq_minmax { + ($word:ty => $($ty:ty),* $(,)?; with $extramin:ident, $extramax:ident) => {{ + $( + let x = <$word>::store(<$ty>::MIN); assert_wordeq!(x, <$ty>::MIN); + $extramin(&x); + let x = <$word>::store(<$ty>::MAX); assert_wordeq!(x, <$ty>::MAX); + $extramax(&x); + )* + }}; +} + +fn check_primitives(extramin: impl Fn(&W), extramax: impl Fn(&W)) +where + W: WordIO + + WordIO + + WordIO + + WordIO + + WordIO + + WordIO + + WordIO + + WordIO + + WordIO + + WordIO + + WordIO + + WordIO<(usize, usize)> + + WordIO<(usize, *mut u8)> + + WordIO<(usize, *const u8)>, +{ + assert_wordeq_minmax!(W => u8, u16, u32, u64, i8, i16, i32, i64, f32, f64; with extramin, extramax); + // bool + let x = W::store(false); + assert_wordeq!(x, false); + extramin(&x); + let x = W::store(true); + assert_wordeq!(x, true); + extramax(&x); + // str + let str = "hello, world"; + let x = W::store((str.len(), str.as_ptr())); + unsafe { + let (len, ptr) = x.load(); + assert_eq!( + str, + str::from_utf8_unchecked(slice::from_raw_parts(ptr, len)) + ); + } + // string (mut) + let mut string = String::from("hello, world"); + let x = W::store((string.len(), string.as_mut_ptr())); + unsafe { + let (len, ptr) = x.load(); + assert_eq!( + string, + str::from_utf8_unchecked(slice::from_raw_parts(ptr, len)) + ); + } +} + +#[test] +fn dwordnn_all() { + check_primitives::(|_| {}, |_| {}); +} + +#[test] +fn dwordqn_all() { + check_primitives::( + |minword| { + let (_a, b) = minword.dwordqn_load_qw_nw(); + assert_eq!(b, ZERO_BLOCK.as_ptr() as usize); + }, + |maxword| { + let (_a, b) = maxword.dwordqn_load_qw_nw(); + assert_eq!(b, ZERO_BLOCK.as_ptr() as usize); + }, + ); +} + +#[test] +fn twordnnn_all() { + check_primitives::(|_| {}, |_| {}); +} + +#[test] +fn qwordnnn_all() { + check_primitives::(|_| {}, |_| {}); +} + +#[test] +fn dwordqn_promotions() { + let x = SpecialPaddedWord::store(u64::MAX); + let y: NativeTword = x.dwordqn_promote(); + let (uint, usize) = y.dwordqn_load_qw_nw(); + assert_eq!(uint, u64::MAX); + assert_eq!(usize, ZERO_BLOCK.as_ptr() as usize); + let z: NativeQword = y.tword_promote(); + let (uint, usize_1, usize_2) = z.qwordnnnn_load_qw_nw_nw(); + assert_eq!(uint, u64::MAX); + assert_eq!(usize_1, ZERO_BLOCK.as_ptr() as usize); + assert_eq!(usize_2, 0); +} + +fn eval_special_case(x: SpecialPaddedWord, qw: u64, nw: usize) { + let y: NativeQword = x.dwordqn_promote(); + assert_eq!(y.dwordqn_load_qw_nw(), (qw, nw)); + let z: SpecialPaddedWord = unsafe { + let (a, b) = y.dwordqn_load_qw_nw(); + SpecialPaddedWord::new(a, b) + }; + assert_eq!(z.dwordqn_load_qw_nw(), (qw, nw)); +} + +#[test] +fn dwordqn_special_case_ldpk() { + let hello = "hello, world"; + eval_special_case( + SpecialPaddedWord::store((hello.len(), hello.as_ptr())), + hello.len() as u64, + hello.as_ptr() as usize, + ); + eval_special_case( + SpecialPaddedWord::store(u64::MAX), + u64::MAX, + ZERO_BLOCK.as_ptr() as usize, + ); +} diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs new file mode 100644 index 00000000..6e71c2f1 --- /dev/null +++ b/server/src/engine/mem/uarray.rs @@ -0,0 +1,300 @@ +/* + * Created on Mon Jan 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use core::{ + fmt, + hash::{Hash, Hasher}, + iter::FusedIterator, + mem::MaybeUninit, + ops::{Deref, DerefMut}, + ptr, slice, +}; + +pub struct UArray { + a: [MaybeUninit; N], + l: usize, +} + +impl UArray { + const NULL: MaybeUninit = MaybeUninit::uninit(); + const NULLED_ARRAY: [MaybeUninit; N] = [Self::NULL; N]; + #[inline(always)] + pub const fn new() -> Self { + Self { + a: Self::NULLED_ARRAY, + l: 0, + } + } + #[inline(always)] + pub const fn len(&self) -> usize { + self.l + } + #[inline(always)] + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } + #[inline(always)] + pub fn push(&mut self, v: T) { + if self.l == N { + panic!("stack,capof"); + } + unsafe { + // UNSAFE(@ohsayan): verified length is smaller + self.push_unchecked(v); + } + } + #[allow(unused)] + pub fn remove(&mut self, idx: usize) -> T { + if idx >= self.len() { + panic!("out of range. idx is `{idx}` but len is `{}`", self.len()); + } + unsafe { + // UNSAFE(@ohsayan): verified idx < l + self.remove_unchecked(idx) + } + } + pub fn pop(&mut self) -> Option { + if self.is_empty() { + None + } else { + unsafe { + // UNSAFE(@ohsayan): Non-empty checked + Some(self.remove_unchecked(self.len() - 1)) + } + } + } + pub fn clear(&mut self) { + unsafe { + let ptr = self.as_slice_mut(); + // UNSAFE(@ohsayan): We know this is the initialized length + ptr::drop_in_place(ptr); + // UNSAFE(@ohsayan): we've destroyed everything, so yeah, all g + self.set_len(0); + } + } + /// SAFETY: idx < self.l + unsafe fn remove_unchecked(&mut self, idx: usize) -> T { + debug_assert!(idx < self.len()); + // UNSAFE(@ohsayan): Verified idx + let target = self.a.as_mut_ptr().add(idx).cast::(); + // UNSAFE(@ohsayan): Verified idx + let ret = ptr::read(target); + // UNSAFE(@ohsayan): ov; not-null; correct len + ptr::copy(target.add(1), target, self.len() - idx - 1); + // UNSAFE(@ohsayan): we just removed something, account for it + self.decr_len(); + ret + } + #[inline(always)] + /// SAFETY: self.l < N + unsafe fn push_unchecked(&mut self, v: T) { + debug_assert!(self.len() < N); + // UNSAFE(@ohsayan): verified correct offsets (N) + self.a.as_mut_ptr().add(self.l).write(MaybeUninit::new(v)); + // UNSAFE(@ohsayan): all G since l =< N + self.incr_len(); + } + pub fn as_slice(&self) -> &[T] { + unsafe { + // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl + slice::from_raw_parts(self.a.as_ptr() as *const T, self.l) + } + } + pub fn as_slice_mut(&mut self) -> &mut [T] { + unsafe { + // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl + slice::from_raw_parts_mut(self.a.as_mut_ptr() as *mut T, self.l) + } + } + #[inline(always)] + unsafe fn set_len(&mut self, l: usize) { + self.l = l; + } + #[inline(always)] + unsafe fn incr_len(&mut self) { + self.set_len(self.len() + 1) + } + #[inline(always)] + unsafe fn decr_len(&mut self) { + self.set_len(self.len() - 1) + } +} + +impl UArray { + pub unsafe fn from_slice(s: &[T]) -> Self { + debug_assert!(s.len() <= N); + let mut new = Self::new(); + unsafe { + // UNSAFE(@ohsayan): the src pointer *will* be correct and the dst is us, and we own our stack here + ptr::copy_nonoverlapping(s.as_ptr(), new.a.as_mut_ptr() as *mut T, s.len()); + // UNSAFE(@ohsayan): and here goes the call; same length as the origin buffer + new.set_len(s.len()); + } + new + } +} + +impl Clone for UArray { + fn clone(&self) -> Self { + self.iter().cloned().collect() + } +} + +impl Eq for UArray {} + +impl PartialEq> for UArray { + fn eq(&self, other: &UArray) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl Drop for UArray { + fn drop(&mut self) { + if !self.is_empty() { + unsafe { + // UNSAFE(@ohsayan): as_slice_mut returns a correct offset + ptr::drop_in_place(self.as_slice_mut()) + } + } + } +} + +impl Deref for UArray { + type Target = [T]; + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} + +impl DerefMut for UArray { + fn deref_mut(&mut self) -> &mut Self::Target { + self.as_slice_mut() + } +} + +impl FromIterator for UArray { + fn from_iter>(iter: I) -> Self { + let mut slf = Self::new(); + iter.into_iter().for_each(|v| slf.push(v)); + slf + } +} + +impl Extend for UArray { + fn extend>(&mut self, iter: I) { + iter.into_iter().for_each(|v| self.push(v)) + } +} + +impl fmt::Debug for UArray { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl Hash for UArray { + fn hash(&self, state: &mut H) { + self.as_slice().hash(state) + } +} + +pub struct IntoIter { + i: usize, + l: usize, + d: UArray, +} + +impl IntoIter { + #[inline(always)] + fn _next(&mut self) -> Option { + if self.i == self.l { + return None; + } + unsafe { + // UNSAFE(@ohsayan): Below length, so this is legal + let target = self.d.a.as_ptr().add(self.i) as *mut T; + // UNSAFE(@ohsayan): Again, non-null and part of our stack + let ret = ptr::read(target); + self.i += 1; + Some(ret) + } + } + #[inline(always)] + fn _next_back(&mut self) -> Option { + if self.i == self.l { + return None; + } + unsafe { + self.l -= 1; + // UNSAFE(@ohsayan): we always ensure EOA condition + Some(ptr::read(self.d.a.as_ptr().add(self.l).cast())) + } + } +} + +impl Drop for IntoIter { + fn drop(&mut self) { + if self.i < self.l { + unsafe { + // UNSAFE(@ohsayan): Len is verified, due to intoiter init + let ptr = self.d.a.as_mut_ptr().add(self.i) as *mut T; + let len = self.l - self.i; + // UNSAFE(@ohsayan): we know the segment to drop + ptr::drop_in_place(ptr::slice_from_raw_parts_mut(ptr, len)) + } + } + } +} + +impl Iterator for IntoIter { + type Item = T; + fn next(&mut self) -> Option { + self._next() + } +} +impl ExactSizeIterator for IntoIter {} +impl FusedIterator for IntoIter {} +impl DoubleEndedIterator for IntoIter { + fn next_back(&mut self) -> Option { + self._next_back() + } +} + +impl IntoIterator for UArray { + type Item = T; + + type IntoIter = IntoIter; + + fn into_iter(mut self) -> Self::IntoIter { + let l = self.len(); + unsafe { + // UNSAFE(@ohsayan): Leave drop to intoiter + // HACK(@ohsayan): sneaky trick to let drop be handled by intoiter + self.set_len(0); + } + Self::IntoIter { d: self, i: 0, l } + } +} diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs new file mode 100644 index 00000000..43a756c0 --- /dev/null +++ b/server/src/engine/mem/vinline.rs @@ -0,0 +1,401 @@ +/* + * Created on Mon Jan 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use std::{ + alloc::{alloc, dealloc, Layout}, + fmt, + iter::FusedIterator, + mem::{self, ManuallyDrop, MaybeUninit}, + ops::{Deref, DerefMut}, + ptr, slice, +}; + +union VData { + s: ManuallyDrop<[MaybeUninit; N]>, + h: *mut T, +} + +pub struct VInline { + d: VData, + l: usize, + c: usize, +} + +impl VInline { + #[inline(always)] + pub const fn new() -> Self { + let _ = Self::_ENSURE_ALIGN; + Self { + d: VData { + s: ManuallyDrop::new(Self::INLINE_NULL_STACK), + }, + l: 0, + c: N, + } + } + #[inline(always)] + pub const fn capacity(&self) -> usize { + self.c + } + #[inline(always)] + pub const fn len(&self) -> usize { + self.l + } + #[inline(always)] + pub fn push(&mut self, v: T) { + self.grow(); + unsafe { + // UNSAFE(@ohsayan): grow allocated the cap we needed + self.push_unchecked(v); + } + } + #[inline(always)] + #[allow(unused)] + pub fn clear(&mut self) { + unsafe { + // UNSAFE(@ohsayan): as_slice_mut will always give a valid ptr + ptr::drop_in_place(self._as_slice_mut()); + } + self.l = 0; + } + #[inline(always)] + #[allow(unused)] + pub fn remove(&mut self, idx: usize) -> T { + if !(idx < self.len()) { + panic!("index out of range"); + } + unsafe { + // UNSAFE(@ohsayan): Verified index is within range + self.remove_unchecked(idx) + } + } + #[inline(always)] + #[allow(unused)] + pub fn remove_compact(&mut self, idx: usize) -> T { + let r = self.remove(idx); + self.optimize_capacity(); + r + } + #[inline(always)] + #[allow(unused)] + /// SAFETY: `idx` must be < l + unsafe fn remove_unchecked(&mut self, idx: usize) -> T { + // UNSAFE(@ohsayan): idx is in range + let ptr = self.as_mut_ptr().add(idx); + // UNSAFE(@ohsayan): idx is in range and is valid + let ret = ptr::read(ptr); + // UNSAFE(@ohsayan): move all elements to the left + ptr::copy(ptr.add(1), ptr, self.len() - idx - 1); + // UNSAFE(@ohsayan): this is our new length + self.set_len(self.len() - 1); + ret + } + #[inline(always)] + unsafe fn set_len(&mut self, len: usize) { + self.l = len; + } +} + +impl VInline { + const INLINE_NULL: MaybeUninit = MaybeUninit::uninit(); + const INLINE_NULL_STACK: [MaybeUninit; N] = [Self::INLINE_NULL; N]; + const ALLOC_MULTIPLIER: usize = 2; + const _ENSURE_ALIGN: () = + debug_assert!(mem::align_of::>() == mem::align_of::>()); + #[inline(always)] + #[cfg(test)] + pub fn on_heap(&self) -> bool { + self.c > N + } + #[inline(always)] + pub fn on_stack(&self) -> bool { + self.c == N + } + #[inline(always)] + fn _as_ptr(&self) -> *const T { + unsafe { + // UNSAFE(@ohsayan): We make legal accesses by checking state + if self.on_stack() { + self.d.s.as_ptr() as *const T + } else { + self.d.h as *const T + } + } + } + #[inline(always)] + fn _as_mut_ptr(&mut self) -> *mut T { + unsafe { + // UNSAFE(@ohsayan): We make legal accesses by checking state + if self.on_stack() { + (&mut self.d).s.as_mut_ptr() as *mut T + } else { + (&mut self.d).h as *mut T + } + } + } + #[inline(always)] + fn _as_slice(&self) -> &[T] { + unsafe { + // UNSAFE(@ohsayan): _as_ptr() will ensure correct addresses + slice::from_raw_parts(self._as_ptr(), self.l) + } + } + #[inline(always)] + fn _as_slice_mut(&mut self) -> &mut [T] { + unsafe { + // UNSAFE(@ohsayan): _as_mut_ptr() will ensure correct addresses + slice::from_raw_parts_mut(self._as_mut_ptr(), self.l) + } + } + #[inline(always)] + fn layout(cap: usize) -> Layout { + Layout::array::(cap).unwrap() + } + #[inline(always)] + fn ncap(&self) -> usize { + self.c * Self::ALLOC_MULTIPLIER + } + fn alloc_block(cap: usize) -> *mut T { + unsafe { + // UNSAFE(@ohsayan): malloc bro + let p = alloc(Self::layout(cap)); + assert!(!p.is_null(), "alloc,0"); + p as *mut T + } + } + pub unsafe fn push_unchecked(&mut self, v: T) { + self._as_mut_ptr().add(self.l).write(v); + self.l += 1; + } + pub fn optimize_capacity(&mut self) { + if self.on_stack() || self.len() == self.capacity() { + return; + } + if self.l <= N { + // the current can be fit into the stack, and we aren't on the stack. so copy data from heap and move it to the stack + unsafe { + // UNSAFE(@ohsayan): non-null heap + self.mv_to_stack(); + } + } else { + // in this case, we can't move to stack but can optimize the heap size. so create a new heap, memcpy old heap and destroy old heap (NO dtor) + let nb = Self::alloc_block(self.len()); + unsafe { + // UNSAFE(@ohsayan): nonov; non-null + ptr::copy_nonoverlapping(self.d.h, nb, self.len()); + // UNSAFE(@ohsayan): non-null heap + self.dealloc_heap(self.d.h); + } + self.d.h = nb; + self.c = self.len(); + } + } + /// SAFETY: (1) non-null heap + unsafe fn mv_to_stack(&mut self) { + let heap = self.d.h; + // UNSAFE(@ohsayan): nonov; non-null (stack lol) + ptr::copy_nonoverlapping(self.d.h, (&mut self.d).s.as_mut_ptr() as *mut T, self.len()); + // UNSAFE(@ohsayan): non-null heap + self.dealloc_heap(heap); + self.c = N; + } + #[inline] + fn grow(&mut self) { + if self.l == self.capacity() { + // allocate new block because we've run out of capacity + let nc = self.ncap(); + let nb = Self::alloc_block(nc); + if self.on_stack() { + // stack -> heap + unsafe { + // UNSAFE(@ohsayan): non-null; valid len + ptr::copy_nonoverlapping(self.d.s.as_ptr() as *const T, nb, self.l); + } + } else { + unsafe { + // UNSAFE(@ohsayan): non-null; valid len + ptr::copy_nonoverlapping(self.d.h.cast_const(), nb, self.l); + // UNSAFE(@ohsayan): non-null heap + self.dealloc_heap(self.d.h); + } + } + self.d.h = nb; + self.c = nc; + } + } + #[inline(always)] + unsafe fn dealloc_heap(&mut self, heap: *mut T) { + dealloc(heap as *mut u8, Self::layout(self.capacity())) + } +} + +impl Deref for VInline { + type Target = [T]; + fn deref(&self) -> &Self::Target { + self._as_slice() + } +} + +impl DerefMut for VInline { + fn deref_mut(&mut self) -> &mut Self::Target { + self._as_slice_mut() + } +} + +impl PartialEq> for VInline { + fn eq(&self, other: &VInline) -> bool { + self._as_slice() == other._as_slice() + } +} + +impl Drop for VInline { + fn drop(&mut self) { + unsafe { + // UNSAFE(@ohsayan): correct ptr guaranteed by safe impl of _as_slice_mut() + ptr::drop_in_place(self._as_slice_mut()); + if !self.on_stack() { + // UNSAFE(@ohsayan): non-null heap + self.dealloc_heap(self.d.h); + } + } + } +} + +impl fmt::Debug for VInline { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl Extend for VInline { + fn extend>(&mut self, iter: I) { + // FIXME(@ohsayan): Optimize capacity match upfront + iter.into_iter().for_each(|item| self.push(item)) + } +} + +#[cfg(test)] +impl From<[T; N]> for VInline { + fn from(a: [T; N]) -> Self { + a.into_iter().collect() + } +} + +impl Clone for VInline { + fn clone(&self) -> Self { + self.iter().cloned().collect() + } +} + +impl FromIterator for VInline { + fn from_iter>(iter: I) -> Self { + let it = iter.into_iter(); + let mut slf = Self::new(); + slf.extend(it); + slf + } +} + +pub struct IntoIter { + v: VInline, + l: usize, + i: usize, +} + +impl IntoIter { + #[inline(always)] + fn _next(&mut self) -> Option { + if self.i == self.l { + return None; + } + unsafe { + let current = self.i; + self.i += 1; + // UNSAFE(@ohsayan): i < l; so in all cases we are behind EOA + ptr::read(self.v._as_ptr().add(current).cast()) + } + } + #[inline(always)] + fn _next_back(&mut self) -> Option { + if self.i == self.l { + return None; + } + unsafe { + // UNSAFE(@ohsayan): we get the back pointer and move back; always behind EOA so we're chill + self.l -= 1; + ptr::read(self.v._as_ptr().add(self.l).cast()) + } + } +} + +impl Drop for IntoIter { + fn drop(&mut self) { + if self.i < self.l { + // sweet + unsafe { + // UNSAFE(@ohsayan): Safe because we maintain the EOA cond; second, the l is the remaining part + ptr::drop_in_place(ptr::slice_from_raw_parts_mut( + self.v._as_mut_ptr().add(self.i), + self.l - self.i, + )) + } + } + } +} + +impl Iterator for IntoIter { + type Item = T; + fn next(&mut self) -> Option { + self._next() + } +} +impl ExactSizeIterator for IntoIter {} +impl FusedIterator for IntoIter {} +impl DoubleEndedIterator for IntoIter { + fn next_back(&mut self) -> Option { + self._next_back() + } +} + +impl IntoIterator for VInline { + type Item = T; + + type IntoIter = IntoIter; + + fn into_iter(mut self) -> Self::IntoIter { + let real = self.len(); + unsafe { + // UNSAFE(@ohsayan): drop work for intoiter + // HACK(@ohsayan): same juicy drop hack + self.set_len(0); + } + Self::IntoIter { + v: self, + l: real, + i: 0, + } + } +} diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs new file mode 100644 index 00000000..8e77a87c --- /dev/null +++ b/server/src/engine/mem/word.rs @@ -0,0 +1,378 @@ +/* + * Created on Wed Mar 01 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{NativeDword, NativeQword, NativeTword, SpecialPaddedWord}, + core::mem::size_of, +}; + +pub static ZERO_BLOCK: [u8; 0] = []; + +#[cfg(target_pointer_width = "32")] +fn quadsplit(q: u64) -> [usize; 2] { + unsafe { + // UNSAFE(@ohsayan): simple numeric ops + core::mem::transmute(q) + } +} +#[cfg(target_pointer_width = "32")] +fn quadmerge(v: [usize; 2]) -> u64 { + unsafe { + // UNSAFE(@ohsayan): simple numeric ops + core::mem::transmute(v) + } +} + +pub trait WordIO { + fn store(v: T) -> Self; + fn load(&self) -> T; +} + +/* + dword + --- + kinds: NN (word * 2), QN (qword, word) + promotions: QN -> NNN +*/ + +pub trait DwordNN: Sized { + const DWORDNN_FROM_UPPER: bool = size_of::() > size_of::<[usize; 2]>(); + fn dwordnn_store_native_full(a: usize, b: usize) -> Self; + fn dwordnn_store_qw(a: u64) -> Self { + debug_assert!(!Self::DWORDNN_FROM_UPPER, "NEED TO OVERRIDE STORE"); + #[cfg(target_pointer_width = "32")] + { + let [a, b] = quadsplit(a); + Self::dwordnn_store_native_full(a, b) + } + #[cfg(target_pointer_width = "64")] + { + Self::dwordnn_store_native_full(a as usize, 0) + } + } + fn dwordnn_load_native_full(&self) -> [usize; 2]; + fn dwordnn_load_qw(&self) -> u64 { + debug_assert!(!Self::DWORDNN_FROM_UPPER, "NEED TO OVERRIDE LOAD"); + #[cfg(target_pointer_width = "32")] + { + quadmerge(self.dwordnn_load_native_full()) + } + #[cfg(target_pointer_width = "64")] + { + self.dwordnn_load_native_full()[0] as u64 + } + } +} + +pub trait DwordQN: Sized { + const DWORDQN_FROM_UPPER: bool = size_of::() > size_of::<(u64, usize)>(); + fn dwordqn_store_qw_nw(a: u64, b: usize) -> Self; + fn dwordqn_load_qw_nw(&self) -> (u64, usize); + // overrides + fn overridable_dwordnn_store_qw(a: u64) -> Self { + Self::dwordqn_store_qw_nw(a, 0) + } + // promotions + fn dwordqn_promote(&self) -> W { + let (a, b) = self.dwordqn_load_qw_nw(); + ::dwordqn_store_qw_nw(a, b) + } +} + +/* + dword: blanket impls +*/ + +impl DwordNN for T { + fn dwordnn_store_native_full(a: usize, b: usize) -> Self { + Self::dwordqn_store_qw_nw(a as u64, b) + } + fn dwordnn_store_qw(a: u64) -> Self { + Self::overridable_dwordnn_store_qw(a) + } + fn dwordnn_load_native_full(&self) -> [usize; 2] { + let (a, b) = self.dwordqn_load_qw_nw(); + debug_assert!(a <= usize::MAX as u64, "overflowed with: `{}`", a); + [a as usize, b] + } + fn dwordnn_load_qw(&self) -> u64 { + DwordQN::dwordqn_load_qw_nw(self).0 + } +} + +/* + dword: impls +*/ + +impl DwordNN for NativeDword { + fn dwordnn_store_native_full(a: usize, b: usize) -> Self { + Self([a, b]) + } + fn dwordnn_load_native_full(&self) -> [usize; 2] { + self.0 + } +} + +impl DwordQN for SpecialPaddedWord { + fn dwordqn_store_qw_nw(a: u64, b: usize) -> Self { + unsafe { + // UNSAFE(@ohsayan): valid construction + Self::new(a, b) + } + } + fn dwordqn_load_qw_nw(&self) -> (u64, usize) { + (self.a, self.b) + } + // overrides + fn overridable_dwordnn_store_qw(a: u64) -> Self { + unsafe { + // UNSAFE(@ohsayan): valid construction + Self::new(a, ZERO_BLOCK.as_ptr() as usize) + } + } +} + +/* + tword + --- + kinds: NNN (word * 3) + promotions: NNN -> NNNN +*/ + +pub trait TwordNNN: Sized { + const TWORDNNN_FROM_UPPER: bool = size_of::() > size_of::<[usize; 3]>(); + fn twordnnn_store_native_full(a: usize, b: usize, c: usize) -> Self; + fn twordnnn_load_native_full(&self) -> [usize; 3]; + // promotions + fn tword_promote(&self) -> W { + let [a, b, c] = self.twordnnn_load_native_full(); + ::twordnnn_store_native_full(a, b, c) + } +} + +/* + tword: blanket impls +*/ + +impl DwordQN for T { + fn dwordqn_store_qw_nw(a: u64, b: usize) -> Self { + #[cfg(target_pointer_width = "32")] + { + let [qw_1, qw_2] = quadsplit(a); + Self::twordnnn_store_native_full(qw_1, qw_2, b) + } + #[cfg(target_pointer_width = "64")] + { + Self::twordnnn_store_native_full(a as usize, b, 0) + } + } + fn dwordqn_load_qw_nw(&self) -> (u64, usize) { + #[cfg(target_pointer_width = "32")] + { + let [w1, w2, b] = self.twordnnn_load_native_full(); + (quadmerge([w1, w2]), b) + } + #[cfg(target_pointer_width = "64")] + { + let [a, b, _] = self.twordnnn_load_native_full(); + (a as u64, b) + } + } +} + +/* + tword: impls +*/ + +impl TwordNNN for NativeTword { + fn twordnnn_store_native_full(a: usize, b: usize, c: usize) -> Self { + Self([a, b, c]) + } + fn twordnnn_load_native_full(&self) -> [usize; 3] { + self.0 + } +} + +/* + qword + --- + kinds: NNNN (word * 4) + promotions: N/A +*/ + +pub trait QwordNNNN: Sized { + const QWORDNNNN_FROM_UPPER: bool = size_of::() > size_of::<[usize; 4]>(); + fn qwordnnnn_store_native_full(a: usize, b: usize, c: usize, d: usize) -> Self; + fn qwordnnnn_store_qw_qw(a: u64, b: u64) -> Self { + #[cfg(target_pointer_width = "32")] + { + let [qw1_a, qw1_b] = quadsplit(a); + let [qw2_a, qw2_b] = quadsplit(b); + Self::qwordnnnn_store_native_full(qw1_a, qw1_b, qw2_a, qw2_b) + } + #[cfg(target_pointer_width = "64")] + { + Self::qwordnnnn_store_native_full(a as usize, b as usize, 0, 0) + } + } + fn qwordnnnn_store_qw_nw_nw(a: u64, b: usize, c: usize) -> Self { + #[cfg(target_pointer_width = "32")] + { + let [qw_a, qw_b] = quadsplit(a); + Self::qwordnnnn_store_native_full(qw_a, qw_b, b, c) + } + #[cfg(target_pointer_width = "64")] + { + Self::qwordnnnn_store_native_full(a as usize, b, c, 0) + } + } + fn qwordnnnn_load_native_full(&self) -> [usize; 4]; + fn qwordnnnn_load_qw_qw(&self) -> [u64; 2] { + let [a, b, c, d] = self.qwordnnnn_load_native_full(); + #[cfg(target_pointer_width = "32")] + { + [quadmerge([a, b]), quadmerge([c, d])] + } + #[cfg(target_pointer_width = "64")] + { + let _ = (c, d); + [a as u64, b as u64] + } + } + fn qwordnnnn_load_qw_nw_nw(&self) -> (u64, usize, usize) { + let [a, b, c, d] = self.qwordnnnn_load_native_full(); + #[cfg(target_pointer_width = "32")] + { + (quadmerge([a, b]), c, d) + } + #[cfg(target_pointer_width = "64")] + { + let _ = d; + (a as u64, b, c) + } + } +} + +/* + qword: blanket impls +*/ + +impl TwordNNN for T { + fn twordnnn_store_native_full(a: usize, b: usize, c: usize) -> Self { + Self::qwordnnnn_store_native_full(a, b, c, 0) + } + fn twordnnn_load_native_full(&self) -> [usize; 3] { + let [a, b, c, _] = self.qwordnnnn_load_native_full(); + [a, b, c] + } +} + +/* + qword: impls +*/ + +impl QwordNNNN for NativeQword { + fn qwordnnnn_store_native_full(a: usize, b: usize, c: usize, d: usize) -> Self { + Self([a, b, c, d]) + } + fn qwordnnnn_load_native_full(&self) -> [usize; 4] { + self.0 + } +} + +/* + impls: WordIO +*/ + +macro_rules! impl_numeric_io { + ($trait:ident => { $($ty:ty),* $(,)? }) => { + $(impl WordIO<$ty> for T { + fn store(v: $ty) -> Self { Self::dwordnn_store_qw(v as _) } + fn load(&self) -> $ty { self.dwordnn_load_qw() as _ } + })* + } +} + +impl_numeric_io!(DwordNN => { u8, u16, u32, u64, i8, i16, i32, i64 }); + +impl WordIO for T { + fn store(v: bool) -> Self { + Self::dwordnn_store_qw(v as _) + } + fn load(&self) -> bool { + self.dwordnn_load_qw() == 1 + } +} + +macro_rules! impl_float_io { + ($($float:ty),* $(,)?) => { + $(impl WordIO<$float> for T { + fn store(v: $float) -> Self { Self::dwordnn_store_qw(v.to_bits() as u64) } + fn load(&self) -> $float { <$float>::from_bits(self.dwordnn_load_qw() as _) } + })* + } +} + +impl_float_io!(f32, f64); + +impl WordIO<(usize, usize)> for T { + fn store((a, b): (usize, usize)) -> Self { + Self::dwordnn_store_native_full(a, b) + } + fn load(&self) -> (usize, usize) { + let [a, b] = self.dwordnn_load_native_full(); + (a, b) + } +} + +impl WordIO<[usize; 2]> for T { + fn store([a, b]: [usize; 2]) -> Self { + Self::dwordnn_store_native_full(a, b) + } + fn load(&self) -> [usize; 2] { + self.dwordnn_load_native_full() + } +} + +impl WordIO<(usize, *mut u8)> for T { + fn store((a, b): (usize, *mut u8)) -> Self { + Self::dwordnn_store_native_full(a, b as usize) + } + fn load(&self) -> (usize, *mut u8) { + let [a, b] = self.dwordnn_load_native_full(); + (a, b as *mut u8) + } +} + +impl WordIO<(usize, *const u8)> for T { + fn store((a, b): (usize, *const u8)) -> Self { + Self::dwordnn_store_native_full(a, b as usize) + } + fn load(&self) -> (usize, *const u8) { + let [a, b] = self.dwordnn_load_native_full(); + (a, b as *const u8) + } +} diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs new file mode 100644 index 00000000..73f75835 --- /dev/null +++ b/server/src/engine/mod.rs @@ -0,0 +1,229 @@ +/* + * Created on Mon Sep 12 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[macro_use] +mod macros; +mod config; +mod core; +mod data; +mod error; +mod fractal; +mod idx; +mod mem; +mod net; +mod ql; +mod storage; +mod sync; +mod txn; +// test +#[cfg(test)] +mod tests; +// re-export +pub use error::RuntimeResult; + +use { + self::{ + config::{ConfigEndpoint, ConfigEndpointTls, ConfigMode, ConfigReturn, Configuration}, + fractal::{ + context::{self, Subsystem}, + sys_store::SystemStore, + }, + storage::v1::{ + loader::{self, SEInitState}, + LocalFS, + }, + }, + crate::util::os::TerminationSignal, + std::process::exit, + tokio::sync::broadcast, +}; + +pub(super) fn set_context_init(msg: &'static str) { + context::set(Subsystem::Init, msg) +} + +/// Initialize all drivers, load all data +/// +/// WARN: Must be in [`tokio::runtime::Runtime`] context! +pub fn load_all() -> RuntimeResult<(Configuration, fractal::GlobalStateStart)> { + // load configuration + info!("checking configuration ..."); + context::set(Subsystem::Init, "loading configuration"); + let config = match config::check_configuration()? { + ConfigReturn::Config(cfg) => cfg, + ConfigReturn::HelpMessage(msg) => { + eprintln!("{msg}"); + exit(0x00); + } + }; + if config.mode == ConfigMode::Dev { + warn!("running in dev mode"); + } + // restore system database + info!("loading system database ..."); + context::set_dmsg("loading system database"); + let (store, state) = SystemStore::::open_or_restore(config.auth.clone(), config.mode)?; + let sysdb_is_new = state.is_created(); + if state.is_existing_updated_root() { + warn!("the root account was updated"); + } + // now load all data + if sysdb_is_new { + info!("initializing storage engine ..."); + } else { + info!("reinitializing storage engine..."); + } + context::set_dmsg("restoring data"); + let SEInitState { + txn_driver, + model_drivers, + gns, + } = loader::SEInitState::try_init(sysdb_is_new)?; + let global = unsafe { + // UNSAFE(@ohsayan): this is the only entrypoint + fractal::load_and_enable_all(gns, store, txn_driver, model_drivers) + }; + Ok((config, global)) +} + +enum EndpointListeners { + Insecure(net::Listener), + Secure { + listener: net::Listener, + ssl: openssl::ssl::SslAcceptor, + }, + Multi { + tcp: net::Listener, + tls: net::Listener, + ssl: openssl::ssl::SslAcceptor, + }, +} + +impl EndpointListeners { + async fn listen(&mut self) { + match self { + Self::Insecure(l) => l.listen_tcp().await, + Self::Secure { listener, ssl } => listener.listen_tls(ssl).await, + Self::Multi { tcp, tls, ssl } => { + tokio::join!(tcp.listen_tcp(), tls.listen_tls(ssl)); + } + } + } + async fn finish(self) { + match self { + Self::Insecure(l) | Self::Secure { listener: l, .. } => l.terminate().await, + Self::Multi { tcp, tls, .. } => { + tokio::join!(tcp.terminate(), tls.terminate()); + } + } + } +} + +pub async fn start( + termsig: TerminationSignal, + Configuration { + endpoints, system, .. + }: Configuration, + fractal::GlobalStateStart { global, boot }: fractal::GlobalStateStart, +) -> RuntimeResult<()> { + // create our system-wide channel + let (signal, _) = broadcast::channel::<()>(1); + // start our services + context::set_dmsg("starting fractal engine"); + let fractal_handle = boot.boot(&signal, system.reliability_system_window); + // create our server + context::set(Subsystem::Network, "initializing endpoints"); + let str; + let mut endpoint_handles = match &endpoints { + ConfigEndpoint::Secure(ConfigEndpointTls { tcp, .. }) | ConfigEndpoint::Insecure(tcp) => { + let listener = + net::Listener::new(tcp.host(), tcp.port(), global.clone(), signal.clone()).await?; + if let ConfigEndpoint::Secure(s) = endpoints { + context::set_dmsg("initializing TLS"); + let acceptor = net::Listener::init_tls(s.cert(), s.private_key(), s.pkey_pass())?; + str = format!("listening on tls@{}:{}", s.tcp().host(), s.tcp().port()); + EndpointListeners::Secure { + listener, + ssl: acceptor, + } + } else { + str = format!("listening on tcp@{}:{}", tcp.host(), tcp.port()); + EndpointListeners::Insecure(listener) + } + } + ConfigEndpoint::Multi(insecure_ep, secure_ep) => { + let tcp_listener = + net::Listener::new_cfg(insecure_ep, global.clone(), signal.clone()).await?; + let tls_listener = + net::Listener::new_cfg(secure_ep.tcp(), global.clone(), signal.clone()).await?; + context::set_dmsg("initializing TLS"); + let acceptor = net::Listener::init_tls( + secure_ep.cert(), + secure_ep.private_key(), + secure_ep.pkey_pass(), + )?; + str = format!( + "listening on tcp@{}:{} and tls@{}:{}", + insecure_ep.host(), + insecure_ep.port(), + secure_ep.tcp().host(), + secure_ep.tcp().port() + ); + EndpointListeners::Multi { + tcp: tcp_listener, + tls: tls_listener, + ssl: acceptor, + } + } + }; + info!("{str}"); + tokio::select! { + _ = endpoint_handles.listen() => {} + _ = termsig => { + info!("received terminate signal. waiting for inflight tasks to complete ..."); + } + } + drop(signal); + endpoint_handles.finish().await; + info!("waiting for fractal engine to exit ..."); + let (hp_handle, lp_handle) = tokio::join!(fractal_handle.hp_handle, fractal_handle.lp_handle); + match (hp_handle, lp_handle) { + (Err(e1), Err(e2)) => { + error!("error while terminating fhp-executor and lhp-executor: {e1};{e2}") + } + (Err(e), _) => error!("error while terminating fhp-executor: {e}"), + (_, Err(e)) => error!("error while terminating flp-executor: {e}"), + _ => {} + } + Ok(()) +} + +pub fn finish(g: fractal::Global) { + unsafe { + // UNSAFE(@ohsayan): the only thing we do before exit + g.unload_all(); + } +} diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs new file mode 100644 index 00000000..4467fd8e --- /dev/null +++ b/server/src/engine/net/mod.rs @@ -0,0 +1,294 @@ +/* + * Created on Fri Sep 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +pub mod protocol; + +use { + crate::engine::{ + config::ConfigEndpointTcp, error::RuntimeResult, fractal::error::ErrorContext, + fractal::Global, + }, + bytes::BytesMut, + openssl::{ + pkey::PKey, + ssl::Ssl, + ssl::{SslAcceptor, SslMethod}, + x509::X509, + }, + std::{cell::Cell, net::SocketAddr, pin::Pin, time::Duration}, + tokio::{ + io::{AsyncRead, AsyncWrite, AsyncWriteExt, BufWriter}, + net::{TcpListener, TcpStream}, + sync::{broadcast, mpsc, Semaphore}, + }, + tokio_openssl::SslStream, +}; + +pub trait Socket: AsyncWrite + AsyncRead + Unpin {} +pub type IoResult = Result; + +const BUF_WRITE_CAP: usize = 16384; +const BUF_READ_CAP: usize = 16384; +const CLIMIT: usize = 50000; + +static CLIM: Semaphore = Semaphore::const_new(CLIMIT); + +enum QueryLoopResult { + Fin, + Rst, + HSFailed, +} + +/* + socket definitions +*/ + +impl Socket for TcpStream {} +impl Socket for SslStream {} + +struct NetBackoff { + at: Cell, +} + +impl NetBackoff { + const BACKOFF_MAX: u8 = 64; + fn new() -> Self { + Self { at: Cell::new(1) } + } + async fn spin(&self) { + let current = self.at.get(); + self.at.set(current << 1); + tokio::time::sleep(Duration::from_secs(current as _)).await + } + fn should_disconnect(&self) -> bool { + self.at.get() >= Self::BACKOFF_MAX + } +} + +unsafe impl Send for NetBackoff {} +unsafe impl Sync for NetBackoff {} + +/* + listener +*/ + +/// Connection handler for a remote connection +pub struct ConnectionHandler { + socket: BufWriter, + buffer: BytesMut, + global: Global, + sig_terminate: broadcast::Receiver<()>, + _sig_inflight_complete: mpsc::Sender<()>, +} + +impl ConnectionHandler { + pub fn new( + socket: S, + global: Global, + term_sig: broadcast::Receiver<()>, + _inflight_complete: mpsc::Sender<()>, + ) -> Self { + Self { + socket: BufWriter::with_capacity(BUF_WRITE_CAP, socket), + buffer: BytesMut::with_capacity(BUF_READ_CAP), + global, + sig_terminate: term_sig, + _sig_inflight_complete: _inflight_complete, + } + } + pub async fn run(&mut self) -> IoResult<()> { + let Self { + socket, + buffer, + global, + .. + } = self; + loop { + tokio::select! { + ret = protocol::query_loop(socket, buffer, global) => { + socket.flush().await?; + match ret { + Ok(QueryLoopResult::Fin) => return Ok(()), + Ok(QueryLoopResult::Rst) => error!("connection reset while talking to client"), + Ok(QueryLoopResult::HSFailed) => error!("failed to handshake with client"), + Err(e) => { + error!("error while handling connection: {e}"); + return Err(e); + } + } + return Ok(()) + }, + _ = self.sig_terminate.recv() => { + return Ok(()); + } + } + } + } +} + +/// A TCP listener bound to a socket +pub struct Listener { + global: Global, + listener: TcpListener, + sig_shutdown: broadcast::Sender<()>, + sig_inflight: mpsc::Sender<()>, + sig_inflight_wait: mpsc::Receiver<()>, +} + +impl Listener { + pub async fn new_cfg( + tcp: &ConfigEndpointTcp, + global: Global, + sig_shutdown: broadcast::Sender<()>, + ) -> RuntimeResult { + Self::new(tcp.host(), tcp.port(), global, sig_shutdown).await + } + pub async fn new( + host: &str, + port: u16, + global: Global, + sig_shutdown: broadcast::Sender<()>, + ) -> RuntimeResult { + let (sig_inflight, sig_inflight_wait) = mpsc::channel(1); + let listener = TcpListener::bind((host, port)) + .await + .set_dmsg(format!("failed to bind to port `{host}:{port}`"))?; + Ok(Self { + global, + listener, + sig_shutdown, + sig_inflight, + sig_inflight_wait, + }) + } + pub async fn terminate(self) { + let Self { + mut sig_inflight_wait, + sig_inflight, + sig_shutdown, + .. + } = self; + drop(sig_shutdown); + drop(sig_inflight); // could be that we are the only ones holding this lol + let _ = sig_inflight_wait.recv().await; // wait + } + async fn accept(&mut self) -> IoResult<(TcpStream, SocketAddr)> { + let backoff = NetBackoff::new(); + loop { + match self.listener.accept().await { + Ok(s) => return Ok(s), + Err(e) => { + if backoff.should_disconnect() { + // that's enough of your crappy connection dear sir + return Err(e.into()); + } + } + } + backoff.spin().await; + } + } + pub async fn listen_tcp(&mut self) { + loop { + // acquire a permit + let permit = CLIM.acquire().await.unwrap(); + let (stream, _) = match self.accept().await { + Ok(s) => s, + Err(e) => { + /* + SECURITY: IGNORE THIS ERROR + */ + warn!("failed to accept connection on TCP socket: `{e}`"); + continue; + } + }; + let mut handler = ConnectionHandler::new( + stream, + self.global.clone(), + self.sig_shutdown.subscribe(), + self.sig_inflight.clone(), + ); + tokio::spawn(async move { + if let Err(e) = handler.run().await { + warn!("error handling client connection: `{e}`"); + } + }); + // return the permit + drop(permit); + } + } + pub fn init_tls( + tls_cert: &str, + tls_priv_key: &str, + tls_key_password: &str, + ) -> RuntimeResult { + let build_acceptor = || { + let cert = X509::from_pem(tls_cert.as_bytes())?; + let priv_key = PKey::private_key_from_pem_passphrase( + tls_priv_key.as_bytes(), + tls_key_password.as_bytes(), + )?; + let mut builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls())?; + builder.set_certificate(&cert)?; + builder.set_private_key(&priv_key)?; + builder.check_private_key()?; + Ok::<_, openssl::error::ErrorStack>(builder.build()) + }; + let acceptor = build_acceptor().set_dmsg("failed to initialize TLS socket")?; + Ok(acceptor) + } + pub async fn listen_tls(&mut self, acceptor: &SslAcceptor) { + loop { + let stream = async { + let (stream, _) = self.accept().await?; + let ssl = Ssl::new(acceptor.context())?; + let mut stream = SslStream::new(ssl, stream)?; + Pin::new(&mut stream).accept().await?; + RuntimeResult::Ok(stream) + }; + let stream = match stream.await { + Ok(s) => s, + Err(e) => { + /* + SECURITY: Once again, ignore this error + */ + warn!("failed to accept connection on TLS socket: `{e}`"); + continue; + } + }; + let mut handler = ConnectionHandler::new( + stream, + self.global.clone(), + self.sig_shutdown.subscribe(), + self.sig_inflight.clone(), + ); + tokio::spawn(async move { + if let Err(e) = handler.run().await { + warn!("error handling client TLS connection: `{e}`"); + } + }); + } + } +} diff --git a/server/src/engine/net/protocol/exchange.rs b/server/src/engine/net/protocol/exchange.rs new file mode 100644 index 00000000..388ac87b --- /dev/null +++ b/server/src/engine/net/protocol/exchange.rs @@ -0,0 +1,273 @@ +/* + * Created on Wed Sep 20 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::mem::BufferedScanner; + +pub(super) unsafe fn resume<'a>( + buf: &'a [u8], + last_cursor: usize, + last_state: QExchangeState, +) -> (usize, QExchangeResult<'a>) { + let mut scanner = BufferedScanner::new_with_cursor(buf, last_cursor); + let ret = last_state.resume(&mut scanner); + (scanner.cursor(), ret) +} + +/* + SQ +*/ + +#[derive(Debug, PartialEq)] +pub(super) enum LFTIntParseResult { + Value(u64), + Partial(u64), + Error, +} + +#[derive(Debug, PartialEq)] +pub struct SQuery<'a> { + q: &'a [u8], + q_window: usize, +} + +impl<'a> SQuery<'a> { + pub(super) fn new(q: &'a [u8], q_window: usize) -> Self { + Self { q, q_window } + } + pub fn payload(&self) -> &'a [u8] { + self.q + } + pub fn q_window(&self) -> usize { + self.q_window + } + pub fn query(&self) -> &'a [u8] { + &self.payload()[..self.q_window()] + } + pub fn params(&self) -> &'a [u8] { + &self.payload()[self.q_window()..] + } + #[cfg(test)] + pub fn query_str(&self) -> &str { + core::str::from_utf8(self.query()).unwrap() + } + #[cfg(test)] + pub fn params_str(&self) -> &str { + core::str::from_utf8(self.params()).unwrap() + } +} + +/* + utils +*/ + +/// scan an integer: +/// - if just an LF: +/// - if disallowed single byte: return an error +/// - else, return value +/// - if no LF: return upto limit +/// - if LF: return value +pub(super) fn scanint( + scanner: &mut BufferedScanner, + first_run: bool, + prev: u64, +) -> LFTIntParseResult { + let mut current = prev; + // guard a case where the buffer might be empty and can potentially have invalid chars + let mut okay = !((scanner.rounded_cursor_value() == b'\n') & first_run); + while scanner.rounded_cursor_not_eof_matches(|b| b'\n'.ne(b)) & okay { + let byte = unsafe { scanner.next_byte() }; + okay &= byte.is_ascii_digit(); + match current + .checked_mul(10) + .map(|new| new.checked_add((byte & 0x0f) as u64)) + { + Some(Some(int)) => { + current = int; + } + _ => { + okay = false; + } + } + } + let lf = scanner.rounded_cursor_not_eof_equals(b'\n'); + unsafe { + // UNSAFE(@ohsayan): within buffer range + scanner.incr_cursor_if(lf); + } + if lf & okay { + LFTIntParseResult::Value(current) + } else { + if okay { + LFTIntParseResult::Partial(current) + } else { + LFTIntParseResult::Error + } + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub(super) enum QExchangeStateInternal { + Initial, + PendingMeta1, + PendingMeta2, + PendingData, +} + +impl Default for QExchangeStateInternal { + fn default() -> Self { + Self::Initial + } +} + +#[derive(Debug, PartialEq)] +pub(super) struct QExchangeState { + state: QExchangeStateInternal, + target: usize, + md_packet_size: u64, + md_q_window: u64, +} + +impl Default for QExchangeState { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, PartialEq)] +/// Result after attempting to complete (or terminate) a query time exchange +pub(super) enum QExchangeResult<'a> { + /// We completed the exchange and yielded a [`SQuery`] + SQCompleted(SQuery<'a>), + /// We're changing states + ChangeState(QExchangeState), + /// We hit an error and need to terminate this exchange + Error, +} + +impl QExchangeState { + fn _new( + state: QExchangeStateInternal, + target: usize, + md_packet_size: u64, + md_q_window: u64, + ) -> Self { + Self { + state, + target, + md_packet_size, + md_q_window, + } + } + #[cfg(test)] + pub(super) fn new_test( + state: QExchangeStateInternal, + target: usize, + md_packet_size: u64, + md_q_window: u64, + ) -> Self { + Self::_new(state, target, md_packet_size, md_q_window) + } +} + +impl QExchangeState { + pub const MIN_READ: usize = b"S\x00\n\x00\n".len(); + pub fn new() -> Self { + Self::_new(QExchangeStateInternal::Initial, Self::MIN_READ, 0, 0) + } + pub fn has_reached_target(&self, new_buffer: &[u8]) -> bool { + new_buffer.len() >= self.target + } + fn resume<'a>(self, scanner: &mut BufferedScanner<'a>) -> QExchangeResult<'a> { + debug_assert!(scanner.has_left(Self::MIN_READ)); + match self.state { + QExchangeStateInternal::Initial => self.start_initial(scanner), + QExchangeStateInternal::PendingMeta1 => self.resume_at_md1(scanner, false), + QExchangeStateInternal::PendingMeta2 => self.resume_at_md2(scanner, false), + QExchangeStateInternal::PendingData => self.resume_data(scanner), + } + } + fn start_initial<'a>(self, scanner: &mut BufferedScanner<'a>) -> QExchangeResult<'a> { + if unsafe { scanner.next_byte() } != b'S' { + // has to be a simple query! + return QExchangeResult::Error; + } + self.resume_at_md1(scanner, true) + } + fn resume_at_md1<'a>( + mut self, + scanner: &mut BufferedScanner<'a>, + first_run: bool, + ) -> QExchangeResult<'a> { + let packet_size = match scanint(scanner, first_run, self.md_packet_size) { + LFTIntParseResult::Value(v) => v, + LFTIntParseResult::Partial(p) => { + // if this is the first run, we read 5 bytes and need atleast one more; if this is a resume we read one or more bytes and + // need atleast one more + self.target += 1; + self.md_packet_size = p; + self.state = QExchangeStateInternal::PendingMeta1; + return QExchangeResult::ChangeState(self); + } + LFTIntParseResult::Error => return QExchangeResult::Error, + }; + self.md_packet_size = packet_size; + self.target = scanner.cursor() + packet_size as usize; + // hand over control to md2 + self.resume_at_md2(scanner, true) + } + fn resume_at_md2<'a>( + mut self, + scanner: &mut BufferedScanner<'a>, + first_run: bool, + ) -> QExchangeResult<'a> { + let q_window = match scanint(scanner, first_run, self.md_q_window) { + LFTIntParseResult::Value(v) => v, + LFTIntParseResult::Partial(p) => { + self.md_q_window = p; + self.state = QExchangeStateInternal::PendingMeta2; + return QExchangeResult::ChangeState(self); + } + LFTIntParseResult::Error => return QExchangeResult::Error, + }; + self.md_q_window = q_window; + // hand over control to data + self.resume_data(scanner) + } + fn resume_data<'a>(mut self, scanner: &mut BufferedScanner<'a>) -> QExchangeResult<'a> { + let df_size = self.target - scanner.cursor(); + if scanner.remaining() == df_size { + unsafe { + QExchangeResult::SQCompleted(SQuery::new( + scanner.next_chunk_variable(df_size), + self.md_q_window as usize, + )) + } + } else { + self.state = QExchangeStateInternal::PendingData; + QExchangeResult::ChangeState(self) + } + } +} diff --git a/server/src/engine/net/protocol/handshake.rs b/server/src/engine/net/protocol/handshake.rs new file mode 100644 index 00000000..c2b28e3f --- /dev/null +++ b/server/src/engine/net/protocol/handshake.rs @@ -0,0 +1,417 @@ +/* + * Created on Mon Sep 18 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::{ + engine::mem::scanner::{BufferedScanner, ScannerDecodeResult}, + util::compiler, +}; + +#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)] +#[repr(u8)] +/// Low-level protocol errors +pub enum ProtocolError { + /// packet has incorrect structure + CorruptedHSPacket = 0, + /// incorrect handshake version + RejectHSVersion = 1, + /// invalid protocol version + RejectProtocol = 2, + /// invalid exchange mode + RejectExchangeMode = 3, + /// invalid query mode + RejectQueryMode = 4, + /// invalid auth details + /// + /// **NB**: this can be due to either an incorrect auth flag, or incorrect auth data or disallowed auth mode. we keep it + /// in one error for purposes of security + RejectAuth = 5, +} + +/* + handshake meta +*/ + +#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)] +#[repr(u8)] +/// the handshake version +pub enum HandshakeVersion { + /// Skyhash/2.0 HS + Original = 0, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)] +#[repr(u8)] +/// the skyhash protocol version +pub enum ProtocolVersion { + /// Skyhash/2.0 protocol + Original = 0, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)] +#[repr(u8)] +/// the data exchange mode +pub enum DataExchangeMode { + /// query-time data exchange mode + QueryTime = 0, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)] +#[repr(u8)] +/// the query mode +pub enum QueryMode { + /// BQL-1 query mode + Bql1 = 0, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)] +#[repr(u8)] +/// the authentication mode +pub enum AuthMode { + Password = 0, +} + +impl AuthMode { + unsafe fn from_raw(v: u8) -> Self { + core::mem::transmute(v) + } + /// returns the minimum number of metadata bytes need to parse the payload for this auth mode + const fn min_payload_bytes(&self) -> usize { + match self { + Self::Password => 4, + } + } +} + +/* + client handshake +*/ + +/// The handshake state +#[derive(Debug, PartialEq, Clone)] +pub enum HandshakeState { + /// we just began the handshake + Initial, + /// we just processed the static block + StaticBlock(CHandshakeStatic), + /// Expecting some more auth meta + ExpectingMetaForVariableBlock { + /// static block + static_hs: CHandshakeStatic, + /// uname len + uname_l: usize, + }, + /// we're expecting to finish the handshake + ExpectingVariableBlock { + /// static block + static_hs: CHandshakeStatic, + /// uname len + uname_l: usize, + /// pwd len + pwd_l: usize, + }, +} + +impl Default for HandshakeState { + fn default() -> Self { + Self::Initial + } +} + +/// The static segment of the handshake +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CHandshakeStatic { + /// the handshake version + hs_version: HandshakeVersion, + /// protocol version + protocol: ProtocolVersion, + /// exchange mode + exchange_mode: DataExchangeMode, + /// query mode + query_mode: QueryMode, + /// authentication mode + auth_mode: AuthMode, +} + +impl CHandshakeStatic { + pub const fn new( + hs_version: HandshakeVersion, + protocol: ProtocolVersion, + exchange_mode: DataExchangeMode, + query_mode: QueryMode, + auth_mode: AuthMode, + ) -> Self { + Self { + hs_version, + protocol, + exchange_mode, + query_mode, + auth_mode, + } + } + pub fn hs_version(&self) -> HandshakeVersion { + self.hs_version + } + pub fn protocol(&self) -> ProtocolVersion { + self.protocol + } + pub fn exchange_mode(&self) -> DataExchangeMode { + self.exchange_mode + } + pub fn query_mode(&self) -> QueryMode { + self.query_mode + } + pub fn auth_mode(&self) -> AuthMode { + self.auth_mode + } +} + +/// handshake authentication +// TODO(@ohsayan): enum? +#[derive(Debug, PartialEq)] +pub struct CHandshakeAuth<'a> { + username: &'a [u8], + password: &'a [u8], +} + +impl<'a> CHandshakeAuth<'a> { + pub fn new(username: &'a [u8], password: &'a [u8]) -> Self { + Self { username, password } + } + pub fn username(&self) -> &[u8] { + self.username + } + pub fn password(&self) -> &[u8] { + self.password + } +} + +#[derive(Debug, PartialEq)] +pub enum HandshakeResult<'a> { + /// Finished handshake + Completed(CHandshake<'a>), + /// Update handshake state + /// + /// **NOTE:** expect does not take into account the current amount of buffered data (hence the unbuffered part must be computed!) + ChangeState { + new_state: HandshakeState, + expect: usize, + }, + /// An error occurred + Error(ProtocolError), +} + +/// The client's handshake record +#[derive(Debug, PartialEq)] +pub struct CHandshake<'a> { + /// the static segment of the handshake + hs_static: CHandshakeStatic, + /// the auth section of the dynamic segment of the handshake + hs_auth: CHandshakeAuth<'a>, +} + +impl<'a> CHandshake<'a> { + pub const INITIAL_READ: usize = 6; + const CLIENT_HELLO: u8 = b'H'; + pub fn new(hs_static: CHandshakeStatic, hs_auth: CHandshakeAuth<'a>) -> Self { + Self { hs_static, hs_auth } + } + /// Resume handshake with the given state and buffer + pub fn resume_with( + scanner: &mut BufferedScanner<'a>, + state: HandshakeState, + ) -> HandshakeResult<'a> { + match state { + // nothing buffered yet + HandshakeState::Initial => Self::resume_initial(scanner), + // buffered static block + HandshakeState::StaticBlock(static_block) => { + Self::resume_at_auth_metadata1(scanner, static_block) + } + // buffered some auth meta + HandshakeState::ExpectingMetaForVariableBlock { static_hs, uname_l } => { + Self::resume_at_auth_metadata2(scanner, static_hs, uname_l) + } + // buffered full auth meta + HandshakeState::ExpectingVariableBlock { + static_hs, + uname_l, + pwd_l, + } => Self::resume_at_variable_block_payload(scanner, static_hs, uname_l, pwd_l), + } + } + pub fn hs_static(&self) -> CHandshakeStatic { + self.hs_static + } + pub fn hs_auth(&self) -> &CHandshakeAuth<'a> { + &self.hs_auth + } +} + +impl<'a> CHandshake<'a> { + /// Resume from the initial state (nothing buffered yet) + fn resume_initial(scanner: &mut BufferedScanner<'a>) -> HandshakeResult<'a> { + // get our block + if cfg!(debug_assertions) { + if scanner.remaining() < Self::INITIAL_READ { + return HandshakeResult::ChangeState { + new_state: HandshakeState::Initial, + expect: Self::INITIAL_READ, + }; + } + } else { + assert!(scanner.remaining() >= Self::INITIAL_READ); + } + let buf: [u8; CHandshake::INITIAL_READ] = unsafe { scanner.next_chunk() }; + let invalid_first_byte = buf[0] != Self::CLIENT_HELLO; + let invalid_hs_version = buf[1] > HandshakeVersion::MAX; + let invalid_proto_version = buf[2] > ProtocolVersion::MAX; + let invalid_exchange_mode = buf[3] > DataExchangeMode::MAX; + let invalid_query_mode = buf[4] > QueryMode::MAX; + let invalid_auth_mode = buf[5] > AuthMode::MAX; + // check block + if compiler::unlikely( + invalid_first_byte + | invalid_hs_version + | invalid_proto_version + | invalid_exchange_mode + | invalid_query_mode + | invalid_auth_mode, + ) { + static ERROR: [ProtocolError; 6] = [ + ProtocolError::CorruptedHSPacket, + ProtocolError::RejectHSVersion, + ProtocolError::RejectProtocol, + ProtocolError::RejectExchangeMode, + ProtocolError::RejectQueryMode, + ProtocolError::RejectAuth, + ]; + return HandshakeResult::Error( + ERROR[((invalid_first_byte as u8 * 1) + | (invalid_hs_version as u8 * 2) + | (invalid_proto_version as u8 * 3) + | (invalid_exchange_mode as u8 * 4) + | (invalid_query_mode as u8 * 5) + | (invalid_auth_mode as u8) * 6) as usize + - 1usize], + ); + } + // init header + let static_header = CHandshakeStatic::new( + HandshakeVersion::Original, + ProtocolVersion::Original, + DataExchangeMode::QueryTime, + QueryMode::Bql1, + unsafe { + // UNSAFE(@ohsayan): already checked + AuthMode::from_raw(buf[5]) + }, + ); + // check if we have auth data + Self::resume_at_auth_metadata1(scanner, static_header) + } + fn resume_at_variable_block_payload( + scanner: &mut BufferedScanner<'a>, + static_hs: CHandshakeStatic, + uname_l: usize, + pwd_l: usize, + ) -> HandshakeResult<'a> { + if scanner.has_left(uname_l + pwd_l) { + // we're done here + return unsafe { + // UNSAFE(@ohsayan): we just checked buffered size + let uname = scanner.next_chunk_variable(uname_l); + let pwd = scanner.next_chunk_variable(pwd_l); + HandshakeResult::Completed(Self::new(static_hs, CHandshakeAuth::new(uname, pwd))) + }; + } + HandshakeResult::ChangeState { + new_state: HandshakeState::ExpectingVariableBlock { + static_hs, + uname_l, + pwd_l, + }, + expect: (uname_l + pwd_l), + } + } +} + +impl<'a> CHandshake<'a> { + /// Resume parsing at the first part of the auth metadata + fn resume_at_auth_metadata1( + scanner: &mut BufferedScanner<'a>, + static_header: CHandshakeStatic, + ) -> HandshakeResult<'a> { + // now let's see if we have buffered enough data for auth + if scanner.remaining() < static_header.auth_mode.min_payload_bytes() { + // we need more data + return HandshakeResult::ChangeState { + new_state: HandshakeState::StaticBlock(static_header), + expect: static_header.auth_mode.min_payload_bytes(), + }; + } + // we seem to have enough data for this auth mode + match static_header.auth_mode { + AuthMode::Password => {} + } + // let us see if we can parse the username length + let uname_l = match scanner.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor() + { + ScannerDecodeResult::NeedMore => { + return HandshakeResult::ChangeState { + new_state: HandshakeState::StaticBlock(static_header), + expect: AuthMode::Password.min_payload_bytes(), // 2 for uname_l and 2 for pwd_l + }; + } + ScannerDecodeResult::Value(v) => v as usize, + ScannerDecodeResult::Error => { + return HandshakeResult::Error(ProtocolError::CorruptedHSPacket) + } + }; + Self::resume_at_auth_metadata2(scanner, static_header, uname_l) + } + /// Resume at trying to get the final part of the auth metadata + fn resume_at_auth_metadata2( + scanner: &mut BufferedScanner<'a>, + static_hs: CHandshakeStatic, + uname_l: usize, + ) -> HandshakeResult<'a> { + // we just have to get the password len + let pwd_l = match scanner.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor() { + ScannerDecodeResult::Value(v) => v as usize, + ScannerDecodeResult::NeedMore => { + // newline missing (or maybe there's more?) + return HandshakeResult::ChangeState { + new_state: HandshakeState::ExpectingMetaForVariableBlock { static_hs, uname_l }, + expect: uname_l + 2, // space for username + password len + }; + } + ScannerDecodeResult::Error => { + return HandshakeResult::Error(ProtocolError::CorruptedHSPacket) + } + }; + Self::resume_at_variable_block_payload(scanner, static_hs, uname_l, pwd_l) + } +} diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs new file mode 100644 index 00000000..1dd6d62c --- /dev/null +++ b/server/src/engine/net/protocol/mod.rs @@ -0,0 +1,296 @@ +/* + * Created on Fri Sep 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod exchange; +mod handshake; +#[cfg(test)] +mod tests; + +// re-export +pub use exchange::SQuery; + +use { + self::{ + exchange::{QExchangeResult, QExchangeState}, + handshake::{ + AuthMode, CHandshake, DataExchangeMode, HandshakeResult, HandshakeState, + HandshakeVersion, ProtocolError, ProtocolVersion, QueryMode, + }, + }, + super::{IoResult, QueryLoopResult, Socket}, + crate::engine::{ + self, + error::QueryError, + fractal::{Global, GlobalInstanceLike}, + mem::{BufferedScanner, IntegerRepr}, + }, + bytes::{Buf, BytesMut}, + tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter}, +}; + +#[repr(u8)] +#[derive(sky_macros::EnumMethods, Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[allow(unused)] +pub enum ResponseType { + Null = 0x00, + Bool = 0x01, + UInt8 = 0x02, + UInt16 = 0x03, + UInt32 = 0x04, + UInt64 = 0x05, + SInt8 = 0x06, + SInt16 = 0x07, + SInt32 = 0x08, + SInt64 = 0x09, + Float32 = 0x0A, + Float64 = 0x0B, + Binary = 0x0C, + String = 0x0D, + List = 0x0E, + Dict = 0x0F, + Error = 0x10, + Row = 0x11, + Empty = 0x12, + MultiRow = 0x13, +} + +#[derive(Debug, PartialEq)] +pub struct ClientLocalState { + username: Box, + root: bool, + hs: handshake::CHandshakeStatic, + cs: Option>, +} + +impl ClientLocalState { + pub fn new(username: Box, root: bool, hs: handshake::CHandshakeStatic) -> Self { + Self { + username, + root, + hs, + cs: None, + } + } + pub fn is_root(&self) -> bool { + self.root + } + pub fn username(&self) -> &str { + &self.username + } + pub fn set_cs(&mut self, new: Box) { + self.cs = Some(new); + } + pub fn unset_cs(&mut self) { + self.cs = None; + } + pub fn get_cs(&self) -> Option<&str> { + self.cs.as_deref() + } +} + +#[derive(Debug, PartialEq)] +pub enum Response { + Empty, + Null, + Serialized { + ty: ResponseType, + size: usize, + data: Vec, + }, + Bool(bool), +} + +pub(super) async fn query_loop( + con: &mut BufWriter, + buf: &mut BytesMut, + global: &Global, +) -> IoResult { + // handshake + let mut client_state = match do_handshake(con, buf, global).await? { + PostHandshake::Okay(hs) => hs, + PostHandshake::ConnectionClosedFin => return Ok(QueryLoopResult::Fin), + PostHandshake::ConnectionClosedRst => return Ok(QueryLoopResult::Rst), + PostHandshake::Error(e) => { + // failed to handshake; we'll close the connection + let hs_err_packet = [b'H', 0, 1, e.value_u8()]; + con.write_all(&hs_err_packet).await?; + return Ok(QueryLoopResult::HSFailed); + } + }; + // done handshaking + con.write_all(b"H\x00\x00\x00").await?; + con.flush().await?; + let mut state = QExchangeState::default(); + let mut cursor = 0; + loop { + if con.read_buf(buf).await? == 0 { + if buf.is_empty() { + return Ok(QueryLoopResult::Fin); + } else { + return Ok(QueryLoopResult::Rst); + } + } + if !state.has_reached_target(buf) { + // we haven't buffered sufficient bytes; keep working + continue; + } + let sq = match unsafe { + // UNSAFE(@ohsayan): we store the cursor from the last run + exchange::resume(buf, cursor, state) + } { + (_, QExchangeResult::SQCompleted(sq)) => sq, + (new_cursor, QExchangeResult::ChangeState(new_state)) => { + cursor = new_cursor; + state = new_state; + continue; + } + (_, QExchangeResult::Error) => { + // respond with error + let [a, b] = (QueryError::SysNetworkSystemIllegalClientPacket.value_u8() as u16) + .to_le_bytes(); + con.write_all(&[ResponseType::Error.value_u8(), a, b]) + .await?; + con.flush().await?; + // reset buffer, cursor and state + buf.clear(); + cursor = 0; + state = QExchangeState::default(); + continue; + } + }; + // now execute query + match engine::core::exec::dispatch_to_executor(global, &mut client_state, sq).await { + Ok(Response::Empty) => { + con.write_all(&[ResponseType::Empty.value_u8()]).await?; + } + Ok(Response::Serialized { ty, size, data }) => { + con.write_u8(ty.value_u8()).await?; + let mut irep = IntegerRepr::new(); + con.write_all(irep.as_bytes(size as u64)).await?; + con.write_u8(b'\n').await?; + con.write_all(&data).await?; + } + Ok(Response::Bool(b)) => { + con.write_all(&[ResponseType::Bool.value_u8(), b as u8]) + .await? + } + Ok(Response::Null) => con.write_u8(ResponseType::Null.value_u8()).await?, + Err(e) => { + let [a, b] = (e.value_u8() as u16).to_le_bytes(); + con.write_all(&[ResponseType::Error.value_u8(), a, b]) + .await?; + } + } + con.flush().await?; + // reset buffer, cursor and state + buf.clear(); + cursor = 0; + state = QExchangeState::default(); + } +} + +#[derive(Debug, PartialEq)] +enum PostHandshake { + Okay(ClientLocalState), + Error(ProtocolError), + ConnectionClosedFin, + ConnectionClosedRst, +} + +async fn do_handshake( + con: &mut BufWriter, + buf: &mut BytesMut, + global: &Global, +) -> IoResult { + let mut expected = CHandshake::INITIAL_READ; + let mut state = HandshakeState::default(); + let mut cursor = 0; + let handshake; + loop { + let read_many = con.read_buf(buf).await?; + if read_many == 0 { + if buf.is_empty() { + return Ok(PostHandshake::ConnectionClosedFin); + } else { + return Ok(PostHandshake::ConnectionClosedRst); + } + } + if buf.len() < expected { + continue; + } + let mut scanner = unsafe { BufferedScanner::new_with_cursor(buf, cursor) }; + match handshake::CHandshake::resume_with(&mut scanner, state) { + HandshakeResult::Completed(hs) => { + handshake = hs; + cursor = scanner.cursor(); + break; + } + HandshakeResult::ChangeState { new_state, expect } => { + expected = expect; + state = new_state; + cursor = scanner.cursor(); + } + HandshakeResult::Error(e) => { + return Ok(PostHandshake::Error(e)); + } + } + } + // check handshake + if cfg!(debug_assertions) { + assert_eq!( + handshake.hs_static().hs_version(), + HandshakeVersion::Original + ); + assert_eq!(handshake.hs_static().protocol(), ProtocolVersion::Original); + assert_eq!( + handshake.hs_static().exchange_mode(), + DataExchangeMode::QueryTime + ); + assert_eq!(handshake.hs_static().query_mode(), QueryMode::Bql1); + assert_eq!(handshake.hs_static().auth_mode(), AuthMode::Password); + } + match core::str::from_utf8(handshake.hs_auth().username()) { + Ok(uname) => { + let auth = global.sys_store().system_store().auth_data().read(); + let r = auth.verify_user_check_root(uname, handshake.hs_auth().password()); + match r { + Ok(is_root) => { + let hs = handshake.hs_static(); + let ret = Ok(PostHandshake::Okay(ClientLocalState::new( + uname.into(), + is_root, + hs, + ))); + buf.advance(cursor); + return ret; + } + Err(_) => {} + } + } + Err(_) => {} + }; + Ok(PostHandshake::Error(ProtocolError::RejectAuth)) +} diff --git a/server/src/engine/net/protocol/tests.rs b/server/src/engine/net/protocol/tests.rs new file mode 100644 index 00000000..9eab9549 --- /dev/null +++ b/server/src/engine/net/protocol/tests.rs @@ -0,0 +1,338 @@ +/* + * Created on Mon Sep 18 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + exchange::{self, scanint, LFTIntParseResult, QExchangeResult, QExchangeState}, + SQuery, + }, + crate::{ + engine::{ + mem::BufferedScanner, + net::protocol::handshake::{ + AuthMode, CHandshake, CHandshakeAuth, CHandshakeStatic, DataExchangeMode, + HandshakeResult, HandshakeState, HandshakeVersion, ProtocolVersion, QueryMode, + }, + }, + util::test_utils, + }, + rand::Rng, +}; + +pub(super) fn create_simple_query(query: &str, params: [&str; N]) -> Vec { + let mut buf = vec![]; + let query_size_as_string = query.len().to_string(); + let total_packet_size = query.len() + + params.iter().map(|l| l.len()).sum::() + + query_size_as_string.len() + + 1; + // segment 1 + buf.push(b'S'); + buf.extend(total_packet_size.to_string().as_bytes()); + buf.push(b'\n'); + // segment + buf.extend(query_size_as_string.as_bytes()); + buf.push(b'\n'); + // dataframe + buf.extend(query.as_bytes()); + params + .into_iter() + .for_each(|param| buf.extend(param.as_bytes())); + buf +} + +/* + client handshake +*/ + +const FULL_HANDSHAKE_WITH_AUTH: [u8; 23] = *b"H\0\0\0\0\x005\n8\nsayanpass1234"; + +const STATIC_HANDSHAKE_WITH_AUTH: CHandshakeStatic = CHandshakeStatic::new( + HandshakeVersion::Original, + ProtocolVersion::Original, + DataExchangeMode::QueryTime, + QueryMode::Bql1, + AuthMode::Password, +); + +/* + handshake with no state changes +*/ + +#[test] +fn parse_staged_with_auth() { + for i in 0..FULL_HANDSHAKE_WITH_AUTH.len() { + let buf = &FULL_HANDSHAKE_WITH_AUTH[..i + 1]; + let mut s = BufferedScanner::new(buf); + let ref mut scanner = s; + let result = CHandshake::resume_with(scanner, HandshakeState::Initial); + match buf.len() { + 1..=5 => { + assert_eq!( + result, + HandshakeResult::ChangeState { + new_state: HandshakeState::Initial, + expect: CHandshake::INITIAL_READ + } + ); + } + 6..=9 => { + // might seem funny that we don't parse the second integer at all, but it's because + // of the relatively small size of the integers + assert_eq!( + result, + HandshakeResult::ChangeState { + new_state: HandshakeState::StaticBlock(STATIC_HANDSHAKE_WITH_AUTH), + expect: 4 + } + ); + } + 10..=22 => { + assert_eq!( + result, + HandshakeResult::ChangeState { + new_state: HandshakeState::ExpectingVariableBlock { + static_hs: STATIC_HANDSHAKE_WITH_AUTH, + uname_l: 5, + pwd_l: 8 + }, + expect: 13, + } + ); + } + 23 => { + assert_eq!( + result, + HandshakeResult::Completed(CHandshake::new( + STATIC_HANDSHAKE_WITH_AUTH, + CHandshakeAuth::new(b"sayan", b"pass1234") + )) + ); + } + _ => unreachable!(), + } + } +} + +/* + handshake with state changes +*/ + +fn run_state_changes_return_rounds(src: &[u8], expected_final_handshake: CHandshake) -> usize { + let mut rounds = 0; + let mut state = HandshakeState::default(); + let mut cursor = 0; + let mut expect_many = CHandshake::INITIAL_READ; + loop { + rounds += 1; + let buf = &src[..cursor + expect_many]; + let mut scanner = unsafe { BufferedScanner::new_with_cursor(buf, cursor) }; + match CHandshake::resume_with(&mut scanner, state) { + HandshakeResult::ChangeState { new_state, expect } => { + state = new_state; + expect_many = expect; + cursor = scanner.cursor(); + } + HandshakeResult::Completed(hs) => { + assert_eq!(hs, expected_final_handshake); + break; + } + HandshakeResult::Error(e) => panic!("unexpected handshake error: {:?}", e), + } + } + rounds +} + +#[test] +fn parse_auth_with_state_updates() { + let rounds = run_state_changes_return_rounds( + &FULL_HANDSHAKE_WITH_AUTH, + CHandshake::new( + STATIC_HANDSHAKE_WITH_AUTH, + CHandshakeAuth::new(b"sayan", b"pass1234"), + ), + ); + assert_eq!(rounds, 3); // r1 = initial read, r2 = lengths, r3 = items +} + +/* + QT-DEX/SQ +*/ + +const SQ: &str = "select * from myspace.mymodel where username = ?"; + +fn parse_staged( + query: &str, + params: [&str; N], + eq: impl Fn(SQuery), + rng: &mut impl Rng, +) { + let __query_buffer = create_simple_query(query, params); + for _ in 0..__query_buffer.len() { + let mut __read_total = 0; + let mut cursor = 0; + let mut state = QExchangeState::default(); + loop { + let remaining = __query_buffer.len() - __read_total; + let read_this_time = { + let mut cnt = 0; + if remaining == 1 { + 1 + } else { + let mut last = test_utils::random_number(1, remaining, rng); + loop { + if cnt >= 10 { + break last; + } + // if we're reading exact, try to keep it low + if last == remaining { + cnt += 1; + last = test_utils::random_number(1, remaining, rng); + } else { + break last; + } + } + } + }; + __read_total += read_this_time; + let buffered = &__query_buffer[..__read_total]; + if !state.has_reached_target(buffered) { + continue; + } + match unsafe { exchange::resume(buffered, cursor, state) } { + (new_cursor, QExchangeResult::ChangeState(new_state)) => { + cursor = new_cursor; + state = new_state; + continue; + } + (_, QExchangeResult::SQCompleted(q)) => { + eq(q); + break; + } + _ => panic!(), + } + } + } +} + +#[test] +fn staged_randomized() { + let mut rng = test_utils::rng(); + parse_staged( + SQ, + ["sayan"], + |q| { + assert_eq!(q.query_str(), SQ); + assert_eq!(q.params_str(), "sayan"); + }, + &mut rng, + ); +} + +#[test] +fn stages_manual() { + let query = create_simple_query("select * from mymodel where username = ?", ["sayan"]); + assert_eq!( + unsafe { exchange::resume(&query[..QExchangeState::MIN_READ], 0, Default::default()) }, + ( + 5, + QExchangeResult::ChangeState(QExchangeState::new_test( + exchange::QExchangeStateInternal::PendingMeta2, + 52, + 48, + 4 + )) + ) + ); + assert_eq!( + unsafe { + exchange::resume( + &query[..QExchangeState::MIN_READ + 1], + 0, + Default::default(), + ) + }, + ( + 6, + QExchangeResult::ChangeState(QExchangeState::new_test( + exchange::QExchangeStateInternal::PendingMeta2, + 52, + 48, + 40 + )) + ) + ); + assert_eq!( + unsafe { + exchange::resume( + &query[..QExchangeState::MIN_READ + 2], + 0, + Default::default(), + ) + }, + ( + 7, + QExchangeResult::ChangeState(QExchangeState::new_test( + exchange::QExchangeStateInternal::PendingData, + 52, + 48, + 40 + )) + ) + ); + // the cursor should never change + for upper_bound in QExchangeState::MIN_READ + 2..query.len() { + assert_eq!( + unsafe { exchange::resume(&query[..upper_bound], 0, Default::default()) }, + ( + 7, + QExchangeResult::ChangeState(QExchangeState::new_test( + exchange::QExchangeStateInternal::PendingData, + 52, + 48, + 40 + )) + ) + ); + } + match unsafe { exchange::resume(&query, 0, Default::default()) } { + (l, QExchangeResult::SQCompleted(q)) if l == query.len() => { + assert_eq!(q.query_str(), "select * from mymodel where username = ?"); + assert_eq!(q.params_str(), "sayan"); + } + e => panic!("expected end, got {e:?}"), + } +} + +#[test] +fn scanint_impl() { + let mut s = BufferedScanner::new(b"\n"); + assert_eq!(scanint(&mut s, true, 0), LFTIntParseResult::Error); + let mut s = BufferedScanner::new(b"12"); + assert_eq!(scanint(&mut s, true, 0), LFTIntParseResult::Partial(12)); + let mut s = BufferedScanner::new(b"12\n"); + assert_eq!(scanint(&mut s, true, 0), LFTIntParseResult::Value(12)); +} diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs new file mode 100644 index 00000000..5446f584 --- /dev/null +++ b/server/src/engine/ql/ast/mod.rs @@ -0,0 +1,381 @@ +/* + * Created on Tue Sep 13 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +pub mod traits; + +#[cfg(test)] +pub use traits::{ + parse_ast_node_full, parse_ast_node_full_with_space, parse_ast_node_multiple_full, +}; + +use { + super::lex::{Keyword, KeywordStmt, Token}, + crate::{ + engine::{ + core::EntityIDRef, + data::{cell::Datacell, lit::Lit}, + error::{QueryError, QueryResult}, + }, + util::{compiler, MaybeInit}, + }, +}; + +#[derive(Debug, PartialEq)] +/// Query parse state +pub struct State<'a, Qd> { + t: &'a [Token<'a>], + d: Qd, + i: usize, + f: bool, + cs: Option<&'static str>, +} + +impl<'a> State<'a, InplaceData> { + pub const fn new_inplace(tok: &'a [Token<'a>]) -> Self { + Self::new(tok, InplaceData::new()) + } +} + +impl<'a, Qd: QueryData<'a>> State<'a, Qd> { + fn _entity_signature_match_self_full(a: &Token<'a>, b: &Token<'a>, c: &Token<'a>) -> bool { + a.is_ident() & Token![.].eq(b) & c.is_ident() + } + fn _entity_signature_match_cs(&self, a: &Token<'a>) -> bool { + a.is_ident() & self.cs.is_some() + } + unsafe fn _entity_new_from_tokens(&mut self) -> EntityIDRef<'a> { + let space = self.fw_read().uck_read_ident(); + self.cursor_ahead(); + let entity = self.fw_read().uck_read_ident(); + EntityIDRef::new(space.as_str(), entity.as_str()) + } + unsafe fn _entity_new_from_cs(&mut self) -> EntityIDRef<'a> { + let entity = self.fw_read().uck_read_ident(); + EntityIDRef::new(self.cs.unwrap_unchecked(), entity.as_str()) + } + pub fn set_space_maybe(&mut self, maybe: Option<&'static str>) { + self.cs = maybe; + } + pub fn unset_space(&mut self) { + self.set_space_maybe(None) + } + #[cfg(test)] + pub fn set_space(&mut self, s: &'static str) { + self.set_space_maybe(Some(s)); + } + pub fn try_entity_buffered_into_state_uninit(&mut self) -> MaybeInit> { + let mut ret = MaybeInit::uninit(); + let self_has_full = Self::_entity_signature_match_self_full( + &self.t[self.cursor()], + &self.t[self.cursor() + 1], + &self.t[self.cursor() + 2], + ); + let self_has_full_cs = self._entity_signature_match_cs(&self.t[self.cursor()]); + unsafe { + if self_has_full { + ret = MaybeInit::new(self._entity_new_from_tokens()); + } else if self_has_full_cs { + ret = MaybeInit::new(self._entity_new_from_cs()); + } + } + self.poison_if_not(self_has_full | self_has_full_cs); + ret + } + pub fn try_entity_ref(&mut self) -> Option> { + let self_has_full = Self::_entity_signature_match_self_full( + self.offset_current_r(0), + self.offset_current_r(1), + self.offset_current_r(2), + ); + let self_has_pre_full = self._entity_signature_match_cs(self.offset_current_r(0)); + if self_has_full { + unsafe { + // UNSAFE(@ohsayan): +branch condition + Some(self._entity_new_from_tokens()) + } + } else { + if self_has_pre_full { + unsafe { + // UNSAFE(@ohsayan): +branch condition + Some(self._entity_new_from_cs()) + } + } else { + None + } + } + } + pub fn try_entity_ref_result(&mut self) -> QueryResult> { + self.try_entity_ref().ok_or(QueryError::QLExpectedEntity) + } +} + +impl<'a, Qd: QueryData<'a>> State<'a, Qd> { + #[inline(always)] + /// Create a new [`State`] instance using the given tokens and data + pub const fn new(t: &'a [Token<'a>], d: Qd) -> Self { + Self { + i: 0, + f: true, + t, + d, + cs: None, + } + } + #[inline(always)] + /// Returns `true` if the state is okay + pub const fn okay(&self) -> bool { + self.f + } + #[inline(always)] + /// Poison the state flag + pub fn poison(&mut self) { + self.f = false; + } + #[inline(always)] + /// Poison the state flag if the expression is satisfied + pub fn poison_if(&mut self, fuse: bool) { + self.f &= !fuse; + } + #[inline(always)] + /// Poison the state flag if the expression is not satisfied + pub fn poison_if_not(&mut self, fuse: bool) { + self.poison_if(!fuse); + } + #[inline(always)] + /// Move the cursor ahead by 1 + pub fn cursor_ahead(&mut self) { + self.cursor_ahead_by(1) + } + #[inline(always)] + /// Move the cursor ahead by the given count + pub fn cursor_ahead_by(&mut self, by: usize) { + self.i += by; + } + #[inline(always)] + /// Move the cursor ahead by 1 if the expression is satisfied + pub fn cursor_ahead_if(&mut self, iff: bool) { + self.cursor_ahead_by(iff as _); + } + #[inline(always)] + /// Read the cursor + pub fn read(&self) -> &'a Token<'a> { + &self.t[self.i] + } + #[inline(always)] + /// Return a subslice of the tokens using the current state + pub fn current(&self) -> &'a [Token<'a>] { + &self.t[self.i..] + } + #[inline(always)] + pub fn offset_current_r(&self, offset: usize) -> &Token<'a> { + &self.t[self.round_cursor_up(offset)] + } + #[inline(always)] + /// Returns a count of the number of consumable tokens remaining + pub fn remaining(&self) -> usize { + self.t.len() - self.i + } + #[inline(always)] + /// Read and forward the cursor + pub fn fw_read(&mut self) -> &'a Token<'a> { + let r = self.read(); + self.cursor_ahead(); + r + } + #[inline(always)] + /// Check if the token stream has alteast `many` count of tokens + pub fn has_remaining(&self, many: usize) -> bool { + self.remaining() >= many + } + #[inline(always)] + /// Returns true if the token stream has been exhausted + pub fn exhausted(&self) -> bool { + self.remaining() == 0 + } + #[inline(always)] + /// Returns true if the token stream has **not** been exhausted + pub fn not_exhausted(&self) -> bool { + self.remaining() != 0 + } + #[inline(always)] + /// Check if the current cursor can read a lit (with context from the data source); rounded + pub fn can_read_lit_rounded(&self) -> bool { + let mx = self.round_cursor(); + Qd::can_read_lit_from(&self.d, &self.t[mx]) & (mx == self.i) + } + #[inline(always)] + /// Check if a lit can be read using the given token with context from the data source + pub fn can_read_lit_from(&self, tok: &'a Token<'a>) -> bool { + Qd::can_read_lit_from(&self.d, tok) + } + #[inline(always)] + /// Read a lit from the cursor and data source + /// + /// ## Safety + /// - Must ensure that `Self::can_read_lit_rounded` is true + pub unsafe fn read_cursor_lit_unchecked(&mut self) -> Lit<'a> { + let tok = self.read(); + Qd::read_lit(&mut self.d, tok) + } + #[inline(always)] + /// Read a lit from the given token + /// + /// ## Safety + /// - Must ensure that `Self::can_read_lit_from` is true for the token + pub unsafe fn read_lit_unchecked_from(&mut self, tok: &'a Token<'a>) -> Lit<'a> { + Qd::read_lit(&mut self.d, tok) + } + #[inline(always)] + /// Check if the cursor equals the given token; rounded + pub fn cursor_rounded_eq(&self, tok: Token<'a>) -> bool { + let mx = self.round_cursor(); + (self.t[mx] == tok) & (mx == self.i) + } + #[inline(always)] + /// Check if the cursor equals the given token + pub(crate) fn cursor_eq(&self, token: Token) -> bool { + self.t[self.i] == token + } + #[inline(always)] + /// Move the cursor back by 1 + pub(crate) fn cursor_back(&mut self) { + self.cursor_back_by(1); + } + #[inline(always)] + /// Move the cursor back by the given count + pub(crate) fn cursor_back_by(&mut self, by: usize) { + self.i -= by; + } + #[inline(always)] + pub(crate) fn cursor_has_ident_rounded(&self) -> bool { + self.offset_current_r(0).is_ident() & self.not_exhausted() + } + #[inline(always)] + /// Check if the current token stream matches the signature of an arity(0) fn; rounded + /// + /// NOTE: Consider using a direct comparison without rounding + pub(crate) fn cursor_signature_match_fn_arity0_rounded(&self) -> bool { + (self.offset_current_r(0).is_ident()) + & (Token![() open].eq(self.offset_current_r(1))) + & (Token![() close].eq(self.offset_current_r(2))) + & self.has_remaining(3) + } + #[inline(always)] + /// Reads a lit using the given token and the internal data source and return a data type + /// + /// ## Safety + /// + /// Caller should have checked that the token matches a lit signature and that enough data is available + /// in the data source. (ideally should run `can_read_lit_from` or `can_read_lit_rounded`) + pub unsafe fn read_lit_into_data_type_unchecked_from(&mut self, tok: &'a Token) -> Datacell { + self.d.read_data_type(tok) + } + #[inline(always)] + /// Loop condition for tt and non-poisoned state only + pub fn loop_tt(&self) -> bool { + self.not_exhausted() & self.okay() + } + #[inline(always)] + /// Returns the position of the cursor + pub(crate) fn cursor(&self) -> usize { + self.i + } + #[inline(always)] + /// Returns true if the cursor is an ident + pub(crate) fn cursor_is_ident(&self) -> bool { + self.read().is_ident() + } + #[inline(always)] + fn round_cursor_up(&self, up: usize) -> usize { + core::cmp::min(self.t.len() - 1, self.i + up) + } + #[inline(always)] + fn round_cursor(&self) -> usize { + self.round_cursor_up(0) + } + pub fn try_statement(&mut self) -> QueryResult { + if compiler::unlikely(self.exhausted()) { + compiler::cold_call(|| Err(QueryError::QLExpectedStatement)) + } else { + match self.fw_read() { + Token::Keyword(Keyword::Statement(stmt)) => Ok(*stmt), + _ => Err(QueryError::QLExpectedStatement), + } + } + } + pub fn ensure_minimum_for_blocking_stmt(&self) -> QueryResult<()> { + if self.remaining() < 2 { + return Err(QueryError::QLExpectedStatement); + } else { + Ok(()) + } + } +} + +pub trait QueryData<'a> { + /// Check if the given token is a lit, while also checking `self`'s data if necessary + fn can_read_lit_from(&self, tok: &Token) -> bool; + /// Read a lit using the given token, using `self`'s data as necessary + /// + /// ## Safety + /// The current token **must match** the signature of a lit + unsafe fn read_lit(&mut self, tok: &'a Token) -> Lit<'a>; + /// Read a lit using the given token and then copy it into a [`DataType`] + /// + /// ## Safety + /// The current token must match the signature of a lit + unsafe fn read_data_type(&mut self, tok: &'a Token) -> Datacell; + /// Returns true if the data source has enough data + fn nonzero(&self) -> bool; +} + +#[derive(Debug)] +pub struct InplaceData; +impl InplaceData { + #[inline(always)] + pub const fn new() -> Self { + Self + } +} + +impl<'a> QueryData<'a> for InplaceData { + #[inline(always)] + fn can_read_lit_from(&self, tok: &Token) -> bool { + tok.is_lit() + } + #[inline(always)] + unsafe fn read_lit(&mut self, tok: &'a Token) -> Lit<'a> { + tok.uck_read_lit().as_ir() + } + #[inline(always)] + unsafe fn read_data_type(&mut self, tok: &'a Token) -> Datacell { + Datacell::from(::read_lit(self, tok)) + } + #[inline(always)] + fn nonzero(&self) -> bool { + true + } +} diff --git a/server/src/engine/ql/ast/traits.rs b/server/src/engine/ql/ast/traits.rs new file mode 100644 index 00000000..5ccb37e8 --- /dev/null +++ b/server/src/engine/ql/ast/traits.rs @@ -0,0 +1,151 @@ +/* + * Created on Thu Feb 02 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(test)] +use crate::engine::ql::{ast::InplaceData, lex::Token}; +use crate::engine::{ + error::{QueryError, QueryResult}, + ql::ast::{QueryData, State}, +}; + +/// An AST node +pub trait ASTNode<'a>: Sized { + /// This AST node MUST use the full token range + const MUST_USE_FULL_TOKEN_RANGE: bool; + /// This AST node MUST use the full token range, and it also verifies that this is the case + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool; + /// This AST node doesn't handle "deep errors" (for example, recursive collections) + const VERIFY_STATE_BEFORE_RETURN: bool = false; + /// A hardened parse that guarantees: + /// - The result is verified (even if it is a deep error) + /// - The result utilizes the full token range + fn parse_from_state_hardened>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let r = Self::__base_impl_parse_from_state(state)?; + if Self::VERIFY_STATE_BEFORE_RETURN { + // must verify + if !state.okay() { + return Err(QueryError::QLInvalidSyntax); + } + } + if Self::MUST_USE_FULL_TOKEN_RANGE { + if !Self::VERIFIES_FULL_TOKEN_RANGE_USAGE { + if state.not_exhausted() { + return Err(QueryError::QLInvalidSyntax); + } + } + } + Ok(r) + } + /// Parse this AST node from the given state + /// + /// Note to implementors: + /// - If the implementor uses a cow style parse, then set [`ASTNode::VERIFY`] to + /// true + /// - Try to propagate errors via [`State`] if possible + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult; + #[cfg(test)] + fn test_parse_from_state>(state: &mut State<'a, Qd>) -> QueryResult { + let r = ::__base_impl_parse_from_state(state); + if Self::VERIFY_STATE_BEFORE_RETURN { + return if state.okay() { + r + } else { + Err(QueryError::QLInvalidSyntax) + }; + } + r + } + #[cfg(test)] + /// Parse multiple nodes of this AST node type. Intended for the test suite. + fn _multiple_from_state>(_: &mut State<'a, Qd>) -> QueryResult> { + unimplemented!() + } + #[cfg(test)] + fn multiple_from_state>(state: &mut State<'a, Qd>) -> QueryResult> { + let r = ::_multiple_from_state(state); + if Self::VERIFY_STATE_BEFORE_RETURN { + return if state.okay() { + r + } else { + Err(QueryError::QLInvalidSyntax) + }; + } + r + } + #[cfg(test)] + /// Parse this AST node utilizing the full token-stream. Intended for the test suite. + fn from_insecure_tokens_full(tok: &'a [Token<'a>]) -> QueryResult { + let mut state = State::new(tok, InplaceData::new()); + let r = ::test_parse_from_state(&mut state)?; + assert!(state.exhausted()); + Ok(r) + } + #[cfg(test)] + fn from_insecure_tokens_full_with_space( + tok: &'a [Token<'a>], + space_name: &'static str, + ) -> QueryResult { + let mut state = State::new(tok, InplaceData::new()); + state.set_space(space_name); + let r = ::test_parse_from_state(&mut state)?; + assert!(state.exhausted()); + Ok(r) + } + #[cfg(test)] + /// Parse multiple nodes of this AST node type, utilizing the full token stream. + /// Intended for the test suite. + fn multiple_from_insecure_tokens_full(tok: &'a [Token<'a>]) -> QueryResult> { + let mut state = State::new(tok, InplaceData::new()); + let r = Self::multiple_from_state(&mut state); + if state.exhausted() && state.okay() { + r + } else { + Err(QueryError::QLInvalidSyntax) + } + } +} + +#[cfg(test)] +pub fn parse_ast_node_full<'a, N: ASTNode<'a>>(tok: &'a [Token<'a>]) -> QueryResult { + N::from_insecure_tokens_full(tok) +} +#[cfg(test)] +pub fn parse_ast_node_full_with_space<'a, N: ASTNode<'a>>( + tok: &'a [Token<'a>], + space_name: &'static str, +) -> QueryResult { + N::from_insecure_tokens_full_with_space(tok, space_name) +} +#[cfg(test)] +pub fn parse_ast_node_multiple_full<'a, N: ASTNode<'a>>( + tok: &'a [Token<'a>], +) -> QueryResult> { + N::multiple_from_insecure_tokens_full(tok) +} diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs new file mode 100644 index 00000000..a8ddcba2 --- /dev/null +++ b/server/src/engine/ql/dcl.rs @@ -0,0 +1,173 @@ +/* + * Created on Thu Sep 21 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + data::DictGeneric, + error::{QueryError, QueryResult}, + ql::{ + ast::{traits, QueryData, State}, + ddl::syn, + lex::Ident, + }, +}; + +#[derive(Debug, PartialEq)] +pub enum SysctlCommand<'a> { + /// `sysctl create user ...` + CreateUser(UserDecl<'a>), + /// `sysctl drop user ...` + DropUser(UserDel<'a>), + /// `systcl alter user ...` + AlterUser(UserDecl<'a>), + /// `sysctl status` + ReportStatus, +} + +impl<'a> SysctlCommand<'a> { + pub fn needs_root(&self) -> bool { + !matches!(self, Self::ReportStatus) + } +} + +impl<'a> traits::ASTNode<'a> for SysctlCommand<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + if state.remaining() < 2 { + return Err(QueryError::QLUnexpectedEndOfStatement); + } + let (a, b) = (state.fw_read(), state.fw_read()); + let alter = Token![alter].eq(a) & b.ident_eq("user"); + let create = Token![create].eq(a) & b.ident_eq("user"); + let drop = Token![drop].eq(a) & b.ident_eq("user"); + let status = a.ident_eq("report") & b.ident_eq("status"); + if !(create | drop | status | alter) { + return Err(QueryError::QLUnknownStatement); + } + if create { + UserDecl::parse(state).map(SysctlCommand::CreateUser) + } else if drop { + UserDel::parse(state).map(SysctlCommand::DropUser) + } else if alter { + UserDecl::parse(state).map(SysctlCommand::AlterUser) + } else { + Ok(SysctlCommand::ReportStatus) + } + } +} + +fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult> { + /* + [username] with { password: [password], ... } + ^cursor + 7 tokens + */ + if state.remaining() < 7 { + return Err(QueryError::QLInvalidSyntax); + } + let token_buffer = state.current(); + // initial sig + let signature_okay = token_buffer[0].is_ident() + & token_buffer[1].eq(&Token![with]) + & token_buffer[2].eq(&Token![open {}]); + // get props + state.poison_if_not(signature_okay); + state.cursor_ahead_by(2); + let Some(dict) = syn::parse_dict(state) else { + return Err(QueryError::QLInvalidCollectionSyntax); + }; + let maybe_username = unsafe { + // UNSAFE(@ohsayan): the dict parse ensures state correctness + token_buffer[0].uck_read_ident() + }; + if state.not_exhausted() | !state.okay() { + // we shouldn't have more tokens + return Err(QueryError::QLInvalidSyntax); + } + Ok(UserMeta { + username: maybe_username, + options: dict, + }) +} + +struct UserMeta<'a> { + username: Ident<'a>, + options: DictGeneric, +} + +#[derive(Debug, PartialEq)] +pub struct UserDecl<'a> { + username: Ident<'a>, + options: DictGeneric, +} + +impl<'a> UserDecl<'a> { + pub(in crate::engine::ql) fn new(username: Ident<'a>, options: DictGeneric) -> Self { + Self { username, options } + } + pub fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + parse(state).map(|UserMeta { username, options }: UserMeta| Self::new(username, options)) + } + pub fn username(&self) -> &str { + self.username.as_str() + } + pub fn options_mut(&mut self) -> &mut DictGeneric { + &mut self.options + } + pub fn options(&self) -> &DictGeneric { + &self.options + } +} + +#[derive(Debug, PartialEq)] +pub struct UserDel<'a> { + username: Ident<'a>, +} + +impl<'a> UserDel<'a> { + pub(in crate::engine::ql) fn new(username: Ident<'a>) -> Self { + Self { username } + } + /// Parse a `user del` DCL command + /// + /// MUSTENDSTREAM: YES + pub fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + if state.cursor_has_ident_rounded() & (state.remaining() == 1) { + let username = unsafe { + // UNSAFE(@ohsayan): +boundck + state.read().uck_read_ident() + }; + state.cursor_ahead(); + return Ok(Self::new(username)); + } + Err(QueryError::QLInvalidSyntax) + } + pub fn username(&self) -> &str { + self.username.as_str() + } +} diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs new file mode 100644 index 00000000..b6069a05 --- /dev/null +++ b/server/src/engine/ql/ddl/alt.rs @@ -0,0 +1,217 @@ +/* + * Created on Thu Feb 02 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::syn::{self, DictFoldState, ExpandedField}, + crate::{ + engine::{ + core::EntityIDRef, + data::DictGeneric, + error::{QueryError, QueryResult}, + ql::{ + ast::{QueryData, State}, + lex::{Ident, Token}, + }, + }, + util::compiler, + }, +}; + +#[derive(Debug, PartialEq)] +/// An alter space query with corresponding data +pub struct AlterSpace<'a> { + pub space_name: Ident<'a>, + pub updated_props: DictGeneric, +} + +impl<'a> AlterSpace<'a> { + #[cfg(test)] + pub fn new(space_name: Ident<'a>, updated_props: DictGeneric) -> Self { + Self { + space_name, + updated_props, + } + } + #[inline(always)] + /// Parse alter space from tokens + fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + if compiler::unlikely(state.remaining() <= 3) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + let space_name = state.fw_read(); + state.poison_if_not(space_name.is_ident()); + state.poison_if_not(state.cursor_eq(Token![with])); + state.cursor_ahead(); // ignore errors + state.poison_if_not(state.cursor_eq(Token![open {}])); + state.cursor_ahead(); // ignore errors + + if compiler::unlikely(!state.okay()) { + return Err(QueryError::QLInvalidSyntax); + } + + let space_name = unsafe { + // UNSAFE(@ohsayan): We just verified that `space_name` is an ident + space_name.uck_read_ident() + }; + let mut d = DictGeneric::new(); + syn::rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut d); + if state.okay() { + Ok(AlterSpace { + space_name, + updated_props: d, + }) + } else { + Err(QueryError::QLInvalidCollectionSyntax) + } + } +} + +#[derive(Debug, PartialEq)] +pub struct AlterModel<'a> { + pub(in crate::engine) model: EntityIDRef<'a>, + pub(in crate::engine) kind: AlterKind<'a>, +} + +impl<'a> AlterModel<'a> { + #[inline(always)] + pub fn new(model: EntityIDRef<'a>, kind: AlterKind<'a>) -> Self { + Self { model, kind } + } +} + +#[derive(Debug, PartialEq)] +/// The alter operation kind +pub enum AlterKind<'a> { + Add(Box<[ExpandedField<'a>]>), + Remove(Box<[Ident<'a>]>), + Update(Box<[ExpandedField<'a>]>), +} + +impl<'a> AlterModel<'a> { + #[inline(always)] + /// Parse an [`AlterKind`] from the given token stream + fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + // alter model mymodel remove x + if state.remaining() <= 2 || !state.cursor_has_ident_rounded() { + return compiler::cold_rerr(QueryError::QLInvalidSyntax); + // FIXME(@ohsayan): bad because no specificity + } + let model_name = state.try_entity_ref_result()?; + let kind = match state.fw_read() { + Token![add] => AlterKind::alter_add(state), + Token![remove] => AlterKind::alter_remove(state), + Token![update] => AlterKind::alter_update(state), + _ => Err(QueryError::QLExpectedStatement), + }; + kind.map(|kind| AlterModel::new(model_name, kind)) + } +} + +impl<'a> AlterKind<'a> { + #[inline(always)] + /// Parse the expression for `alter model <> add (..)` + fn alter_add>(state: &mut State<'a, Qd>) -> QueryResult { + ExpandedField::parse_multiple(state).map(Self::Add) + } + #[inline(always)] + /// Parse the expression for `alter model <> add (..)` + fn alter_update>(state: &mut State<'a, Qd>) -> QueryResult { + ExpandedField::parse_multiple(state).map(Self::Update) + } + #[inline(always)] + /// Parse the expression for `alter model <> remove (..)` + fn alter_remove>(state: &mut State<'a, Qd>) -> QueryResult { + const DEFAULT_REMOVE_COL_CNT: usize = 4; + /* + WARNING: No trailing commas allowed + ::= | ( )* + */ + if compiler::unlikely(state.exhausted()) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + + let r = match state.fw_read() { + Token::Ident(id) => Box::new([*id]), + Token![() open] => { + let mut stop = false; + let mut cols = Vec::with_capacity(DEFAULT_REMOVE_COL_CNT); + while state.loop_tt() && !stop { + match state.fw_read() { + Token::Ident(ref ident) => { + cols.push(*ident); + let nx_comma = state.cursor_rounded_eq(Token![,]); + let nx_close = state.cursor_rounded_eq(Token![() close]); + state.poison_if_not(nx_comma | nx_close); + stop = nx_close; + state.cursor_ahead_if(state.okay()); + } + _ => { + state.cursor_back(); + state.poison(); + break; + } + } + } + state.poison_if_not(stop); + if state.okay() { + cols.into_boxed_slice() + } else { + return Err(QueryError::QLInvalidSyntax); + } + } + _ => return Err(QueryError::QLInvalidSyntax), + }; + Ok(Self::Remove(r)) + } +} + +mod impls { + use { + super::{AlterModel, AlterSpace}, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for AlterModel<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + } + impl<'a> ASTNode<'a> for AlterSpace<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + } +} diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs new file mode 100644 index 00000000..693cc8b9 --- /dev/null +++ b/server/src/engine/ql/ddl/crt.rs @@ -0,0 +1,209 @@ +/* + * Created on Thu Feb 02 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::syn::{self, DictFoldState, FieldSpec}, + crate::{ + engine::{ + core::EntityIDRef, + data::DictGeneric, + error::{QueryError, QueryResult}, + ql::{ + ast::{QueryData, State}, + lex::Ident, + }, + }, + util::compiler, + }, +}; + +fn sig_if_not_exists<'a, Qd: QueryData<'a>>(state: &State<'a, Qd>) -> bool { + Token![if].eq(state.offset_current_r(0)) + & Token![not].eq(state.offset_current_r(1)) + & Token![exists].eq(state.offset_current_r(2)) + & (state.remaining() >= 3) +} + +#[derive(Debug, PartialEq)] +/// A space +pub struct CreateSpace<'a> { + /// the space name + pub space_name: Ident<'a>, + /// properties + pub props: DictGeneric, + pub if_not_exists: bool, +} + +impl<'a> CreateSpace<'a> { + #[inline(always)] + /// Parse space data from the given tokens + fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + // smallest declaration: `create space myspace` -> >= 1 token + if compiler::unlikely(state.remaining() < 1) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + // check for `if not exists` + let if_not_exists = sig_if_not_exists(state); + state.cursor_ahead_by(if_not_exists as usize * 3); + // get space name + if state.exhausted() { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + let space_name = state.fw_read(); + state.poison_if_not(space_name.is_ident()); + // either we have `with` or nothing. don't be stupid + let has_more_properties = state.cursor_rounded_eq(Token![with]); + state.poison_if_not(has_more_properties | state.exhausted()); + state.cursor_ahead_if(has_more_properties); // +WITH + let mut d = DictGeneric::new(); + // properties + if has_more_properties & state.okay() { + syn::rfold_dict(DictFoldState::OB, state, &mut d); + } + if state.okay() { + Ok(CreateSpace { + space_name: unsafe { + // UNSAFE(@ohsayan): we checked if `space_name` with `is_ident` above + space_name.uck_read_ident() + }, + props: d, + if_not_exists, + }) + } else { + Err(QueryError::QLInvalidSyntax) + } + } +} + +#[derive(Debug, PartialEq)] +/// A model definition +pub struct CreateModel<'a> { + /// the model name + pub(in crate::engine) model_name: EntityIDRef<'a>, + /// the fields + pub(in crate::engine) fields: Vec>, + /// properties + pub(in crate::engine) props: DictGeneric, + /// if not exists + pub(in crate::engine) if_not_exists: bool, +} + +/* + model definition: + create model mymodel( + [primary|null] ident: type, + ) +*/ + +impl<'a> CreateModel<'a> { + #[cfg(test)] + pub fn new( + model_name: EntityIDRef<'a>, + fields: Vec>, + props: DictGeneric, + if_not_exists: bool, + ) -> Self { + Self { + model_name, + fields, + props, + if_not_exists, + } + } + fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + if compiler::unlikely(state.remaining() < 10) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + // if not exists? + let if_not_exists = sig_if_not_exists(state); + state.cursor_ahead_by(if_not_exists as usize * 3); + // model name; ignore errors + let model_uninit = state.try_entity_buffered_into_state_uninit(); + state.poison_if_not(state.cursor_eq(Token![() open])); + state.cursor_ahead(); + // fields + let mut stop = false; + let mut fields = Vec::with_capacity(2); + while state.loop_tt() && !stop { + fields.push(FieldSpec::parse(state)?); + let nx_close = state.cursor_rounded_eq(Token![() close]); + let nx_comma = state.cursor_rounded_eq(Token![,]); + state.poison_if_not(nx_close | nx_comma); + state.cursor_ahead_if(nx_close | nx_comma); + stop = nx_close; + } + state.poison_if_not(stop); + // check props + let mut props = DictGeneric::new(); + if state.cursor_rounded_eq(Token![with]) { + state.cursor_ahead(); + // parse props + syn::rfold_dict(DictFoldState::OB, state, &mut props); + } + // we're done + if state.okay() { + Ok(Self { + model_name: unsafe { + // UNSAFE(@ohsayan): we verified if `model_name` is initialized through the state + model_uninit.assume_init() + }, + fields, + props, + if_not_exists, + }) + } else { + Err(QueryError::QLInvalidSyntax) + } + } +} + +mod impls { + use { + super::{CreateModel, CreateSpace}, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for CreateSpace<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + } + impl<'a> ASTNode<'a> for CreateModel<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + } +} diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs new file mode 100644 index 00000000..b164b16c --- /dev/null +++ b/server/src/engine/ql/ddl/drop.rs @@ -0,0 +1,167 @@ +/* + * Created on Wed Feb 01 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::EntityIDRef, + error::{QueryError, QueryResult}, + ql::{ + ast::{QueryData, State}, + lex::Ident, + }, +}; + +fn sig_if_exists<'a, Qd: QueryData<'a>>(state: &State<'a, Qd>) -> bool { + Token![if].eq(state.offset_current_r(0)) & Token![exists].eq(state.offset_current_r(1)) +} + +#[derive(Debug, PartialEq)] +/// A generic representation of `drop` query +pub struct DropSpace<'a> { + pub(in crate::engine) space: Ident<'a>, + pub(in crate::engine) force: bool, + pub(in crate::engine) if_exists: bool, +} + +impl<'a> DropSpace<'a> { + #[inline(always)] + /// Instantiate + pub const fn new(space: Ident<'a>, force: bool, if_exists: bool) -> Self { + Self { + space, + force, + if_exists, + } + } + fn parse>(state: &mut State<'a, Qd>) -> QueryResult> { + /* + either drop space OR drop space allow not empty + */ + let if_exists = check_if_exists(state)?; + if state.cursor_is_ident() { + let ident = state.fw_read(); + // either `force` or nothing + return Ok(DropSpace::new( + unsafe { + // UNSAFE(@ohsayan): Safe because the if predicate ensures that tok[0] (relative) is indeed an ident + ident.uck_read_ident() + }, + false, + if_exists, + )); + } else { + if ddl_allow_non_empty(state) { + state.cursor_ahead_by(3); + let space_name = unsafe { + // UNSAFE(@ohsayan): verified in branch + state.fw_read().uck_read_ident() + }; + return Ok(DropSpace::new(space_name, true, if_exists)); + } + } + Err(QueryError::QLInvalidSyntax) + } +} + +fn check_if_exists<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Result { + if state.exhausted() { + return Err(QueryError::QLUnexpectedEndOfStatement); + } + let if_exists = sig_if_exists(state); + state.cursor_ahead_by((if_exists as usize) << 1); + if state.exhausted() { + return Err(QueryError::QLUnexpectedEndOfStatement); + } + Ok(if_exists) +} + +#[inline(always)] +fn ddl_allow_non_empty<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> bool { + let tok_allow = Token![allow].eq(state.offset_current_r(0)); + let tok_not = Token![not].eq(state.offset_current_r(1)); + let tok_empty = state.offset_current_r(2).ident_eq("empty"); + let name = state.offset_current_r(3).is_ident(); + (tok_allow & tok_not & tok_empty & name) & (state.remaining() >= 4) +} + +#[derive(Debug, PartialEq)] +pub struct DropModel<'a> { + pub(in crate::engine) entity: EntityIDRef<'a>, + pub(in crate::engine) force: bool, + pub(in crate::engine) if_exists: bool, +} + +impl<'a> DropModel<'a> { + #[inline(always)] + pub fn new(entity: EntityIDRef<'a>, force: bool, if_exists: bool) -> Self { + Self { + entity, + force, + if_exists, + } + } + fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + let if_exists = check_if_exists(state)?; + if state.cursor_is_ident() { + let e = state.try_entity_ref_result()?; + return Ok(DropModel::new(e, false, if_exists)); + } else { + if ddl_allow_non_empty(state) { + state.cursor_ahead_by(3); // allow not empty + let e = state.try_entity_ref_result()?; + return Ok(DropModel::new(e, true, if_exists)); + } + } + Err(QueryError::QLInvalidSyntax) + } +} + +mod impls { + use { + super::{DropModel, DropSpace}, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for DropModel<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + } + impl<'a> ASTNode<'a> for DropSpace<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + } +} diff --git a/server/src/engine/ql/ddl/mod.rs b/server/src/engine/ql/ddl/mod.rs new file mode 100644 index 00000000..a1dd6643 --- /dev/null +++ b/server/src/engine/ql/ddl/mod.rs @@ -0,0 +1,115 @@ +/* + * Created on Wed Nov 16 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[macro_use] +pub(in crate::engine) mod syn; +pub(in crate::engine) mod alt; +pub(in crate::engine) mod crt; +pub(in crate::engine) mod drop; + +use { + super::{ + ast::{traits::ASTNode, QueryData, State}, + lex::{Ident, Token}, + }, + crate::engine::{ + core::EntityIDRef, + error::{QueryError, QueryResult}, + }, +}; + +#[derive(Debug, PartialEq)] +pub enum Use<'a> { + Space(Ident<'a>), + RefreshCurrent, + Null, +} + +impl<'a> ASTNode<'a> for Use<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + /* + should have either an ident or null + */ + if state.exhausted() | (state.remaining() > 2) { + return Err(QueryError::QLInvalidSyntax); + } + Ok(match state.fw_read() { + Token::Ident(new_space) => Self::Space(*new_space), + Token![null] => Self::Null, + Token![$] => { + if state.exhausted() { + return Err(QueryError::QLInvalidSyntax); + } + match state.fw_read() { + Token::Ident(id) if id.eq_ignore_ascii_case("current") => Self::RefreshCurrent, + _ => return Err(QueryError::QLInvalidSyntax), + } + } + _ => return Err(QueryError::QLInvalidSyntax), + }) + } +} + +#[derive(Debug, PartialEq)] +pub enum Inspect<'a> { + Global, + Space(Ident<'a>), + Model(EntityIDRef<'a>), +} + +impl<'a> ASTNode<'a> for Inspect<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + if state.exhausted() { + return Err(QueryError::QLUnexpectedEndOfStatement); + } + let me = match state.fw_read() { + Token::Ident(id) if id.eq_ignore_ascii_case("global") => Self::Global, + Token![space] => { + if state.exhausted() { + return Err(QueryError::QLUnexpectedEndOfStatement); + } + match state.fw_read() { + Token::Ident(space) => Self::Space(*space), + _ => return Err(QueryError::QLInvalidSyntax), + } + } + Token![model] => { + let entity = state.try_entity_ref_result()?; + Self::Model(entity) + } + _ => return Err(QueryError::QLInvalidSyntax), + }; + Ok(me) + } +} diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs new file mode 100644 index 00000000..1c1a9388 --- /dev/null +++ b/server/src/engine/ql/ddl/syn.rs @@ -0,0 +1,626 @@ +/* + * Created on Wed Feb 01 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +/* + Most grammar tools are pretty much "off the shelf" which makes some things incredibly hard to achieve (such + as custom error injection logic). To make things icier, Rust's integration with these tools (like lex) is not + very "refined." Hence, it is best for us to implement our own parsers. In the future, I plan to optimize our + rule checkers but that's not a concern at the moment. + + This module makes use of DFAs with additional flags, accepting a token stream as input to generate appropriate + structures, and have provable correctness. Hence, the unsafe code used here is correct, because the states are + only transitioned to if the input is accepted. If you do find otherwise, please file a bug report. The + transitions are currently very inefficient but can be made much faster. + + TODO: The SMs can be reduced greatly, enocded to fixed-sized structures even, so do that + NOTE: The `ASTNode` impls are test-only. most of the time they do stupid things. we should never rely on `ASTNode` + impls for `syn` elements + + -- + Sayan (@ohsayan) + Feb. 2, 2023 +*/ + +use crate::{ + engine::{ + data::{ + cell::Datacell, + dict::{DictEntryGeneric, DictGeneric}, + }, + error::{QueryError, QueryResult}, + ql::{ + ast::{QueryData, State}, + lex::{Ident, Token}, + }, + }, + util::{compiler, MaybeInit}, +}; + +/// This macro constructs states for our machine +/// +/// **DO NOT** construct states manually +macro_rules! states { + ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])*$v:vis$state:ident = $statexp:expr),+ $(,)?}) => { + #[::core::prelude::v1::derive(::core::cmp::PartialEq,::core::cmp::Eq,::core::clone::Clone,::core::marker::Copy)] + $(#[$attr])+$vis struct $stateid {__base: $statebase} + impl $stateid {$($(#[$tyattr])*$v const $state:Self=$stateid{__base: $statexp,};)*} + impl ::core::fmt::Debug for $stateid { + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + let r = match self.__base {$($statexp => ::core::stringify!($state),)* _ => panic!("invalid state"),}; + ::core::write!(f, "{}::{}", ::core::stringify!($stateid), r) + } + } + } +} + +/* + Context-free dict +*/ + +states! { + /// The dict fold state + pub struct DictFoldState: u8 { + FINAL = 0xFF, + pub(super) OB = 0x00, + pub(super) CB_OR_IDENT = 0x01, + COLON = 0x02, + LIT_OR_OB = 0x03, + COMMA_OR_CB = 0x04, + } +} + +trait Breakpoint<'a> { + const HAS_BREAKPOINT: bool; + fn check_breakpoint(state: DictFoldState, tok: &'a Token<'a>) -> bool; +} + +struct NoBreakpoint; +impl<'a> Breakpoint<'a> for NoBreakpoint { + const HAS_BREAKPOINT: bool = false; + fn check_breakpoint(_: DictFoldState, _: &'a Token<'a>) -> bool { + false + } +} +struct TypeBreakpoint; +impl<'a> Breakpoint<'a> for TypeBreakpoint { + const HAS_BREAKPOINT: bool = true; + fn check_breakpoint(state: DictFoldState, tok: &'a Token<'a>) -> bool { + (state == DictFoldState::CB_OR_IDENT) & matches!(tok, Token![type]) + } +} + +/// Fold a dictionary +fn _rfold_dict<'a, Qd, Bp>( + mut mstate: DictFoldState, + state: &mut State<'a, Qd>, + dict: &mut DictGeneric, +) -> bool +where + Qd: QueryData<'a>, + Bp: Breakpoint<'a>, +{ + /* + NOTE: Assume rules wherever applicable + + ::= "{" + ::= "}" + ::= "," + ::= ":" + ::= ( ( | ) )* * + */ + let mut key = MaybeInit::uninit(); + while state.loop_tt() { + match (state.fw_read(), mstate) { + (Token![open {}], DictFoldState::OB) => { + // open + mstate = DictFoldState::CB_OR_IDENT; + } + (Token![close {}], DictFoldState::CB_OR_IDENT | DictFoldState::COMMA_OR_CB) => { + // well, that's the end of the dict + mstate = DictFoldState::FINAL; + break; + } + (Token::Ident(id), DictFoldState::CB_OR_IDENT) => { + key = MaybeInit::new(*id); + // found a key, now expect colon + mstate = DictFoldState::COLON; + } + (Token![:], DictFoldState::COLON) => { + // found colon, now lit or ob + mstate = DictFoldState::LIT_OR_OB; + } + (tok, DictFoldState::LIT_OR_OB) if state.can_read_lit_from(tok) => { + // found lit + let v = unsafe { + // UNSAFE(@ohsayan): verified at guard + state.read_lit_unchecked_from(tok).into() + }; + state.poison_if_not( + dict.insert( + unsafe { + // UNSAFE(@ohsayan): we switch to this state only when we are in the LIT_OR_OB state. this means that we've already read in a key + key.take().as_str().into() + }, + v, + ) + .is_none(), + ); + // after lit we're either done or expect something else + mstate = DictFoldState::COMMA_OR_CB; + } + (Token![null], DictFoldState::LIT_OR_OB) => { + // found a null + state.poison_if_not( + dict.insert( + unsafe { + // UNSAFE(@ohsayan): we only switch to this when we've already read in a key + key.take().as_str().into() + }, + DictEntryGeneric::Data(Datacell::null()), + ) + .is_none(), + ); + // after a null (essentially counts as a lit) we're either done or expect something else + mstate = DictFoldState::COMMA_OR_CB; + } + (Token![open {}], DictFoldState::LIT_OR_OB) => { + // found a nested dict + let mut ndict = DictGeneric::new(); + _rfold_dict::(DictFoldState::CB_OR_IDENT, state, &mut ndict); + state.poison_if_not( + dict.insert( + unsafe { + // UNSAFE(@ohsayan): correct again because whenever we hit an expression position, we've already read in a key (ident) + key.take().as_str().into() + }, + DictEntryGeneric::Map(ndict), + ) + .is_none(), + ); + mstate = DictFoldState::COMMA_OR_CB; + } + (Token![,], DictFoldState::COMMA_OR_CB) => { + // expecting a comma, found it. now expect a close brace or an ident + mstate = DictFoldState::CB_OR_IDENT; + } + (this_tok, this_key) + if Bp::HAS_BREAKPOINT && Bp::check_breakpoint(this_key, this_tok) => + { + // reached custom breakpoint + return true; + } + _ => { + state.cursor_back(); + state.poison(); + break; + } + } + } + state.poison_if_not(mstate == DictFoldState::FINAL); + false +} + +pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>( + mstate: DictFoldState, + state: &mut State<'a, Qd>, + dict: &mut DictGeneric, +) { + _rfold_dict::(mstate, state, dict); +} + +pub fn parse_dict<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option { + let mut d = DictGeneric::new(); + rfold_dict(DictFoldState::OB, state, &mut d); + if state.okay() { + Some(d) + } else { + None + } +} + +pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>>( + mstate: DictFoldState, + state: &mut State<'a, Qd>, + dict: &mut DictGeneric, +) -> bool { + _rfold_dict::(mstate, state, dict) +} + +#[derive(Debug, PartialEq)] +/// A layer contains a type and corresponding metadata +pub struct LayerSpec<'a> { + pub(in crate::engine) ty: Ident<'a>, + pub(in crate::engine) props: DictGeneric, +} + +impl<'a> LayerSpec<'a> { + //// Create a new layer + #[cfg(test)] + pub const fn new(ty: Ident<'a>, props: DictGeneric) -> Self { + Self { ty, props } + } +} + +states! { + /// Layer fold state + pub struct LayerFoldState: u8 { + BEGIN_IDENT = 0x01, + FOLD_INCOMPLETE = 0x03, + FINAL_OR_OB = 0x04, + FINAL = 0xFF, + } +} + +fn rfold_layers<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>, layers: &mut Vec>) { + let mut mstate = LayerFoldState::BEGIN_IDENT; + let mut ty = MaybeInit::uninit(); + let mut props = Default::default(); + while state.loop_tt() { + match (state.fw_read(), mstate) { + (Token::Ident(id), LayerFoldState::BEGIN_IDENT) => { + ty = MaybeInit::new(*id); + mstate = LayerFoldState::FINAL_OR_OB; + } + (Token![open {}], LayerFoldState::FINAL_OR_OB) => { + // we were done ... but we found some props + if rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut props) { + // we have more layers + // but we first need a colon + state.poison_if_not(state.cursor_rounded_eq(Token![:])); + state.cursor_ahead_if(state.okay()); + rfold_layers(state, layers); + // we are yet to parse the remaining props + mstate = LayerFoldState::FOLD_INCOMPLETE; + } else { + // didn't hit bp; so we should be done here + mstate = LayerFoldState::FINAL; + break; + } + } + (Token![close {}], LayerFoldState::FOLD_INCOMPLETE) => { + // found end of the dict. roger the terminal! + mstate = LayerFoldState::FINAL; + break; + } + (Token![,], LayerFoldState::FOLD_INCOMPLETE) => { + // we found a comma, but we should finish parsing the dict + rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut props); + // we're done parsing + mstate = LayerFoldState::FINAL; + break; + } + // FIXME(@ohsayan): if something falls apart, it's the arm below + (_, LayerFoldState::FINAL_OR_OB) => { + state.cursor_back(); + mstate = LayerFoldState::FINAL; + break; + } + _ => { + state.cursor_back(); + state.poison(); + break; + } + } + } + if ((mstate == LayerFoldState::FINAL) | (mstate == LayerFoldState::FINAL_OR_OB)) & state.okay() + { + layers.push(LayerSpec { + ty: unsafe { + // UNSAFE(@ohsayan): our start state always looks for an ident + ty.take() + }, + props, + }); + } else { + state.poison(); + } +} + +#[derive(Debug, PartialEq)] +/// A field definition +pub struct FieldSpec<'a> { + /// the field name + pub(in crate::engine) field_name: Ident<'a>, + /// layers + pub(in crate::engine) layers: Vec>, + /// is null + pub(in crate::engine) null: bool, + /// is primary + pub(in crate::engine) primary: bool, +} + +impl<'a> FieldSpec<'a> { + #[cfg(test)] + pub fn new( + field_name: Ident<'a>, + layers: Vec>, + null: bool, + primary: bool, + ) -> Self { + Self { + field_name, + layers, + null, + primary, + } + } + pub fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + if compiler::unlikely(state.remaining() < 2) { + // smallest field: `ident: type` + return Err(QueryError::QLUnexpectedEndOfStatement); + } + // check if primary or null + let is_primary = state.cursor_eq(Token![primary]); + state.cursor_ahead_if(is_primary); + let is_null = state.cursor_eq(Token![null]); + state.cursor_ahead_if(is_null); + state.poison_if(is_primary & is_null); + // parse layers + // field name + let field_name = match (state.fw_read(), state.fw_read()) { + (Token::Ident(id), Token![:]) => id, + _ => return Err(QueryError::QLInvalidSyntax), + }; + // layers + let mut layers = Vec::new(); + rfold_layers(state, &mut layers); + if state.okay() { + Ok(FieldSpec { + field_name: *field_name, + layers, + null: is_null, + primary: is_primary, + }) + } else { + Err(QueryError::QLInvalidTypeDefinitionSyntax) + } + } +} + +#[derive(Debug, PartialEq)] +/// An [`ExpandedField`] is a full field definition with advanced metadata +pub struct ExpandedField<'a> { + pub(in crate::engine) field_name: Ident<'a>, + pub(in crate::engine) layers: Vec>, + pub(in crate::engine) props: DictGeneric, +} + +impl<'a> ExpandedField<'a> { + #[cfg(test)] + pub fn new(field_name: Ident<'a>, layers: Vec>, props: DictGeneric) -> Self { + Self { + field_name, + layers, + props, + } + } + #[inline(always)] + /// Parse a field declared using the field syntax + pub(super) fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + if compiler::unlikely(state.remaining() < 6) { + // smallest: fieldname { type: ident } + return Err(QueryError::QLUnexpectedEndOfStatement); + } + let field_name = state.fw_read(); + state.poison_if_not(field_name.is_ident()); + state.poison_if_not(state.cursor_eq(Token![open {}])); + state.cursor_ahead(); + // ignore errors; now attempt a tymeta-like parse + let mut props = DictGeneric::new(); + let mut layers = Vec::new(); + if rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut props) { + // this has layers. fold them; but don't forget the colon + if compiler::unlikely(state.exhausted()) { + // we need more tokens + return Err(QueryError::QLUnexpectedEndOfStatement); + } + state.poison_if_not(state.cursor_eq(Token![:])); + state.cursor_ahead(); + rfold_layers(state, &mut layers); + match state.fw_read() { + Token![,] => { + rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut props); + } + Token![close {}] => { + // hit end + } + _ => { + state.poison(); + } + } + } + if state.okay() { + Ok(Self { + field_name: unsafe { + // UNSAFE(@ohsayan): We just verified if `field_name` returns `is_ident` + field_name.uck_read_ident() + }, + props, + layers, + }) + } else { + Err(QueryError::QLInvalidSyntax) + } + } + #[inline(always)] + /// Parse multiple fields declared using the field syntax. Flag setting allows or disallows reset syntax + pub fn parse_multiple>( + state: &mut State<'a, Qd>, + ) -> QueryResult> { + const DEFAULT_ADD_COL_CNT: usize = 4; + /* + WARNING: No trailing commas allowed + + ::= ( )* + + Smallest length: + alter model add myfield { type string } + */ + if compiler::unlikely(state.remaining() < 5) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + match state.read() { + Token::Ident(_) => { + let ef = Self::parse(state)?; + Ok([ef].into()) + } + Token![() open] => { + state.cursor_ahead(); + let mut stop = false; + let mut cols = Vec::with_capacity(DEFAULT_ADD_COL_CNT); + while state.loop_tt() && !stop { + match state.read() { + Token::Ident(_) => { + let ef = Self::parse(state)?; + cols.push(ef); + let nx_comma = state.cursor_rounded_eq(Token![,]); + let nx_close = state.cursor_rounded_eq(Token![() close]); + stop = nx_close; + state.poison_if_not(nx_comma | nx_close); + state.cursor_ahead_if(state.okay()); + } + _ => { + state.poison(); + break; + } + } + } + state.poison_if_not(stop); + if state.okay() { + Ok(cols.into_boxed_slice()) + } else { + Err(QueryError::QLInvalidSyntax) + } + } + _ => Err(QueryError::QLExpectedStatement), + } + } +} + +#[cfg(test)] +pub use impls::{DictBasic, DictTypeMeta, DictTypeMetaSplit}; +#[cfg(test)] +mod impls { + use { + super::{ + rfold_dict, rfold_layers, rfold_tymeta, DictFoldState, DictGeneric, ExpandedField, + FieldSpec, LayerSpec, + }, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for ExpandedField<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + fn _multiple_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult> { + Self::parse_multiple(state).map(Vec::from) + } + } + impl<'a> ASTNode<'a> for LayerSpec<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let mut layers = Vec::new(); + rfold_layers(state, &mut layers); + assert!(layers.len() == 1); + Ok(layers.swap_remove(0)) + } + fn _multiple_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult> { + let mut l = Vec::new(); + rfold_layers(state, &mut l); + Ok(l) + } + } + #[derive(sky_macros::Wrapper, Debug)] + pub struct DictBasic(DictGeneric); + impl<'a> ASTNode<'a> for DictBasic { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let mut dict = DictGeneric::new(); + rfold_dict(DictFoldState::OB, state, &mut dict); + Ok(Self(dict)) + } + } + #[derive(sky_macros::Wrapper, Debug)] + pub struct DictTypeMetaSplit(DictGeneric); + impl<'a> ASTNode<'a> for DictTypeMetaSplit { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let mut dict = DictGeneric::new(); + rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut dict); + Ok(Self(dict)) + } + } + #[derive(sky_macros::Wrapper, Debug)] + pub struct DictTypeMeta(DictGeneric); + impl<'a> ASTNode<'a> for DictTypeMeta { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let mut dict = DictGeneric::new(); + rfold_tymeta(DictFoldState::OB, state, &mut dict); + Ok(Self(dict)) + } + } + impl<'a> ASTNode<'a> for FieldSpec<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + } +} diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs new file mode 100644 index 00000000..7ecb0127 --- /dev/null +++ b/server/src/engine/ql/dml/del.rs @@ -0,0 +1,124 @@ +/* + * Created on Fri Jan 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(test)] +use super::WhereClauseCollection; +use { + super::WhereClause, + crate::{ + engine::{ + core::EntityIDRef, + error::{QueryError, QueryResult}, + ql::ast::{QueryData, State}, + }, + util::compiler, + }, +}; + +/* + Impls for delete + --- + Smallest statement: + delete model:primary_key +*/ + +#[derive(Debug, PartialEq)] +pub struct DeleteStatement<'a> { + pub(super) entity: EntityIDRef<'a>, + pub(super) wc: WhereClause<'a>, +} + +impl<'a> DeleteStatement<'a> { + pub const fn entity(&self) -> EntityIDRef<'a> { + self.entity + } + pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> { + &mut self.wc + } +} + +impl<'a> DeleteStatement<'a> { + #[inline(always)] + #[cfg(test)] + pub(super) fn new(entity: EntityIDRef<'a>, wc: WhereClause<'a>) -> Self { + Self { entity, wc } + } + #[inline(always)] + #[cfg(test)] + pub fn new_test(entity: EntityIDRef<'a>, wc: WhereClauseCollection<'a>) -> Self { + Self::new(entity, WhereClause::new(wc)) + } + #[inline(always)] + pub fn parse_delete>(state: &mut State<'a, Qd>) -> QueryResult { + /* + TODO(@ohsayan): Volcano + smallest tt: + delete from model where x = 1 + ^1 ^2 ^3 ^4 ^5 + */ + if compiler::unlikely(state.remaining() < 5) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + // from + entity + state.poison_if_not(state.cursor_eq(Token![from])); + state.cursor_ahead(); // ignore errors (if any) + let entity = state.try_entity_buffered_into_state_uninit(); + // where + clauses + state.poison_if_not(state.cursor_eq(Token![where])); + state.cursor_ahead(); // ignore errors + let wc = WhereClause::parse_where(state); + if compiler::likely(state.okay()) { + Ok(Self { + entity: unsafe { + // UNSAFE(@ohsayan): Safety guaranteed by state + entity.assume_init() + }, + wc, + }) + } else { + compiler::cold_rerr(QueryError::QLInvalidSyntax) + } + } +} + +mod impls { + use { + super::DeleteStatement, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for DeleteStatement<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse_delete(state) + } + } +} diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs new file mode 100644 index 00000000..9bd4069b --- /dev/null +++ b/server/src/engine/ql/dml/ins.rs @@ -0,0 +1,470 @@ +/* + * Created on Fri Jan 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::{ + engine::{ + core::EntityIDRef, + data::cell::Datacell, + error::{QueryError, QueryResult}, + ql::{ + ast::{QueryData, State}, + lex::{Ident, Token}, + }, + }, + util::compiler, + }, + std::{ + collections::HashMap, + time::{Duration, SystemTime, UNIX_EPOCH}, + }, + uuid::Uuid, +}; + +/* + Impls for insert +*/ + +pub const T_UUIDSTR: &str = "4593264b-0231-43e9-b0aa-50784f14e204"; +pub const T_TIMESEC: u64 = 1673187839_u64; + +type ProducerFn = fn() -> Datacell; + +// base +#[inline(always)] +fn pfnbase_time() -> Duration { + if cfg!(debug_assertions) { + Duration::from_secs(T_TIMESEC) + } else { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + } +} +#[inline(always)] +fn pfnbase_uuid() -> Uuid { + if cfg!(debug_assertions) { + Uuid::parse_str(T_UUIDSTR).unwrap() + } else { + Uuid::new_v4() + } +} +// impl +#[inline(always)] +fn pfn_timesec() -> Datacell { + Datacell::new_uint_default(pfnbase_time().as_secs()) +} +#[inline(always)] +fn pfn_uuidstr() -> Datacell { + Datacell::new_str(pfnbase_uuid().to_string().into_boxed_str()) +} +#[inline(always)] +fn pfn_uuidbin() -> Datacell { + Datacell::new_bin(pfnbase_uuid().as_bytes().to_vec().into_boxed_slice()) +} + +static PRODUCER_G: [u8; 4] = [0, 2, 3, 0]; +static PRODUCER_F: [(&[u8], ProducerFn); 3] = [ + (b"uuidstr", pfn_uuidstr), + (b"uuidbin", pfn_uuidbin), + (b"timesec", pfn_timesec), +]; +const MAGIC_1: [u8; 7] = *b"cp21rLd"; +const MAGIC_2: [u8; 7] = *b"zS8zgaK"; +const MAGIC_L: usize = MAGIC_1.len(); + +#[inline(always)] +fn hashf(key: &[u8], m: &[u8]) -> u32 { + let mut i = 0; + let mut s = 0; + while i < key.len() { + s += m[i % MAGIC_L] as u32 * key[i] as u32; + i += 1; + } + s % PRODUCER_G.len() as u32 +} +#[inline(always)] +fn hashp(key: &[u8]) -> u32 { + (PRODUCER_G[hashf(key, &MAGIC_1) as usize] + PRODUCER_G[hashf(key, &MAGIC_2) as usize]) as u32 + % PRODUCER_G.len() as u32 +} +#[inline(always)] +fn ldfunc(func: Ident<'_>) -> Option { + let func = func.as_bytes(); + let ph = hashp(func) as usize; + let min = ph.min(PRODUCER_F.len() - 1); + let data = PRODUCER_F[min]; + if data.0 == func { + Some(data.1) + } else { + None + } +} + +/// ## Panics +/// - If tt length is less than 1 +pub(super) fn parse_list<'a, Qd: QueryData<'a>>( + state: &mut State<'a, Qd>, + list: &mut Vec, +) -> Option { + let mut stop = state.cursor_eq(Token![close []]); + state.cursor_ahead_if(stop); + let mut overall_dscr = None; + let mut prev_nlist_dscr = None; + while state.not_exhausted() && state.okay() && !stop { + let d = match state.fw_read() { + tok if state.can_read_lit_from(tok) => { + unsafe { + // UNSAFE(@ohsayan): the if guard guarantees correctness + state.read_lit_into_data_type_unchecked_from(tok) + } + } + Token![open []] => { + // a nested list + let mut nested_list = Vec::new(); + let nlist_dscr = parse_list(state, &mut nested_list); + // check type return + state.poison_if_not( + prev_nlist_dscr.is_none() + || nlist_dscr.is_none() + || prev_nlist_dscr == nlist_dscr, + ); + if prev_nlist_dscr.is_none() && nlist_dscr.is_some() { + prev_nlist_dscr = nlist_dscr; + } + Datacell::new_list(nested_list) + } + Token![@] if state.cursor_signature_match_fn_arity0_rounded() => match unsafe { + // UNSAFE(@ohsayan): Just verified at guard + handle_func_sub(state) + } { + Some(value) => value, + None => { + state.poison(); + break; + } + }, + _ => { + state.cursor_back(); + state.poison(); + break; + } + }; + state.poison_if_not(list.is_empty() || d.kind() == list[0].kind()); + overall_dscr = Some(d.kind()); + list.push(d); + let nx_comma = state.cursor_rounded_eq(Token![,]); + let nx_csqrb = state.cursor_rounded_eq(Token![close []]); + state.poison_if_not(nx_comma | nx_csqrb); + state.cursor_ahead_if(state.okay()); + stop = nx_csqrb; + } + overall_dscr +} + +#[inline(always)] +/// ## Safety +/// - Cursor must match arity(0) function signature +unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option { + let func = state.fw_read().uck_read_ident(); + state.cursor_ahead_by(2); // skip tt:paren + ldfunc(func).map(move |f| f()) +} + +/// ## Panics +/// - If tt is empty +pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>( + state: &mut State<'a, Qd>, +) -> Vec { + let mut stop = state.cursor_eq(Token![() close]); + state.cursor_ahead_if(stop); + let mut data = Vec::new(); + while state.not_exhausted() && state.okay() && !stop { + match state.fw_read() { + tok if state.can_read_lit_from(tok) => unsafe { + // UNSAFE(@ohsayan): if guard guarantees correctness + data.push(state.read_lit_into_data_type_unchecked_from(tok)); + }, + Token![open []] if state.not_exhausted() => { + let mut l = Vec::new(); + let _ = parse_list(state, &mut l); + data.push(l.into()); + } + Token![null] => data.push(Datacell::null()), + Token![@] if state.cursor_signature_match_fn_arity0_rounded() => match unsafe { + // UNSAFE(@ohsayan): Just verified at guard + handle_func_sub(state) + } { + Some(value) => data.push(value), + None => { + state.poison(); + break; + } + }, + _ => { + state.cursor_back(); + state.poison(); + break; + } + } + let nx_comma = state.cursor_rounded_eq(Token![,]); + let nx_csprn = state.cursor_rounded_eq(Token![() close]); + state.poison_if_not(nx_comma | nx_csprn); + state.cursor_ahead_if(state.okay()); + stop = nx_csprn; + } + data +} + +/// ## Panics +/// Panics if tt is empty +pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>( + state: &mut State<'a, Qd>, +) -> HashMap, Datacell> { + let mut stop = state.cursor_eq(Token![close {}]); + state.cursor_ahead_if(stop); + let mut data = HashMap::with_capacity(2); + while state.has_remaining(3) && state.okay() && !stop { + let field = state.fw_read(); + let colon = state.fw_read(); + let expr = state.fw_read(); + state.poison_if_not(Token![:].eq(colon)); + match (field, expr) { + (Token::Ident(id), tok) if state.can_read_lit_from(tok) => { + let ldata = unsafe { + // UNSAFE(@ohsayan): The if guard guarantees correctness + state.read_lit_into_data_type_unchecked_from(tok) + }; + state.poison_if_not(data.insert(*id, ldata).is_none()); + } + (Token::Ident(id), Token![null]) => { + state.poison_if_not(data.insert(*id, Datacell::null()).is_none()); + } + (Token::Ident(id), Token![open []]) if state.not_exhausted() => { + let mut l = Vec::new(); + let _ = parse_list(state, &mut l); + state.poison_if_not(data.insert(*id, l.into()).is_none()); + } + (Token::Ident(id), Token![@]) if state.cursor_signature_match_fn_arity0_rounded() => { + match unsafe { + // UNSAFE(@ohsayan): Just verified at guard + handle_func_sub(state) + } { + Some(value) => state.poison_if_not(data.insert(*id, value).is_none()), + None => { + state.poison(); + break; + } + } + } + _ => { + state.cursor_back_by(3); + state.poison(); + break; + } + } + let nx_comma = state.cursor_rounded_eq(Token![,]); + let nx_csbrc = state.cursor_rounded_eq(Token![close {}]); + state.poison_if_not(nx_comma | nx_csbrc); + state.cursor_ahead_if(state.okay()); + stop = nx_csbrc; + } + data +} + +#[derive(Debug, PartialEq)] +pub enum InsertData<'a> { + Ordered(Vec), + Map(HashMap, Datacell>), +} + +impl<'a> InsertData<'a> { + pub fn column_count(&self) -> usize { + match self { + Self::Ordered(ord) => ord.len(), + Self::Map(m) => m.len(), + } + } +} + +impl<'a> From> for InsertData<'a> { + fn from(v: Vec) -> Self { + Self::Ordered(v) + } +} + +impl<'a> From, Datacell>> for InsertData<'a> { + fn from(m: HashMap, Datacell>) -> Self { + Self::Map(m) + } +} + +#[derive(Debug, PartialEq)] +pub struct InsertStatement<'a> { + pub(super) entity: EntityIDRef<'a>, + pub(super) data: InsertData<'a>, +} + +impl<'a> InsertStatement<'a> { + #[inline(always)] + #[cfg(test)] + pub fn new(entity: EntityIDRef<'a>, data: InsertData<'a>) -> Self { + Self { entity, data } + } + pub fn entity(&self) -> EntityIDRef<'a> { + self.entity + } + pub fn data(self) -> InsertData<'a> { + self.data + } +} + +impl<'a> InsertStatement<'a> { + pub fn parse_insert>(state: &mut State<'a, Qd>) -> QueryResult { + /* + smallest: + insert into model (primarykey) + ^1 ^2 ^3 ^4 ^5 + */ + if compiler::unlikely(state.remaining() < 5) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + state.poison_if_not(state.cursor_eq(Token![into])); + state.cursor_ahead(); // ignore errors + + // entity + let entity = state.try_entity_buffered_into_state_uninit(); + let mut data = None; + match state.fw_read() { + Token![() open] if state.not_exhausted() => { + let this_data = parse_data_tuple_syntax(state); + data = Some(InsertData::Ordered(this_data)); + } + Token![open {}] if state.not_exhausted() => { + let this_data = parse_data_map_syntax(state); + data = Some(InsertData::Map(this_data)); + } + _ => { + state.poison(); + } + } + if state.okay() { + let data = unsafe { + // UNSAFE(@ohsayan): state's flag guarantees correctness (see wildcard branch) + data.unwrap_unchecked() + }; + Ok(InsertStatement { + entity: unsafe { + // UNSAFE(@ohsayan): state's flag ensures correctness (see Entity::parse_entity) + entity.assume_init() + }, + data, + }) + } else { + compiler::cold_rerr(QueryError::QLInvalidSyntax) + } + } +} + +#[cfg(test)] +pub use impls::test::{DataMap, DataTuple, List}; + +use crate::engine::data::tag::TagClass; +mod impls { + use { + super::InsertStatement, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for InsertStatement<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse_insert(state) + } + } + #[cfg(test)] + pub mod test { + use { + super::super::{ + parse_data_map_syntax, parse_data_tuple_syntax, parse_list, Datacell, HashMap, + }, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + #[derive(sky_macros::Wrapper, Debug)] + pub struct List(Vec); + impl<'a> ASTNode<'a> for List { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let mut l = Vec::new(); + parse_list(state, &mut l); + Ok(List(l)) + } + } + #[derive(sky_macros::Wrapper, Debug)] + pub struct DataTuple(Vec); + impl<'a> ASTNode<'a> for DataTuple { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let r = parse_data_tuple_syntax(state); + Ok(Self(r)) + } + } + #[derive(sky_macros::Wrapper, Debug)] + pub struct DataMap(HashMap, Datacell>); + impl<'a> ASTNode<'a> for DataMap { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let r = parse_data_map_syntax(state); + Ok(Self( + r.into_iter() + .map(|(ident, val)| (ident.to_string().into_boxed_str(), val)) + .collect(), + )) + } + } + } +} diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs new file mode 100644 index 00000000..2c23a825 --- /dev/null +++ b/server/src/engine/ql/dml/mod.rs @@ -0,0 +1,193 @@ +/* + * Created on Fri Oct 14 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +/* + TODO(@ohsayan): For now we've settled for an imprecise error site reporting for simplicity, which we + should augment in future revisions of the QL engine +*/ + +pub mod del; +pub mod ins; +pub mod sel; +pub mod upd; + +use { + super::{ + ast::{QueryData, State}, + lex::Ident, + }, + crate::{engine::data::lit::Lit, util::compiler}, + std::collections::HashMap, +}; + +#[inline(always)] +fn u(b: bool) -> u8 { + b as _ +} + +/* + Misc +*/ + +/* + Contexts +*/ + +#[derive(Debug, PartialEq)] +pub struct RelationalExpr<'a> { + pub(super) lhs: Ident<'a>, + pub(super) rhs: Lit<'a>, + pub(super) opc: u8, +} + +impl<'a> RelationalExpr<'a> { + #[inline(always)] + pub(super) fn new(lhs: Ident<'a>, rhs: Lit<'a>, opc: u8) -> RelationalExpr<'a> { + Self { lhs, rhs, opc } + } + pub(super) const OP_EQ: u8 = 1; + pub(super) const OP_NE: u8 = 2; + pub(super) const OP_GT: u8 = 3; + pub(super) const OP_GE: u8 = 4; + pub(super) const OP_LT: u8 = 5; + pub(super) const OP_LE: u8 = 6; + pub fn filter_hint_none(&self) -> bool { + self.opc == Self::OP_EQ + } + pub fn rhs(&self) -> Lit<'a> { + self.rhs.clone() + } + #[inline(always)] + fn parse_operator>(state: &mut State<'a, Qd>) -> u8 { + let tok = state.current(); + let op_eq = u(tok[0] == Token![=]) * Self::OP_EQ; + let op_ne = u(tok[0] == Token![!] && tok[1] == Token![=]) * Self::OP_NE; + let op_ge = u(tok[0] == Token![>] && tok[1] == Token![=]) * Self::OP_GE; + let op_gt = u(tok[0] == Token![>] && op_ge == 0) * Self::OP_GT; + let op_le = u(tok[0] == Token![<] && tok[1] == Token![=]) * Self::OP_LE; + let op_lt = u(tok[0] == Token![<] && op_le == 0) * Self::OP_LT; + let opc = op_eq + op_ne + op_ge + op_gt + op_le + op_lt; + state.poison_if_not(opc != 0); + state.cursor_ahead_by(1 + (opc & 1 == 0) as usize); + opc + } + #[inline(always)] + fn try_parse>(state: &mut State<'a, Qd>) -> Option { + if compiler::likely(state.remaining() < 3) { + return compiler::cold_val(None); + } + let ident = state.read(); + state.poison_if_not(ident.is_ident()); + state.cursor_ahead(); // ignore any errors + let operator = Self::parse_operator(state); + state.poison_if_not(state.can_read_lit_rounded()); + if compiler::likely(state.okay()) { + unsafe { + // UNSAFE(@ohsayan): we verified this above + let lit = state.read_cursor_lit_unchecked(); + state.cursor_ahead(); + // UNSAFE(@ohsayan): we checked if `ident` returns `is_ident` and updated state + Some(Self::new(ident.uck_read_ident(), lit, operator)) + } + } else { + None + } + } +} + +#[derive(Debug, PartialEq)] +pub struct WhereClause<'a> { + c: WhereClauseCollection<'a>, +} + +type WhereClauseCollection<'a> = HashMap, RelationalExpr<'a>>; + +impl<'a> WhereClause<'a> { + #[inline(always)] + pub(super) fn new(c: WhereClauseCollection<'a>) -> Self { + Self { c } + } + pub fn clauses_mut(&mut self) -> &mut WhereClauseCollection<'a> { + &mut self.c + } + #[inline(always)] + fn parse_where_and_append_to>( + state: &mut State<'a, Qd>, + c: &mut WhereClauseCollection<'a>, + ) { + let mut has_more = true; + while has_more && state.not_exhausted() && state.okay() { + if let Some(expr) = RelationalExpr::try_parse(state) { + state.poison_if_not(c.insert(expr.lhs, expr).is_none()); + } + has_more = state.cursor_rounded_eq(Token![and]); + state.cursor_ahead_if(has_more); + } + } + #[inline(always)] + /// Parse a where context + /// + /// Notes: + /// - Enforce a minimum of 1 clause + pub(super) fn parse_where>(state: &mut State<'a, Qd>) -> Self { + let mut c = HashMap::with_capacity(2); + Self::parse_where_and_append_to(state, &mut c); + state.poison_if(c.is_empty()); + Self { c } + } +} + +#[cfg(test)] +mod impls { + use { + super::{RelationalExpr, WhereClause}, + crate::engine::{ + error::{QueryError, QueryResult}, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for WhereClause<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let wh = Self::parse_where(state); + Ok(wh) + } + } + impl<'a> ASTNode<'a> for RelationalExpr<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::try_parse(state).ok_or(QueryError::QLInvalidSyntax) + } + } +} diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs new file mode 100644 index 00000000..e379db28 --- /dev/null +++ b/server/src/engine/ql/dml/sel.rs @@ -0,0 +1,259 @@ +/* + * Created on Fri Jan 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(test)] +use super::WhereClauseCollection; +use { + super::WhereClause, + crate::{ + engine::{ + core::EntityIDRef, + error::{QueryError, QueryResult}, + ql::{ + ast::{QueryData, State}, + lex::{Ident, Token}, + }, + }, + util::compiler, + }, +}; + +/* + Impls for select +*/ + +#[derive(Debug, PartialEq)] +pub struct SelectStatement<'a> { + /// the entity + pub(super) entity: EntityIDRef<'a>, + /// fields in order of querying. will be zero when wildcard is set + pub(super) fields: Vec>, + /// whether a wildcard was passed + pub(super) wildcard: bool, + /// where clause + pub(super) clause: WhereClause<'a>, +} + +impl<'a> SelectStatement<'a> { + #[inline(always)] + #[cfg(test)] + pub(crate) fn new_test( + entity: EntityIDRef<'a>, + fields: Vec>, + wildcard: bool, + clauses: WhereClauseCollection<'a>, + ) -> SelectStatement<'a> { + Self::new(entity, fields, wildcard, clauses) + } + #[inline(always)] + #[cfg(test)] + fn new( + entity: EntityIDRef<'a>, + fields: Vec>, + wildcard: bool, + clauses: WhereClauseCollection<'a>, + ) -> SelectStatement<'a> { + Self { + entity, + fields, + wildcard, + clause: WhereClause::new(clauses), + } + } + pub fn entity(&self) -> EntityIDRef<'a> { + self.entity + } + pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> { + &mut self.clause + } + pub fn is_wildcard(&self) -> bool { + self.wildcard + } + pub fn into_fields(self) -> Vec> { + self.fields + } +} + +impl<'a> SelectStatement<'a> { + pub fn parse_select>(state: &mut State<'a, Qd>) -> QueryResult { + /* + Smallest query: + select * from model + ^ ^ ^ + 1 2 3 + */ + if compiler::unlikely(state.remaining() < 3) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + let mut select_fields = Vec::new(); + let is_wildcard = state.cursor_eq(Token![*]); + state.cursor_ahead_if(is_wildcard); + while state.not_exhausted() && state.okay() && !is_wildcard { + match state.read() { + Token::Ident(id) => select_fields.push(*id), + _ => break, + } + state.cursor_ahead(); + let nx_comma = state.cursor_rounded_eq(Token![,]); + let nx_from = state.cursor_rounded_eq(Token![from]); + state.poison_if_not(nx_comma | nx_from); + state.cursor_ahead_if(nx_comma); + } + state.poison_if_not(is_wildcard | !select_fields.is_empty()); + // we should have from + model + if compiler::unlikely(state.remaining() < 2 || !state.okay()) { + return compiler::cold_rerr(QueryError::QLInvalidSyntax); + } + state.poison_if_not(state.cursor_eq(Token![from])); + state.cursor_ahead(); // ignore errors + let entity = state.try_entity_buffered_into_state_uninit(); + let mut clauses = <_ as Default>::default(); + if state.cursor_rounded_eq(Token![where]) { + state.cursor_ahead(); + WhereClause::parse_where_and_append_to(state, &mut clauses); + state.poison_if(clauses.is_empty()); + } + if compiler::likely(state.okay()) { + Ok(SelectStatement { + entity: unsafe { + // UNSAFE(@ohsayan): `process_entity` and `okay` assert correctness + entity.assume_init() + }, + fields: select_fields, + wildcard: is_wildcard, + clause: WhereClause::new(clauses), + }) + } else { + compiler::cold_rerr(QueryError::QLInvalidSyntax) + } + } +} + +#[derive(Debug, PartialEq)] +pub struct SelectAllStatement<'a> { + pub entity: EntityIDRef<'a>, + pub fields: Vec>, + pub wildcard: bool, + pub limit: u64, +} + +impl<'a> SelectAllStatement<'a> { + #[cfg(test)] + pub fn test_new( + entity: EntityIDRef<'a>, + fields: Vec>, + wildcard: bool, + limit: u64, + ) -> Self { + Self::new(entity, fields, wildcard, limit) + } + fn new(entity: EntityIDRef<'a>, fields: Vec>, wildcard: bool, limit: u64) -> Self { + Self { + entity, + fields, + wildcard, + limit, + } + } + fn parse>(state: &mut State<'a, Qd>) -> QueryResult { + /* + smallest query: select all * from mymodel limit 10 + */ + if state.remaining() < 5 { + return Err(QueryError::QLUnexpectedEndOfStatement); + } + let mut select_fields = Vec::new(); + let is_wildcard = state.cursor_eq(Token![*]); + state.cursor_ahead_if(is_wildcard); + while state.not_exhausted() && state.okay() && !is_wildcard { + match state.read() { + Token::Ident(id) => select_fields.push(*id), + _ => break, + } + state.cursor_ahead(); + let nx_comma = state.cursor_rounded_eq(Token![,]); + let nx_from = state.cursor_rounded_eq(Token![from]); + state.poison_if_not(nx_comma | nx_from); + state.cursor_ahead_if(nx_comma); + } + state.poison_if_not(is_wildcard | !select_fields.is_empty()); + if state.remaining() < 4 { + return Err(QueryError::QLUnexpectedEndOfStatement); + } + state.poison_if_not(state.cursor_eq(Token![from])); + state.cursor_ahead(); // ignore error + let entity = state.try_entity_buffered_into_state_uninit(); + state.poison_if_not(state.cursor_rounded_eq(Token![limit])); + state.cursor_ahead_if(state.okay()); // we did read limit + state.poison_if(state.exhausted()); // we MUST have the limit + if state.okay() { + let lit = unsafe { state.fw_read().uck_read_lit() }; + match lit.try_uint() { + Some(limit) => { + return unsafe { + // UNSAFE(@ohsayan): state guarantees this works + Ok(Self::new( + entity.assume_init(), + select_fields, + is_wildcard, + limit, + )) + }; + } + _ => {} + } + } + Err(QueryError::QLInvalidSyntax) + } +} + +mod impls { + use { + super::{SelectAllStatement, SelectStatement}, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for SelectStatement<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse_select(state) + } + } + impl<'a> ASTNode<'a> for SelectAllStatement<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse(state) + } + } +} diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs new file mode 100644 index 00000000..8eaf8ea3 --- /dev/null +++ b/server/src/engine/ql/dml/upd.rs @@ -0,0 +1,254 @@ +/* + * Created on Fri Jan 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{u, WhereClause}, + crate::{ + engine::{ + core::{query_meta::AssignmentOperator, EntityIDRef}, + data::lit::Lit, + error::{QueryError, QueryResult}, + ql::{ + ast::{QueryData, State}, + lex::Ident, + }, + }, + util::compiler, + }, +}; + +/* + Impls for update +*/ + +static OPERATOR: [AssignmentOperator; 6] = [ + AssignmentOperator::Assign, + AssignmentOperator::Assign, + AssignmentOperator::AddAssign, + AssignmentOperator::SubAssign, + AssignmentOperator::MulAssign, + AssignmentOperator::DivAssign, +]; + +#[derive(Debug, PartialEq)] +pub struct AssignmentExpression<'a> { + /// the LHS ident + pub lhs: Ident<'a>, + /// the RHS lit + pub rhs: Lit<'a>, + /// operator + pub operator_fn: AssignmentOperator, +} + +impl<'a> AssignmentExpression<'a> { + pub fn new(lhs: Ident<'a>, rhs: Lit<'a>, operator_fn: AssignmentOperator) -> Self { + Self { + lhs, + rhs, + operator_fn, + } + } + fn parse_and_append_expression>( + state: &mut State<'a, Qd>, + expressions: &mut Vec, + ) { + /* + smallest expr: + x = y + */ + if compiler::unlikely(state.remaining() < 3) { + state.poison(); + return; + } + let lhs = state.fw_read(); + state.poison_if_not(lhs.is_ident()); + let op_ass = u(state.cursor_eq(Token![=])); + let op_add = u(state.cursor_eq(Token![+])) * 2; + let op_sub = u(state.cursor_eq(Token![-])) * 3; + let op_mul = u(state.cursor_eq(Token![*])) * 4; + let op_div = u(state.cursor_eq(Token![/])) * 5; + let operator_code = op_ass + op_add + op_sub + op_mul + op_div; + unsafe { + // UNSAFE(@ohsayan): A hint, obvious from above + if operator_code > 5 { + impossible!(); + } + } + state.cursor_ahead(); + state.poison_if(operator_code == 0); + let has_double_assign = state.cursor_rounded_eq(Token![=]); + let double_assign_okay = operator_code != 1 && has_double_assign; + let single_assign_okay = operator_code == 1 && !double_assign_okay; + state.poison_if_not(single_assign_okay | double_assign_okay); + state.cursor_ahead_if(double_assign_okay); + state.poison_if_not(state.can_read_lit_rounded()); + + if state.okay() { + unsafe { + // UNSAFE(@ohsayan): Checked lit, state flag ensures we have ident for lhs + let rhs = state.read_cursor_lit_unchecked(); + state.cursor_ahead(); + expressions.push(AssignmentExpression::new( + // UNSAFE(@ohsayan): we verified if `lhs` returns `is_ident` + lhs.uck_read_ident(), + rhs, + OPERATOR[operator_code as usize], + )) + } + } + } +} + +#[derive(Debug, PartialEq)] +pub struct UpdateStatement<'a> { + pub(super) entity: EntityIDRef<'a>, + pub(super) expressions: Vec>, + pub(super) wc: WhereClause<'a>, +} + +impl<'a> UpdateStatement<'a> { + pub fn entity(&self) -> EntityIDRef<'a> { + self.entity + } + pub fn expressions(&self) -> &[AssignmentExpression<'a>] { + &self.expressions + } + pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> { + &mut self.wc + } + pub fn into_expressions(self) -> Vec> { + self.expressions + } +} + +impl<'a> UpdateStatement<'a> { + #[inline(always)] + #[cfg(test)] + pub fn new( + entity: EntityIDRef<'a>, + expressions: Vec>, + wc: WhereClause<'a>, + ) -> Self { + Self { + entity, + expressions, + wc, + } + } + #[inline(always)] + pub fn parse_update>(state: &mut State<'a, Qd>) -> QueryResult { + /* + TODO(@ohsayan): Allow volcanoes + smallest tt: + update model SET x = 1 where x = 1 + ^1 ^2 ^3 ^4 ^5^6 ^7^8^9 + */ + if compiler::unlikely(state.remaining() < 9) { + return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement); + } + // parse entity + let entity = state.try_entity_buffered_into_state_uninit(); + if !(state.has_remaining(6)) { + unsafe { + // UNSAFE(@ohsayan): Obvious from above, max 3 fw + impossible!(); + } + } + state.poison_if_not(state.cursor_eq(Token![set])); + state.cursor_ahead(); // ignore errors if any + let mut nx_where = false; + let mut expressions = Vec::new(); + while state.not_exhausted() && state.okay() && !nx_where { + AssignmentExpression::parse_and_append_expression(state, &mut expressions); + let nx_comma = state.cursor_rounded_eq(Token![,]); + nx_where = state.cursor_rounded_eq(Token![where]); // NOTE: volcano + state.poison_if_not(nx_comma | nx_where); + state.cursor_ahead_if(nx_comma); + } + state.poison_if_not(nx_where); + state.cursor_ahead_if(state.okay()); + // check where clauses + let mut clauses = <_ as Default>::default(); + WhereClause::parse_where_and_append_to(state, &mut clauses); + state.poison_if(clauses.is_empty()); // NOTE: volcano + if compiler::likely(state.okay()) { + Ok(Self { + entity: unsafe { + // UNSAFE(@ohsayan): This is safe because of `parse_entity` and `okay` + entity.assume_init() + }, + expressions, + wc: WhereClause::new(clauses), + }) + } else { + compiler::cold_rerr(QueryError::QLInvalidSyntax) + } + } +} + +mod impls { + use { + super::UpdateStatement, + crate::engine::{ + error::QueryResult, + ql::ast::{traits::ASTNode, QueryData, State}, + }, + }; + impl<'a> ASTNode<'a> for UpdateStatement<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = true; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + Self::parse_update(state) + } + } + #[cfg(test)] + mod test { + use super::{super::AssignmentExpression, ASTNode, QueryData, QueryResult, State}; + impl<'a> ASTNode<'a> for AssignmentExpression<'a> { + const MUST_USE_FULL_TOKEN_RANGE: bool = false; + const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false; + // important: upstream must verify this + const VERIFY_STATE_BEFORE_RETURN: bool = true; + fn __base_impl_parse_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult { + let mut expr = Vec::new(); + AssignmentExpression::parse_and_append_expression(state, &mut expr); + state.poison_if_not(expr.len() == 1); + Ok(expr.remove(0)) + } + fn _multiple_from_state>( + state: &mut State<'a, Qd>, + ) -> QueryResult> { + let mut expr = Vec::new(); + AssignmentExpression::parse_and_append_expression(state, &mut expr); + Ok(expr) + } + } + } +} diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs new file mode 100644 index 00000000..c4dac3aa --- /dev/null +++ b/server/src/engine/ql/lex/mod.rs @@ -0,0 +1,505 @@ +/* + * Created on Tue Sep 13 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod raw; +#[cfg(test)] +pub use insecure_impl::InsecureLexer; +pub use raw::{Ident, Keyword, KeywordMisc, KeywordStmt, Symbol, Token}; + +use { + crate::engine::{ + data::lit::Lit, + error::{QueryError, QueryResult}, + mem::BufferedScanner, + }, + core::slice, +}; + +/* + basic lexer definition +*/ + +type Slice<'a> = &'a [u8]; + +#[derive(Debug, PartialEq)] +/// The internal lexer impl +pub struct Lexer<'a> { + token_buffer: BufferedScanner<'a>, + tokens: Vec>, + last_error: Option, +} + +impl<'a> Lexer<'a> { + /// Initialize a new lexer + fn new(src: &'a [u8]) -> Self { + Self { + token_buffer: BufferedScanner::new(src), + tokens: Vec::new(), + last_error: None, + } + } + /// set an error + #[inline(never)] + #[cold] + fn set_error(&mut self, e: QueryError) { + self.last_error = Some(e); + } + /// push in a new token + fn push_token(&mut self, t: impl Into>) { + self.tokens.push(t.into()) + } + fn no_error(&self) -> bool { + self.last_error.is_none() + } +} + +impl<'a> Lexer<'a> { + /// Scan an identifier + fn scan_ident(&mut self) -> Slice<'a> { + let s = self.token_buffer.cursor_ptr(); + unsafe { + while self + .token_buffer + .rounded_cursor_not_eof_matches(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + // UNSAFE(@ohsayan): increment cursor, this is valid + self.token_buffer.incr_cursor(); + } + // UNSAFE(@ohsayan): valid slice and ptrs + slice::from_raw_parts( + s, + self.token_buffer.current_buffer().as_ptr().offset_from(s) as usize, + ) + } + } + /// Scan an identifier or keyword + fn scan_ident_or_keyword(&mut self) { + let s = self.scan_ident(); + match Keyword::get(s) { + Some(kw) => self.tokens.push(kw.into()), + // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small + None if s.eq_ignore_ascii_case(b"true") || s.eq_ignore_ascii_case(b"false") => { + self.push_token(Lit::new_bool(s.eq_ignore_ascii_case(b"true"))) + } + None => self.tokens.push(unsafe { + // UNSAFE(@ohsayan): scan_ident only returns a valid ident which is always a string + Token::Ident(Ident::new(s)) + }), + } + } + fn scan_byte(&mut self, byte: u8) { + match Symbol::get(byte) { + Some(tok) => self.push_token(tok), + None => return self.set_error(QueryError::LexUnexpectedByte), + } + unsafe { + // UNSAFE(@ohsayan): we are sent a byte, so fw cursor + self.token_buffer.incr_cursor(); + } + } +} + +impl<'a> Lexer<'a> { + fn trim_ahead(&mut self) { + self.token_buffer + .trim_ahead(|b| (b == b' ') | (b == b'\n') | (b == b'\t')) + } +} + +/* + Insecure lexer +*/ + +mod insecure_impl { + #![allow(unused)] // TODO(@ohsayan): yank this + use { + super::Lexer, + crate::{ + engine::{ + data::lit::Lit, + error::{QueryError, QueryResult}, + ql::lex::Token, + }, + util::compiler, + }, + }; + + pub struct InsecureLexer<'a> { + pub(crate) l: Lexer<'a>, + } + + impl<'a> InsecureLexer<'a> { + pub fn lex(src: &'a [u8]) -> QueryResult>> { + let slf = Self { l: Lexer::new(src) }; + slf._lex() + } + pub(crate) fn _lex(mut self) -> QueryResult>> { + while !self.l.token_buffer.eof() & self.l.no_error() { + let byte = unsafe { + // UNSAFE(@ohsayan): loop invariant + self.l.token_buffer.deref_cursor() + }; + match byte { + #[cfg(test)] + byte if byte == b'\x01' => { + self.l.push_token(Token::IgnorableComma); + unsafe { + // UNSAFE(@ohsayan): All good here. Already read the token + self.l.token_buffer.incr_cursor(); + } + } + // ident + byte if byte.is_ascii_alphabetic() | (byte == b'_') => { + self.l.scan_ident_or_keyword() + } + // uint + byte if byte.is_ascii_digit() => self.scan_unsigned_integer(), + // sint + b'-' => { + unsafe { + // UNSAFE(@ohsayan): loop invariant + self.l.token_buffer.incr_cursor() + }; + self.scan_signed_integer(); + } + // binary + b'\r' => { + unsafe { + // UNSAFE(@ohsayan): loop invariant + self.l.token_buffer.incr_cursor() + } + self.scan_binary() + } + // string + quote_style @ (b'"' | b'\'') => { + unsafe { + // UNSAFE(@ohsayan): loop invariant + self.l.token_buffer.incr_cursor() + } + self.scan_quoted_string(quote_style) + } + // whitespace + b' ' | b'\n' | b'\t' => self.l.trim_ahead(), + // some random byte + byte => self.l.scan_byte(byte), + } + } + match self.l.last_error { + None => Ok(self.l.tokens), + Some(e) => Err(e), + } + } + } + + impl<'a> InsecureLexer<'a> { + pub(crate) fn scan_binary(&mut self) { + let Some(len) = self + .l + .token_buffer + .try_next_ascii_u64_lf_separated_or_restore_cursor() + else { + self.l.set_error(QueryError::LexInvalidInput); + return; + }; + let len = len as usize; + match self.l.token_buffer.try_next_variable_block(len) { + Some(block) => self.l.push_token(Lit::new_bin(block)), + None => self.l.set_error(QueryError::LexInvalidInput), + } + } + pub(crate) fn scan_quoted_string(&mut self, quote_style: u8) { + // cursor is at beginning of `"`; we need to scan until the end of quote or an escape + let mut buf = Vec::new(); + while self + .l + .token_buffer + .rounded_cursor_not_eof_matches(|b| *b != quote_style) + { + let byte = unsafe { + // UNSAFE(@ohsayan): loop invariant + self.l.token_buffer.next_byte() + }; + match byte { + b'\\' => { + // hmm, this might be an escape (either `\\` or `\"`) + if self + .l + .token_buffer + .rounded_cursor_not_eof_matches(|b| *b == quote_style || *b == b'\\') + { + // ignore escaped byte + unsafe { + buf.push(self.l.token_buffer.next_byte()); + } + } else { + // this is not allowed + unsafe { + // UNSAFE(@ohsayan): we move the cursor ahead, now we're moving it back + self.l.token_buffer.decr_cursor() + } + self.l.set_error(QueryError::LexInvalidInput); + return; + } + } + _ => buf.push(byte), + } + } + let ended_with_quote = self + .l + .token_buffer + .rounded_cursor_not_eof_equals(quote_style); + // skip quote + unsafe { + // UNSAFE(@ohsayan): not eof + self.l.token_buffer.incr_cursor_if(ended_with_quote) + } + match String::from_utf8(buf) { + Ok(s) if ended_with_quote => self.l.push_token(Lit::new_string(s)), + Err(_) | Ok(_) => self.l.set_error(QueryError::LexInvalidInput), + } + } + pub(crate) fn scan_unsigned_integer(&mut self) { + let mut okay = true; + // extract integer + let int = self + .l + .token_buffer + .try_next_ascii_u64_stop_at::(&mut okay, |b| b.is_ascii_digit()); + /* + see if we ended at a correct byte: + iff the integer has an alphanumeric byte at the end is the integer invalid + */ + if compiler::unlikely( + !okay + | self + .l + .token_buffer + .rounded_cursor_not_eof_matches(u8::is_ascii_alphanumeric), + ) { + self.l.set_error(QueryError::LexInvalidInput); + } else { + self.l.push_token(Lit::new_uint(int)) + } + } + pub(crate) fn scan_signed_integer(&mut self) { + if self.l.token_buffer.rounded_cursor_value().is_ascii_digit() { + unsafe { + // UNSAFE(@ohsayan): the cursor was moved ahead, now we're moving it back + self.l.token_buffer.decr_cursor() + } + let (okay, int) = self + .l + .token_buffer + .try_next_ascii_i64_stop_at(|b| !b.is_ascii_digit()); + if okay + & !self + .l + .token_buffer + .rounded_cursor_value() + .is_ascii_alphabetic() + { + self.l.push_token(Lit::new_sint(int)) + } else { + self.l.set_error(QueryError::LexInvalidInput) + } + } else { + self.l.push_token(Token![-]); + } + } + } +} + +/* + secure +*/ + +#[derive(Debug)] +pub struct SecureLexer<'a> { + l: Lexer<'a>, + param_buffer: BufferedScanner<'a>, +} + +impl<'a> SecureLexer<'a> { + pub fn new_with_segments(q: &'a [u8], p: &'a [u8]) -> Self { + Self { + l: Lexer::new(q), + param_buffer: BufferedScanner::new(p), + } + } + pub fn lex(self) -> QueryResult>> { + self._lex() + } + #[cfg(test)] + pub fn lex_with_window(src: &'a [u8], query_window: usize) -> QueryResult>> { + Self { + l: Lexer::new(&src[..query_window]), + param_buffer: BufferedScanner::new(&src[query_window..]), + } + .lex() + } +} + +impl<'a> SecureLexer<'a> { + fn _lex(mut self) -> QueryResult>> { + while self.l.no_error() & !self.l.token_buffer.eof() { + let b = unsafe { + // UNSAFE(@ohsayan): loop invariant + self.l.token_buffer.deref_cursor() + }; + match b { + b if b.is_ascii_alphabetic() | (b == b'_') => self.l.scan_ident_or_keyword(), + b'?' => { + // skip the param byte + unsafe { + // UNSAFE(@ohsayan): loop invariant + self.l.token_buffer.incr_cursor() + } + // find target + let ecc_code = SCAN_PARAM.len() - 1; + let target_code = self.param_buffer.rounded_cursor_value(); + let target_fn = target_code.min(ecc_code as u8); + // forward if we have target + unsafe { + self.param_buffer + .incr_cursor_by((target_code == target_fn) as _) + } + // check requirements + let has_enough = self + .param_buffer + .has_left(SCAN_PARAM_EXPECT[target_fn as usize] as _); + let final_target = + (has_enough as u8 * target_fn) | (!has_enough as u8 * ecc_code as u8); + // exec + let final_target = final_target as usize; + unsafe { + if final_target >= SCAN_PARAM.len() { + impossible!() + } + } + unsafe { + // UNSAFE(@ohsayan): our computation above ensures that we're meeting the expected target + SCAN_PARAM[final_target](&mut self) + } + } + b' ' | b'\t' | b'\n' => self.l.trim_ahead(), + sym => self.l.scan_byte(sym), + } + } + match self.l.last_error { + None => Ok(self.l.tokens), + Some(e) => Err(e), + } + } +} + +const SCAN_PARAM_EXPECT: [u8; 8] = [0, 1, 2, 2, 2, 2, 2, 0]; +static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe { + [ + // null + |s| s.l.push_token(Token![null]), + // bool + |slf| { + let nb = slf.param_buffer.next_byte(); + slf.l.push_token(Token::Lit(Lit::new_bool(nb == 1))); + if nb > 1 { + slf.l.set_error(QueryError::LexInvalidInput); + } + }, + // uint + |slf| match slf + .param_buffer + .try_next_ascii_u64_lf_separated_or_restore_cursor() + { + Some(int) => slf.l.push_token(Lit::new_uint(int)), + None => slf.l.set_error(QueryError::LexInvalidInput), + }, + // sint + |slf| { + let (okay, int) = slf.param_buffer.try_next_ascii_i64_separated_by::(); + if okay { + slf.l.push_token(Lit::new_sint(int)) + } else { + slf.l.set_error(QueryError::LexInvalidInput) + } + }, + // float + |slf| { + let start = slf.param_buffer.cursor(); + while !slf.param_buffer.eof() { + let cursor = slf.param_buffer.cursor(); + let byte = slf.param_buffer.next_byte(); + if byte == b'\n' { + match core::str::from_utf8(&slf.param_buffer.inner_buffer()[start..cursor]) + .map(core::str::FromStr::from_str) + { + Ok(Ok(f)) => slf.l.push_token(Lit::new_float(f)), + _ => slf.l.set_error(QueryError::LexInvalidInput), + } + return; + } + } + slf.l.set_error(QueryError::LexInvalidInput) + }, + // binary + |slf| { + let Some(size_of_body) = slf + .param_buffer + .try_next_ascii_u64_lf_separated_or_restore_cursor() + else { + slf.l.set_error(QueryError::LexInvalidInput); + return; + }; + match slf + .param_buffer + .try_next_variable_block(size_of_body as usize) + { + Some(block) => slf.l.push_token(Lit::new_bin(block)), + None => slf.l.set_error(QueryError::LexInvalidInput), + } + }, + // string + |slf| { + let Some(size_of_body) = slf + .param_buffer + .try_next_ascii_u64_lf_separated_or_restore_cursor() + else { + slf.l.set_error(QueryError::LexInvalidInput); + return; + }; + match slf + .param_buffer + .try_next_variable_block(size_of_body as usize) + .map(core::str::from_utf8) + { + // TODO(@ohsayan): obliterate this alloc + Some(Ok(s)) => slf.l.push_token(Lit::new_string(s.to_owned())), + _ => slf.l.set_error(QueryError::LexInvalidInput), + } + }, + // ecc + |s| s.l.set_error(QueryError::LexInvalidInput), + ] +}; diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs new file mode 100644 index 00000000..15c45eac --- /dev/null +++ b/server/src/engine/ql/lex/raw.rs @@ -0,0 +1,468 @@ +/* + * Created on Wed Feb 01 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::data::lit::Lit, + core::{borrow::Borrow, fmt, ops::Deref, str}, +}; + +/* + ident +*/ + +#[repr(transparent)] +#[derive(PartialEq, Eq, Clone, Copy, Hash)] +pub struct Ident<'a>(&'a [u8]); +impl<'a> Ident<'a> { + pub const unsafe fn new(v: &'a [u8]) -> Self { + Self(v) + } + pub const fn new_str(v: &'a str) -> Self { + Self(v.as_bytes()) + } + pub fn as_slice(&self) -> &'a [u8] { + self.0 + } + pub fn as_str(&self) -> &'a str { + unsafe { + // UNSAFE(@ohsayan): it's the ctor + str::from_utf8_unchecked(self.0) + } + } + pub fn boxed_str(&self) -> Box { + self.as_str().to_string().into_boxed_str() + } +} +impl<'a> fmt::Debug for Ident<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} +impl<'a> Deref for Ident<'a> { + type Target = str; + fn deref(&self) -> &Self::Target { + self.as_str() + } +} +impl<'a> PartialEq<[u8]> for Ident<'a> { + fn eq(&self, other: &[u8]) -> bool { + self.0 == other + } +} +impl<'a> PartialEq> for [u8] { + fn eq(&self, other: &Ident<'a>) -> bool { + self == other.as_bytes() + } +} +impl<'a> PartialEq for Ident<'a> { + fn eq(&self, other: &str) -> bool { + self.0 == other.as_bytes() + } +} +impl<'a> PartialEq> for str { + fn eq(&self, other: &Ident<'a>) -> bool { + self == other.as_str() + } +} +impl<'a> From<&'a str> for Ident<'a> { + fn from(s: &'a str) -> Self { + Self::new_str(s) + } +} +impl<'a> AsRef<[u8]> for Ident<'a> { + fn as_ref(&self) -> &'a [u8] { + self.0 + } +} +impl<'a> AsRef for Ident<'a> { + fn as_ref(&self) -> &'a str { + self.as_str() + } +} +impl<'a> Default for Ident<'a> { + fn default() -> Self { + Self::new_str("") + } +} +impl<'a> Borrow<[u8]> for Ident<'a> { + fn borrow(&self) -> &[u8] { + self.0 + } +} + +/* + token +*/ + +#[derive(Debug, PartialEq, Clone)] +pub enum Token<'a> { + Symbol(Symbol), + Keyword(Keyword), + Ident(Ident<'a>), + #[cfg(test)] + /// A comma that can be ignored (used for fuzzing) + IgnorableComma, + Lit(Lit<'a>), // literal +} + +impl<'a> Token<'a> { + pub unsafe fn uck_read_ident(&self) -> Ident<'a> { + extract!(self, Self::Ident(id) => *id) + } + pub unsafe fn uck_read_lit(&self) -> &Lit<'a> { + extract!(self, Self::Lit(l) => l) + } + pub fn ident_eq(&self, ident: &str) -> bool { + matches!(self, Token::Ident(id) if id.eq_ignore_ascii_case(ident)) + } +} + +impl<'a> ToString for Token<'a> { + fn to_string(&self) -> String { + match self { + Self::Symbol(s) => s.to_string(), + Self::Keyword(k) => k.to_string(), + Self::Ident(id) => id.to_string(), + Self::Lit(l) => l.to_string(), + #[cfg(test)] + Self::IgnorableComma => "[IGNORE_COMMA]".to_owned(), + } + } +} + +impl<'a> PartialEq for Token<'a> { + fn eq(&self, other: &Symbol) -> bool { + match self { + Self::Symbol(s) => s == other, + _ => false, + } + } +} + +direct_from! { + Token<'a> => { + Keyword as Keyword, + Symbol as Symbol, + Lit<'a> as Lit, + } +} + +impl<'a> Token<'a> { + #[inline(always)] + pub(crate) const fn is_ident(&self) -> bool { + matches!(self, Token::Ident(_)) + } + #[inline(always)] + pub const fn is_lit(&self) -> bool { + matches!(self, Self::Lit(_)) + } +} + +impl<'a> AsRef> for Token<'a> { + #[inline(always)] + fn as_ref(&self) -> &Token<'a> { + self + } +} + +/* + symbols +*/ + +build_lut!( + static SYM_LUT in symlut; + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + #[repr(u8)] + pub enum Symbol { + OpArithmeticAdd = b'+', + OpArithmeticSub = b'-', + OpArithmeticMul = b'*', + OpArithmeticDiv = b'/', + OpLogicalNot = b'!', + OpLogicalAnd = b'&', + OpLogicalXor = b'^', + OpLogicalOr = b'|', + OpAssign = b'=', + TtOpenParen = b'(', + TtCloseParen = b')', + TtOpenSqBracket = b'[', + TtCloseSqBracket = b']', + TtOpenBrace = b'{', + TtCloseBrace = b'}', + OpComparatorLt = b'<', + OpComparatorGt = b'>', + QuoteS = b'\'', + QuoteD = b'"', + SymAt = b'@', + SymHash = b'#', + SymDollar = b'$', + SymPercent = b'%', + SymUnderscore = b'_', + SymBackslash = b'\\', + SymColon = b':', + SymSemicolon = b';', + SymComma = b',', + SymPeriod = b'.', + SymQuestion = b'?', + SymTilde = b'~', + SymAccent = b'`', + } + |s: u8| -> u8 { s }, + |c: u8| -> String { char::from(c).to_string() } +); + +impl Symbol { + pub fn get(k: u8) -> Option { + const SYM_MAGIC_A: u8 = b'w'; + const SYM_MAGIC_B: u8 = b'E'; + static G: [u8; 69] = [ + 0, 0, 25, 0, 3, 0, 21, 0, 6, 13, 0, 0, 0, 0, 8, 0, 0, 0, 17, 0, 0, 30, 0, 28, 0, 20, + 19, 12, 0, 0, 2, 0, 0, 15, 0, 0, 0, 5, 0, 31, 14, 0, 1, 0, 18, 29, 24, 0, 0, 10, 0, 0, + 26, 0, 0, 0, 22, 0, 23, 7, 0, 27, 0, 4, 16, 11, 0, 0, 9, + ]; + let symfh = |magic, k| (magic as u16 * k as u16) % G.len() as u16; + let hf = + (G[symfh(k, SYM_MAGIC_A) as usize] + G[symfh(k, SYM_MAGIC_B) as usize]) % G.len() as u8; + if hf < SYM_LUT.len() as u8 && SYM_LUT[hf as usize].0 == k { + Some(SYM_LUT[hf as usize].1) + } else { + None + } + } +} + +/* + keywords +*/ + +macro_rules! flattened_lut { + ( + $staticvis:vis static $staticname:ident in $staticpriv:ident; + $(#[$enumattr:meta])* + $vis:vis enum $enum:ident { + $($(#[$variant_attr:meta])* $variant:ident => { + $(#[$nested_enum_attr:meta])* + $nested_enum_vis:vis enum $nested_enum_name:ident {$($(#[$nested_variant_attr:meta])* $nested_enum_variant_name:ident $(= $nested_enum_variant_dscr:expr)?,)*} + }),* $(,)? + } + ) => { + $( + $(#[$nested_enum_attr])* + $nested_enum_vis enum $nested_enum_name {$($(#[$nested_variant_attr])* $nested_enum_variant_name $(= $nested_enum_variant_dscr)*),*} + impl $nested_enum_name { + const __LEN: usize = {let mut i = 0; $( let _ = Self::$nested_enum_variant_name; i += 1; )*i}; + const __SL: [usize; 2] = { + let mut largest = 0; + let mut smallest = usize::MAX; + $( + let this = stringify!($nested_enum_variant_name).len(); + if this > largest { largest = this } if this < smallest { smallest = this } + )* + [smallest, largest] + }; + const __SMALLEST: usize = Self::__SL[0]; + const __LARGEST: usize = Self::__SL[1]; + const fn __max() -> usize { Self::__LEN } + pub const fn as_str(&self) -> &'static str {match self {$( + Self::$nested_enum_variant_name => { + const NAME_STR: &'static str = stringify!($nested_enum_variant_name); + const NAME_BUF: [u8; { NAME_STR.len() }] = { + let mut buf = [0u8; { NAME_STR.len() }]; let name = NAME_STR.as_bytes(); + buf[0] = name[0].to_ascii_lowercase(); let mut i = 1; + while i < NAME_STR.len() { buf[i] = name[i]; i += 1; } + buf + }; const NAME: &'static str = unsafe { core::str::from_utf8_unchecked(&NAME_BUF) }; NAME + } + )*}} + } + impl ToString for $nested_enum_name { fn to_string(&self) -> String { self.as_str().to_owned() } } + )* + $(#[$enumattr])* + $vis enum $enum {$($(#[$variant_attr])* $variant($nested_enum_name)),*} + impl $enum { pub const fn as_str(&self) -> &'static str { match self {$(Self::$variant(v) => { $nested_enum_name::as_str(v) })*} } } + impl $enum { + const SL: [usize; 2] = { + let mut largest = 0; let mut smallest = usize::MAX; + $( + if $nested_enum_name::__LARGEST > largest { largest = $nested_enum_name::__LARGEST; } + if $nested_enum_name::__SMALLEST < smallest { smallest = $nested_enum_name::__SMALLEST; } + )* + [smallest, largest] + }; + const SIZE_MIN: usize = Self::SL[0]; + const SIZE_MAX: usize = Self::SL[1]; + } + impl ToString for $enum { fn to_string(&self) -> String { self.as_str().to_owned() } } + mod $staticpriv { pub const LEN: usize = { let mut i = 0; $(i += super::$nested_enum_name::__max();)* i }; } + $staticvis static $staticname: [(&'static [u8], $enum); { $staticpriv::LEN }] = [ + $($(($nested_enum_name::$nested_enum_variant_name.as_str().as_bytes() ,$enum::$variant($nested_enum_name::$nested_enum_variant_name)),)*)* + ]; + } +} + +flattened_lut! { + static KW in kw; + #[derive(Debug, PartialEq, Clone, Copy)] + #[repr(u8)] + pub enum Keyword { + Statement => { + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)] + #[repr(u8)] + /// A statement keyword + pub enum KeywordStmt { + // system + Sysctl = 0, + // DDL + Create = 1, + Alter = 2, + Drop = 3, + // system/DDL misc + Use = 4, + Inspect = 5, + Describe = 6, + // DML + Insert = 7, + Select = 8, + Update = 9, + Delete = 10, + Exists = 11, + } + }, + /// Hi + Misc => { + #[derive(Debug, PartialEq, Clone, Copy)] + #[repr(u8)] + /// Misc. keywords + pub enum KeywordMisc { + // item definitions + Table, + Model, + Space, + Index, + Type, + Function, + // operations + Rename, + Add, + Remove, + Transform, + Set, + Return, + // sort related + Order, + Sort, + Group, + Limit, + Asc, + Desc, + All, + // container relational specifier + By, + With, + On, + From, + Into, + As, + To, + In, + Of, + // logical + And, + Or, + Not, + // conditional + If, + Else, + Where, + When, + Allow, + // value + Auto, + Default, + Null, + // transaction related + Transaction, + Batch, + Lock, + Read, + Write, + Begin, + End, + // misc + Key, + Value, + Primary, + // temporarily reserved (will probably be removed in the future) + Truncate, // TODO: decide what we want to do with this + } + } + } +} + +impl Keyword { + #[inline(always)] + pub fn get(k: &[u8]) -> Option { + if (k.len() > Self::SIZE_MAX) | (k.len() < Self::SIZE_MIN) { + None + } else { + Self::compute(k) + } + } + fn compute(key: &[u8]) -> Option { + static G: [u8; 69] = [ + 0, 0, 9, 64, 16, 43, 7, 49, 24, 8, 41, 37, 19, 66, 18, 0, 17, 0, 12, 63, 34, 56, 3, 24, + 55, 14, 0, 67, 7, 0, 39, 60, 56, 0, 51, 23, 31, 19, 30, 12, 10, 58, 20, 39, 32, 0, 6, + 30, 26, 58, 52, 62, 39, 27, 24, 9, 4, 21, 24, 68, 10, 38, 40, 21, 62, 27, 53, 27, 44, + ]; + static M1: [u8; 11] = *b"D8N5FwqrxdA"; + static M2: [u8; 11] = *b"FsIPJv9hsXx"; + let h1 = Self::_sum(key, M1) % G.len(); + let h2 = Self::_sum(key, M2) % G.len(); + let h = (G[h1] + G[h2]) as usize % G.len(); + if h < KW.len() && KW[h].0.eq_ignore_ascii_case(key) { + Some(KW[h].1) + } else { + None + } + } + #[inline(always)] + fn _sum(key: &[u8], block: [u8; 11]) -> usize { + let mut sum = 0; + let mut i = 0; + while i < key.len() { + let char = block[i % 11]; + sum += char as usize * (key[i] | 0x20) as usize; + i += 1; + } + sum + } +} + +impl KeywordStmt { + pub const fn is_blocking(&self) -> bool { + self.value_u8() <= Self::Drop.value_u8() + } +} diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs new file mode 100644 index 00000000..78ef4830 --- /dev/null +++ b/server/src/engine/ql/macros.rs @@ -0,0 +1,120 @@ +/* + * Created on Fri Sep 16 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +macro_rules! __sym_token { + ($ident:ident) => { + $crate::engine::ql::lex::Token::Symbol($crate::engine::ql::lex::Symbol::$ident) + }; +} + +#[cfg(test)] +macro_rules! dict { + () => { + <::std::collections::HashMap<_, _> as ::core::default::Default>::default() + }; + ($($key:expr => $value:expr),* $(,)?) => {{ + let mut hm: ::std::collections::HashMap<_, _> = ::core::default::Default::default(); + $(hm.insert($key.into(), $value.into());)* + hm + }}; +} + +#[cfg(test)] +macro_rules! null_dict { + () => { + dict! {} + }; + ($($key:expr => $value:expr),* $(,)?) => { + dict! { + $( + $key => $crate::engine::ql::tests::NullableDictEntry::data($value), + )* + } + }; +} + +#[cfg(test)] +macro_rules! dict_nullable { + () => { + <::std::collections::HashMap<_, _> as ::core::default::Default>::default() + }; + ($($key:expr => $value:expr),* $(,)?) => {{ + let mut hm: ::std::collections::HashMap<_, _> = ::core::default::Default::default(); + $(hm.insert($key.into(), $crate::engine::ql::tests::nullable_datatype($value));)* + hm + }}; +} + +#[cfg(test)] +macro_rules! into_array_nullable { + ($($e:expr),* $(,)?) => { [$($crate::engine::ql::tests::nullable_datatype($e)),*] }; +} + +#[allow(unused_macros)] +macro_rules! statictbl { + ($name:ident: $kind:ty => [$($expr:expr),*]) => {{ + const LEN: usize = {let mut i = 0;$(let _ = $expr; i += 1;)*i}; + static $name: [$kind; LEN] = [$($expr),*]; + &'static $name + }}; +} + +macro_rules! build_lut { + ( + $(#[$attr_s:meta])* $vis_s:vis static $LUT:ident in $lut:ident; $(#[$attr_e:meta])* $vis_e:vis enum $SYM:ident {$($variant:ident = $match:literal),*$(,)?} + |$arg:ident: $inp:ty| -> $ret:ty $block:block, + |$arg2:ident: $inp2:ty| -> String $block2:block + ) => { + mod $lut { + pub const L: usize = { let mut i = 0; $(let _ = $match;i += 1;)*i }; + pub const fn f($arg: $inp) -> $ret $block + pub fn s($arg2: $inp2) -> String $block2 + } + $(#[$attr_e])* $vis_e enum $SYM {$($variant),*} + $(#[$attr_s])* $vis_s static $LUT: [($ret, $SYM); $lut::L] = {[$(($lut::f($match), $SYM::$variant)),*]}; + impl ::std::string::ToString for $SYM { + fn to_string(&self) -> ::std::string::String {match self {$(Self::$variant => {$lut::s($match)},)*}} + } + } +} + +#[cfg(test)] +macro_rules! into_vec { + ($ty:ty => ($($v:expr),* $(,)?)) => {{ + let v: Vec<$ty> = std::vec![$($v.into(),)*]; + v + }}; + ($($v:expr),*) => {{ + std::vec![$($v.into(),)*] + }} +} + +#[cfg(test)] +macro_rules! lit { + ($lit:expr) => { + $crate::engine::data::lit::Lit::from($lit) + }; +} diff --git a/sky-bench/src/bench/validation.rs b/server/src/engine/ql/mod.rs similarity index 75% rename from sky-bench/src/bench/validation.rs rename to server/src/engine/ql/mod.rs index c93b35f0..a05c924c 100644 --- a/sky-bench/src/bench/validation.rs +++ b/server/src/engine/ql/mod.rs @@ -1,5 +1,5 @@ /* - * Created on Tue Aug 09 2022 + * Created on Tue Sep 13 2022 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -24,16 +24,12 @@ * */ -pub const RESPCODE_OKAY: &[u8] = b"*!0\n"; - -pub fn calculate_response_size(keylen: usize) -> usize { - /* - *+5\n - hello - */ - let mut size = 2; // simple query byte + tsymbol - size += keylen.to_string().len(); // bytes in length - size += 1; // LF - size += keylen; // payload - size -} +#[macro_use] +mod macros; +pub(super) mod ast; +pub(super) mod dcl; +pub(super) mod ddl; +pub(super) mod dml; +pub(super) mod lex; +#[cfg(test)] +pub(in crate::engine) mod tests; diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs new file mode 100644 index 00000000..ecdd34f4 --- /dev/null +++ b/server/src/engine/ql/tests.rs @@ -0,0 +1,178 @@ +/* + * Created on Tue Sep 13 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + ast::{self, traits::ASTNode}, + lex::{InsecureLexer, SecureLexer, Symbol, Token}, + }, + crate::{ + engine::{data::cell::Datacell, error::QueryResult}, + util::test_utils, + }, + rand::{self, Rng}, +}; + +mod dcl; +mod dml_tests; +mod lexer_tests; +mod misc; +mod schema_tests; +mod structure_syn; + +#[inline(always)] +/// Uses the [`InsecureLexer`] to lex the given input +pub fn lex_insecure(src: &[u8]) -> QueryResult>> { + InsecureLexer::lex(src) +} +pub fn lex_secure<'a>(src: &'a [u8], query_window: usize) -> QueryResult>> { + SecureLexer::lex_with_window(src, query_window) +} + +pub trait NullableData { + fn data(self) -> T; +} + +impl NullableData for T +where + T: Into, +{ + fn data(self) -> Datacell { + self.into() + } +} + +struct Null; + +impl NullableData for Null { + fn data(self) -> Datacell { + Datacell::null() + } +} + +fn nullable_datatype(v: impl NullableData) -> Datacell { + v.data() +} + +pub trait NullableDictEntry { + fn data(self) -> crate::engine::data::DictEntryGeneric; +} + +impl NullableDictEntry for Null { + fn data(self) -> crate::engine::data::DictEntryGeneric { + crate::engine::data::DictEntryGeneric::Data(Datacell::null()) + } +} + +impl<'a> NullableDictEntry for crate::engine::data::lit::Lit<'a> { + fn data(self) -> crate::engine::data::DictEntryGeneric { + crate::engine::data::DictEntryGeneric::from(self.as_ir()) + } +} + +impl NullableDictEntry for crate::engine::data::DictGeneric { + fn data(self) -> crate::engine::data::DictEntryGeneric { + crate::engine::data::DictEntryGeneric::Map(self) + } +} + +/// A very "basic" fuzzer that will randomly inject tokens wherever applicable +fn fuzz_tokens(src: &[u8], fuzzverify: impl Fn(bool, &[Token]) -> bool) { + let src_tokens = lex_insecure(src).unwrap(); + static FUZZ_TARGETS: [Token; 2] = [Token::Symbol(Symbol::SymComma), Token::IgnorableComma]; + let mut rng = rand::thread_rng(); + #[inline(always)] + fn inject(new_src: &mut Vec, rng: &mut impl Rng) -> usize { + let start = new_src.len(); + (0..test_utils::random_number(0, 5, rng)) + .for_each(|_| new_src.push(Token::Symbol(Symbol::SymComma))); + new_src.len() - start + } + let fuzz_amount = src_tokens + .iter() + .filter(|tok| FUZZ_TARGETS.contains(tok)) + .count(); + for _ in 0..(fuzz_amount.pow(3)) { + let mut new_src = Vec::with_capacity(src_tokens.len()); + let mut should_pass = true; + src_tokens.iter().for_each(|tok| match tok { + Token::IgnorableComma => { + let added = inject(&mut new_src, &mut rng); + should_pass &= added <= 1; + } + Token::Symbol(Symbol::SymComma) => { + let added = inject(&mut new_src, &mut rng); + should_pass &= added == 1; + } + tok => new_src.push(tok.clone()), + }); + if fuzzverify(should_pass, &new_src) && !should_pass { + panic!( + "expected failure for `{}`, but it passed", + new_src + .iter() + .flat_map(|tok| format!("{} ", tok.to_string()).into_bytes()) + .map(char::from) + .collect::() + ) + } + } +} + +pub(self) fn fullparse_verify<'a, A: ASTNode<'a> + 'a>(q: &'a str, offset: usize, v: impl Fn(A)) { + let tok = lex_insecure(q.as_bytes()).unwrap(); + unsafe { + let q: &'a [_] = core::mem::transmute(tok.as_slice()); + let a: A = ASTNode::from_insecure_tokens_full(&q[offset..]).unwrap(); + v(a); + } +} + +pub(self) fn fullparse_verify_substmt<'a, A: ASTNode<'a> + 'a>(q: &'a str, v: impl Fn(A)) { + fullparse_verify(q, 2, v) +} + +pub(self) fn fullparse_verify_with_space<'a, A: ASTNode<'a> + 'a>( + q: &'a str, + space_name: &'static str, + offset: usize, + v: impl Fn(A), +) { + let tok = lex_insecure(q.as_bytes()).unwrap(); + unsafe { + let q: &'static [_] = core::mem::transmute(&tok.as_slice()[offset..]); + let a: A = ast::parse_ast_node_full_with_space(q, space_name).unwrap(); + v(a); + } +} + +pub(self) fn fullparse_verify_substmt_with_space<'a, A: ASTNode<'a> + 'a>( + q: &'a str, + space_name: &'static str, + v: impl Fn(A), +) { + fullparse_verify_with_space(q, space_name, 2, v) +} diff --git a/server/src/engine/ql/tests/dcl.rs b/server/src/engine/ql/tests/dcl.rs new file mode 100644 index 00000000..d7c8a169 --- /dev/null +++ b/server/src/engine/ql/tests/dcl.rs @@ -0,0 +1,74 @@ +/* + * Created on Fri Sep 22 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::ql::{ + ast, + dcl::{self, SysctlCommand}, + tests::lex_insecure, +}; + +#[test] +fn report_status_simple() { + let query = lex_insecure(b"sysctl report status").unwrap(); + let q = ast::parse_ast_node_full::(&query[1..]).unwrap(); + assert_eq!(q, SysctlCommand::ReportStatus) +} + +#[test] +fn create_user_simple() { + let query = lex_insecure(b"sysctl create user sayan with { password: 'mypass123' }").unwrap(); + let q = ast::parse_ast_node_full::(&query[1..]).unwrap(); + assert_eq!( + q, + SysctlCommand::CreateUser(dcl::UserDecl::new( + "sayan".into(), + into_dict!("password" => lit!("mypass123")) + )) + ) +} + +#[test] +fn alter_user_simple() { + let query = lex_insecure(b"sysctl alter user sayan with { password: 'mypass123' }").unwrap(); + let q = ast::parse_ast_node_full::(&query[1..]).unwrap(); + assert_eq!( + q, + SysctlCommand::AlterUser(dcl::UserDecl::new( + "sayan".into(), + into_dict!("password" => lit!("mypass123")) + )) + ) +} + +#[test] +fn delete_user_simple() { + let query = lex_insecure(b"sysctl drop user monster").unwrap(); + let q = ast::parse_ast_node_full::(&query[1..]).unwrap(); + assert_eq!( + q, + SysctlCommand::DropUser(dcl::UserDel::new("monster".into())) + ); +} diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs new file mode 100644 index 00000000..4be9b1ae --- /dev/null +++ b/server/src/engine/ql/tests/dml_tests.rs @@ -0,0 +1,1006 @@ +/* + * Created on Sun Dec 18 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use super::*; +mod list_parse { + use super::*; + use crate::engine::ql::{ast::parse_ast_node_full, dml::ins::List}; + + #[test] + fn list_mini() { + let tok = lex_insecure( + b" + [] + ", + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!(r, vec![]) + } + #[test] + fn list() { + let tok = lex_insecure( + b" + [1, 2, 3, 4] + ", + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!(r.as_slice(), into_array![1, 2, 3, 4]) + } + #[test] + fn list_pro() { + let tok = lex_insecure( + b" + [ + [1, 2], + [3, 4], + [5, 6], + [] + ] + ", + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!( + r.as_slice(), + into_array![ + into_array![1, 2], + into_array![3, 4], + into_array![5, 6], + into_array![] + ] + ) + } + #[test] + fn list_pro_max() { + let tok = lex_insecure( + b" + [ + [[1, 1], [2, 2]], + [[], [4, 4]], + [[5, 5], [6, 6]], + [[7, 7], []] + ] + ", + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!( + r.as_slice(), + into_array![ + into_array![into_array![1, 1], into_array![2, 2]], + into_array![into_array![], into_array![4, 4]], + into_array![into_array![5, 5], into_array![6, 6]], + into_array![into_array![7, 7], into_array![]], + ] + ) + } +} + +mod tuple_syntax { + use super::*; + use crate::engine::ql::{ast::parse_ast_node_full, dml::ins::DataTuple}; + + #[test] + fn tuple_mini() { + let tok = lex_insecure(b"()").unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!(r, vec![]); + } + + #[test] + fn tuple() { + let tok = lex_insecure( + br#" + (1234, "email@example.com", true) + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!( + r.as_slice(), + into_array_nullable![1234, "email@example.com", true] + ); + } + + #[test] + fn tuple_pro() { + let tok = lex_insecure( + br#" + ( + 1234, + "email@example.com", + true, + ["hello", "world", "and", "the", "universe"] + ) + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!( + r.as_slice(), + into_array_nullable![ + 1234, + "email@example.com", + true, + into_array!["hello", "world", "and", "the", "universe"] + ] + ); + } + + #[test] + fn tuple_pro_max() { + let tok = lex_insecure( + br#" + ( + 1234, + "email@example.com", + true, + [ + ["h", "hello"], + ["w", "world"], + ["a", "and"], + ["the"], + ["universe"], + [] + ] + ) + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!( + r.as_slice(), + into_array_nullable![ + 1234, + "email@example.com", + true, + into_array![ + into_array!["h", "hello"], + into_array!["w", "world"], + into_array!["a", "and"], + into_array!["the"], + into_array!["universe"], + into_array![], + ] + ] + ); + } +} +mod map_syntax { + use super::*; + use crate::engine::ql::{ast::parse_ast_node_full, dml::ins::DataMap}; + + #[test] + fn map_mini() { + let tok = lex_insecure(b"{}").unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!(r, null_dict! {}) + } + + #[test] + fn map() { + let tok = lex_insecure( + br#" + { + name: "John Appletree", + email: "john@example.com", + verified: false, + followers: 12345 + } + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!( + r, + dict_nullable! { + "name" => "John Appletree", + "email" => "john@example.com", + "verified" => false, + "followers" => 12345, + } + ) + } + + #[test] + fn map_pro() { + let tok = lex_insecure( + br#" + { + name: "John Appletree", + email: "john@example.com", + verified: false, + followers: 12345, + tweets_by_day: [] + } + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!( + r, + dict_nullable! { + "name" => "John Appletree", + "email" => "john@example.com", + "verified" => false, + "followers" => 12345, + "tweets_by_day" => [] + } + ) + } + + #[test] + fn map_pro_max() { + let tok = lex_insecure(br#" + { + name: "John Appletree", + email: "john@example.com", + verified: false, + followers: 12345, + tweets_by_day: [ + ["it's a fresh monday", "monday was tiring"], + ["already bored with tuesday", "nope. gotta change stuff, life's getting boring"], + ["sunday, going to bed"] + ] + } + "#) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + assert_eq!( + r, + dict_nullable! { + "name" => "John Appletree", + "email" => "john@example.com", + "verified" => false, + "followers" => 12345, + "tweets_by_day" => into_array![ + into_array![ + "it's a fresh monday", "monday was tiring" + ], + into_array![ + "already bored with tuesday", "nope. gotta change stuff, life's getting boring" + ], + into_array!["sunday, going to bed"] + ] + } + ) + } +} +mod stmt_insert { + use { + super::*, + crate::engine::ql::{ + ast::parse_ast_node_full, + dml::{self, ins::InsertStatement}, + lex::Ident, + }, + }; + + #[test] + fn insert_tuple_mini() { + let x = lex_insecure( + br#" + insert into twitter.users ("sayan") + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&x[1..]).unwrap(); + let e = InsertStatement::new( + ("twitter", "users").into(), + into_array_nullable!["sayan"].to_vec().into(), + ); + assert_eq!(e, r); + } + #[test] + fn insert_tuple() { + let x = lex_insecure( + br#" + insert into twitter.users ( + "sayan", + "Sayan", + "sayan@example.com", + true, + 12345, + 67890 + ) + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&x[1..]).unwrap(); + let e = InsertStatement::new( + ("twitter", "users").into(), + into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890] + .to_vec() + .into(), + ); + assert_eq!(e, r); + } + #[test] + fn insert_tuple_pro() { + let x = lex_insecure( + br#" + insert into twitter.users ( + "sayan", + "Sayan", + "sayan@example.com", + true, + 12345, + 67890, + null, + 12345, + null + ) + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&x[1..]).unwrap(); + let e = InsertStatement::new( + ("twitter", "users").into(), + into_array_nullable![ + "sayan", + "Sayan", + "sayan@example.com", + true, + 12345, + 67890, + Null, + 12345, + Null + ] + .to_vec() + .into(), + ); + assert_eq!(e, r); + } + #[test] + fn insert_map_mini() { + let tok = lex_insecure( + br#" + insert into jotsy.app { username: "sayan" } + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + let e = InsertStatement::new( + ("jotsy", "app").into(), + dict_nullable! { + Ident::from("username") => "sayan" + } + .into(), + ); + assert_eq!(e, r); + } + #[test] + fn insert_map() { + let tok = lex_insecure( + br#" + insert into jotsy.app { + username: "sayan", + name: "Sayan", + email: "sayan@example.com", + verified: true, + following: 12345, + followers: 67890 + } + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + let e = InsertStatement::new( + ("jotsy", "app").into(), + dict_nullable! { + Ident::from("username") => "sayan", + Ident::from("name") => "Sayan", + Ident::from("email") => "sayan@example.com", + Ident::from("verified") => true, + Ident::from("following") => 12345, + Ident::from("followers") => 67890 + } + .into(), + ); + assert_eq!(e, r); + } + #[test] + fn insert_map_pro() { + let tok = lex_insecure( + br#" + insert into jotsy.app { + username: "sayan", + password: "pass123", + email: "sayan@example.com", + verified: true, + following: 12345, + followers: 67890, + linked_smart_devices: null, + bookmarks: 12345, + other_linked_accounts: null + } + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + let e = InsertStatement::new( + ("jotsy", "app").into(), + dict_nullable! { + Ident::from("username") => "sayan", + "password" => "pass123", + "email" => "sayan@example.com", + "verified" => true, + "following" => 12345, + "followers" => 67890, + "linked_smart_devices" => Null, + "bookmarks" => 12345, + "other_linked_accounts" => Null + } + .into(), + ); + assert_eq!(r, e); + } + #[test] + fn insert_tuple_fnsub() { + let tok = + lex_insecure(br#"insert into jotsy.app(@uuidstr(), "sayan", @timesec())"#).unwrap(); + let ret = parse_ast_node_full::(&tok[1..]).unwrap(); + let expected = InsertStatement::new( + ("jotsy", "app").into(), + into_array_nullable![dml::ins::T_UUIDSTR, "sayan", dml::ins::T_TIMESEC] + .to_vec() + .into(), + ); + assert_eq!(ret, expected); + } + #[test] + fn insert_map_fnsub() { + let tok = lex_insecure( + br#"insert into jotsy.app { uuid: @uuidstr(), username: "sayan", signup_time: @timesec() }"# + ).unwrap(); + let ret = parse_ast_node_full::(&tok[1..]).unwrap(); + let expected = InsertStatement::new( + ("jotsy", "app").into(), + dict_nullable! { + "uuid" => dml::ins::T_UUIDSTR, + Ident::from("username") => "sayan", + "signup_time" => dml::ins::T_TIMESEC, + } + .into(), + ); + assert_eq!(ret, expected); + } +} + +mod stmt_select { + use { + super::*, + crate::engine::{ + data::lit::Lit, + ql::{ + ast::{parse_ast_node_full, parse_ast_node_full_with_space}, + dml::{sel::SelectStatement, RelationalExpr}, + lex::Ident, + }, + }, + }; + #[test] + fn select_mini() { + let tok = lex_insecure( + br#" + select * from users where username = "sayan" + "#, + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[1..], "apps").unwrap(); + let e = SelectStatement::new_test( + ("apps", "users").into(), + [].to_vec(), + true, + dict! { + Ident::from("username") => RelationalExpr::new( + Ident::from("username"), Lit::new_str("sayan"), RelationalExpr::OP_EQ + ), + }, + ); + assert_eq!(r, e); + } + #[test] + fn select() { + let tok = lex_insecure( + br#" + select field1 from users where username = "sayan" + "#, + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[1..], "apps").unwrap(); + let e = SelectStatement::new_test( + ("apps", "users").into(), + [Ident::from("field1")].to_vec(), + false, + dict! { + Ident::from("username") => RelationalExpr::new( + Ident::from("username"), Lit::new_str("sayan"), RelationalExpr::OP_EQ + ), + }, + ); + assert_eq!(r, e); + } + #[test] + fn select_pro() { + let tok = lex_insecure( + br#" + select field1 from twitter.users where username = "sayan" + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + let e = SelectStatement::new_test( + ("twitter", "users").into(), + [Ident::from("field1")].to_vec(), + false, + dict! { + Ident::from("username") => RelationalExpr::new( + Ident::from("username"), Lit::new_str("sayan"), RelationalExpr::OP_EQ + ), + }, + ); + assert_eq!(r, e); + } + #[test] + fn select_pro_max() { + let tok = lex_insecure( + br#" + select field1, field2 from twitter.users where username = "sayan" + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + let e = SelectStatement::new_test( + ("twitter", "users").into(), + [Ident::from("field1"), Ident::from("field2")].to_vec(), + false, + dict! { + Ident::from("username") => RelationalExpr::new( + Ident::from("username"), Lit::new_str("sayan"), RelationalExpr::OP_EQ + ), + }, + ); + assert_eq!(r, e); + } +} +mod expression_tests { + use { + super::*, + crate::engine::{ + core::query_meta::AssignmentOperator, + data::lit::Lit, + ql::{ast::parse_ast_node_full, dml::upd::AssignmentExpression, lex::Ident}, + }, + }; + #[test] + fn expr_assign() { + let src = lex_insecure(b"username = 'sayan'").unwrap(); + let r = parse_ast_node_full::(&src).unwrap(); + assert_eq!( + r, + AssignmentExpression::new( + Ident::from("username"), + Lit::new_str("sayan"), + AssignmentOperator::Assign + ) + ); + } + #[test] + fn expr_add_assign() { + let src = lex_insecure(b"followers += 100").unwrap(); + let r = parse_ast_node_full::(&src).unwrap(); + assert_eq!( + r, + AssignmentExpression::new( + Ident::from("followers"), + Lit::new_uint(100), + AssignmentOperator::AddAssign + ) + ); + } + #[test] + fn expr_sub_assign() { + let src = lex_insecure(b"following -= 150").unwrap(); + let r = parse_ast_node_full::(&src).unwrap(); + assert_eq!( + r, + AssignmentExpression::new( + Ident::from("following"), + Lit::new_uint(150), + AssignmentOperator::SubAssign + ) + ); + } + #[test] + fn expr_mul_assign() { + let src = lex_insecure(b"product_qty *= 2").unwrap(); + let r = parse_ast_node_full::(&src).unwrap(); + assert_eq!( + r, + AssignmentExpression::new( + Ident::from("product_qty"), + Lit::new_uint(2), + AssignmentOperator::MulAssign + ) + ); + } + #[test] + fn expr_div_assign() { + let src = lex_insecure(b"image_crop_factor /= 2").unwrap(); + let r = parse_ast_node_full::(&src).unwrap(); + assert_eq!( + r, + AssignmentExpression::new( + Ident::from("image_crop_factor"), + Lit::new_uint(2), + AssignmentOperator::DivAssign + ) + ); + } +} +mod update_statement { + use { + super::*, + crate::engine::{ + core::query_meta::AssignmentOperator, + data::lit::Lit, + ql::{ + ast::{parse_ast_node_full, parse_ast_node_full_with_space}, + dml::{ + upd::{AssignmentExpression, UpdateStatement}, + RelationalExpr, WhereClause, + }, + lex::Ident, + }, + }, + }; + #[test] + fn update_mini() { + let tok = lex_insecure( + br#" + update app SET notes += "this is my new note" where username = "sayan" + "#, + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[1..], "apps").unwrap(); + let e = UpdateStatement::new( + ("apps", "app").into(), + vec![AssignmentExpression::new( + Ident::from("notes"), + Lit::new_str("this is my new note"), + AssignmentOperator::AddAssign, + )], + WhereClause::new(dict! { + Ident::from("username") => RelationalExpr::new( + Ident::from("username"), + Lit::new_str("sayan"), + RelationalExpr::OP_EQ + ) + }), + ); + assert_eq!(r, e); + } + #[test] + fn update() { + let tok = lex_insecure( + br#" + update + jotsy.app + SET + notes += "this is my new note", + email = "sayan@example.com" + WHERE + username = "sayan" + "#, + ) + .unwrap(); + let r = parse_ast_node_full::(&tok[1..]).unwrap(); + let e = UpdateStatement::new( + ("jotsy", "app").into(), + vec![ + AssignmentExpression::new( + Ident::from("notes"), + Lit::new_str("this is my new note"), + AssignmentOperator::AddAssign, + ), + AssignmentExpression::new( + Ident::from("email"), + Lit::new_str("sayan@example.com"), + AssignmentOperator::Assign, + ), + ], + WhereClause::new(dict! { + Ident::from("username") => RelationalExpr::new( + Ident::from("username"), + Lit::new_str("sayan"), + RelationalExpr::OP_EQ + ) + }), + ); + assert_eq!(r, e); + } +} +mod delete_stmt { + use { + super::*, + crate::engine::{ + data::lit::Lit, + ql::{ + ast::{parse_ast_node_full, parse_ast_node_full_with_space}, + dml::{del::DeleteStatement, RelationalExpr}, + lex::Ident, + }, + }, + }; + + #[test] + fn delete_mini() { + let tok = lex_insecure( + br#" + delete from users where username = "sayan" + "#, + ) + .unwrap(); + let e = DeleteStatement::new_test( + ("apps", "users").into(), + dict! { + Ident::from("username") => RelationalExpr::new( + Ident::from("username"), + Lit::new_str("sayan"), + RelationalExpr::OP_EQ + ) + }, + ); + assert_eq!( + parse_ast_node_full_with_space::(&tok[1..], "apps").unwrap(), + e + ); + } + #[test] + fn delete() { + let tok = lex_insecure( + br#" + delete from twitter.users where username = "sayan" + "#, + ) + .unwrap(); + let e = DeleteStatement::new_test( + ("twitter", "users").into(), + dict! { + Ident::from("username") => RelationalExpr::new( + Ident::from("username"), + Lit::new_str("sayan"), + RelationalExpr::OP_EQ + ) + }, + ); + assert_eq!( + parse_ast_node_full::(&tok[1..]).unwrap(), + e + ); + } +} +mod relational_expr { + use { + super::*, + crate::engine::{ + data::lit::Lit, + ql::{ast::parse_ast_node_full, dml::RelationalExpr, lex::Ident}, + }, + }; + + #[test] + fn expr_eq() { + let expr = lex_insecure(b"primary_key = 10").unwrap(); + let r = parse_ast_node_full::(&expr).unwrap(); + assert_eq!( + r, + RelationalExpr { + rhs: Lit::new_uint(10), + lhs: Ident::from("primary_key"), + opc: RelationalExpr::OP_EQ + } + ); + } + #[test] + fn expr_ne() { + let expr = lex_insecure(b"primary_key != 10").unwrap(); + let r = parse_ast_node_full::(&expr).unwrap(); + assert_eq!( + r, + RelationalExpr { + rhs: Lit::new_uint(10), + lhs: Ident::from("primary_key"), + opc: RelationalExpr::OP_NE + } + ); + } + #[test] + fn expr_gt() { + let expr = lex_insecure(b"primary_key > 10").unwrap(); + let r = parse_ast_node_full::(&expr).unwrap(); + assert_eq!( + r, + RelationalExpr { + rhs: Lit::new_uint(10), + lhs: Ident::from("primary_key"), + opc: RelationalExpr::OP_GT + } + ); + } + #[test] + fn expr_ge() { + let expr = lex_insecure(b"primary_key >= 10").unwrap(); + let r = parse_ast_node_full::(&expr).unwrap(); + assert_eq!( + r, + RelationalExpr { + rhs: Lit::new_uint(10), + lhs: Ident::from("primary_key"), + opc: RelationalExpr::OP_GE + } + ); + } + #[test] + fn expr_lt() { + let expr = lex_insecure(b"primary_key < 10").unwrap(); + let r = parse_ast_node_full::(&expr).unwrap(); + assert_eq!( + r, + RelationalExpr { + rhs: Lit::new_uint(10), + lhs: Ident::from("primary_key"), + opc: RelationalExpr::OP_LT + } + ); + } + #[test] + fn expr_le() { + let expr = lex_insecure(b"primary_key <= 10").unwrap(); + let r = parse_ast_node_full::(&expr).unwrap(); + assert_eq!( + r, + RelationalExpr::new( + Ident::from("primary_key"), + Lit::new_uint(10), + RelationalExpr::OP_LE + ) + ); + } +} +mod where_clause { + use { + super::*, + crate::engine::{ + data::lit::Lit, + ql::{ + ast::parse_ast_node_full, + dml::{RelationalExpr, WhereClause}, + lex::Ident, + }, + }, + }; + #[test] + fn where_single() { + let tok = lex_insecure( + br#" + x = 100 + "#, + ) + .unwrap(); + let expected = WhereClause::new(dict! { + Ident::from("x") => RelationalExpr::new( + Ident::from("x"), + Lit::new_uint(100), + RelationalExpr::OP_EQ + ) + }); + assert_eq!(expected, parse_ast_node_full::(&tok).unwrap()); + } + #[test] + fn where_double() { + let tok = lex_insecure( + br#" + userid = 100 and pass = "password" + "#, + ) + .unwrap(); + let expected = WhereClause::new(dict! { + Ident::from("userid") => RelationalExpr::new( + Ident::from("userid"), + Lit::new_uint(100), + RelationalExpr::OP_EQ + ), + Ident::from("pass") => RelationalExpr::new( + Ident::from("pass"), + Lit::new_str("password"), + RelationalExpr::OP_EQ + ) + }); + assert_eq!(expected, parse_ast_node_full::(&tok).unwrap()); + } + #[test] + fn where_duplicate_condition() { + let tok = lex_insecure( + br#" + userid = 100 and userid > 200 + "#, + ) + .unwrap(); + assert!(parse_ast_node_full::(&tok).is_err()); + } +} + +mod select_all { + use { + super::lex_insecure, + crate::engine::{ + error::QueryError, + ql::{ast::parse_ast_node_full_with_space, dml::sel::SelectAllStatement}, + }, + }; + + #[test] + fn select_all_wildcard() { + let tok = lex_insecure(b"select all * from mymodel limit 100").unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&tok[2..], "myspace").unwrap(), + SelectAllStatement::test_new(("myspace", "mymodel").into(), vec![], true, 100) + ); + } + + #[test] + fn select_all_multiple_fields() { + let tok = lex_insecure(b"select all username, password from mymodel limit 100").unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&tok[2..], "myspace").unwrap(), + SelectAllStatement::test_new( + ("myspace", "mymodel").into(), + into_vec!["username", "password"], + false, + 100 + ) + ); + } + + #[test] + fn select_all_missing_limit() { + let tok = lex_insecure(b"select all * from mymodel").unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&tok[2..], "myspace").unwrap_err(), + QueryError::QLUnexpectedEndOfStatement + ); + let tok = lex_insecure(b"select all username, password from mymodel").unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&tok[2..], "myspace").unwrap_err(), + QueryError::QLUnexpectedEndOfStatement + ); + } +} diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs new file mode 100644 index 00000000..e1aeec6e --- /dev/null +++ b/server/src/engine/ql/tests/lexer_tests.rs @@ -0,0 +1,321 @@ +/* + * Created on Sun Dec 18 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + super::lex::{Ident, Token}, + lex_insecure, lex_secure, + }, + crate::engine::{data::lit::Lit, error::QueryError}, +}; + +macro_rules! v( + ($e:literal) => {{ + $e.as_bytes().to_vec() + }}; + ($($e:literal),* $(,)?) => {{ + ($(v!($e)),*) + }}; +); + +#[test] +fn lex_ident() { + let src = v!("hello"); + assert_eq!( + lex_insecure(&src).unwrap(), + vec![Token::Ident(Ident::from("hello"))] + ); +} + +// literals +#[test] +fn lex_unsigned_int() { + let number = v!("123456"); + assert_eq!( + lex_insecure(&number).unwrap(), + vec![Token::Lit(Lit::new_uint(123456))] + ); +} +#[test] +fn lex_signed_int() { + let number = v!("-123456"); + assert_eq!( + lex_insecure(&number).unwrap(), + vec![Token::Lit(Lit::new_sint(-123456))] + ); +} +#[test] +fn lex_bool() { + let (t, f) = v!("true", "false"); + assert_eq!( + lex_insecure(&t).unwrap(), + vec![Token::Lit(Lit::new_bool(true))] + ); + assert_eq!( + lex_insecure(&f).unwrap(), + vec![Token::Lit(Lit::new_bool(false))] + ); +} +#[test] +fn lex_string() { + let s = br#" "hello, world" "#; + assert_eq!( + lex_insecure(s).unwrap(), + vec![Token::Lit(Lit::new_string("hello, world".into()))] + ); + let s = br#" 'hello, world' "#; + assert_eq!( + lex_insecure(s).unwrap(), + vec![Token::Lit(Lit::new_string("hello, world".into()))] + ); +} +#[test] +fn lex_string_test_escape_quote() { + let s = br#" "\"hello world\"" "#; // == "hello world" + assert_eq!( + lex_insecure(s).unwrap(), + vec![Token::Lit(Lit::new_string("\"hello world\"".into()))] + ); + let s = br#" '\'hello world\'' "#; // == 'hello world' + assert_eq!( + lex_insecure(s).unwrap(), + vec![Token::Lit(Lit::new_string("'hello world'".into()))] + ); +} +#[test] +fn lex_string_use_different_quote_style() { + let s = br#" "he's on it" "#; + assert_eq!( + lex_insecure(s).unwrap(), + vec![Token::Lit(Lit::new_string("he's on it".into()))] + ); + let s = br#" 'he thinks that "that girl" fixed it' "#; + assert_eq!( + lex_insecure(s).unwrap(), + vec![Token::Lit(Lit::new_string( + "he thinks that \"that girl\" fixed it".into() + ))] + ) +} +#[test] +fn lex_string_escape_bs() { + let s = v!(r#" "windows has c:\\" "#); + assert_eq!( + lex_insecure(&s).unwrap(), + vec![Token::Lit(Lit::new_string("windows has c:\\".into()))] + ); + let s = v!(r#" 'windows has c:\\' "#); + assert_eq!( + lex_insecure(&s).unwrap(), + vec![Token::Lit(Lit::new_string("windows has c:\\".into()))] + ); + let lol = v!(r#"'\\\\\\\\\\'"#); + let lexed = lex_insecure(&lol).unwrap(); + assert_eq!( + lexed, + vec![Token::Lit(Lit::new_string("\\".repeat(5)))], + "lol" + ) +} +#[test] +fn lex_string_bad_escape() { + let wth = br#" '\a should be an alert on windows apparently' "#; + assert_eq!(lex_insecure(wth).unwrap_err(), QueryError::LexInvalidInput); +} +#[test] +fn lex_string_unclosed() { + let wth = br#" 'omg where did the end go "#; + assert_eq!(lex_insecure(wth).unwrap_err(), QueryError::LexInvalidInput); + let wth = br#" 'see, we escaped the end\' "#; + assert_eq!(lex_insecure(wth).unwrap_err(), QueryError::LexInvalidInput); +} +#[test] +fn lex_unsafe_literal_mini() { + let usl = lex_insecure("\r0\n".as_bytes()).unwrap(); + assert_eq!(usl.len(), 1); + assert_eq!(Token::Lit(Lit::new_bin(b"")), usl[0]); +} +#[test] +fn lex_unsafe_literal() { + let usl = lex_insecure("\r9\nabcdefghi".as_bytes()).unwrap(); + assert_eq!(usl.len(), 1); + assert_eq!(Token::Lit(Lit::new_bin(b"abcdefghi")), usl[0]); +} +#[test] +fn lex_unsafe_literal_pro() { + let usl = lex_insecure("\r18\nabcdefghi123456789".as_bytes()).unwrap(); + assert_eq!(usl.len(), 1); + assert_eq!(Token::Lit(Lit::new_bin(b"abcdefghi123456789")), usl[0]); +} + +/* + safe query tests +*/ + +fn make_safe_query(a: &[u8], b: &[u8]) -> (Vec, usize) { + let mut s = Vec::with_capacity(a.len() + b.len()); + s.extend(a); + s.extend(b); + (s, a.len()) +} + +#[test] +fn safe_query_all_literals() { + let (query, query_window) = make_safe_query( + b"? ? ? ? ? ? ?", + b"\x00\x01\x01\x021234\n\x03-1234\n\x041234.5678\n\x0513\nbinarywithlf\n\x065\nsayan", + ); + let ret = lex_secure(&query, query_window).unwrap(); + assert_eq!( + ret, + into_vec![Token<'static> => ( + Token![null], + Lit::new_bool(true), + Lit::new_uint(1234), + Lit::new_sint(-1234), + Lit::new_float(1234.5678), + Lit::new_bin(b"binarywithlf\n"), + Lit::new_string("sayan".into()), + )], + ); +} + +const SFQ_NULL: &[u8] = b"\x00"; +const SFQ_BOOL_FALSE: &[u8] = b"\x01\0"; +const SFQ_BOOL_TRUE: &[u8] = b"\x01\x01"; +const SFQ_UINT: &[u8] = b"\x0218446744073709551615\n"; +const SFQ_SINT: &[u8] = b"\x03-9223372036854775808\n"; +const SFQ_FLOAT: &[u8] = b"\x043.141592654\n"; +const SFQ_BINARY: &[u8] = "\x0546\ncringe😃😄😁😆😅😂🤣😊😸😺".as_bytes(); +const SFQ_STRING: &[u8] = "\x0646\ncringe😃😄😁😆😅😂🤣😊😸😺".as_bytes(); + +#[test] +fn safe_query_null() { + let (query, query_window) = make_safe_query(b"?", SFQ_NULL); + let r = lex_secure(&query, query_window).unwrap(); + assert_eq!(r, vec![Token![null]]) +} + +#[test] +fn safe_query_bool() { + let (query, query_window) = make_safe_query(b"?", SFQ_BOOL_FALSE); + let b_false = lex_secure(&query, query_window).unwrap(); + let (query, query_window) = make_safe_query(b"?", SFQ_BOOL_TRUE); + let b_true = lex_secure(&query, query_window).unwrap(); + assert_eq!( + [b_false, b_true].concat(), + vec![ + Token::from(Lit::new_bool(false)), + Token::from(Lit::new_bool(true)) + ] + ); +} + +#[test] +fn safe_query_uint() { + let (query, query_window) = make_safe_query(b"?", SFQ_UINT); + let int = lex_secure(&query, query_window).unwrap(); + assert_eq!(int, vec![Token::Lit(Lit::new_uint(u64::MAX))]); +} + +#[test] +fn safe_query_sint() { + let (query, query_window) = make_safe_query(b"?", SFQ_SINT); + let int = lex_secure(&query, query_window).unwrap(); + assert_eq!(int, vec![Token::Lit(Lit::new_sint(i64::MIN))]); +} + +#[test] +fn safe_query_float() { + let (query, query_window) = make_safe_query(b"?", SFQ_FLOAT); + let float = lex_secure(&query, query_window).unwrap(); + assert_eq!(float, vec![Token::Lit(Lit::new_float(3.141592654))]); +} + +#[test] +fn safe_query_binary() { + let (query, query_window) = make_safe_query(b"?", SFQ_BINARY); + let binary = lex_secure(&query, query_window).unwrap(); + assert_eq!( + binary, + vec![Token::Lit(Lit::new_bin( + "cringe😃😄😁😆😅😂🤣😊😸😺".as_bytes() + ))] + ); +} + +#[test] +fn safe_query_string() { + let (query, query_window) = make_safe_query(b"?", SFQ_STRING); + let binary = lex_secure(&query, query_window).unwrap(); + assert_eq!( + binary, + vec![Token::Lit(Lit::new_string( + "cringe😃😄😁😆😅😂🤣😊😸😺".to_owned().into() + ))] + ); +} + +#[test] +fn safe_params_shuffled() { + let expected = [ + (SFQ_NULL, Token![null]), + (SFQ_BOOL_FALSE, Token::Lit(Lit::new_bool(false))), + (SFQ_BOOL_TRUE, Token::Lit(Lit::new_bool(true))), + (SFQ_UINT, Token::Lit(Lit::new_uint(u64::MAX))), + (SFQ_SINT, Token::Lit(Lit::new_sint(i64::MIN))), + (SFQ_FLOAT, Token::Lit(Lit::new_float(3.141592654))), + ( + SFQ_BINARY, + Token::Lit(Lit::new_bin("cringe😃😄😁😆😅😂🤣😊😸😺".as_bytes())), + ), + ( + SFQ_STRING, + Token::Lit(Lit::new_string( + "cringe😃😄😁😆😅😂🤣😊😸😺".to_owned().into(), + )), + ), + ]; + let mut rng = crate::util::test_utils::randomizer(); + for _ in 0..expected.len().pow(2) { + let mut this_expected = expected.clone(); + crate::util::test_utils::shuffle_slice(&mut this_expected, &mut rng); + let param_segment: Vec = this_expected + .iter() + .map(|(raw, _)| raw.to_vec()) + .flatten() + .collect(); + let (query, query_window) = make_safe_query(b"? ? ? ? ? ? ? ?", ¶m_segment); + let ret = lex_secure(&query, query_window).unwrap(); + assert_eq!( + ret, + this_expected + .into_iter() + .map(|(_, expected)| expected) + .collect::>() + ) + } +} diff --git a/server/src/engine/ql/tests/misc.rs b/server/src/engine/ql/tests/misc.rs new file mode 100644 index 00000000..032f3e0a --- /dev/null +++ b/server/src/engine/ql/tests/misc.rs @@ -0,0 +1,115 @@ +/* + * Created on Sun Dec 18 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use super::*; +use crate::engine::ql::{ + ast::{traits::ASTNode, State}, + ddl::{Inspect, Use}, +}; + +/* + entity +*/ + +#[test] +fn entity_current() { + let t = lex_insecure(b"hello").unwrap(); + let mut state = State::new_inplace(&t); + state.set_space("apps"); + let r = state.try_entity_ref().unwrap(); + assert_eq!(r, ("apps", "hello").into()); +} + +#[test] +fn entity_full() { + let t = lex_insecure(b"hello.world").unwrap(); + let mut state = State::new_inplace(&t); + assert_eq!( + state.try_entity_ref().unwrap(), + (("hello"), ("world")).into() + ) +} + +/* + use +*/ + +#[test] +fn use_new() { + let t = lex_insecure(b"use myspace").unwrap(); + let mut state = State::new_inplace(&t[1..]); + assert_eq!( + Use::test_parse_from_state(&mut state).unwrap(), + Use::Space("myspace".into()) + ); +} + +#[test] +fn use_null() { + let t = lex_insecure(b"use null").unwrap(); + let mut state = State::new_inplace(&t[1..]); + assert_eq!(Use::test_parse_from_state(&mut state).unwrap(), Use::Null); +} + +#[test] +fn use_current() { + let t = lex_insecure(b"use $current").unwrap(); + let mut state = State::new_inplace(&t[1..]); + assert_eq!( + Use::test_parse_from_state(&mut state).unwrap(), + Use::RefreshCurrent + ); +} + +#[test] +fn inspect_global() { + let t = lex_insecure(b"inspect global").unwrap(); + let mut state = State::new_inplace(&t[1..]); + assert_eq!( + Inspect::test_parse_from_state(&mut state).unwrap(), + Inspect::Global + ); +} + +#[test] +fn inspect_space() { + let t = lex_insecure(b"inspect space myspace").unwrap(); + let mut state = State::new_inplace(&t[1..]); + assert_eq!( + Inspect::test_parse_from_state(&mut state).unwrap(), + Inspect::Space("myspace".into()) + ); +} + +#[test] +fn inspect_model() { + let t = lex_insecure(b"inspect model myspace.mymodel").unwrap(); + let mut state = State::new_inplace(&t[1..]); + assert_eq!( + Inspect::test_parse_from_state(&mut state).unwrap(), + Inspect::Model(("myspace", "mymodel").into()) + ); +} diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs new file mode 100644 index 00000000..2ecf25e3 --- /dev/null +++ b/server/src/engine/ql/tests/schema_tests.rs @@ -0,0 +1,1153 @@ +/* + * Created on Sun Dec 18 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{super::lex::Ident, lex_insecure, *}, + crate::engine::data::lit::Lit, +}; + +mod alter_space { + use { + super::*, + crate::engine::{data::lit::Lit, ql::ddl::alt::AlterSpace}, + }; + #[test] + fn alter_space_mini() { + fullparse_verify_substmt("alter model mymodel with {}", |r: AlterSpace| { + assert_eq!(r, AlterSpace::new(Ident::from("mymodel"), null_dict! {})); + }) + } + #[test] + fn alter_space() { + fullparse_verify_substmt( + r#" + alter model mymodel with { + max_entry: 1000, + driver: "ts-0.8" + }"#, + |r: AlterSpace| { + assert_eq!( + r, + AlterSpace::new( + Ident::from("mymodel"), + null_dict! { + "max_entry" => Lit::new_uint(1000), + "driver" => Lit::new_string("ts-0.8".into()) + } + ) + ); + }, + ); + } +} +mod tymeta { + use super::*; + use crate::engine::ql::{ + ast::{parse_ast_node_full, traits::ASTNode, State}, + ddl::syn::{DictTypeMeta, DictTypeMetaSplit}, + }; + #[test] + fn tymeta_mini() { + let tok = lex_insecure(b"{}").unwrap(); + let tymeta = parse_ast_node_full::(&tok).unwrap(); + assert_eq!(tymeta, null_dict!()); + } + #[test] + #[should_panic] + fn tymeta_mini_fail() { + let tok = lex_insecure(b"{,}").unwrap(); + parse_ast_node_full::(&tok).unwrap(); + } + #[test] + fn tymeta() { + let tok = lex_insecure(br#"{hello: "world", loading: true, size: 100 }"#).unwrap(); + let tymeta = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + tymeta, + null_dict! { + "hello" => Lit::new_string("world".into()), + "loading" => Lit::new_bool(true), + "size" => Lit::new_uint(100) + } + ); + } + #[test] + fn tymeta_pro() { + // list { maxlen: 100, type: string, unique: true } + // ^^^^^^^^^^^^^^^^^^ cursor should be at string + let tok = lex_insecure(br#"{maxlen: 100, type: string, unique: true }"#).unwrap(); + let mut state = State::new_inplace(&tok); + let tymeta: DictTypeMeta = ASTNode::test_parse_from_state(&mut state).unwrap(); + assert_eq!(state.cursor(), 6); + assert!(Token![:].eq(state.fw_read())); + assert!(Token::Ident(Ident::from("string")).eq(state.fw_read())); + assert!(Token![,].eq(state.fw_read())); + let tymeta2: DictTypeMetaSplit = ASTNode::test_parse_from_state(&mut state).unwrap(); + assert!(state.exhausted()); + let mut final_ret = tymeta.into_inner(); + final_ret.extend(tymeta2.into_inner()); + assert_eq!( + final_ret, + null_dict! { + "maxlen" => Lit::new_uint(100), + "unique" => Lit::new_bool(true) + } + ) + } + #[test] + fn tymeta_pro_max() { + // list { maxlen: 100, this: { is: "cool" }, type: string, unique: true } + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cursor should be at string + let tok = + lex_insecure(br#"{maxlen: 100, this: { is: "cool" }, type: string, unique: true }"#) + .unwrap(); + let mut state = State::new_inplace(&tok); + let tymeta: DictTypeMeta = ASTNode::test_parse_from_state(&mut state).unwrap(); + assert_eq!(state.cursor(), 14); + assert!(Token![:].eq(state.fw_read())); + assert!(Token::Ident(Ident::from("string")).eq(state.fw_read())); + assert!(Token![,].eq(state.fw_read())); + let tymeta2: DictTypeMetaSplit = ASTNode::test_parse_from_state(&mut state).unwrap(); + assert!(state.exhausted()); + let mut final_ret = tymeta.into_inner(); + final_ret.extend(tymeta2.into_inner()); + assert_eq!( + final_ret, + null_dict! { + "maxlen" => Lit::new_uint(100), + "unique" => Lit::new_bool(true), + "this" => null_dict! { + "is" => Lit::new_string("cool".into()) + } + } + ) + } +} +mod layer { + use super::*; + use crate::engine::ql::{ast::parse_ast_node_multiple_full, ddl::syn::LayerSpec}; + #[test] + fn layer_mini() { + let tok = lex_insecure(b"string").unwrap(); + let layers = parse_ast_node_multiple_full::(&tok).unwrap(); + assert_eq!( + layers, + vec![LayerSpec::new(Ident::from("string"), null_dict! {})] + ); + } + #[test] + fn layer() { + let tok = lex_insecure(b"string { maxlen: 100 }").unwrap(); + let layers = parse_ast_node_multiple_full::(&tok).unwrap(); + assert_eq!( + layers, + vec![LayerSpec::new( + Ident::from("string"), + null_dict! { + "maxlen" => Lit::new_uint(100) + } + )] + ); + } + #[test] + fn layer_plus() { + let tok = lex_insecure(b"list { type: string }").unwrap(); + let layers = parse_ast_node_multiple_full::(&tok).unwrap(); + assert_eq!( + layers, + vec![ + LayerSpec::new(Ident::from("string"), null_dict! {}), + LayerSpec::new(Ident::from("list"), null_dict! {}) + ] + ); + } + #[test] + fn layer_pro() { + let tok = lex_insecure(b"list { unique: true, type: string, maxlen: 10 }").unwrap(); + let layers = parse_ast_node_multiple_full::(&tok).unwrap(); + assert_eq!( + layers, + vec![ + LayerSpec::new(Ident::from("string"), null_dict! {}), + LayerSpec::new( + Ident::from("list"), + null_dict! { + "unique" => Lit::new_bool(true), + "maxlen" => Lit::new_uint(10), + } + ) + ] + ); + } + #[test] + fn layer_pro_max() { + let tok = lex_insecure( + b"list { unique: true, type: string { ascii_only: true, maxlen: 255 }, maxlen: 10 }", + ) + .unwrap(); + let layers = parse_ast_node_multiple_full::(&tok).unwrap(); + assert_eq!( + layers, + vec![ + LayerSpec::new( + Ident::from("string"), + null_dict! { + "ascii_only" => Lit::new_bool(true), + "maxlen" => Lit::new_uint(255) + } + ), + LayerSpec::new( + Ident::from("list"), + null_dict! { + "unique" => Lit::new_bool(true), + "maxlen" => Lit::new_uint(10), + } + ) + ] + ); + } + + #[test] + #[cfg(not(miri))] + fn fuzz_layer() { + let tok = b" + list { + type: list { + maxlen: 100, + type: string\x01 + }, + unique: true\x01 + } + "; + let expected = vec![ + LayerSpec::new(Ident::from("string"), null_dict!()), + LayerSpec::new( + Ident::from("list"), + null_dict! { + "maxlen" => Lit::new_uint(100), + }, + ), + LayerSpec::new( + Ident::from("list"), + null_dict!("unique" => Lit::new_bool(true)), + ), + ]; + fuzz_tokens(tok.as_slice(), |should_pass, new_tok| { + let layers = parse_ast_node_multiple_full::(&new_tok); + let ok = layers.is_ok(); + if should_pass { + assert_eq!(layers.unwrap(), expected); + } + ok + }); + } +} +mod fields { + use { + super::*, + crate::engine::ql::{ + ast::parse_ast_node_full, + ddl::syn::{FieldSpec, LayerSpec}, + lex::Ident, + }, + }; + #[test] + fn field_mini() { + let tok = lex_insecure(b"username: string").unwrap(); + let f = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + f, + FieldSpec::new( + Ident::from("username"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + false, + false + ) + ) + } + #[test] + fn field() { + let tok = lex_insecure(b"primary username: string").unwrap(); + let f = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + f, + FieldSpec::new( + Ident::from("username"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + false, + true + ) + ) + } + #[test] + fn field_pro() { + let tok = lex_insecure( + b" + primary username: string { + maxlen: 10, + ascii_only: true, + } + ", + ) + .unwrap(); + let f = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + f, + FieldSpec::new( + Ident::from("username"), + [LayerSpec::new( + Ident::from("string"), + null_dict! { + "maxlen" => Lit::new_uint(10), + "ascii_only" => Lit::new_bool(true), + } + )] + .into(), + false, + true, + ) + ) + } + #[test] + fn field_pro_max() { + let tok = lex_insecure( + b" + null notes: list { + type: string { + maxlen: 255, + ascii_only: true, + }, + unique: true, + } + ", + ) + .unwrap(); + let f = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + f, + FieldSpec::new( + Ident::from("notes"), + [ + LayerSpec::new( + Ident::from("string"), + null_dict! { + "maxlen" => Lit::new_uint(255), + "ascii_only" => Lit::new_bool(true), + } + ), + LayerSpec::new( + Ident::from("list"), + null_dict! { + "unique" => Lit::new_bool(true) + } + ), + ] + .into(), + true, + false, + ) + ) + } +} +mod schemas { + use super::*; + use crate::engine::ql::ddl::{ + crt::CreateModel, + syn::{FieldSpec, LayerSpec}, + }; + #[test] + fn schema_mini() { + let mut ret = CreateModel::new( + ("apps", "mymodel").into(), + vec![ + FieldSpec::new( + Ident::from("username"), + vec![LayerSpec::new(Ident::from("string"), null_dict! {})], + false, + true, + ), + FieldSpec::new( + Ident::from("password"), + vec![LayerSpec::new(Ident::from("binary"), null_dict! {})], + false, + false, + ), + ], + null_dict! {}, + false, + ); + fullparse_verify_substmt_with_space( + "create model mymodel( + primary username: string, + password: binary + )", + "apps", + |r: CreateModel| assert_eq!(r, ret), + ); + ret.if_not_exists = true; + fullparse_verify_substmt_with_space( + "create model if not exists mymodel( + primary username: string, + password: binary + )", + "apps", + |r: CreateModel| assert_eq!(r, ret), + ); + } + #[test] + fn schema() { + let mut ret = CreateModel::new( + ("apps", "mymodel").into(), + vec![ + FieldSpec::new( + Ident::from("username"), + vec![LayerSpec::new(Ident::from("string"), null_dict! {})], + false, + true, + ), + FieldSpec::new( + Ident::from("password"), + vec![LayerSpec::new(Ident::from("binary"), null_dict! {})], + false, + false, + ), + FieldSpec::new( + Ident::from("profile_pic"), + vec![LayerSpec::new(Ident::from("binary"), null_dict! {})], + true, + false, + ), + ], + null_dict! {}, + false, + ); + fullparse_verify_substmt_with_space( + "create model mymodel( + primary username: string, + password: binary, + null profile_pic: binary + )", + "apps", + |r: CreateModel| assert_eq!(r, ret), + ); + ret.if_not_exists = true; + fullparse_verify_substmt_with_space( + "create model if not exists mymodel( + primary username: string, + password: binary, + null profile_pic: binary + )", + "apps", + |r: CreateModel| assert_eq!(r, ret), + ); + } + + #[test] + fn schema_pro() { + let mut ret = CreateModel::new( + ("apps", "mymodel").into(), + vec![ + FieldSpec::new( + Ident::from("username"), + vec![LayerSpec::new(Ident::from("string"), null_dict! {})], + false, + true, + ), + FieldSpec::new( + Ident::from("password"), + vec![LayerSpec::new(Ident::from("binary"), null_dict! {})], + false, + false, + ), + FieldSpec::new( + Ident::from("profile_pic"), + vec![LayerSpec::new(Ident::from("binary"), null_dict! {})], + true, + false, + ), + FieldSpec::new( + Ident::from("notes"), + vec![ + LayerSpec::new(Ident::from("string"), null_dict! {}), + LayerSpec::new( + Ident::from("list"), + null_dict! { + "unique" => Lit::new_bool(true) + }, + ), + ], + true, + false, + ), + ], + null_dict! {}, + false, + ); + ret.if_not_exists = true; + fullparse_verify_substmt_with_space( + " + create model if not exists mymodel( + primary username: string, + password: binary, + null profile_pic: binary, + null notes: list { + type: string, + unique: true, + } + ) + ", + "apps", + |r: CreateModel| assert_eq!(ret, r), + ); + } + #[test] + fn schema_pro_max() { + let mut ret = CreateModel::new( + ("apps", "mymodel").into(), + vec![ + FieldSpec::new( + Ident::from("username"), + vec![LayerSpec::new(Ident::from("string"), null_dict! {})], + false, + true, + ), + FieldSpec::new( + Ident::from("password"), + vec![LayerSpec::new(Ident::from("binary"), null_dict! {})], + false, + false, + ), + FieldSpec::new( + Ident::from("profile_pic"), + vec![LayerSpec::new(Ident::from("binary"), null_dict! {})], + true, + false, + ), + FieldSpec::new( + Ident::from("notes"), + vec![ + LayerSpec::new(Ident::from("string"), null_dict! {}), + LayerSpec::new( + Ident::from("list"), + null_dict! { + "unique" => Lit::new_bool(true) + }, + ), + ], + true, + false, + ), + ], + null_dict! { + "env" => null_dict! { + "free_user_limit" => Lit::new_uint(100), + }, + "storage_driver" => Lit::new_string("skyheap".into()), + }, + false, + ); + ret.if_not_exists = true; + fullparse_verify_substmt_with_space( + " + create model if not exists mymodel( + primary username: string, + password: binary, + null profile_pic: binary, + null notes: list { + type: string, + unique: true, + } + ) with { + env: { + free_user_limit: 100, + }, + storage_driver: \"skyheap\" + }", + "apps", + |r: CreateModel| assert_eq!(r, ret), + ); + } +} +mod dict_field_syntax { + use super::*; + use crate::engine::ql::{ + ast::parse_ast_node_full, + ddl::syn::{ExpandedField, LayerSpec}, + }; + #[test] + fn field_syn_mini() { + let tok = lex_insecure(b"username { type: string }").unwrap(); + let ef = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + ef, + ExpandedField::new( + Ident::from("username"), + vec![LayerSpec::new(Ident::from("string"), null_dict! {})], + null_dict! {} + ) + ) + } + #[test] + fn field_syn() { + let tok = lex_insecure( + b" + username { + nullable: false, + type: string, + } + ", + ) + .unwrap(); + let ef = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + ef, + ExpandedField::new( + Ident::from("username"), + vec![LayerSpec::new(Ident::from("string"), null_dict! {})], + null_dict! { + "nullable" => Lit::new_bool(false), + }, + ) + ); + } + #[test] + fn field_syn_pro() { + let tok = lex_insecure( + b" + username { + nullable: false, + type: string { + minlen: 6, + maxlen: 255, + }, + jingle_bells: \"snow\" + } + ", + ) + .unwrap(); + let ef = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + ef, + ExpandedField::new( + Ident::from("username"), + vec![LayerSpec::new( + Ident::from("string"), + null_dict! { + "minlen" => Lit::new_uint(6), + "maxlen" => Lit::new_uint(255), + } + )], + null_dict! { + "nullable" => Lit::new_bool(false), + "jingle_bells" => Lit::new_string("snow".into()), + }, + ) + ); + } + #[test] + fn field_syn_pro_max() { + let tok = lex_insecure( + b" + notes { + nullable: true, + type: list { + type: string { + ascii_only: true, + }, + unique: true, + }, + jingle_bells: \"snow\" + } + ", + ) + .unwrap(); + let ef = parse_ast_node_full::(&tok).unwrap(); + assert_eq!( + ef, + ExpandedField::new( + Ident::from("notes"), + vec![ + LayerSpec::new( + Ident::from("string"), + null_dict! { + "ascii_only" => Lit::new_bool(true), + } + ), + LayerSpec::new( + Ident::from("list"), + null_dict! { + "unique" => Lit::new_bool(true), + } + ) + ], + null_dict! { + "nullable" => Lit::new_bool(true), + "jingle_bells" => Lit::new_string("snow".into()), + }, + ) + ); + } +} +mod alter_model_remove { + use super::*; + use crate::engine::ql::{ + ast::parse_ast_node_full_with_space, + ddl::alt::{AlterKind, AlterModel}, + lex::Ident, + }; + #[test] + fn alter_mini() { + let tok = lex_insecure(b"alter model mymodel remove myfield").unwrap(); + let remove = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + remove, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Remove(Box::from([Ident::from("myfield")])) + ) + ); + } + #[test] + fn alter_mini_2() { + let tok = lex_insecure(b"alter model mymodel remove (myfield)").unwrap(); + let remove = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + remove, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Remove(Box::from([Ident::from("myfield")])) + ) + ); + } + #[test] + fn alter() { + let tok = + lex_insecure(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)") + .unwrap(); + let remove = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + remove, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Remove(Box::from([ + Ident::from("myfield1"), + Ident::from("myfield2"), + Ident::from("myfield3"), + Ident::from("myfield4"), + ])) + ) + ); + } +} +mod alter_model_add { + use super::*; + use crate::engine::ql::{ + ast::parse_ast_node_full_with_space, + ddl::{ + alt::{AlterKind, AlterModel}, + syn::{ExpandedField, LayerSpec}, + }, + }; + #[test] + fn add_mini() { + let tok = lex_insecure( + b" + alter model mymodel add myfield { type: string } + ", + ) + .unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(), + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Add( + [ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! {}, + )] + .into() + ) + ) + ); + } + #[test] + fn add() { + let tok = lex_insecure( + b" + alter model mymodel add myfield { type: string, nullable: true } + ", + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + r, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Add( + [ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! { + "nullable" => Lit::new_bool(true) + }, + )] + .into() + ) + ) + ); + } + #[test] + fn add_pro() { + let tok = lex_insecure( + b" + alter model mymodel add (myfield { type: string, nullable: true }) + ", + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + r, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Add( + [ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! { + "nullable" => Lit::new_bool(true) + }, + )] + .into() + ) + ) + ); + } + #[test] + fn add_pro_max() { + let tok = lex_insecure( + b" + alter model mymodel add ( + myfield { + type: string, + nullable: true + }, + another { + type: list { + type: string { + maxlen: 255 + }, + unique: true + }, + nullable: false, + } + ) + ", + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + r, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Add( + [ + ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! { + "nullable" => Lit::new_bool(true) + }, + ), + ExpandedField::new( + Ident::from("another"), + [ + LayerSpec::new( + Ident::from("string"), + null_dict! { + "maxlen" => Lit::new_uint(255) + } + ), + LayerSpec::new( + Ident::from("list"), + null_dict! { + "unique" => Lit::new_bool(true) + }, + ) + ] + .into(), + null_dict! { + "nullable" => Lit::new_bool(false) + }, + ) + ] + .into() + ) + ) + ); + } +} +mod alter_model_update { + use super::*; + use crate::engine::ql::{ + ast::parse_ast_node_full_with_space, + ddl::{ + alt::{AlterKind, AlterModel}, + syn::{ExpandedField, LayerSpec}, + }, + }; + + #[test] + fn alter_mini() { + let tok = lex_insecure( + b" + alter model mymodel update myfield { type: string } + ", + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + r, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Update( + [ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! {}, + )] + .into() + ) + ) + ); + } + #[test] + fn alter_mini_2() { + let tok = lex_insecure( + b" + alter model mymodel update (myfield { type: string }) + ", + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + r, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Update( + [ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! {}, + )] + .into() + ) + ) + ); + } + #[test] + fn alter() { + let tok = lex_insecure( + b" + alter model mymodel update ( + myfield { + type: string, + nullable: true, + } + ) + ", + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + r, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Update( + [ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! { + "nullable" => Lit::new_bool(true) + }, + )] + .into() + ) + ) + ); + } + #[test] + fn alter_pro() { + let tok = lex_insecure( + b" + alter model mymodel update ( + myfield { + type: string, + nullable: true, + }, + myfield2 { + type: string, + } + ) + ", + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + r, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Update( + [ + ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! { + "nullable" => Lit::new_bool(true) + }, + ), + ExpandedField::new( + Ident::from("myfield2"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! {}, + ) + ] + .into() + ) + ) + ); + } + #[test] + fn alter_pro_max() { + let tok = lex_insecure( + b" + alter model mymodel update ( + myfield { + type: string {}, + nullable: true, + }, + myfield2 { + type: string { + maxlen: 255, + }, + } + ) + ", + ) + .unwrap(); + let r = parse_ast_node_full_with_space::(&tok[2..], "apps").unwrap(); + assert_eq!( + r, + AlterModel::new( + ("apps", "mymodel").into(), + AlterKind::Update( + [ + ExpandedField::new( + Ident::from("myfield"), + [LayerSpec::new(Ident::from("string"), null_dict! {})].into(), + null_dict! { + "nullable" => Lit::new_bool(true) + }, + ), + ExpandedField::new( + Ident::from("myfield2"), + [LayerSpec::new( + Ident::from("string"), + null_dict! {"maxlen" => Lit::new_uint(255)} + )] + .into(), + null_dict! {}, + ) + ] + .into() + ) + ) + ); + } +} + +mod ddl_other_query_tests { + use { + super::*, + crate::engine::ql::{ + ast::{parse_ast_node_full, parse_ast_node_full_with_space}, + ddl::drop::{DropModel, DropSpace}, + lex::Ident, + }, + }; + #[test] + fn drop_space() { + let src = lex_insecure(br"drop space myspace").unwrap(); + assert_eq!( + parse_ast_node_full::(&src[2..]).unwrap(), + DropSpace::new(Ident::from("myspace"), false, false) + ); + let src = lex_insecure(br"drop space if exists myspace").unwrap(); + assert_eq!( + parse_ast_node_full::(&src[2..]).unwrap(), + DropSpace::new(Ident::from("myspace"), false, true) + ); + } + #[test] + fn drop_space_force() { + let src = lex_insecure(br"drop space allow not empty myspace").unwrap(); + assert_eq!( + parse_ast_node_full::(&src[2..]).unwrap(), + DropSpace::new(Ident::from("myspace"), true, false) + ); + let src = lex_insecure(br"drop space if exists allow not empty myspace").unwrap(); + assert_eq!( + parse_ast_node_full::(&src[2..]).unwrap(), + DropSpace::new(Ident::from("myspace"), true, true) + ); + } + #[test] + fn drop_model() { + let src = lex_insecure(br"drop model mymodel").unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&src[2..], "apps").unwrap(), + DropModel::new(("apps", "mymodel").into(), false, false) + ); + let src = lex_insecure(br"drop model if exists mymodel").unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&src[2..], "apps").unwrap(), + DropModel::new(("apps", "mymodel").into(), false, true) + ); + } + #[test] + fn drop_model_force() { + let src = lex_insecure(br"drop model allow not empty mymodel").unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&src[2..], "apps").unwrap(), + DropModel::new(("apps", "mymodel").into(), true, false) + ); + let src = lex_insecure(br"drop model if exists allow not empty mymodel").unwrap(); + assert_eq!( + parse_ast_node_full_with_space::(&src[2..], "apps").unwrap(), + DropModel::new(("apps", "mymodel").into(), true, true) + ); + } +} diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs new file mode 100644 index 00000000..960a0cac --- /dev/null +++ b/server/src/engine/ql/tests/structure_syn.rs @@ -0,0 +1,318 @@ +/* + * Created on Sun Dec 18 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::*, + crate::engine::{ + data::{lit::Lit, DictGeneric}, + ql::{ast::parse_ast_node_full, ddl::syn::DictBasic}, + }, +}; + +macro_rules! fold_dict { + ($($dict:expr),+ $(,)?) => { + ($( + fold_dict($dict).unwrap() + ),+) + } +} + +fn fold_dict(raw: &[u8]) -> Option { + let lexed = lex_insecure(raw).unwrap(); + parse_ast_node_full::(&lexed) + .map(|v| v.into_inner()) + .ok() +} + +mod dict { + use super::*; + + #[test] + fn dict_read_mini() { + let (d1, d2) = fold_dict! { + br#"{name: "sayan"}"#, + br#"{name: "sayan",}"#, + }; + let r = null_dict!("name" => Lit::new_string("sayan".into())); + multi_assert_eq!(d1, d2 => r); + } + #[test] + fn dict_read() { + let (d1, d2) = fold_dict! { + br#" + { + name: "sayan", + verified: true, + burgers: 152 + } + "#, + br#" + { + name: "sayan", + verified: true, + burgers: 152, + } + "#, + }; + let r = null_dict! ( + "name" => Lit::new_string("sayan".into()), + "verified" => Lit::new_bool(true), + "burgers" => Lit::new_uint(152), + ); + multi_assert_eq!(d1, d2 => r); + } + #[test] + fn dict_read_pro() { + let (d1, d2, d3) = fold_dict! { + br#" + { + name: "sayan", + notes: { + burgers: "all the time, extra mayo", + taco: true, + pretzels: 1 + } + } + "#, + br#" + { + name: "sayan", + notes: { + burgers: "all the time, extra mayo", + taco: true, + pretzels: 1, + } + } + "#, + br#" + { + name: "sayan", + notes: { + burgers: "all the time, extra mayo", + taco: true, + pretzels: 1, + }, + }"# + }; + multi_assert_eq!( + d1, d2, d3 => null_dict! { + "name" => Lit::new_string("sayan".into()), + "notes" => null_dict! { + "burgers" => Lit::new_string("all the time, extra mayo".into()), + "taco" => Lit::new_bool(true), + "pretzels" => Lit::new_uint(1), + } + } + ); + } + + #[test] + fn dict_read_pro_max() { + let (d1, d2, d3) = fold_dict! { + br#" + { + well: { + now: { + this: { + is: { + ridiculous: true + } + } + } + } + } + "#, + br#" + { + well: { + now: { + this: { + is: { + ridiculous: true, + } + } + } + } + } + "#, + br#" + { + well: { + now: { + this: { + is: { + ridiculous: true, + }, + }, + }, + }, + } + "# + }; + multi_assert_eq!( + d1, d2, d3 => null_dict! { + "well" => null_dict! { + "now" => null_dict! { + "this" => null_dict! { + "is" => null_dict! { + "ridiculous" => Lit::new_bool(true), + } + } + } + } + } + ); + } + + #[test] + #[cfg(not(miri))] + fn fuzz_dict() { + let tok = b" + { + the_tradition_is: \"hello, world\", + could_have_been: { + this: true, + or_maybe_this: 100, + even_this: \"hello, universe!\"\x01 + }, + but_oh_well: \"it continues to be the 'annoying' phrase\", + lorem: { + ipsum: { + dolor: \"sit amet\"\x01 + }\x01 + }\x01 + } + "; + let ret_dict = null_dict! { + "the_tradition_is" => Lit::new_string("hello, world".into()), + "could_have_been" => null_dict! { + "this" => Lit::new_bool(true), + "or_maybe_this" => Lit::new_uint(100), + "even_this" => Lit::new_string("hello, universe!".into()), + }, + "but_oh_well" => Lit::new_string("it continues to be the 'annoying' phrase".into()), + "lorem" => null_dict! { + "ipsum" => null_dict! { + "dolor" => Lit::new_string("sit amet".into()) + } + } + }; + fuzz_tokens(&tok[..], |should_pass, new_src| { + let r = parse_ast_node_full::(new_src); + let okay = r.is_ok(); + if should_pass { + assert_eq!(r.unwrap(), ret_dict) + } + okay + }); + } +} +mod null_dict_tests { + use super::*; + mod dict { + use super::*; + + #[test] + fn null_mini() { + let d = fold_dict!(br"{ x: null }"); + assert_eq!( + d, + null_dict! { + "x" => Null, + } + ); + } + #[test] + fn null() { + let d = fold_dict! { + br#" + { + this_is_non_null: "hello", + but_this_is_null: null, + } + "# + }; + assert_eq!( + d, + null_dict! { + "this_is_non_null" => Lit::new_string("hello".into()), + "but_this_is_null" => Null, + } + ) + } + #[test] + fn null_pro() { + let d = fold_dict! { + br#" + { + a_string: "this is a string", + num: 1234, + a_dict: { + a_null: null, + } + } + "# + }; + assert_eq!( + d, + null_dict! { + "a_string" => Lit::new_string("this is a string".into()), + "num" => Lit::new_uint(1234), + "a_dict" => null_dict! { + "a_null" => Null, + } + } + ) + } + #[test] + fn null_pro_max() { + let d = fold_dict! { + br#" + { + a_string: "this is a string", + num: 1234, + a_dict: { + a_null: null, + }, + another_null: null, + } + "# + }; + assert_eq!( + d, + null_dict! { + "a_string" => Lit::new_string("this is a string".into()), + "num" => Lit::new_uint(1234), + "a_dict" => null_dict! { + "a_null" => Null, + }, + "another_null" => Null, + } + ) + } + } + // TODO(@ohsayan): Add null tests +} diff --git a/server/src/engine/storage/checksum.rs b/server/src/engine/storage/checksum.rs new file mode 100644 index 00000000..dc14856e --- /dev/null +++ b/server/src/engine/storage/checksum.rs @@ -0,0 +1,52 @@ +/* + * Created on Sun Sep 03 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crc::{Crc, Digest, CRC_64_XZ}; + +/* + NOTE(@ohsayan): we're currently using crc's impl. but the reason I decided to make a wrapper is because I have a + different impl in mind +*/ + +const CRC64: Crc = Crc::::new(&CRC_64_XZ); + +pub struct SCrc { + digest: Digest<'static, u64>, +} + +impl SCrc { + pub const fn new() -> Self { + Self { + digest: CRC64.digest(), + } + } + pub fn recompute_with_new_var_block(&mut self, b: &[u8]) { + self.digest.update(b) + } + pub fn finish(self) -> u64 { + self.digest.finalize() + } +} diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs new file mode 100644 index 00000000..e69de29b diff --git a/server/src/engine/storage/mod.rs b/server/src/engine/storage/mod.rs new file mode 100644 index 00000000..540fb990 --- /dev/null +++ b/server/src/engine/storage/mod.rs @@ -0,0 +1,34 @@ +/* + * Created on Mon May 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +//! Implementations of the Skytable Disk Storage Subsystem (SDSS) + +mod checksum; +mod versions; +// impls +pub mod v1; + +pub use checksum::SCrc; diff --git a/server/src/engine/storage/v1/batch_jrnl/mod.rs b/server/src/engine/storage/v1/batch_jrnl/mod.rs new file mode 100644 index 00000000..2013acbf --- /dev/null +++ b/server/src/engine/storage/v1/batch_jrnl/mod.rs @@ -0,0 +1,66 @@ +/* + * Created on Sun Sep 03 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod persist; +mod restore; + +/// the data batch file was reopened +const MARKER_BATCH_REOPEN: u8 = 0xFB; +/// the data batch file was closed +const MARKER_BATCH_CLOSED: u8 = 0xFC; +/// end of batch marker +const MARKER_END_OF_BATCH: u8 = 0xFD; +/// "real" batch event marker +const MARKER_ACTUAL_BATCH_EVENT: u8 = 0xFE; +/// recovery batch event marker +const MARKER_RECOVERY_EVENT: u8 = 0xFF; + +#[cfg(test)] +pub(super) use restore::{DecodedBatchEvent, DecodedBatchEventKind, NormalBatch}; +pub use {persist::DataBatchPersistDriver, restore::DataBatchRestoreDriver}; + +use { + super::{rw::SDSSFileIO, spec, RawFSInterface}, + crate::engine::{core::model::Model, error::RuntimeResult}, +}; + +/// Re-initialize an existing batch journal and read all its data into model +pub fn reinit( + name: &str, + model: &Model, +) -> RuntimeResult> { + let (f, _header) = SDSSFileIO::::open::(name)?; + // restore + let mut restore_driver = DataBatchRestoreDriver::new(f)?; + restore_driver.read_data_batch_into_model(model)?; + DataBatchPersistDriver::new(restore_driver.into_file()?, false) +} + +/// Create a new batch journal +pub fn create(path: &str) -> RuntimeResult> { + let f = SDSSFileIO::::create::(path)?; + DataBatchPersistDriver::new(f, true) +} diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs new file mode 100644 index 00000000..852411d1 --- /dev/null +++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs @@ -0,0 +1,249 @@ +/* + * Created on Tue Sep 05 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::storage::v1::inf::obj::cell; + +use { + super::{ + MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH, + MARKER_RECOVERY_EVENT, + }, + crate::{ + engine::{ + core::{ + index::{PrimaryIndexKey, RowData}, + model::{ + delta::{DataDelta, DataDeltaKind, DeltaVersion}, + Model, + }, + }, + data::{ + cell::Datacell, + tag::{DataTag, TagUnique}, + }, + error::{RuntimeResult, StorageError}, + idx::STIndexSeq, + storage::v1::rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedWriter}, + }, + util::EndianQW, + }, + crossbeam_epoch::pin, +}; + +pub struct DataBatchPersistDriver { + f: SDSSFileTrackedWriter, +} + +impl DataBatchPersistDriver { + pub fn new(mut file: SDSSFileIO, is_new: bool) -> RuntimeResult { + if !is_new { + file.fsynced_write(&[MARKER_BATCH_REOPEN])?; + } + Ok(Self { + f: SDSSFileTrackedWriter::new(file)?, + }) + } + pub fn close(self) -> RuntimeResult<()> { + let mut slf = self.f.into_inner_file()?; + if slf.fsynced_write(&[MARKER_BATCH_CLOSED]).is_ok() { + return Ok(()); + } else { + return Err(StorageError::DataBatchCloseError.into()); + } + } + pub fn write_new_batch(&mut self, model: &Model, observed_len: usize) -> RuntimeResult<()> { + // pin model + let schema_version = model.delta_state().schema_current_version(); + let g = pin(); + // init restore list + let mut restore_list = Vec::new(); + // prepare computations + let mut i = 0; + let mut inconsistent_reads = 0; + let mut exec = || -> RuntimeResult<()> { + // write batch start + self.write_batch_start( + observed_len, + schema_version, + model.p_tag().tag_unique(), + model.fields().len() - 1, + )?; + while i < observed_len { + let delta = model.delta_state().__data_delta_dequeue(&g).unwrap(); + restore_list.push(delta.clone()); // TODO(@ohsayan): avoid this + match delta.change() { + DataDeltaKind::Delete => { + self.write_batch_item_common_row_data(&delta)?; + self.encode_pk_only(delta.row().d_key())?; + } + DataDeltaKind::Insert | DataDeltaKind::Update => { + // resolve deltas (this is yet another opportunity for us to reclaim memory from deleted items) + let row_data = delta + .row() + .resolve_schema_deltas_and_freeze_if(&model.delta_state(), |row| { + row.get_txn_revised() <= delta.data_version() + }); + if row_data.get_txn_revised() > delta.data_version() { + // we made an inconsistent (stale) read; someone updated the state after our snapshot + inconsistent_reads += 1; + i += 1; + continue; + } + self.write_batch_item_common_row_data(&delta)?; + // encode data + self.encode_pk_only(delta.row().d_key())?; + self.encode_row_data(model, &row_data)?; + } + } + i += 1; + } + return self.append_batch_summary_and_sync(observed_len, inconsistent_reads); + }; + match exec() { + Ok(()) => Ok(()), + Err(e) => { + // republish changes since we failed to commit + restore_list.into_iter().for_each(|delta| { + model.delta_state().append_new_data_delta(delta, &g); + }); + // now attempt to fix the file + self.attempt_fix_data_batchfile()?; + // IMPORTANT: return an error because even though we recovered the journal we still didn't succeed in + // writing the batch + return Err(e); + } + } + } + /// Write the batch start block: + /// - Batch start magic + /// - Primary key type + /// - Expected commit + /// - Schema version + /// - Column count + fn write_batch_start( + &mut self, + observed_len: usize, + schema_version: DeltaVersion, + pk_tag: TagUnique, + col_cnt: usize, + ) -> RuntimeResult<()> { + self.f + .tracked_write_unfsynced(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.value_u8()])?; + let observed_len_bytes = observed_len.u64_bytes_le(); + self.f.tracked_write_unfsynced(&observed_len_bytes)?; + self.f + .tracked_write_unfsynced(&schema_version.value_u64().to_le_bytes())?; + self.f.tracked_write_unfsynced(&col_cnt.u64_bytes_le())?; + Ok(()) + } + /// Append a summary of this batch and most importantly, **sync everything to disk** + fn append_batch_summary_and_sync( + &mut self, + observed_len: usize, + inconsistent_reads: usize, + ) -> RuntimeResult<()> { + // [0xFD][actual_commit][checksum] + self.f.tracked_write_unfsynced(&[MARKER_END_OF_BATCH])?; + let actual_commit = (observed_len - inconsistent_reads).u64_bytes_le(); + self.f.tracked_write_unfsynced(&actual_commit)?; + let cs = self.f.reset_and_finish_checksum().to_le_bytes(); + self.f.untracked_write(&cs)?; + // IMPORTANT: now that all data has been written, we need to actually ensure that the writes pass through the cache + self.f.sync_writes()?; + Ok(()) + } + /// Attempt to fix the batch journal + // TODO(@ohsayan): declare an "international system disaster" when this happens + fn attempt_fix_data_batchfile(&mut self) -> RuntimeResult<()> { + /* + attempt to append 0xFF to the part of the file where a corruption likely occurred, marking + it recoverable + */ + if self.f.untracked_write(&[MARKER_RECOVERY_EVENT]).is_ok() { + return Ok(()); + } + Err(StorageError::DataBatchRecoveryFailStageOne.into()) + } +} + +impl DataBatchPersistDriver { + /// encode the primary key only. this means NO TAG is encoded. + fn encode_pk_only(&mut self, pk: &PrimaryIndexKey) -> RuntimeResult<()> { + let buf = &mut self.f; + match pk.tag() { + TagUnique::UnsignedInt | TagUnique::SignedInt => { + let data = unsafe { + // UNSAFE(@ohsayan): +tagck + pk.read_uint() + } + .to_le_bytes(); + buf.tracked_write_unfsynced(&data)?; + } + TagUnique::Str | TagUnique::Bin => { + let slice = unsafe { + // UNSAFE(@ohsayan): +tagck + pk.read_bin() + }; + let slice_l = slice.len().u64_bytes_le(); + buf.tracked_write_unfsynced(&slice_l)?; + buf.tracked_write_unfsynced(slice)?; + } + TagUnique::Illegal => unsafe { + // UNSAFE(@ohsayan): a pk can't be constructed with illegal + impossible!() + }, + } + Ok(()) + } + /// Encode a single cell + fn encode_cell(&mut self, value: &Datacell) -> RuntimeResult<()> { + let mut buf = vec![]; + cell::encode(&mut buf, value); + self.f.tracked_write_unfsynced(&buf)?; + Ok(()) + } + /// Encode row data + fn encode_row_data(&mut self, model: &Model, row_data: &RowData) -> RuntimeResult<()> { + for field_name in model.fields().stseq_ord_key() { + match row_data.fields().get(field_name) { + Some(cell) => { + self.encode_cell(cell)?; + } + None if field_name.as_str() == model.p_key() => {} + None => self.f.tracked_write_unfsynced(&[0])?, + } + } + Ok(()) + } + /// Write the change type and txnid + fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> RuntimeResult<()> { + let change_type = [delta.change().value_u8()]; + self.f.tracked_write_unfsynced(&change_type)?; + let txn_id = delta.data_version().value_u64().to_le_bytes(); + self.f.tracked_write_unfsynced(&txn_id)?; + Ok(()) + } +} diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs new file mode 100644 index 00000000..bcc99534 --- /dev/null +++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs @@ -0,0 +1,544 @@ +/* + * Created on Tue Sep 05 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::storage::v1::inf::{ + obj::cell::{self, StorageCellTypeID}, + DataSource, +}; + +use { + super::{ + MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH, + MARKER_RECOVERY_EVENT, + }, + crate::engine::{ + core::{ + index::{DcFieldIndex, PrimaryIndexKey, Row}, + model::{delta::DeltaVersion, Model}, + }, + data::{cell::Datacell, tag::TagUnique}, + error::{RuntimeResult, StorageError}, + idx::{MTIndex, STIndex, STIndexSeq}, + storage::v1::rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader}, + }, + std::{ + collections::{hash_map::Entry as HMEntry, HashMap}, + mem::ManuallyDrop, + }, +}; + +#[derive(Debug, PartialEq)] +pub(in crate::engine::storage::v1) struct DecodedBatchEvent { + txn_id: DeltaVersion, + pk: PrimaryIndexKey, + kind: DecodedBatchEventKind, +} + +impl DecodedBatchEvent { + pub(in crate::engine::storage::v1) const fn new( + txn_id: u64, + pk: PrimaryIndexKey, + kind: DecodedBatchEventKind, + ) -> Self { + Self { + txn_id: DeltaVersion::__new(txn_id), + pk, + kind, + } + } +} + +#[derive(Debug, PartialEq)] +pub(in crate::engine::storage::v1) enum DecodedBatchEventKind { + Delete, + Insert(Vec), + Update(Vec), +} + +#[derive(Debug, PartialEq)] +pub(in crate::engine::storage::v1) struct NormalBatch { + events: Vec, + schema_version: u64, +} + +impl NormalBatch { + pub(in crate::engine::storage::v1) fn new( + events: Vec, + schema_version: u64, + ) -> Self { + Self { + events, + schema_version, + } + } +} + +enum Batch { + RecoveredFromerror, + Normal(NormalBatch), + FinishedEarly(NormalBatch), + BatchClosed, +} + +pub struct DataBatchRestoreDriver { + f: SDSSFileTrackedReader, +} + +impl DataBatchRestoreDriver { + pub fn new(f: SDSSFileIO) -> RuntimeResult { + Ok(Self { + f: SDSSFileTrackedReader::new(f)?, + }) + } + pub fn into_file(self) -> RuntimeResult> { + self.f.into_inner_file() + } + pub(in crate::engine::storage::v1) fn read_data_batch_into_model( + &mut self, + model: &Model, + ) -> RuntimeResult<()> { + self.read_all_batches_and_for_each(|batch| { + // apply the batch + Self::apply_batch(model, batch) + }) + } + #[cfg(test)] + pub(in crate::engine::storage::v1) fn read_all_batches( + &mut self, + ) -> RuntimeResult> { + let mut all_batches = vec![]; + self.read_all_batches_and_for_each(|batch| { + all_batches.push(batch); + Ok(()) + })?; + Ok(all_batches) + } +} + +impl DataBatchRestoreDriver { + fn read_all_batches_and_for_each( + &mut self, + mut f: impl FnMut(NormalBatch) -> RuntimeResult<()>, + ) -> RuntimeResult<()> { + // begin + let mut closed = false; + while !self.f.is_eof() && !closed { + self.f.__reset_checksum(); + // try to decode this batch + let Ok(batch) = self.read_batch() else { + self.attempt_recover_data_batch()?; + continue; + }; + // see what happened when decoding it + let finished_early = matches!(batch, Batch::FinishedEarly { .. }); + let batch = match batch { + Batch::RecoveredFromerror => { + // there was an error, but it was safely "handled" because of a recovery byte mark + continue; + } + Batch::FinishedEarly(batch) | Batch::Normal(batch) => batch, + Batch::BatchClosed => { + // the batch was closed; this means that we probably are done with this round; but was it re-opened? + closed = self.handle_reopen_is_actual_close()?; + continue; + } + }; + // now we need to read the batch summary + let Ok(actual_commit) = self.read_batch_summary(finished_early) else { + self.attempt_recover_data_batch()?; + continue; + }; + // check if we have the expected batch size + if batch.events.len() as u64 != actual_commit { + // corrupted + self.attempt_recover_data_batch()?; + continue; + } + f(batch)?; + // apply the batch + } + if closed { + if self.f.is_eof() { + // that was the last batch + return Ok(()); + } + } + // nope, this is a corrupted file + Err(StorageError::DataBatchRestoreCorruptedBatchFile.into()) + } + fn handle_reopen_is_actual_close(&mut self) -> RuntimeResult { + if self.f.is_eof() { + // yup, it was closed + Ok(true) + } else { + // maybe not + if self.f.read_byte()? == MARKER_BATCH_REOPEN { + // driver was closed, but reopened + Ok(false) + } else { + // that's just a nice bug + Err(StorageError::DataBatchRestoreCorruptedBatchFile.into()) + } + } + } +} + +impl DataBatchRestoreDriver { + fn apply_batch( + m: &Model, + NormalBatch { + events, + schema_version, + }: NormalBatch, + ) -> RuntimeResult<()> { + // NOTE(@ohsayan): current complexity is O(n) which is good enough (in the future I might revise this to a fancier impl) + // pin model + let g = unsafe { crossbeam_epoch::unprotected() }; + let mut pending_delete = HashMap::new(); + let p_index = m.primary_index().__raw_index(); + // scan rows + for DecodedBatchEvent { txn_id, pk, kind } in events { + match kind { + DecodedBatchEventKind::Insert(new_row) | DecodedBatchEventKind::Update(new_row) => { + // this is more like a "newrow" + match p_index.mt_get_element(&pk, &g) { + Some(row) if row.d_data().read().get_restored_txn_revised() > txn_id => { + // skewed + // resolve deltas if any + let _ = row.resolve_schema_deltas_and_freeze(m.delta_state()); + continue; + } + Some(_) | None => { + // new row (logically) + let _ = p_index.mt_delete(&pk, &g); + let mut data = DcFieldIndex::default(); + for (field_name, new_data) in m + .fields() + .stseq_ord_key() + .filter(|key| key.as_str() != m.p_key()) + .zip(new_row) + { + data.st_insert( + unsafe { + // UNSAFE(@ohsayan): model in scope, we're good + field_name.clone() + }, + new_data, + ); + } + let row = Row::new_restored( + pk, + data, + DeltaVersion::__new(schema_version), + DeltaVersion::__new(0), + txn_id, + ); + // resolve any deltas + let _ = row.resolve_schema_deltas_and_freeze(m.delta_state()); + // put it back in (lol); blame @ohsayan for this joke + p_index.mt_insert(row, &g); + } + } + } + DecodedBatchEventKind::Delete => { + match pending_delete.entry(pk) { + HMEntry::Occupied(mut existing_delete) => { + if *existing_delete.get() > txn_id { + // the existing delete "happened after" our delete, so it takes precedence + continue; + } + // the existing delete happened before our delete, so our delete takes precedence + // we have a newer delete for the same key + *existing_delete.get_mut() = txn_id; + } + HMEntry::Vacant(new) => { + // we never deleted this + new.insert(txn_id); + } + } + } + } + } + for (pk, txn_id) in pending_delete { + match p_index.mt_get(&pk, &g) { + Some(row) => { + if row.read().get_restored_txn_revised() > txn_id { + // our delete "happened before" this row was inserted + continue; + } + // yup, go ahead and chuck it + let _ = p_index.mt_delete(&pk, &g); + } + None => { + // since we never delete rows until here, this is quite impossible + unreachable!() + } + } + } + Ok(()) + } +} + +impl DataBatchRestoreDriver { + fn read_batch_summary(&mut self, finished_early: bool) -> RuntimeResult { + if !finished_early { + // we must read the batch termination signature + let b = self.f.read_byte()?; + if b != MARKER_END_OF_BATCH { + return Err(StorageError::DataBatchRestoreCorruptedBatch.into()); + } + } + // read actual commit + let actual_commit = self.f.read_u64_le()?; + // find actual checksum + let actual_checksum = self.f.__reset_checksum(); + // find hardcoded checksum + let mut hardcoded_checksum = [0; sizeof!(u64)]; + self.f.untracked_read(&mut hardcoded_checksum)?; + if actual_checksum == u64::from_le_bytes(hardcoded_checksum) { + Ok(actual_commit) + } else { + Err(StorageError::DataBatchRestoreCorruptedBatch.into()) + } + } + fn read_batch(&mut self) -> RuntimeResult { + let mut this_batch = vec![]; + // check batch type + let batch_type = self.f.read_byte()?; + match batch_type { + MARKER_ACTUAL_BATCH_EVENT => {} + MARKER_RECOVERY_EVENT => { + // while attempting to write this batch, some sort of an error occurred but we got a nice recovery byte + // so proceed that way + return Ok(Batch::RecoveredFromerror); + } + MARKER_BATCH_CLOSED => { + // this isn't a batch; it has been closed + return Ok(Batch::BatchClosed); + } + _ => { + // this is the only singular byte that is expected to be intact. If this isn't intact either, I'm sorry + return Err(StorageError::DataBatchRestoreCorruptedBatch.into()); + } + } + // decode batch start block + let batch_start_block = self.read_start_batch_block()?; + + let mut processed_in_this_batch = 0; + while (processed_in_this_batch != batch_start_block.expected_commit()) & !self.f.is_eof() { + // decode common row data + let change_type = self.f.read_byte()?; + // now decode event + match change_type { + MARKER_END_OF_BATCH => { + // the file tells us that we've reached the end of this batch; hmmm + return Ok(Batch::FinishedEarly(NormalBatch::new( + this_batch, + batch_start_block.schema_version(), + ))); + } + normal_event => { + let txnid = self.f.read_u64_le()?; + match normal_event { + 0 => { + // delete + let pk = self.decode_primary_key(batch_start_block.pk_tag())?; + this_batch.push(DecodedBatchEvent::new( + txnid, + pk, + DecodedBatchEventKind::Delete, + )); + processed_in_this_batch += 1; + } + 1 | 2 => { + // insert or update + // get pk + let pk = self.decode_primary_key(batch_start_block.pk_tag())?; + // prepare row + let mut row = vec![]; + let mut this_col_cnt = batch_start_block.column_cnt(); + while this_col_cnt != 0 && !self.f.is_eof() { + row.push(self.decode_cell()?); + this_col_cnt -= 1; + } + if this_col_cnt != 0 { + return Err(StorageError::DataBatchRestoreCorruptedEntry.into()); + } + if change_type == 1 { + this_batch.push(DecodedBatchEvent::new( + txnid, + pk, + DecodedBatchEventKind::Insert(row), + )); + } else { + this_batch.push(DecodedBatchEvent::new( + txnid, + pk, + DecodedBatchEventKind::Update(row), + )); + } + processed_in_this_batch += 1; + } + _ => { + return Err(StorageError::DataBatchRestoreCorruptedBatch.into()); + } + } + } + } + } + Ok(Batch::Normal(NormalBatch::new( + this_batch, + batch_start_block.schema_version(), + ))) + } + fn attempt_recover_data_batch(&mut self) -> RuntimeResult<()> { + let mut buf = [0u8; 1]; + self.f.untracked_read(&mut buf)?; + if let [MARKER_RECOVERY_EVENT] = buf { + return Ok(()); + } + Err(StorageError::DataBatchRestoreCorruptedBatch.into()) + } + fn read_start_batch_block(&mut self) -> RuntimeResult { + let pk_tag = self.f.read_byte()?; + let expected_commit = self.f.read_u64_le()?; + let schema_version = self.f.read_u64_le()?; + let column_cnt = self.f.read_u64_le()?; + Ok(BatchStartBlock::new( + pk_tag, + expected_commit, + schema_version, + column_cnt, + )) + } +} + +#[derive(Debug, PartialEq)] +struct BatchStartBlock { + pk_tag: u8, + expected_commit: u64, + schema_version: u64, + column_cnt: u64, +} + +impl BatchStartBlock { + const fn new(pk_tag: u8, expected_commit: u64, schema_version: u64, column_cnt: u64) -> Self { + Self { + pk_tag, + expected_commit, + schema_version, + column_cnt, + } + } + fn pk_tag(&self) -> u8 { + self.pk_tag + } + fn expected_commit(&self) -> u64 { + self.expected_commit + } + fn schema_version(&self) -> u64 { + self.schema_version + } + fn column_cnt(&self) -> u64 { + self.column_cnt + } +} + +impl DataBatchRestoreDriver { + fn decode_primary_key(&mut self, pk_type: u8) -> RuntimeResult { + let Some(pk_type) = TagUnique::try_from_raw(pk_type) else { + return Err(StorageError::DataBatchRestoreCorruptedEntry.into()); + }; + Ok(match pk_type { + TagUnique::SignedInt | TagUnique::UnsignedInt => { + let qw = self.f.read_u64_le()?; + unsafe { + // UNSAFE(@ohsayan): +tagck + PrimaryIndexKey::new_from_qw(pk_type, qw) + } + } + TagUnique::Str | TagUnique::Bin => { + let len = self.f.read_u64_le()?; + let mut data = vec![0; len as usize]; + self.f.read_into_buffer(&mut data)?; + if pk_type == TagUnique::Str { + if core::str::from_utf8(&data).is_err() { + return Err(StorageError::DataBatchRestoreCorruptedEntry.into()); + } + } + unsafe { + // UNSAFE(@ohsayan): +tagck +verityck + let mut md = ManuallyDrop::new(data); + PrimaryIndexKey::new_from_dual(pk_type, len, md.as_mut_ptr() as usize) + } + } + _ => unsafe { + // UNSAFE(@ohsayan): TagUnique::try_from_raw rejects an construction with Invalid as the dscr + impossible!() + }, + }) + } + fn decode_cell(&mut self) -> RuntimeResult { + let Some(dscr) = StorageCellTypeID::try_from_raw(self.f.read_byte()?) else { + return Err(StorageError::DataBatchRestoreCorruptedEntry.into()); + }; + unsafe { cell::decode_element::>(&mut self.f, dscr) } + .map_err(|e| e.0) + } +} + +pub struct ErrorHack(crate::engine::fractal::error::Error); +impl From for ErrorHack { + fn from(value: crate::engine::fractal::error::Error) -> Self { + Self(value) + } +} +impl From<()> for ErrorHack { + fn from(_: ()) -> Self { + Self(StorageError::DataBatchRestoreCorruptedEntry.into()) + } +} +impl DataSource for SDSSFileTrackedReader { + const RELIABLE_SOURCE: bool = false; + type Error = ErrorHack; + fn has_remaining(&self, cnt: usize) -> bool { + self.remaining() >= cnt as u64 + } + unsafe fn read_next_byte(&mut self) -> Result { + Ok(self.read_byte()?) + } + unsafe fn read_next_block(&mut self) -> Result<[u8; N], Self::Error> { + Ok(self.read_block()?) + } + unsafe fn read_next_u64_le(&mut self) -> Result { + Ok(self.read_u64_le()?) + } + unsafe fn read_next_variable_block(&mut self, size: usize) -> Result, Self::Error> { + let mut buf = vec![0; size]; + self.read_into_buffer(&mut buf)?; + Ok(buf) + } +} diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs new file mode 100644 index 00000000..1914eee0 --- /dev/null +++ b/server/src/engine/storage/v1/inf/map.rs @@ -0,0 +1,383 @@ +/* + * Created on Wed Aug 16 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + obj::{ + cell::{self, CanYieldDict, StorageCellTypeID}, + FieldMD, + }, + AbstractMap, MapStorageSpec, PersistObject, VecU8, + }, + crate::{ + engine::{ + core::model::Field, + data::dict::DictEntryGeneric, + error::{RuntimeResult, StorageError}, + idx::{IndexSTSeqCns, STIndexSeq}, + mem::{BufferedScanner, StatelessLen}, + storage::v1::inf, + }, + util::{copy_slice_to_array as memcpy, EndianQW}, + }, + std::{collections::HashMap, marker::PhantomData}, +}; + +#[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)] +pub struct MapIndexSizeMD(pub usize); + +/// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations +pub struct PersistMapImpl<'a, M: MapStorageSpec>(PhantomData<&'a M::InMemoryMap>); + +impl<'a, M: MapStorageSpec> PersistObject for PersistMapImpl<'a, M> { + const METADATA_SIZE: usize = sizeof!(u64); + type InputType = &'a M::InMemoryMap; + type OutputType = M::RestoredMap; + type Metadata = MapIndexSizeMD; + fn pretest_can_dec_object( + s: &BufferedScanner, + MapIndexSizeMD(dict_size): &Self::Metadata, + ) -> bool { + M::decode_pretest_for_map(s, *dict_size) + } + fn meta_enc(buf: &mut VecU8, data: Self::InputType) { + buf.extend(data.stateless_len().u64_bytes_le()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + Ok(MapIndexSizeMD(scanner.next_u64_le() as usize)) + } + fn obj_enc(buf: &mut VecU8, map: Self::InputType) { + for (key, val) in M::get_iter_from_memory(map) { + M::encode_entry_meta(buf, key, val); + if M::ENC_AS_ENTRY { + M::encode_entry_data(buf, key, val); + } else { + M::encode_entry_key(buf, key); + M::encode_entry_val(buf, val); + } + } + } + unsafe fn obj_dec( + scanner: &mut BufferedScanner, + MapIndexSizeMD(dict_size): Self::Metadata, + ) -> RuntimeResult { + let mut dict = M::RestoredMap::map_new(); + let decode_pretest_for_entry_meta = M::decode_pretest_for_entry_meta(scanner); + while decode_pretest_for_entry_meta & (dict.map_length() != dict_size) { + let md = unsafe { + // UNSAFE(@ohsayan): +pretest + M::decode_entry_meta(scanner).ok_or::( + StorageError::InternalDecodeStructureCorruptedPayload.into(), + )? + }; + if !M::decode_pretest_for_entry_data(scanner, &md) { + return Err(StorageError::InternalDecodeStructureCorruptedPayload.into()); + } + let key; + let val; + unsafe { + if M::DEC_AS_ENTRY { + match M::decode_entry_data(scanner, md) { + Some((_k, _v)) => { + key = _k; + val = _v; + } + None => { + return Err(StorageError::InternalDecodeStructureCorruptedPayload.into()) + } + } + } else { + let _k = M::decode_entry_key(scanner, &md); + let _v = M::decode_entry_val(scanner, &md); + match (_k, _v) { + (Some(_k), Some(_v)) => { + key = _k; + val = _v; + } + _ => { + return Err(StorageError::InternalDecodeStructureCorruptedPayload.into()) + } + } + } + } + if !dict.map_insert(key, val) { + return Err(StorageError::InternalDecodeStructureIllegalData.into()); + } + } + if dict.map_length() == dict_size { + Ok(dict) + } else { + Err(StorageError::InternalDecodeStructureIllegalData.into()) + } + } +} + +/// generic dict spec (simple spec for [DictGeneric](crate::engine::data::dict::DictGeneric)) +pub struct GenericDictSpec; + +/// generic dict entry metadata +pub struct GenericDictEntryMetadata { + pub(crate) klen: usize, + pub(crate) dscr: u8, +} + +impl GenericDictEntryMetadata { + /// decode md (no need for any validation since that has to be handled later and can only produce incorrect results + /// if unsafe code is used to translate an incorrect dscr) + pub(crate) fn decode(data: [u8; 9]) -> Self { + Self { + klen: u64::from_le_bytes(memcpy(&data[..8])) as usize, + dscr: data[8], + } + } +} + +impl MapStorageSpec for GenericDictSpec { + type InMemoryMap = HashMap; + type InMemoryKey = Box; + type InMemoryVal = DictEntryGeneric; + type InMemoryMapIter<'a> = + std::collections::hash_map::Iter<'a, Self::InMemoryKey, Self::InMemoryVal>; + type RestoredKey = Self::InMemoryKey; + type RestoredMap = Self::InMemoryMap; + type RestoredVal = Self::InMemoryVal; + type EntryMetadata = GenericDictEntryMetadata; + const DEC_AS_ENTRY: bool = false; + const ENC_AS_ENTRY: bool = true; + fn get_iter_from_memory<'a>(map: &'a Self::InMemoryMap) -> Self::InMemoryMapIter<'a> { + map.iter() + } + fn encode_entry_meta(buf: &mut VecU8, key: &Self::InMemoryKey, _: &Self::InMemoryVal) { + buf.extend(key.len().u64_bytes_le()); + } + fn encode_entry_data(buf: &mut VecU8, key: &Self::InMemoryKey, val: &Self::InMemoryVal) { + match val { + DictEntryGeneric::Map(map) => { + buf.push(StorageCellTypeID::Dict.value_u8()); + buf.extend(key.as_bytes()); + as PersistObject>::default_full_enc(buf, map); + } + DictEntryGeneric::Data(dc) => { + buf.push(cell::encode_tag(dc)); + buf.extend(key.as_bytes()); + cell::encode_cell(buf, dc); + } + } + } + fn encode_entry_key(_: &mut VecU8, _: &Self::InMemoryKey) { + unimplemented!() + } + fn encode_entry_val(_: &mut VecU8, _: &Self::InMemoryVal) { + unimplemented!() + } + fn decode_pretest_for_entry_meta(scanner: &mut BufferedScanner) -> bool { + // we just need to see if we can decode the entry metadata + scanner.has_left(9) + } + fn decode_pretest_for_entry_data(s: &mut BufferedScanner, md: &Self::EntryMetadata) -> bool { + StorageCellTypeID::is_valid(md.dscr) + & s.has_left(StorageCellTypeID::expect_atleast(md.dscr)) + } + unsafe fn decode_entry_meta(s: &mut BufferedScanner) -> Option { + Some(Self::EntryMetadata::decode(s.next_chunk())) + } + unsafe fn decode_entry_data( + _: &mut BufferedScanner, + _: Self::EntryMetadata, + ) -> Option<(Self::RestoredKey, Self::RestoredVal)> { + unimplemented!() + } + unsafe fn decode_entry_key( + s: &mut BufferedScanner, + md: &Self::EntryMetadata, + ) -> Option { + inf::dec::utils::decode_string(s, md.klen as usize) + .map(|s| s.into_boxed_str()) + .ok() + } + unsafe fn decode_entry_val( + scanner: &mut BufferedScanner, + md: &Self::EntryMetadata, + ) -> Option { + Some( + match cell::decode_element::( + scanner, + StorageCellTypeID::from_raw(md.dscr), + ) + .ok()? + { + CanYieldDict::Data(d) => DictEntryGeneric::Data(d), + CanYieldDict::Dict => DictEntryGeneric::Map( + as PersistObject>::default_full_dec(scanner) + .ok()?, + ), + }, + ) + } +} + +pub struct FieldMapEntryMetadata { + field_id_l: u64, + field_prop_c: u64, + field_layer_c: u64, + null: u8, +} + +impl FieldMapEntryMetadata { + const fn new(field_id_l: u64, field_prop_c: u64, field_layer_c: u64, null: u8) -> Self { + Self { + field_id_l, + field_prop_c, + field_layer_c, + null, + } + } +} + +pub trait FieldMapAny: StatelessLen { + type Iterator<'a>: Iterator + where + Self: 'a; + fn get_iter<'a>(&'a self) -> Self::Iterator<'a> + where + Self: 'a; +} + +impl FieldMapAny for HashMap, Field> { + type Iterator<'a> = std::iter::Map< + std::collections::hash_map::Iter<'a, Box, Field>, + fn((&Box, &Field)) -> (&'a str, &'a Field), + >; + fn get_iter<'a>(&'a self) -> Self::Iterator<'a> + where + Self: 'a, + { + self.iter() + .map(|(a, b)| unsafe { core::mem::transmute((a.as_ref(), b)) }) + } +} +impl FieldMapAny for IndexSTSeqCns { + type Iterator<'a> = std::iter::Map< + crate::engine::idx::stdord_iter::IndexSTSeqDllIterOrdKV<'a, crate::engine::mem::RawStr, Field>, + fn((&crate::engine::mem::RawStr, &Field)) -> (&'a str, &'a Field)> + where + Self: 'a; + + fn get_iter<'a>(&'a self) -> Self::Iterator<'a> + where + Self: 'a, + { + self.stseq_ord_kv() + .map(|(k, v)| unsafe { core::mem::transmute((k.as_str(), v)) }) + } +} +impl FieldMapAny for IndexSTSeqCns, Field> { + type Iterator<'a> = std::iter::Map< + crate::engine::idx::stdord_iter::IndexSTSeqDllIterOrdKV<'a, Box, Field>, + fn((&Box, &Field)) -> (&'a str, &'a Field)> + where + Self: 'a; + + fn get_iter<'a>(&'a self) -> Self::Iterator<'a> + where + Self: 'a, + { + self.stseq_ord_kv() + .map(|(k, v)| unsafe { core::mem::transmute((k.as_ref(), v)) }) + } +} + +pub struct FieldMapSpec(PhantomData); +impl MapStorageSpec for FieldMapSpec { + type InMemoryMap = FM; + type InMemoryKey = str; + type InMemoryVal = Field; + type InMemoryMapIter<'a> = FM::Iterator<'a> where FM: 'a; + type RestoredKey = Box; + type RestoredVal = Field; + type RestoredMap = IndexSTSeqCns, Field>; + type EntryMetadata = FieldMapEntryMetadata; + const ENC_AS_ENTRY: bool = false; + const DEC_AS_ENTRY: bool = false; + fn get_iter_from_memory<'a>(map: &'a Self::InMemoryMap) -> Self::InMemoryMapIter<'a> { + map.get_iter() + } + fn encode_entry_meta(buf: &mut VecU8, key: &Self::InMemoryKey, val: &Self::InMemoryVal) { + buf.extend(key.len().u64_bytes_le()); + buf.extend(0u64.to_le_bytes()); // TODO(@ohsayan): props + buf.extend(val.layers().len().u64_bytes_le()); + buf.push(val.is_nullable() as u8); + } + fn encode_entry_data(_: &mut VecU8, _: &Self::InMemoryKey, _: &Self::InMemoryVal) { + unimplemented!() + } + fn encode_entry_key(buf: &mut VecU8, key: &Self::InMemoryKey) { + buf.extend(key.as_bytes()); + } + fn encode_entry_val(buf: &mut VecU8, val: &Self::InMemoryVal) { + for layer in val.layers() { + super::obj::LayerRef::default_full_enc(buf, super::obj::LayerRef(layer)) + } + } + fn decode_pretest_for_entry_meta(scanner: &mut BufferedScanner) -> bool { + scanner.has_left(sizeof!(u64, 3) + 1) + } + fn decode_pretest_for_entry_data(s: &mut BufferedScanner, md: &Self::EntryMetadata) -> bool { + s.has_left(md.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc + } + unsafe fn decode_entry_meta(scanner: &mut BufferedScanner) -> Option { + Some(FieldMapEntryMetadata::new( + scanner.next_u64_le(), + scanner.next_u64_le(), + scanner.next_u64_le(), + scanner.next_byte(), + )) + } + unsafe fn decode_entry_data( + _: &mut BufferedScanner, + _: Self::EntryMetadata, + ) -> Option<(Self::RestoredKey, Self::RestoredVal)> { + unimplemented!() + } + unsafe fn decode_entry_key( + scanner: &mut BufferedScanner, + md: &Self::EntryMetadata, + ) -> Option { + inf::dec::utils::decode_string(scanner, md.field_id_l as usize) + .map(|s| s.into_boxed_str()) + .ok() + } + unsafe fn decode_entry_val( + scanner: &mut BufferedScanner, + md: &Self::EntryMetadata, + ) -> Option { + super::obj::FieldRef::obj_dec( + scanner, + FieldMD::new(md.field_prop_c, md.field_layer_c, md.null), + ) + .ok() + } +} diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs new file mode 100644 index 00000000..8a367b90 --- /dev/null +++ b/server/src/engine/storage/v1/inf/mod.rs @@ -0,0 +1,275 @@ +/* + * Created on Fri Aug 04 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +//! High level interfaces + +pub mod map; +pub mod obj; +// tests +#[cfg(test)] +mod tests; + +use crate::engine::{ + error::{RuntimeResult, StorageError}, + idx::{AsKey, AsValue, STIndex}, + mem::{BufferedScanner, StatelessLen}, +}; + +type VecU8 = Vec; + +pub trait DataSource { + type Error; + const RELIABLE_SOURCE: bool = true; + fn has_remaining(&self, cnt: usize) -> bool; + unsafe fn read_next_byte(&mut self) -> Result; + unsafe fn read_next_block(&mut self) -> Result<[u8; N], Self::Error>; + unsafe fn read_next_u64_le(&mut self) -> Result; + unsafe fn read_next_variable_block(&mut self, size: usize) -> Result, Self::Error>; +} + +impl<'a> DataSource for BufferedScanner<'a> { + type Error = (); + fn has_remaining(&self, cnt: usize) -> bool { + self.has_left(cnt) + } + unsafe fn read_next_byte(&mut self) -> Result { + Ok(self.next_byte()) + } + unsafe fn read_next_block(&mut self) -> Result<[u8; N], Self::Error> { + Ok(self.next_chunk()) + } + unsafe fn read_next_u64_le(&mut self) -> Result { + Ok(self.next_u64_le()) + } + unsafe fn read_next_variable_block(&mut self, size: usize) -> Result, Self::Error> { + Ok(self.next_chunk_variable(size).into()) + } +} + +/* + obj spec +*/ + +/// Any object that can be persisted +pub trait PersistObject { + // const + /// Size of the metadata region + const METADATA_SIZE: usize; + // types + /// Input type for enc operations + type InputType: Copy; + /// Output type for dec operations + type OutputType; + /// Metadata type + type Metadata; + // pretest + /// Pretest to see if the src has the required data for metadata dec. Defaults to the metadata size + fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool { + scanner.has_left(Self::METADATA_SIZE) + } + /// Pretest to see if the src has the required data for object dec + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool; + // meta + /// metadata enc + fn meta_enc(buf: &mut VecU8, data: Self::InputType); + /// metadata dec + /// + /// ## Safety + /// + /// Must pass the [`PersistObject::pretest_can_dec_metadata`] assertion + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult; + // obj + /// obj enc + fn obj_enc(buf: &mut VecU8, data: Self::InputType); + /// obj dec + /// + /// ## Safety + /// + /// Must pass the [`PersistObject::pretest_can_dec_object`] assertion + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult; + // default + /// Default routine to encode an object + its metadata + fn default_full_enc(buf: &mut VecU8, data: Self::InputType) { + Self::meta_enc(buf, data); + Self::obj_enc(buf, data); + } + /// Default routine to decode an object + its metadata (however, the metadata is used and not returned) + fn default_full_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + if !Self::pretest_can_dec_metadata(scanner) { + return Err(StorageError::InternalDecodeStructureCorrupted.into()); + } + let md = unsafe { + // UNSAFE(@ohsayan): +pretest + Self::meta_dec(scanner)? + }; + if !Self::pretest_can_dec_object(scanner, &md) { + return Err(StorageError::InternalDecodeStructureCorruptedPayload.into()); + } + unsafe { + // UNSAFE(@ohsayan): +obj pretest + Self::obj_dec(scanner, md) + } + } +} + +/* + map spec +*/ + +pub trait AbstractMap { + fn map_new() -> Self; + fn map_insert(&mut self, k: K, v: V) -> bool; + fn map_length(&self) -> usize; +} + +impl> AbstractMap for M { + fn map_new() -> Self { + Self::idx_init() + } + fn map_insert(&mut self, k: K, v: V) -> bool { + self.st_insert(k, v) + } + fn map_length(&self) -> usize { + self.st_len() + } +} + +pub trait MapStorageSpec { + // in memory + type InMemoryMap: StatelessLen; + type InMemoryKey: ?Sized; + type InMemoryVal; + type InMemoryMapIter<'a>: Iterator + where + Self: 'a, + Self::InMemoryKey: 'a, + Self::InMemoryVal: 'a; + // from disk + type RestoredKey: AsKey; + type RestoredVal: AsValue; + type RestoredMap: AbstractMap; + // metadata + type EntryMetadata; + // settings + const ENC_AS_ENTRY: bool; + const DEC_AS_ENTRY: bool; + // iterator + fn get_iter_from_memory<'a>(map: &'a Self::InMemoryMap) -> Self::InMemoryMapIter<'a>; + // encode + fn encode_entry_meta(buf: &mut VecU8, key: &Self::InMemoryKey, val: &Self::InMemoryVal); + fn encode_entry_data(buf: &mut VecU8, key: &Self::InMemoryKey, val: &Self::InMemoryVal); + fn encode_entry_key(buf: &mut VecU8, key: &Self::InMemoryKey); + fn encode_entry_val(buf: &mut VecU8, val: &Self::InMemoryVal); + // decode + fn decode_pretest_for_map(_: &BufferedScanner, _: usize) -> bool { + true + } + fn decode_pretest_for_entry_meta(scanner: &mut BufferedScanner) -> bool; + fn decode_pretest_for_entry_data(s: &mut BufferedScanner, md: &Self::EntryMetadata) -> bool; + unsafe fn decode_entry_meta(s: &mut BufferedScanner) -> Option; + unsafe fn decode_entry_data( + s: &mut BufferedScanner, + md: Self::EntryMetadata, + ) -> Option<(Self::RestoredKey, Self::RestoredVal)>; + unsafe fn decode_entry_key( + s: &mut BufferedScanner, + md: &Self::EntryMetadata, + ) -> Option; + unsafe fn decode_entry_val( + s: &mut BufferedScanner, + md: &Self::EntryMetadata, + ) -> Option; +} + +// enc +pub mod enc { + use super::{map, MapStorageSpec, PersistObject, VecU8}; + // obj + #[cfg(test)] + pub fn enc_full(obj: Obj::InputType) -> Vec { + let mut v = vec![]; + enc_full_into_buffer::(&mut v, obj); + v + } + pub fn enc_full_into_buffer(buf: &mut VecU8, obj: Obj::InputType) { + Obj::default_full_enc(buf, obj) + } + #[cfg(test)] + pub fn enc_full_self>(obj: Obj) -> Vec { + enc_full::(obj) + } + // dict + pub fn enc_dict_full(dict: &PM::InMemoryMap) -> Vec { + let mut v = vec![]; + enc_dict_full_into_buffer::(&mut v, dict); + v + } + pub fn enc_dict_full_into_buffer(buf: &mut VecU8, dict: &PM::InMemoryMap) { + as PersistObject>::default_full_enc(buf, dict) + } +} + +// dec +pub mod dec { + use { + super::{map, MapStorageSpec, PersistObject}, + crate::engine::{error::RuntimeResult, mem::BufferedScanner}, + }; + // obj + #[cfg(test)] + pub fn dec_full(data: &[u8]) -> RuntimeResult { + let mut scanner = BufferedScanner::new(data); + dec_full_from_scanner::(&mut scanner) + } + pub fn dec_full_from_scanner( + scanner: &mut BufferedScanner, + ) -> RuntimeResult { + Obj::default_full_dec(scanner) + } + // dec + pub fn dec_dict_full(data: &[u8]) -> RuntimeResult { + let mut scanner = BufferedScanner::new(data); + dec_dict_full_from_scanner::(&mut scanner) + } + fn dec_dict_full_from_scanner( + scanner: &mut BufferedScanner, + ) -> RuntimeResult { + as PersistObject>::default_full_dec(scanner) + } + pub mod utils { + use crate::engine::{ + error::{RuntimeResult, StorageError}, + mem::BufferedScanner, + }; + pub unsafe fn decode_string(s: &mut BufferedScanner, len: usize) -> RuntimeResult { + String::from_utf8(s.next_chunk_variable(len).to_owned()) + .map_err(|_| StorageError::InternalDecodeStructureCorruptedPayload.into()) + } + } +} diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs new file mode 100644 index 00000000..62cb7679 --- /dev/null +++ b/server/src/engine/storage/v1/inf/obj.rs @@ -0,0 +1,522 @@ +/* + * Created on Wed Aug 16 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{PersistObject, VecU8}, + crate::{ + engine::{ + core::{ + model::{Field, Layer, Model}, + space::Space, + }, + data::{ + tag::{DataTag, TagClass, TagSelector}, + uuid::Uuid, + DictGeneric, + }, + error::{RuntimeResult, StorageError}, + idx::IndexSTSeqCns, + mem::{BufferedScanner, VInline}, + storage::v1::inf, + }, + util::EndianQW, + }, +}; + +/* + generic cells +*/ + +pub mod cell { + use crate::{ + engine::{ + data::{ + cell::Datacell, + tag::{DataTag, TagClass, TagSelector}, + }, + storage::v1::inf::{DataSource, VecU8}, + }, + util::EndianQW, + }; + + #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord, Hash, sky_macros::EnumMethods)] + #[repr(u8)] + #[allow(dead_code)] + pub enum StorageCellTypeID { + Null = 0x00, + Bool = 0x01, + UInt8 = 0x02, + UInt16 = 0x03, + UInt32 = 0x04, + UInt64 = 0x05, + SInt8 = 0x06, + SInt16 = 0x07, + SInt32 = 0x08, + SInt64 = 0x09, + Float32 = 0x0A, + Float64 = 0x0B, + Bin = 0x0C, + Str = 0x0D, + List = 0x0E, + Dict = 0x0F, + } + impl StorageCellTypeID { + pub const unsafe fn from_raw(v: u8) -> Self { + core::mem::transmute(v) + } + pub const fn try_from_raw(v: u8) -> Option { + if Self::is_valid(v) { + Some(unsafe { Self::from_raw(v) }) + } else { + None + } + } + #[inline(always)] + pub const fn is_valid(d: u8) -> bool { + d <= Self::MAX + } + const unsafe fn into_selector(self) -> TagSelector { + debug_assert!(self.value_u8() != Self::Null.value_u8()); + core::mem::transmute(self.value_u8() - 1) + } + #[inline(always)] + pub fn expect_atleast(d: u8) -> usize { + [0u8, 1, 8, 8][d.min(3) as usize] as usize + } + } + pub fn encode(buf: &mut VecU8, dc: &Datacell) { + buf.push(encode_tag(dc)); + encode_cell(buf, dc) + } + pub fn encode_tag(dc: &Datacell) -> u8 { + (dc.tag().tag_selector().value_u8() + 1) * (dc.is_init() as u8) + } + pub fn encode_cell(buf: &mut VecU8, dc: &Datacell) { + if dc.is_null() { + return; + } + unsafe { + use TagClass::*; + match dc.tag().tag_class() { + Bool if dc.is_init() => buf.push(dc.read_bool() as u8), + Bool => {} + UnsignedInt | SignedInt | Float => buf.extend(dc.read_uint().to_le_bytes()), + Str | Bin => { + let slc = dc.read_bin(); + buf.extend(slc.len().u64_bytes_le()); + buf.extend(slc); + } + List => { + let lst = dc.read_list().read(); + buf.extend(lst.len().u64_bytes_le()); + for item in lst.iter() { + encode(buf, item); + } + } + } + } + } + pub trait ElementYield { + type Yield; + type Error; + const CAN_YIELD_DICT: bool = false; + fn yield_data(dc: Datacell) -> Result; + fn yield_dict() -> Result { + panic!() + } + fn error() -> Result; + } + impl ElementYield for Datacell { + type Yield = Self; + type Error = (); + fn yield_data(dc: Datacell) -> Result { + Ok(dc) + } + fn error() -> Result { + Err(()) + } + } + #[derive(Debug, PartialEq)] + pub enum CanYieldDict { + Data(Datacell), + Dict, + } + impl ElementYield for CanYieldDict { + type Yield = Self; + type Error = (); + const CAN_YIELD_DICT: bool = true; + fn error() -> Result { + Err(()) + } + fn yield_data(dc: Datacell) -> Result { + Ok(CanYieldDict::Data(dc)) + } + fn yield_dict() -> Result { + Ok(CanYieldDict::Dict) + } + } + pub unsafe fn decode_element( + s: &mut DS, + dscr: StorageCellTypeID, + ) -> Result + where + DS::Error: From, + DS::Error: From<()>, + { + if dscr == StorageCellTypeID::Dict { + if EY::CAN_YIELD_DICT { + return Ok(EY::yield_dict()?); + } else { + return Ok(EY::error()?); + } + } + if dscr == StorageCellTypeID::Null { + return Ok(EY::yield_data(Datacell::null())?); + } + let tag = dscr.into_selector().into_full(); + let d = match tag.tag_class() { + TagClass::Bool => { + let nx = s.read_next_byte()?; + if nx > 1 { + return Ok(EY::error()?); + } + Datacell::new_bool(nx == 1) + } + TagClass::UnsignedInt | TagClass::SignedInt | TagClass::Float => { + let nx = s.read_next_u64_le()?; + Datacell::new_qw(nx, tag) + } + TagClass::Bin | TagClass::Str => { + let len = s.read_next_u64_le()? as usize; + let block = s.read_next_variable_block(len)?; + if tag.tag_class() == TagClass::Str { + match String::from_utf8(block).map(|s| Datacell::new_str(s.into_boxed_str())) { + Ok(s) => s, + Err(_) => return Ok(EY::error()?), + } + } else { + Datacell::new_bin(block.into()) + } + } + TagClass::List => { + let len = s.read_next_u64_le()? as usize; + let mut l = vec![]; + while (l.len() != len) & s.has_remaining(1) { + let Some(dscr) = StorageCellTypeID::try_from_raw(s.read_next_byte()?) else { + return Ok(EY::error()?); + }; + // FIXME(@ohsayan): right now, a list cannot contain a dict! + if !s.has_remaining(StorageCellTypeID::expect_atleast(dscr.value_u8())) { + return Ok(EY::error()?); + } + l.push(self::decode_element::(s, dscr)?); + } + if l.len() != len { + return Ok(EY::error()?); + } + Datacell::new_list(l) + } + }; + Ok(EY::yield_data(d)?) + } +} + +/* + layer +*/ + +#[derive(Debug)] +pub struct LayerMD { + type_selector: u64, + prop_set_arity: u64, +} + +impl LayerMD { + const fn new(type_selector: u64, prop_set_arity: u64) -> Self { + Self { + type_selector, + prop_set_arity, + } + } +} + +#[derive(Clone, Copy)] +pub struct LayerRef<'a>(pub &'a Layer); +impl<'a> From<&'a Layer> for LayerRef<'a> { + fn from(value: &'a Layer) -> Self { + Self(value) + } +} +impl<'a> PersistObject for LayerRef<'a> { + const METADATA_SIZE: usize = sizeof!(u64, 2); + type InputType = LayerRef<'a>; + type OutputType = Layer; + type Metadata = LayerMD; + fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool { + true + } + fn meta_enc(buf: &mut VecU8, LayerRef(layer): Self::InputType) { + buf.extend(layer.tag().tag_selector().value_qword().to_le_bytes()); + buf.extend(0u64.to_le_bytes()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + Ok(LayerMD::new(scanner.next_u64_le(), scanner.next_u64_le())) + } + fn obj_enc(_: &mut VecU8, _: Self::InputType) {} + unsafe fn obj_dec( + _: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + if (md.type_selector > TagSelector::List.value_qword()) | (md.prop_set_arity != 0) { + return Err(StorageError::InternalDecodeStructureCorruptedPayload.into()); + } + Ok(Layer::new_empty_props( + TagSelector::from_raw(md.type_selector as u8).into_full(), + )) + } +} + +/* + field +*/ + +pub struct FieldMD { + prop_c: u64, + layer_c: u64, + null: u8, +} + +impl FieldMD { + pub(super) const fn new(prop_c: u64, layer_c: u64, null: u8) -> Self { + Self { + prop_c, + layer_c, + null, + } + } +} + +pub struct FieldRef<'a>(&'a Field); +impl<'a> From<&'a Field> for FieldRef<'a> { + fn from(f: &'a Field) -> Self { + Self(f) + } +} +impl<'a> PersistObject for FieldRef<'a> { + const METADATA_SIZE: usize = sizeof!(u64, 2) + 1; + type InputType = &'a Field; + type OutputType = Field; + type Metadata = FieldMD; + fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool { + true + } + fn meta_enc(buf: &mut VecU8, slf: Self::InputType) { + // [prop_c][layer_c][null] + buf.extend(0u64.to_le_bytes()); + buf.extend(slf.layers().len().u64_bytes_le()); + buf.push(slf.is_nullable() as u8); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + Ok(FieldMD::new( + scanner.next_u64_le(), + scanner.next_u64_le(), + scanner.next_byte(), + )) + } + fn obj_enc(buf: &mut VecU8, slf: Self::InputType) { + for layer in slf.layers() { + LayerRef::default_full_enc(buf, LayerRef(layer)); + } + } + unsafe fn obj_dec( + scanner: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let mut layers = VInline::new(); + let mut fin = false; + while (!scanner.eof()) + & (layers.len() as u64 != md.layer_c) + & (LayerRef::pretest_can_dec_metadata(scanner)) + & !fin + { + let layer_md = unsafe { + // UNSAFE(@ohsayan): pretest + LayerRef::meta_dec(scanner)? + }; + let l = LayerRef::obj_dec(scanner, layer_md)?; + fin = l.tag().tag_class() != TagClass::List; + layers.push(l); + } + let field = Field::new(layers, md.null == 1); + if (field.layers().len() as u64 == md.layer_c) & (md.null <= 1) & (md.prop_c == 0) & fin { + Ok(field) + } else { + Err(StorageError::InternalDecodeStructureCorrupted.into()) + } + } +} + +#[derive(Debug)] +pub struct ModelLayoutMD { + model_uuid: Uuid, + p_key_len: u64, + p_key_tag: u64, + field_c: u64, +} + +impl ModelLayoutMD { + pub(super) const fn new( + model_uuid: Uuid, + p_key_len: u64, + p_key_tag: u64, + field_c: u64, + ) -> Self { + Self { + model_uuid, + p_key_len, + p_key_tag, + field_c, + } + } + pub fn p_key_len(&self) -> u64 { + self.p_key_len + } +} + +#[derive(Clone, Copy)] +pub struct ModelLayoutRef<'a>(pub(super) &'a Model); +impl<'a> From<&'a Model> for ModelLayoutRef<'a> { + fn from(mdl: &'a Model) -> Self { + Self(mdl) + } +} +impl<'a> PersistObject for ModelLayoutRef<'a> { + const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64, 3); + type InputType = ModelLayoutRef<'a>; + type OutputType = Model; + type Metadata = ModelLayoutMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left(md.p_key_len as usize) + } + fn meta_enc(buf: &mut VecU8, ModelLayoutRef(model_def): Self::InputType) { + buf.extend(model_def.get_uuid().to_le_bytes()); + buf.extend(model_def.p_key().len().u64_bytes_le()); + buf.extend(model_def.p_tag().tag_selector().value_qword().to_le_bytes()); + buf.extend(model_def.fields().len().u64_bytes_le()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + Ok(ModelLayoutMD::new( + Uuid::from_bytes(scanner.next_chunk()), + scanner.next_u64_le(), + scanner.next_u64_le(), + scanner.next_u64_le(), + )) + } + fn obj_enc(buf: &mut VecU8, ModelLayoutRef(model_definition): Self::InputType) { + buf.extend(model_definition.p_key().as_bytes()); + > as PersistObject>::obj_enc( + buf, + model_definition.fields(), + ) + } + unsafe fn obj_dec( + scanner: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let key = inf::dec::utils::decode_string(scanner, md.p_key_len as usize)?; + let fieldmap = , _>>, + > as PersistObject>::obj_dec( + scanner, super::map::MapIndexSizeMD(md.field_c as usize) + )?; + let ptag = if md.p_key_tag > TagSelector::MAX as u64 { + return Err(StorageError::InternalDecodeStructureCorruptedPayload.into()); + } else { + TagSelector::from_raw(md.p_key_tag as u8) + }; + Ok(Model::new_restore( + md.model_uuid, + key.into_boxed_str(), + ptag.into_full(), + fieldmap, + )) + } +} + +#[derive(Debug)] +pub struct SpaceLayoutMD { + uuid: Uuid, + prop_c: usize, +} + +impl SpaceLayoutMD { + pub fn new(uuid: Uuid, prop_c: usize) -> Self { + Self { uuid, prop_c } + } +} + +#[derive(Clone, Copy)] +pub struct SpaceLayoutRef<'a>(&'a Space, &'a DictGeneric); +impl<'a> From<(&'a Space, &'a DictGeneric)> for SpaceLayoutRef<'a> { + fn from((spc, spc_meta): (&'a Space, &'a DictGeneric)) -> Self { + Self(spc, spc_meta) + } +} +impl<'a> PersistObject for SpaceLayoutRef<'a> { + const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64); + type InputType = SpaceLayoutRef<'a>; + type OutputType = Space; + type Metadata = SpaceLayoutMD; + fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool { + true + } + fn meta_enc(buf: &mut VecU8, SpaceLayoutRef(space, space_meta): Self::InputType) { + buf.extend(space.get_uuid().to_le_bytes()); + buf.extend(space_meta.len().u64_bytes_le()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + Ok(SpaceLayoutMD::new( + Uuid::from_bytes(scanner.next_chunk()), + scanner.next_u64_le() as usize, + )) + } + fn obj_enc(buf: &mut VecU8, SpaceLayoutRef(_, space_meta): Self::InputType) { + as PersistObject>::obj_enc( + buf, space_meta, + ) + } + unsafe fn obj_dec( + scanner: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let space_meta = + as PersistObject>::obj_dec( + scanner, + super::map::MapIndexSizeMD(md.prop_c), + )?; + Ok(Space::new_restore_empty(md.uuid, space_meta)) + } +} diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs new file mode 100644 index 00000000..c8e2f7b0 --- /dev/null +++ b/server/src/engine/storage/v1/inf/tests.rs @@ -0,0 +1,123 @@ +/* + * Created on Sun Aug 13 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::obj, + crate::engine::{ + core::{ + model::{Field, Layer, Model}, + space::Space, + }, + data::{ + cell::Datacell, + dict::{DictEntryGeneric, DictGeneric}, + tag::TagSelector, + uuid::Uuid, + }, + idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq}, + }, +}; + +#[test] +fn dict() { + let dict: DictGeneric = into_dict! { + "hello" => Datacell::new_str("world".into()), + "omg a null?" => Datacell::null(), + "a big fat dict" => DictEntryGeneric::Map(into_dict!( + "with a value" => Datacell::new_uint_default(1002), + "and a null" => Datacell::null(), + )) + }; + let encoded = super::enc::enc_dict_full::(&dict); + let decoded = super::dec::dec_dict_full::(&encoded).unwrap(); + assert_eq!(dict, decoded); +} + +#[test] +fn layer() { + let layer = Layer::list(); + let encoded = super::enc::enc_full::(obj::LayerRef(&layer)); + let dec = super::dec::dec_full::(&encoded).unwrap(); + assert_eq!(layer, dec); +} + +#[test] +fn field() { + let field = Field::new([Layer::list(), Layer::uint64()].into(), true); + let encoded = super::enc::enc_full::((&field).into()); + let dec = super::dec::dec_full::(&encoded).unwrap(); + assert_eq!(field, dec); +} + +#[test] +fn fieldmap() { + let mut fields = IndexSTSeqCns::, Field>::idx_init(); + fields.st_insert("password".into(), Field::new([Layer::bin()].into(), false)); + fields.st_insert( + "profile_pic".into(), + Field::new([Layer::bin()].into(), true), + ); + let enc = super::enc::enc_dict_full::>(&fields); + let dec = super::dec::dec_dict_full::< + super::map::FieldMapSpec, _>>, + >(&enc) + .unwrap(); + for ((orig_field_id, orig_field), (restored_field_id, restored_field)) in + fields.stseq_ord_kv().zip(dec.stseq_ord_kv()) + { + assert_eq!(orig_field_id, restored_field_id); + assert_eq!(orig_field, restored_field); + } +} + +#[test] +fn model() { + let uuid = Uuid::new(); + let model = Model::new_restore( + uuid, + "username".into(), + TagSelector::String.into_full(), + into_dict! { + "password" => Field::new([Layer::bin()].into(), false), + "profile_pic" => Field::new([Layer::bin()].into(), true), + }, + ); + let enc = super::enc::enc_full::(obj::ModelLayoutRef(&model)); + let dec = super::dec::dec_full::(&enc).unwrap(); + assert_eq!(model, dec); +} + +#[test] +fn space() { + let uuid = Uuid::new(); + let space = Space::new_restore_empty(uuid, Default::default()); + let enc = super::enc::enc_full::(obj::SpaceLayoutRef::from(( + &space, + space.props(), + ))); + let dec = super::dec::dec_full::(&enc).unwrap(); + assert_eq!(space, dec); +} diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs new file mode 100644 index 00000000..a1815b6c --- /dev/null +++ b/server/src/engine/storage/v1/journal.rs @@ -0,0 +1,455 @@ +/* + * Created on Sat Jul 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +/* + +----------------+------------------------------+------------------+------------------+--------------------+ + | EVENT ID (16B) | EVENT SOURCE + METADATA (8B) | EVENT CRC32 (4B) | PAYLOAD LEN (8B) | EVENT PAYLOAD (?B) | + +----------------+------------------------------+------------------+------------------+--------------------+ + Event ID: + - The atomically incrementing event ID (for future scale we have 16B; it's like the ZFS situation haha) + - Event source (1B) + 7B padding (for future metadata) + - Event CRC32 + - Payload len: the size of the pyload + - Payload: the payload + + + Notes on error tolerance: + - FIXME(@ohsayan): we currently expect atleast 36 bytes of the signature to be present. this is not great + - FIXME(@ohsayan): we will probably (naively) need to dynamically reposition the cursor in case the metadata is corrupted as well +*/ + +use { + super::{ + rw::{RawFSInterface, SDSSFileIO}, + spec, + }, + crate::{ + engine::error::{RuntimeResult, StorageError}, + util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy}, + }, + std::marker::PhantomData, +}; + +const CRC: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); + +#[cfg(test)] +pub fn open_or_create_journal( + log_file_name: &str, + gs: &TA::GlobalState, +) -> RuntimeResult>> { + use super::rw::FileOpen; + let file = match SDSSFileIO::::open_or_create_perm_rw::(log_file_name)? { + FileOpen::Created(f) => return Ok(FileOpen::Created(JournalWriter::new(f, 0, true)?)), + FileOpen::Existing((file, _header)) => file, + }; + let (file, last_txn) = JournalReader::::scroll(file, gs)?; + Ok(FileOpen::Existing(JournalWriter::new( + file, last_txn, false, + )?)) +} + +pub fn create_journal( + log_file_name: &str, +) -> RuntimeResult> { + JournalWriter::new(SDSSFileIO::create::(log_file_name)?, 0, true) +} + +pub fn load_journal( + log_file_name: &str, + gs: &TA::GlobalState, +) -> RuntimeResult> { + let (file, _) = SDSSFileIO::::open::(log_file_name)?; + let (file, last_txn_id) = JournalReader::::scroll(file, gs)?; + JournalWriter::new(file, last_txn_id, false) +} + +/// The journal adapter +pub trait JournalAdapter { + /// deny any SDSS file level operations that require non-append mode writes (for example, updating the SDSS header's modify count) + const DENY_NONAPPEND: bool = true; + /// enable/disable automated recovery algorithms + const RECOVERY_PLUGIN: bool; + /// The journal event + type JournalEvent; + /// The global state, which we want to modify on decoding the event + type GlobalState; + /// The transactional impl that makes use of this journal, should define it's error type + type Error; + /// Encode a journal event into a blob + fn encode(event: Self::JournalEvent) -> Box<[u8]>; + /// Decode a journal event and apply it to the global state + fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> Result<(), Self::Error>; +} + +#[derive(Debug)] +pub struct JournalEntryMetadata { + event_id: u128, + event_source_md: u64, + event_crc: u32, + event_payload_len: u64, +} + +impl JournalEntryMetadata { + const SIZE: usize = sizeof!(u128) + sizeof!(u64) + sizeof!(u32) + sizeof!(u64); + const P0: usize = 0; + const P1: usize = sizeof!(u128); + const P2: usize = Self::P1 + sizeof!(u64); + const P3: usize = Self::P2 + sizeof!(u32); + pub const fn new( + event_id: u128, + event_source_md: u64, + event_crc: u32, + event_payload_len: u64, + ) -> Self { + Self { + event_id, + event_source_md, + event_crc, + event_payload_len, + } + } + /// Encodes the log entry metadata + pub const fn encoded(&self) -> [u8; JournalEntryMetadata::SIZE] { + let mut encoded = [0u8; JournalEntryMetadata::SIZE]; + encoded = copy_a_into_b(self.event_id.to_le_bytes(), encoded, Self::P0); + encoded = copy_a_into_b(self.event_source_md.to_le_bytes(), encoded, Self::P1); + encoded = copy_a_into_b(self.event_crc.to_le_bytes(), encoded, Self::P2); + encoded = copy_a_into_b(self.event_payload_len.to_le_bytes(), encoded, Self::P3); + encoded + } + /// Decodes the log entry metadata (essentially a simply type transmutation) + pub fn decode(data: [u8; JournalEntryMetadata::SIZE]) -> Self { + Self::new( + u128::from_le_bytes(memcpy(&data[..Self::P1])), + u64::from_le_bytes(memcpy(&data[Self::P1..Self::P2])), + u32::from_le_bytes(memcpy(&data[Self::P2..Self::P3])), + u64::from_le_bytes(memcpy(&data[Self::P3..])), + ) + } +} + +/* + Event source: + * * * * _ * * * * + + b1 (s+d): event source (unset -> driver, set -> server) + b* -> unused. MUST be unset + b7 (d): + - set: [recovery] reverse journal event + b8 (d): + - unset: closed log + - set: reopened log +*/ + +pub enum EventSourceMarker { + ServerStandard, + DriverClosed, + RecoveryReverseLastJournal, + DriverReopened, +} + +impl EventSourceMarker { + const SERVER_STD: u64 = 1 << 63; + const DRIVER_CLOSED: u64 = 0; + const DRIVER_REOPENED: u64 = 1; + const RECOVERY_REVERSE_LAST_JOURNAL: u64 = 2; +} + +impl JournalEntryMetadata { + pub const fn event_source_marker(&self) -> Option { + Some(match self.event_source_md { + EventSourceMarker::SERVER_STD => EventSourceMarker::ServerStandard, + EventSourceMarker::DRIVER_CLOSED => EventSourceMarker::DriverClosed, + EventSourceMarker::DRIVER_REOPENED => EventSourceMarker::DriverReopened, + EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL => { + EventSourceMarker::RecoveryReverseLastJournal + } + _ => return None, + }) + } +} + +pub struct JournalReader { + log_file: SDSSFileIO, + evid: u64, + closed: bool, + remaining_bytes: u64, + _m: PhantomData, +} + +impl JournalReader { + pub fn new(log_file: SDSSFileIO) -> RuntimeResult { + let log_size = log_file.file_length()? - spec::SDSSStaticHeaderV1Compact::SIZE as u64; + Ok(Self { + log_file, + evid: 0, + closed: false, + remaining_bytes: log_size, + _m: PhantomData, + }) + } + /// Read the next event and apply it to the global state + pub fn rapply_next_event(&mut self, gs: &TA::GlobalState) -> RuntimeResult<()> { + // read metadata + let mut en_jrnl_md = [0u8; JournalEntryMetadata::SIZE]; + self.logfile_read_into_buffer(&mut en_jrnl_md)?; // FIXME(@ohsayan): increase tolerance to not just payload + let entry_metadata = JournalEntryMetadata::decode(en_jrnl_md); + /* + validate metadata: + - evid + - sourcemd + - sum + - len < alloc cap; FIXME(@ohsayan): more sensitive via alloc? + */ + if self.evid != entry_metadata.event_id as u64 { + // the only case when this happens is when the journal faults at runtime with a write zero (or some other error when no bytes were written) + self.remaining_bytes += JournalEntryMetadata::SIZE as u64; + // move back cursor to see if we have a recovery block + let new_cursor = self.log_file.retrieve_cursor()? - JournalEntryMetadata::SIZE as u64; + self.log_file.seek_from_start(new_cursor)?; + return self.try_recover_journal_strategy_simple_reverse(); + } + match entry_metadata + .event_source_marker() + .ok_or(StorageError::JournalLogEntryCorrupted)? + { + EventSourceMarker::ServerStandard => {} + EventSourceMarker::DriverClosed => { + // is this a real close? + if self.end_of_file() { + self.closed = true; + return Ok(()); + } else { + return self.handle_driver_reopen(); + } + } + EventSourceMarker::DriverReopened | EventSourceMarker::RecoveryReverseLastJournal => { + // these two are only taken in close and error paths (respectively) so we shouldn't see them here; this is bad + // two special directives in the middle of nowhere? incredible + return Err(StorageError::JournalCorrupted.into()); + } + } + // read payload + if compiler::unlikely(!self.has_remaining_bytes(entry_metadata.event_payload_len)) { + return compiler::cold_call(|| self.try_recover_journal_strategy_simple_reverse()); + } + let mut payload = vec![0; entry_metadata.event_payload_len as usize]; + self.logfile_read_into_buffer(&mut payload)?; // exit jump -> we checked if enough data is there, but the read failed so this is not our business + if compiler::unlikely(CRC.checksum(&payload) != entry_metadata.event_crc) { + return compiler::cold_call(|| self.try_recover_journal_strategy_simple_reverse()); + } + if compiler::unlikely(TA::decode_and_update_state(&payload, gs).is_err()) { + return compiler::cold_call(|| self.try_recover_journal_strategy_simple_reverse()); + } + self._incr_evid(); + Ok(()) + } + /// handle a driver reopen (IMPORTANT: every event is unique so this must be called BEFORE the ID is incremented) + fn handle_driver_reopen(&mut self) -> RuntimeResult<()> { + if self.has_remaining_bytes(JournalEntryMetadata::SIZE as _) { + let mut reopen_block = [0u8; JournalEntryMetadata::SIZE]; + self.logfile_read_into_buffer(&mut reopen_block)?; // exit jump -> not our business since we have checked flen and if it changes due to user intervention, that's a you problem + let md = JournalEntryMetadata::decode(reopen_block); + if (md.event_id as u64 == self.evid) + & (md.event_crc == 0) + & (md.event_payload_len == 0) + & (md.event_source_md == EventSourceMarker::DRIVER_REOPENED) + { + self._incr_evid(); + Ok(()) + } else { + // FIXME(@ohsayan): tolerate loss in this directive too + Err(StorageError::JournalCorrupted.into()) + } + } else { + Err(StorageError::JournalCorrupted.into()) + } + } + #[cold] // FIXME(@ohsayan): how bad can prod systems be? (clue: pretty bad, so look for possible changes) + #[inline(never)] + /// attempt to recover the journal using the reverse directive (simple strategy) + /// IMPORTANT: every event is unique so this must be called BEFORE the ID is incremented (remember that we only increment + /// once we **sucessfully** finish processing a normal (aka server event origin) event and not a non-normal branch) + fn try_recover_journal_strategy_simple_reverse(&mut self) -> RuntimeResult<()> { + debug_assert!(TA::RECOVERY_PLUGIN, "recovery plugin not enabled"); + self.__record_read_bytes(JournalEntryMetadata::SIZE); // FIXME(@ohsayan): don't assume read length? + let mut entry_buf = [0u8; JournalEntryMetadata::SIZE]; + if self.log_file.read_to_buffer(&mut entry_buf).is_err() { + return Err(StorageError::JournalCorrupted.into()); + } + let entry = JournalEntryMetadata::decode(entry_buf); + let okay = (entry.event_id == self.evid as u128) + & (entry.event_crc == 0) + & (entry.event_payload_len == 0) + & (entry.event_source_md == EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL); + self._incr_evid(); + if okay { + return Ok(()); + } else { + Err(StorageError::JournalCorrupted.into()) + } + } + /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID) + pub fn scroll( + file: SDSSFileIO, + gs: &TA::GlobalState, + ) -> RuntimeResult<(SDSSFileIO, u64)> { + let mut slf = Self::new(file)?; + while !slf.end_of_file() { + slf.rapply_next_event(gs)?; + } + if slf.closed { + Ok((slf.log_file, slf.evid)) + } else { + Err(StorageError::JournalCorrupted.into()) + } + } +} + +impl JournalReader { + fn _incr_evid(&mut self) { + self.evid += 1; + } + fn __record_read_bytes(&mut self, cnt: usize) { + self.remaining_bytes -= cnt as u64; + } + fn has_remaining_bytes(&self, size: u64) -> bool { + self.remaining_bytes >= size + } + fn end_of_file(&self) -> bool { + self.remaining_bytes == 0 + } +} + +impl JournalReader { + fn logfile_read_into_buffer(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + if !self.has_remaining_bytes(buf.len() as _) { + // do this right here to avoid another syscall + return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into()); + } + self.log_file.read_to_buffer(buf)?; + self.__record_read_bytes(buf.len()); + Ok(()) + } +} + +pub struct JournalWriter { + /// the txn log file + log_file: SDSSFileIO, + /// the id of the **next** journal + id: u64, + _m: PhantomData, + closed: bool, +} + +impl JournalWriter { + pub fn new(mut log_file: SDSSFileIO, last_txn_id: u64, new: bool) -> RuntimeResult { + let log_size = log_file.file_length()?; + log_file.seek_from_start(log_size)?; // avoid jumbling with headers + let mut slf = Self { + log_file, + id: last_txn_id, + _m: PhantomData, + closed: false, + }; + if !new { + // IMPORTANT: don't forget this; otherwise the journal reader will report a false error! + slf.append_journal_reopen()?; + } + Ok(slf) + } + pub fn append_event(&mut self, event: TA::JournalEvent) -> RuntimeResult<()> { + let encoded = TA::encode(event); + let md = JournalEntryMetadata::new( + self._incr_id() as u128, + EventSourceMarker::SERVER_STD, + CRC.checksum(&encoded), + encoded.len() as u64, + ) + .encoded(); + self.log_file.unfsynced_write(&md)?; + self.log_file.unfsynced_write(&encoded)?; + self.log_file.fsync_all()?; + Ok(()) + } + pub fn append_event_with_recovery_plugin( + &mut self, + event: TA::JournalEvent, + ) -> RuntimeResult<()> { + debug_assert!(TA::RECOVERY_PLUGIN); + match self.append_event(event) { + Ok(()) => Ok(()), + Err(e) => compiler::cold_call(move || { + // IMPORTANT: we still need to return an error so that the caller can retry if deemed appropriate + self.appendrec_journal_reverse_entry()?; + Err(e) + }), + } + } +} + +impl JournalWriter { + pub fn appendrec_journal_reverse_entry(&mut self) -> RuntimeResult<()> { + let mut entry = + JournalEntryMetadata::new(0, EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL, 0, 0); + entry.event_id = self._incr_id() as u128; + if self.log_file.fsynced_write(&entry.encoded()).is_ok() { + return Ok(()); + } + Err(StorageError::JournalWRecoveryStageOneFailCritical.into()) + } + pub fn append_journal_reopen(&mut self) -> RuntimeResult<()> { + let id = self._incr_id() as u128; + self.log_file.fsynced_write( + &JournalEntryMetadata::new(id, EventSourceMarker::DRIVER_REOPENED, 0, 0).encoded(), + ) + } + pub fn __close_mut(&mut self) -> RuntimeResult<()> { + self.closed = true; + let id = self._incr_id() as u128; + self.log_file.fsynced_write( + &JournalEntryMetadata::new(id, EventSourceMarker::DRIVER_CLOSED, 0, 0).encoded(), + )?; + Ok(()) + } + pub fn close(mut self) -> RuntimeResult<()> { + self.__close_mut() + } +} + +impl JournalWriter { + fn _incr_id(&mut self) -> u64 { + let current = self.id; + self.id += 1; + current + } +} + +impl Drop for JournalWriter { + fn drop(&mut self) { + assert!(self.closed, "log not closed"); + } +} diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs new file mode 100644 index 00000000..065a3f49 --- /dev/null +++ b/server/src/engine/storage/v1/loader.rs @@ -0,0 +1,148 @@ +/* + * Created on Sun Sep 10 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(test)] +use crate::engine::storage::v1::{ + rw::{FileOpen, RawFSInterface}, + JournalWriter, +}; +use crate::engine::{ + core::{EntityIDRef, GlobalNS}, + data::uuid::Uuid, + error::RuntimeResult, + fractal::error::ErrorContext, + fractal::{FractalModelDriver, ModelDrivers, ModelUniqueID}, + storage::v1::{batch_jrnl, journal, spec, LocalFS}, + txn::gns::{GNSAdapter, GNSTransactionDriverAnyFS}, +}; + +const GNS_FILE_PATH: &str = "gns.db-tlog"; +const DATA_DIR: &str = "data"; + +pub struct SEInitState { + pub txn_driver: GNSTransactionDriverAnyFS, + pub model_drivers: ModelDrivers, + pub gns: GlobalNS, +} + +impl SEInitState { + pub fn new( + txn_driver: GNSTransactionDriverAnyFS, + model_drivers: ModelDrivers, + gns: GlobalNS, + ) -> Self { + Self { + txn_driver, + model_drivers, + gns, + } + } + pub fn try_init(is_new: bool) -> RuntimeResult { + let gns = GlobalNS::empty(); + let gns_txn_driver = if is_new { + journal::create_journal::(GNS_FILE_PATH) + } else { + journal::load_journal::( + GNS_FILE_PATH, + &gns, + ) + }?; + let mut model_drivers = ModelDrivers::new(); + let mut driver_guard = || { + if is_new { + std::fs::create_dir(DATA_DIR).inherit_set_dmsg("creating data directory")?; + } + if !is_new { + let mut models = gns.idx_models().write(); + // this is an existing instance, so read in all data + for (space_name, space) in gns.idx().read().iter() { + let space_uuid = space.get_uuid(); + for model_name in space.models().iter() { + let model = models + .get_mut(&EntityIDRef::new(&space_name, &model_name)) + .unwrap(); + let path = + Self::model_path(space_name, space_uuid, model_name, model.get_uuid()); + let persist_driver = batch_jrnl::reinit(&path, model).inherit_set_dmsg( + format!("failed to restore model data from journal in `{path}`"), + )?; + unsafe { + // UNSAFE(@ohsayan): all pieces of data are upgraded by now, so vacuum + model.model_mutator().vacuum_stashed(); + } + let _ = model_drivers.insert( + ModelUniqueID::new(space_name, model_name, model.get_uuid()), + FractalModelDriver::init(persist_driver), + ); + } + } + } + RuntimeResult::Ok(()) + }; + if let Err(e) = driver_guard() { + gns_txn_driver.close().unwrap(); + for (_, driver) in model_drivers { + driver.close().unwrap(); + } + return Err(e); + } + Ok(SEInitState::new( + GNSTransactionDriverAnyFS::new(gns_txn_driver), + model_drivers, + gns, + )) + } + pub fn model_path( + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + ) -> String { + format!( + "{}/data.db-btlog", + Self::model_dir(space_name, space_uuid, model_name, model_uuid) + ) + } + pub fn model_dir( + space_name: &str, + space_uuid: Uuid, + model_name: &str, + model_uuid: Uuid, + ) -> String { + format!("data/{space_name}-{space_uuid}/mdl_{model_name}-{model_uuid}") + } + pub fn space_dir(space_name: &str, space_uuid: Uuid) -> String { + format!("data/{space_name}-{space_uuid}") + } +} + +#[cfg(test)] +pub fn open_gns_driver( + path: &str, + gns: &GlobalNS, +) -> RuntimeResult>> { + journal::open_or_create_journal::(path, gns) +} diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs new file mode 100644 index 00000000..d7d5ade5 --- /dev/null +++ b/server/src/engine/storage/v1/memfs.rs @@ -0,0 +1,581 @@ +/* + * Created on Fri Sep 08 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::engine::{ + error::RuntimeResult, + storage::v1::rw::{ + FileOpen, RawFSInterface, RawFileInterface, RawFileInterfaceBufferedWriter, + RawFileInterfaceExt, RawFileInterfaceRead, RawFileInterfaceWrite, + RawFileInterfaceWriteExt, + }, + sync::cell::Lazy, + }, + parking_lot::RwLock, + std::{ + collections::{ + hash_map::{Entry, OccupiedEntry}, + HashMap, + }, + io::{Error, ErrorKind}, + }, +}; + +static VFS: Lazy, VNode>>, fn() -> RwLock, VNode>>> = + Lazy::new(|| Default::default()); + +type ComponentIter<'a> = std::iter::Take>; + +/* + vnode + --- + either a vfile or a vdir +*/ + +#[derive(Debug)] +pub(super) enum VNode { + Dir(HashMap, Self>), + File(VFile), +} + +impl VNode { + fn as_dir_mut(&mut self) -> Option<&mut HashMap, Self>> { + match self { + Self::Dir(d) => Some(d), + Self::File(_) => None, + } + } +} + +/* + errors +*/ + +fn err_item_is_not_file() -> RuntimeResult { + Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into()) +} +fn err_file_in_dir_path() -> RuntimeResult { + Err(Error::new(ErrorKind::InvalidInput, "found file in directory path").into()) +} +fn err_dir_missing_in_path() -> RuntimeResult { + Err(Error::new(ErrorKind::InvalidInput, "could not find directory in path").into()) +} +fn err_could_not_find_item() -> RuntimeResult { + Err(Error::new(ErrorKind::NotFound, "could not find item").into()) +} + +/* + vfs impl: + - nested directory structure + - make parents + - make child +*/ + +fn split_parts(fpath: &str) -> Vec<&str> { + fpath.split("/").collect() +} + +fn split_target_and_components(fpath: &str) -> (&str, ComponentIter) { + let parts = split_parts(fpath); + let target = parts.last().unwrap(); + let component_len = parts.len() - 1; + (target, parts.into_iter().take(component_len)) +} + +#[derive(Debug)] +pub struct VirtualFS; + +impl RawFSInterface for VirtualFS { + type File = VFileDescriptor; + fn fs_rename_file(from: &str, to: &str) -> RuntimeResult<()> { + // get file data + let data = with_file(from, |f| Ok(f.data.clone()))?; + // create new file + let file = VirtualFS::fs_fopen_or_create_rw(to)?; + match file { + FileOpen::Created(mut c) => { + c.fw_write_all(&data)?; + } + FileOpen::Existing(mut e) => { + e.fwext_truncate_to(0)?; + e.fw_write_all(&data)?; + } + } + // delete old file + Self::fs_remove_file(from) + } + fn fs_remove_file(fpath: &str) -> RuntimeResult<()> { + handle_item_mut(fpath, |e| match e.get() { + VNode::File(_) => { + e.remove(); + Ok(()) + } + _ => return err_item_is_not_file(), + }) + } + fn fs_create_dir(fpath: &str) -> RuntimeResult<()> { + // get vfs + let mut vfs = VFS.write(); + // get root dir + let mut current = &mut *vfs; + // process components + let (target, mut components) = split_target_and_components(fpath); + while let Some(component) = components.next() { + match current.get_mut(component) { + Some(VNode::Dir(d)) => { + current = d; + } + Some(VNode::File(_)) => return err_file_in_dir_path(), + None => return err_dir_missing_in_path(), + } + } + match current.entry(target.into()) { + Entry::Occupied(_) => return Err(Error::from(ErrorKind::AlreadyExists).into()), + Entry::Vacant(ve) => { + ve.insert(VNode::Dir(into_dict!())); + Ok(()) + } + } + } + fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> { + let mut vfs = VFS.write(); + fn create_ahead( + mut ahead: &[&str], + current: &mut HashMap, VNode>, + ) -> RuntimeResult<()> { + if ahead.is_empty() { + return Ok(()); + } + let this = ahead[0]; + ahead = &ahead[1..]; + match current.get_mut(this) { + Some(VNode::Dir(d)) => { + if ahead.is_empty() { + // hmm, this was the list dir that was to be created, but it already exists + return Err(Error::from(ErrorKind::AlreadyExists).into()); + } + return create_ahead(ahead, d); + } + Some(VNode::File(_)) => return err_file_in_dir_path(), + None => { + let _ = current.insert(this.into(), VNode::Dir(into_dict!())); + let dir = current.get_mut(this).unwrap().as_dir_mut().unwrap(); + return create_ahead(ahead, dir); + } + } + } + let pieces = split_parts(fpath); + create_ahead(&pieces, &mut *vfs) + } + fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> { + delete_dir(fpath, false) + } + fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> { + delete_dir(fpath, true) + } + fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult> { + let mut vfs = VFS.write(); + // components + let (target_file, components) = split_target_and_components(fpath); + let target_dir = find_target_dir_mut(components, &mut vfs)?; + match target_dir.entry(target_file.into()) { + Entry::Occupied(mut oe) => match oe.get_mut() { + VNode::File(f) => { + f.read = true; + f.write = true; + Ok(FileOpen::Existing(VFileDescriptor(fpath.into()))) + } + VNode::Dir(_) => return err_item_is_not_file(), + }, + Entry::Vacant(v) => { + v.insert(VNode::File(VFile::new(true, true, vec![], 0))); + Ok(FileOpen::Created(VFileDescriptor(fpath.into()))) + } + } + } + fn fs_fcreate_rw(fpath: &str) -> RuntimeResult { + let mut vfs = VFS.write(); + let (target_file, components) = split_target_and_components(fpath); + let target_dir = find_target_dir_mut(components, &mut vfs)?; + match target_dir.entry(target_file.into()) { + Entry::Occupied(k) => { + match k.get() { + VNode::Dir(_) => { + return Err(Error::new( + ErrorKind::AlreadyExists, + "found directory with same name where file was to be created", + ) + .into()); + } + VNode::File(_) => { + // the file already exists + return Err(Error::new( + ErrorKind::AlreadyExists, + "the file already exists", + ) + .into()); + } + } + } + Entry::Vacant(v) => { + // no file exists, we can create this + v.insert(VNode::File(VFile::new(true, true, vec![], 0))); + Ok(VFileDescriptor(fpath.into())) + } + } + } + fn fs_fopen_rw(fpath: &str) -> RuntimeResult { + with_file_mut(fpath, |f| { + f.read = true; + f.write = true; + Ok(VFileDescriptor(fpath.into())) + }) + } +} + +fn find_target_dir_mut<'a>( + components: ComponentIter, + mut current: &'a mut HashMap, VNode>, +) -> RuntimeResult<&'a mut HashMap, VNode>> { + for component in components { + match current.get_mut(component) { + Some(VNode::Dir(d)) => current = d, + Some(VNode::File(_)) => return err_file_in_dir_path(), + None => return err_dir_missing_in_path(), + } + } + Ok(current) +} + +fn find_target_dir<'a>( + components: ComponentIter, + mut current: &'a HashMap, VNode>, +) -> RuntimeResult<&'a HashMap, VNode>> { + for component in components { + match current.get(component) { + Some(VNode::Dir(d)) => current = d, + Some(VNode::File(_)) => return err_file_in_dir_path(), + None => return err_dir_missing_in_path(), + } + } + Ok(current) +} + +fn handle_item_mut( + fpath: &str, + f: impl Fn(OccupiedEntry, VNode>) -> RuntimeResult, +) -> RuntimeResult { + let mut vfs = VFS.write(); + let mut current = &mut *vfs; + // process components + let (target, components) = split_target_and_components(fpath); + for component in components { + match current.get_mut(component) { + Some(VNode::Dir(dir)) => { + current = dir; + } + Some(VNode::File(_)) => return err_file_in_dir_path(), + None => return err_dir_missing_in_path(), + } + } + match current.entry(target.into()) { + Entry::Occupied(item) => return f(item), + Entry::Vacant(_) => return err_could_not_find_item(), + } +} +fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> RuntimeResult<()> { + handle_item_mut(fpath, |node| match node.get() { + VNode::Dir(d) => { + if allow_if_non_empty || d.is_empty() { + node.remove(); + return Ok(()); + } + return Err(Error::new(ErrorKind::InvalidInput, "directory is not empty").into()); + } + VNode::File(_) => return err_file_in_dir_path(), + }) +} + +/* + vfile impl + --- + - all r/w operations + - all seek operations + - dummy sync operations +*/ + +#[derive(Debug)] +pub struct VFile { + read: bool, + write: bool, + data: Vec, + pos: usize, +} + +impl VFile { + fn new(read: bool, write: bool, data: Vec, pos: usize) -> Self { + Self { + read, + write, + data, + pos, + } + } + fn current(&self) -> &[u8] { + &self.data[self.pos..] + } +} + +pub struct VFileDescriptor(Box); +impl Drop for VFileDescriptor { + fn drop(&mut self) { + let _ = with_file_mut(&self.0, |f| { + f.read = false; + f.write = false; + f.pos = 0; + Ok(()) + }); + } +} + +fn with_file_mut( + fpath: &str, + mut f: impl FnMut(&mut VFile) -> RuntimeResult, +) -> RuntimeResult { + let mut vfs = VFS.write(); + let (target_file, components) = split_target_and_components(fpath); + let target_dir = find_target_dir_mut(components, &mut vfs)?; + match target_dir.get_mut(target_file) { + Some(VNode::File(file)) => f(file), + Some(VNode::Dir(_)) => return err_item_is_not_file(), + None => return Err(Error::from(ErrorKind::NotFound).into()), + } +} + +fn with_file(fpath: &str, mut f: impl FnMut(&VFile) -> RuntimeResult) -> RuntimeResult { + let vfs = VFS.read(); + let (target_file, components) = split_target_and_components(fpath); + let target_dir = find_target_dir(components, &vfs)?; + match target_dir.get(target_file) { + Some(VNode::File(file)) => f(file), + Some(VNode::Dir(_)) => return err_item_is_not_file(), + None => return Err(Error::from(ErrorKind::NotFound).into()), + } +} + +impl RawFileInterface for VFileDescriptor { + type BufReader = Self; + type BufWriter = Self; + fn into_buffered_reader(self) -> RuntimeResult { + Ok(self) + } + fn downgrade_reader(r: Self::BufReader) -> RuntimeResult { + Ok(r) + } + fn into_buffered_writer(self) -> RuntimeResult { + Ok(self) + } + fn downgrade_writer(w: Self::BufWriter) -> RuntimeResult { + Ok(w) + } +} + +impl RawFileInterfaceBufferedWriter for VFileDescriptor {} + +impl RawFileInterfaceRead for VFileDescriptor { + fn fr_read_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + with_file_mut(&self.0, |file| { + if !file.read { + return Err( + Error::new(ErrorKind::PermissionDenied, "Read permission denied").into(), + ); + } + let available_bytes = file.current().len(); + if available_bytes < buf.len() { + return Err(Error::from(ErrorKind::UnexpectedEof).into()); + } + buf.copy_from_slice(&file.data[file.pos..file.pos + buf.len()]); + file.pos += buf.len(); + Ok(()) + }) + } +} + +impl RawFileInterfaceWrite for VFileDescriptor { + fn fw_write_all(&mut self, bytes: &[u8]) -> RuntimeResult<()> { + with_file_mut(&self.0, |file| { + if !file.write { + return Err( + Error::new(ErrorKind::PermissionDenied, "Write permission denied").into(), + ); + } + if file.pos + bytes.len() > file.data.len() { + file.data.resize(file.pos + bytes.len(), 0); + } + file.data[file.pos..file.pos + bytes.len()].copy_from_slice(bytes); + file.pos += bytes.len(); + Ok(()) + }) + } +} + +impl RawFileInterfaceWriteExt for VFileDescriptor { + fn fwext_fsync_all(&mut self) -> RuntimeResult<()> { + with_file(&self.0, |_| Ok(())) + } + fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> { + with_file_mut(&self.0, |file| { + if !file.write { + return Err( + Error::new(ErrorKind::PermissionDenied, "Write permission denied").into(), + ); + } + if to as usize > file.data.len() { + file.data.resize(to as usize, 0); + } else { + file.data.truncate(to as usize); + } + if file.pos > file.data.len() { + file.pos = file.data.len(); + } + Ok(()) + }) + } +} + +impl RawFileInterfaceExt for VFileDescriptor { + fn fext_file_length(&self) -> RuntimeResult { + with_file(&self.0, |f| Ok(f.data.len() as u64)) + } + fn fext_cursor(&mut self) -> RuntimeResult { + with_file(&self.0, |f| Ok(f.pos as u64)) + } + fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> { + with_file_mut(&self.0, |file| { + if by > file.data.len() as u64 { + return Err( + Error::new(ErrorKind::InvalidInput, "Can't seek beyond file's end").into(), + ); + } + file.pos = by as usize; + Ok(()) + }) + } +} + +/* + nullfs + --- + - equivalent of `/dev/null` + - all calls are no-ops + - infallible +*/ + +/// An infallible `/dev/null` implementation. Whatever you run on this, it will always be a no-op since nothing +/// is actually happening +#[derive(Debug)] +pub struct NullFS; +pub struct NullFile; +impl RawFSInterface for NullFS { + const NOT_NULL: bool = false; + type File = NullFile; + fn fs_rename_file(_: &str, _: &str) -> RuntimeResult<()> { + Ok(()) + } + fn fs_remove_file(_: &str) -> RuntimeResult<()> { + Ok(()) + } + fn fs_create_dir(_: &str) -> RuntimeResult<()> { + Ok(()) + } + fn fs_create_dir_all(_: &str) -> RuntimeResult<()> { + Ok(()) + } + fn fs_delete_dir(_: &str) -> RuntimeResult<()> { + Ok(()) + } + fn fs_delete_dir_all(_: &str) -> RuntimeResult<()> { + Ok(()) + } + fn fs_fopen_or_create_rw(_: &str) -> RuntimeResult> { + Ok(FileOpen::Created(NullFile)) + } + fn fs_fopen_rw(_: &str) -> RuntimeResult { + Ok(NullFile) + } + fn fs_fcreate_rw(_: &str) -> RuntimeResult { + Ok(NullFile) + } +} +impl RawFileInterfaceRead for NullFile { + fn fr_read_exact(&mut self, _: &mut [u8]) -> RuntimeResult<()> { + Ok(()) + } +} +impl RawFileInterfaceWrite for NullFile { + fn fw_write_all(&mut self, _: &[u8]) -> RuntimeResult<()> { + Ok(()) + } +} +impl RawFileInterfaceWriteExt for NullFile { + fn fwext_fsync_all(&mut self) -> RuntimeResult<()> { + Ok(()) + } + fn fwext_truncate_to(&mut self, _: u64) -> RuntimeResult<()> { + Ok(()) + } +} +impl RawFileInterfaceExt for NullFile { + fn fext_file_length(&self) -> RuntimeResult { + Ok(0) + } + + fn fext_cursor(&mut self) -> RuntimeResult { + Ok(0) + } + + fn fext_seek_ahead_from_start_by(&mut self, _: u64) -> RuntimeResult<()> { + Ok(()) + } +} +impl RawFileInterface for NullFile { + type BufReader = Self; + type BufWriter = Self; + fn into_buffered_reader(self) -> RuntimeResult { + Ok(self) + } + fn downgrade_reader(r: Self::BufReader) -> RuntimeResult { + Ok(r) + } + fn into_buffered_writer(self) -> RuntimeResult { + Ok(self) + } + fn downgrade_writer(w: Self::BufWriter) -> RuntimeResult { + Ok(w) + } +} + +impl RawFileInterfaceBufferedWriter for NullFile {} diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs new file mode 100644 index 00000000..3a1bb406 --- /dev/null +++ b/server/src/engine/storage/v1/mod.rs @@ -0,0 +1,48 @@ +/* + * Created on Mon May 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +// impls +mod batch_jrnl; +mod journal; +pub(in crate::engine) mod loader; +mod rw; +pub mod spec; +pub mod sysdb; +// hl +pub mod inf; +// test +pub mod memfs; +#[cfg(test)] +mod tests; + +// re-exports +pub use { + journal::{JournalAdapter, JournalWriter}, + rw::{LocalFS, RawFSInterface, SDSSFileIO}, +}; +pub mod data_batch { + pub use super::batch_jrnl::{create, DataBatchPersistDriver}; +} diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs new file mode 100644 index 00000000..83b6d087 --- /dev/null +++ b/server/src/engine/storage/v1/rw.rs @@ -0,0 +1,516 @@ +/* + * Created on Tue Jul 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::spec::{FileSpec, Header}, + crate::{ + engine::{error::RuntimeResult, storage::SCrc}, + util::os::SysIOError, + }, + std::{ + fs::{self, File}, + io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write}, + marker::PhantomData, + }, +}; + +#[derive(Debug)] +/// Log whether +pub enum FileOpen { + Created(CF), + Existing(EF), +} + +#[cfg(test)] +impl FileOpen { + pub fn into_existing(self) -> Option { + match self { + Self::Existing(e) => Some(e), + Self::Created(_) => None, + } + } + pub fn into_created(self) -> Option { + match self { + Self::Created(f) => Some(f), + Self::Existing(_) => None, + } + } +} + +#[cfg(test)] +impl FileOpen { + pub fn into_inner(self) -> F { + match self { + Self::Created(f) => f, + Self::Existing(f) => f, + } + } +} + +/// The specification for a file system interface (our own abstraction over the fs) +pub trait RawFSInterface { + /// asserts that the file system is not a null filesystem (like `/dev/null` for example) + const NOT_NULL: bool = true; + /// the file descriptor that is returned by the file system when a file is opened + type File: RawFileInterface; + /// Remove a file + fn fs_remove_file(fpath: &str) -> RuntimeResult<()>; + /// Rename a file + fn fs_rename_file(from: &str, to: &str) -> RuntimeResult<()>; + /// Create a directory + fn fs_create_dir(fpath: &str) -> RuntimeResult<()>; + /// Create a directory and all corresponding path components + fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()>; + /// Delete a directory + fn fs_delete_dir(fpath: &str) -> RuntimeResult<()>; + /// Delete a directory and recursively remove all (if any) children + fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()>; + /// Open or create a file in R/W mode + /// + /// This will: + /// - Create a file if it doesn't exist + /// - Open a file it it does exist + fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult>; + /// Open an existing file + fn fs_fopen_rw(fpath: &str) -> RuntimeResult; + /// Create a new file + fn fs_fcreate_rw(fpath: &str) -> RuntimeResult; +} + +/// A file (well, probably) that can be used for RW operations along with advanced write and extended operations (such as seeking) +pub trait RawFileInterface: Sized +where + Self: RawFileInterfaceRead + + RawFileInterfaceWrite + + RawFileInterfaceWriteExt + + RawFileInterfaceExt, +{ + type BufReader: RawFileInterfaceBufferedReader; + type BufWriter: RawFileInterfaceBufferedWriter; + fn into_buffered_reader(self) -> RuntimeResult; + fn downgrade_reader(r: Self::BufReader) -> RuntimeResult; + fn into_buffered_writer(self) -> RuntimeResult; + fn downgrade_writer(w: Self::BufWriter) -> RuntimeResult; +} + +pub trait RawFileInterfaceBufferedReader: RawFileInterfaceRead + RawFileInterfaceExt {} +impl RawFileInterfaceBufferedReader for R {} + +pub trait RawFileInterfaceBufferedWriter: RawFileInterfaceWrite + RawFileInterfaceExt { + fn sync_write_cache(&mut self) -> RuntimeResult<()> { + Ok(()) + } +} + +/// A file interface that supports read operations +pub trait RawFileInterfaceRead { + fn fr_read_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()>; +} + +impl RawFileInterfaceRead for R { + fn fr_read_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + self.read_exact(buf).map_err(From::from) + } +} + +/// A file interface that supports write operations +pub trait RawFileInterfaceWrite { + fn fw_write_all(&mut self, buf: &[u8]) -> RuntimeResult<()>; +} + +impl RawFileInterfaceWrite for W { + fn fw_write_all(&mut self, buf: &[u8]) -> RuntimeResult<()> { + self.write_all(buf).map_err(From::from) + } +} + +/// A file interface that supports advanced write operations +pub trait RawFileInterfaceWriteExt { + fn fwext_fsync_all(&mut self) -> RuntimeResult<()>; + fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()>; +} + +/// A file interface that supports advanced file operations +pub trait RawFileInterfaceExt { + fn fext_file_length(&self) -> RuntimeResult; + fn fext_cursor(&mut self) -> RuntimeResult; + fn fext_seek_ahead_from_start_by(&mut self, ahead_by: u64) -> RuntimeResult<()>; +} + +fn cvt(v: std::io::Result) -> RuntimeResult { + let r = v?; + Ok(r) +} + +/// The actual local host file system (as an abstraction [`fs`]) +#[derive(Debug)] +pub struct LocalFS; + +impl RawFSInterface for LocalFS { + type File = File; + fn fs_remove_file(fpath: &str) -> RuntimeResult<()> { + cvt(fs::remove_file(fpath)) + } + fn fs_rename_file(from: &str, to: &str) -> RuntimeResult<()> { + cvt(fs::rename(from, to)) + } + fn fs_create_dir(fpath: &str) -> RuntimeResult<()> { + cvt(fs::create_dir(fpath)) + } + fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> { + cvt(fs::create_dir_all(fpath)) + } + fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> { + cvt(fs::remove_dir(fpath)) + } + fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> { + cvt(fs::remove_dir_all(fpath)) + } + fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult> { + let f = File::options() + .create(true) + .read(true) + .write(true) + .open(fpath)?; + let md = f.metadata()?; + if md.len() == 0 { + Ok(FileOpen::Created(f)) + } else { + Ok(FileOpen::Existing(f)) + } + } + fn fs_fcreate_rw(fpath: &str) -> RuntimeResult { + let f = File::options() + .create_new(true) + .read(true) + .write(true) + .open(fpath)?; + Ok(f) + } + fn fs_fopen_rw(fpath: &str) -> RuntimeResult { + let f = File::options().read(true).write(true).open(fpath)?; + Ok(f) + } +} + +impl RawFileInterface for File { + type BufReader = BufReader; + type BufWriter = BufWriter; + fn into_buffered_reader(self) -> RuntimeResult { + Ok(BufReader::new(self)) + } + fn downgrade_reader(r: Self::BufReader) -> RuntimeResult { + Ok(r.into_inner()) + } + fn into_buffered_writer(self) -> RuntimeResult { + Ok(BufWriter::new(self)) + } + fn downgrade_writer(mut w: Self::BufWriter) -> RuntimeResult { + w.flush()?; // TODO(@ohsayan): handle rare case where writer does panic + let (w, _) = w.into_parts(); + Ok(w) + } +} + +impl RawFileInterfaceBufferedWriter for BufWriter { + fn sync_write_cache(&mut self) -> RuntimeResult<()> { + self.flush()?; + self.get_mut().sync_all()?; + Ok(()) + } +} + +impl RawFileInterfaceWriteExt for File { + fn fwext_fsync_all(&mut self) -> RuntimeResult<()> { + cvt(self.sync_all()) + } + fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> { + cvt(self.set_len(to)) + } +} + +trait LocalFSFile { + fn file_mut(&mut self) -> &mut File; + fn file(&self) -> &File; +} + +impl LocalFSFile for File { + fn file_mut(&mut self) -> &mut File { + self + } + fn file(&self) -> &File { + self + } +} + +impl LocalFSFile for BufReader { + fn file_mut(&mut self) -> &mut File { + self.get_mut() + } + fn file(&self) -> &File { + self.get_ref() + } +} + +impl LocalFSFile for BufWriter { + fn file_mut(&mut self) -> &mut File { + self.get_mut() + } + fn file(&self) -> &File { + self.get_ref() + } +} + +impl RawFileInterfaceExt for F { + fn fext_file_length(&self) -> RuntimeResult { + Ok(self.file().metadata()?.len()) + } + fn fext_cursor(&mut self) -> RuntimeResult { + cvt(self.file_mut().stream_position()) + } + fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> { + cvt(self.file_mut().seek(SeekFrom::Start(by)).map(|_| ())) + } +} + +pub struct SDSSFileTrackedWriter { + f: SDSSFileIO::BufWriter>, + cs: SCrc, +} + +impl SDSSFileTrackedWriter { + pub fn new(f: SDSSFileIO) -> RuntimeResult { + Ok(Self { + f: f.into_buffered_sdss_writer()?, + cs: SCrc::new(), + }) + } + pub fn tracked_write_unfsynced(&mut self, block: &[u8]) -> RuntimeResult<()> { + self.untracked_write(block) + .map(|_| self.cs.recompute_with_new_var_block(block)) + } + pub fn untracked_write(&mut self, block: &[u8]) -> RuntimeResult<()> { + match self.f.unfsynced_write(block) { + Ok(()) => Ok(()), + e => e, + } + } + pub fn sync_writes(&mut self) -> RuntimeResult<()> { + self.f.f.sync_write_cache() + } + pub fn reset_and_finish_checksum(&mut self) -> u64 { + let scrc = core::mem::replace(&mut self.cs, SCrc::new()); + scrc.finish() + } + pub fn into_inner_file(self) -> RuntimeResult> { + self.f.downgrade_writer() + } +} + +/// [`SDSSFileLenTracked`] simply maintains application level length and checksum tracking to avoid frequent syscalls because we +/// do not expect (even though it's very possible) users to randomly modify file lengths while we're reading them +pub struct SDSSFileTrackedReader { + f: SDSSFileIO::BufReader>, + len: u64, + pos: u64, + cs: SCrc, +} + +impl SDSSFileTrackedReader { + /// Important: this will only look at the data post the current cursor! + pub fn new(mut f: SDSSFileIO) -> RuntimeResult { + let len = f.file_length()?; + let pos = f.retrieve_cursor()?; + let f = f.into_buffered_sdss_reader()?; + Ok(Self { + f, + len, + pos, + cs: SCrc::new(), + }) + } + pub fn remaining(&self) -> u64 { + self.len - self.pos + } + pub fn is_eof(&self) -> bool { + self.len == self.pos + } + pub fn has_left(&self, v: u64) -> bool { + self.remaining() >= v + } + pub fn read_into_buffer(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + self.untracked_read(buf) + .map(|_| self.cs.recompute_with_new_var_block(buf)) + } + pub fn read_byte(&mut self) -> RuntimeResult { + let mut buf = [0u8; 1]; + self.read_into_buffer(&mut buf).map(|_| buf[0]) + } + pub fn __reset_checksum(&mut self) -> u64 { + let mut crc = SCrc::new(); + core::mem::swap(&mut crc, &mut self.cs); + crc.finish() + } + pub fn untracked_read(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + if self.remaining() >= buf.len() as u64 { + match self.f.read_to_buffer(buf) { + Ok(()) => { + self.pos += buf.len() as u64; + Ok(()) + } + Err(e) => return Err(e), + } + } else { + Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into()) + } + } + pub fn into_inner_file(self) -> RuntimeResult> { + self.f.downgrade_reader() + } + pub fn read_block(&mut self) -> RuntimeResult<[u8; N]> { + if !self.has_left(N as _) { + return Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into()); + } + let mut buf = [0; N]; + self.read_into_buffer(&mut buf)?; + Ok(buf) + } + pub fn read_u64_le(&mut self) -> RuntimeResult { + Ok(u64::from_le_bytes(self.read_block()?)) + } +} + +#[derive(Debug)] +pub struct SDSSFileIO::File> { + f: F, + _fs: PhantomData, +} + +impl SDSSFileIO { + pub fn open(fpath: &str) -> RuntimeResult<(Self, F::Header)> { + let mut f = Self::_new(Fs::fs_fopen_rw(fpath)?); + let header = F::Header::decode_verify(&mut f, F::DECODE_DATA, F::VERIFY_DATA)?; + Ok((f, header)) + } + pub fn create(fpath: &str) -> RuntimeResult { + let mut f = Self::_new(Fs::fs_fcreate_rw(fpath)?); + F::Header::encode(&mut f, F::ENCODE_DATA)?; + Ok(f) + } + pub fn open_or_create_perm_rw( + fpath: &str, + ) -> RuntimeResult> { + match Fs::fs_fopen_or_create_rw(fpath)? { + FileOpen::Created(c) => { + let mut f = Self::_new(c); + F::Header::encode(&mut f, F::ENCODE_DATA)?; + Ok(FileOpen::Created(f)) + } + FileOpen::Existing(e) => { + let mut f = Self::_new(e); + let header = F::Header::decode_verify(&mut f, F::DECODE_DATA, F::VERIFY_DATA)?; + Ok(FileOpen::Existing((f, header))) + } + } + } + pub fn into_buffered_sdss_reader( + self, + ) -> RuntimeResult::BufReader>> { + self.f.into_buffered_reader().map(SDSSFileIO::_new) + } + pub fn into_buffered_sdss_writer( + self, + ) -> RuntimeResult::BufWriter>> { + self.f.into_buffered_writer().map(SDSSFileIO::_new) + } +} + +impl SDSSFileIO::BufReader> { + pub fn downgrade_reader(self) -> RuntimeResult> { + let me = ::downgrade_reader(self.f)?; + Ok(SDSSFileIO::_new(me)) + } +} + +impl SDSSFileIO::BufWriter> { + pub fn downgrade_writer(self) -> RuntimeResult> { + let me = ::downgrade_writer(self.f)?; + Ok(SDSSFileIO::_new(me)) + } +} + +impl SDSSFileIO { + pub fn _new(f: F) -> Self { + Self { + f, + _fs: PhantomData, + } + } +} + +impl SDSSFileIO { + pub fn read_to_buffer(&mut self, buffer: &mut [u8]) -> RuntimeResult<()> { + self.f.fr_read_exact(buffer) + } +} + +impl SDSSFileIO { + pub fn retrieve_cursor(&mut self) -> RuntimeResult { + self.f.fext_cursor() + } + pub fn file_length(&self) -> RuntimeResult { + self.f.fext_file_length() + } + pub fn seek_from_start(&mut self, by: u64) -> RuntimeResult<()> { + self.f.fext_seek_ahead_from_start_by(by) + } +} + +impl SDSSFileIO { + pub fn load_remaining_into_buffer(&mut self) -> RuntimeResult> { + let len = self.file_length()? - self.retrieve_cursor()?; + let mut buf = vec![0; len as usize]; + self.read_to_buffer(&mut buf)?; + Ok(buf) + } +} + +impl SDSSFileIO { + pub fn unfsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> { + self.f.fw_write_all(data) + } +} + +impl SDSSFileIO { + pub fn fsync_all(&mut self) -> RuntimeResult<()> { + self.f.fwext_fsync_all()?; + Ok(()) + } + pub fn fsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> { + self.f.fw_write_all(data)?; + self.f.fwext_fsync_all() + } +} diff --git a/server/src/engine/storage/v1/spec.rs b/server/src/engine/storage/v1/spec.rs new file mode 100644 index 00000000..9f556c1d --- /dev/null +++ b/server/src/engine/storage/v1/spec.rs @@ -0,0 +1,610 @@ +/* + * Created on Mon Sep 25 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +/* + Header specification + --- + We utilize two different kinds of headers: + - Static header - Mostly to avoid data corruption + - Variable header - For preserving dynamic information +*/ + +use { + super::rw::{RawFSInterface, SDSSFileIO}, + crate::{ + engine::{ + error::{RuntimeResult, StorageError}, + storage::versions::{self, DriverVersion, HeaderVersion, ServerVersion}, + }, + util::os, + }, + std::{ + mem::{transmute, ManuallyDrop}, + ops::Range, + }, +}; + +/* + meta +*/ + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +/// Host architecture enumeration for common platforms +pub enum HostArch { + X86 = 0, + X86_64 = 1, + ARM = 2, + ARM64 = 3, + MIPS = 4, + PowerPC = 5, +} + +impl HostArch { + pub const fn new() -> Self { + if cfg!(target_arch = "x86") { + HostArch::X86 + } else if cfg!(target_arch = "x86_64") { + HostArch::X86_64 + } else if cfg!(target_arch = "arm") { + HostArch::ARM + } else if cfg!(target_arch = "aarch64") { + HostArch::ARM64 + } else if cfg!(target_arch = "mips") { + HostArch::MIPS + } else if cfg!(target_arch = "powerpc") { + HostArch::PowerPC + } else { + panic!("Unsupported target architecture") + } + } +} + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +/// Host OS enumeration for common operating systems +pub enum HostOS { + // T1 + Linux = 0, + Windows = 1, + MacOS = 2, + // T2 + Android = 3, + AppleiOS = 4, + FreeBSD = 5, + OpenBSD = 6, + NetBSD = 7, + WASI = 8, + Emscripten = 9, + // T3 + Solaris = 10, + Fuchsia = 11, + Redox = 12, + DragonFly = 13, +} + +impl HostOS { + pub const fn new() -> Self { + if cfg!(target_os = "linux") { + HostOS::Linux + } else if cfg!(target_os = "windows") { + HostOS::Windows + } else if cfg!(target_os = "macos") { + HostOS::MacOS + } else if cfg!(target_os = "android") { + HostOS::Android + } else if cfg!(target_os = "ios") { + HostOS::AppleiOS + } else if cfg!(target_os = "freebsd") { + HostOS::FreeBSD + } else if cfg!(target_os = "openbsd") { + HostOS::OpenBSD + } else if cfg!(target_os = "netbsd") { + HostOS::NetBSD + } else if cfg!(target_os = "dragonfly") { + HostOS::DragonFly + } else if cfg!(target_os = "redox") { + HostOS::Redox + } else if cfg!(target_os = "fuchsia") { + HostOS::Fuchsia + } else if cfg!(target_os = "solaris") { + HostOS::Solaris + } else if cfg!(target_os = "emscripten") { + HostOS::Emscripten + } else if cfg!(target_os = "wasi") { + HostOS::WASI + } else { + panic!("unknown os") + } + } +} + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +/// Host endian enumeration +pub enum HostEndian { + Big = 0, + Little = 1, +} + +impl HostEndian { + pub const fn new() -> Self { + if cfg!(target_endian = "little") { + Self::Little + } else { + Self::Big + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +#[repr(u8)] +/// Host pointer width enumeration +pub enum HostPointerWidth { + P32 = 0, + P64 = 1, +} + +impl HostPointerWidth { + pub const fn new() -> Self { + match sizeof!(usize) { + 4 => Self::P32, + 8 => Self::P64, + _ => panic!("unknown pointer width"), + } + } +} + +/// The file scope +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +pub enum FileScope { + Journal = 0, + DataBatch = 1, + FlatmapData = 2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +#[repr(u8)] +pub enum FileSpecifier { + GNSTxnLog = 0, + TableDataBatch = 1, + SysDB = 2, + #[cfg(test)] + TestTransactionLog = 0xFF, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct FileSpecifierVersion(u16); +impl FileSpecifierVersion { + pub const fn __new(v: u16) -> Self { + Self(v) + } +} + +const SDSS_MAGIC: u64 = 0x4F48534159414E21; + +/// Specification for a SDSS file +pub trait FileSpec { + /// The header spec for the file + type Header: Header; + /// Encode data + const ENCODE_DATA: ::EncodeArgs; + /// Decode data + const DECODE_DATA: ::DecodeArgs; + /// Verify data + const VERIFY_DATA: ::DecodeVerifyArgs; +} + +/* + file spec impls +*/ + +#[cfg(test)] +pub struct TestFile; +#[cfg(test)] +impl FileSpec for TestFile { + type Header = SDSSStaticHeaderV1Compact; + const ENCODE_DATA: ::EncodeArgs = ( + FileScope::FlatmapData, + FileSpecifier::TestTransactionLog, + FileSpecifierVersion::__new(0), + ); + const DECODE_DATA: ::DecodeArgs = (); + const VERIFY_DATA: ::DecodeVerifyArgs = Self::ENCODE_DATA; +} + +/// The file specification for the GNS transaction log (impl v1) +pub struct GNSTransactionLogV1; +impl FileSpec for GNSTransactionLogV1 { + type Header = SDSSStaticHeaderV1Compact; + const ENCODE_DATA: ::EncodeArgs = ( + FileScope::Journal, + FileSpecifier::GNSTxnLog, + FileSpecifierVersion::__new(0), + ); + const DECODE_DATA: ::DecodeArgs = (); + const VERIFY_DATA: ::DecodeVerifyArgs = Self::ENCODE_DATA; +} + +/// The file specification for a journal batch +pub struct DataBatchJournalV1; +impl FileSpec for DataBatchJournalV1 { + type Header = SDSSStaticHeaderV1Compact; + const ENCODE_DATA: ::EncodeArgs = ( + FileScope::DataBatch, + FileSpecifier::TableDataBatch, + FileSpecifierVersion::__new(0), + ); + const DECODE_DATA: ::DecodeArgs = (); + const VERIFY_DATA: ::DecodeVerifyArgs = Self::ENCODE_DATA; +} + +/// The file specification for the system db +pub struct SysDBV1; +impl FileSpec for SysDBV1 { + type Header = SDSSStaticHeaderV1Compact; + const ENCODE_DATA: ::EncodeArgs = ( + FileScope::FlatmapData, + FileSpecifier::SysDB, + FileSpecifierVersion::__new(0), + ); + const DECODE_DATA: ::DecodeArgs = (); + const VERIFY_DATA: ::DecodeVerifyArgs = Self::ENCODE_DATA; +} + +/* + header spec +*/ + +/// SDSS Header specification +pub trait Header: Sized { + /// Encode arguments + type EncodeArgs; + /// Decode arguments + type DecodeArgs; + /// Decode verify arguments + type DecodeVerifyArgs; + /// Encode the header + fn encode( + f: &mut SDSSFileIO, + args: Self::EncodeArgs, + ) -> RuntimeResult<()>; + /// Decode the header + fn decode( + f: &mut SDSSFileIO, + args: Self::DecodeArgs, + ) -> RuntimeResult; + /// Verify the header + fn verify(&self, args: Self::DecodeVerifyArgs) -> RuntimeResult<()>; + /// Decode and verify the header + fn decode_verify( + f: &mut SDSSFileIO, + d_args: Self::DecodeArgs, + v_args: Self::DecodeVerifyArgs, + ) -> RuntimeResult { + let h = Self::decode(f, d_args)?; + h.verify(v_args)?; + Ok(h) + } +} + +/* + header impls +*/ + +unsafe fn memcpy(src: &[u8]) -> [u8; N] { + let mut dst = [0u8; N]; + src.as_ptr().copy_to_nonoverlapping(dst.as_mut_ptr(), N); + dst +} + +macro_rules! var { + (let $($name:ident),* $(,)?) => { + $(let $name;)* + } +} + +/* + Compact SDSS Header v1 + --- + - 1: Magic block (16B): magic + header version + - 2: Static block (40B): + - 2.1: Genesis static record (24B) + - 2.1.1: Software information (16B) + - Server version (8B) + - Driver version (8B) + - 2.1.2: Host information (4B): + - OS (1B) + - Arch (1B) + - Pointer width (1B) + - Endian (1B) + - 2.1.3: File information (4B): + - File class (1B) + - File specifier (1B) + - File specifier version (2B) + - 2.2: Genesis runtime record (16B) + - Host epoch (16B) + - 3: Padding block (8B) +*/ + +#[repr(align(8))] +#[derive(Debug, PartialEq)] +pub struct SDSSStaticHeaderV1Compact { + // 1 magic block + magic_header_version: HeaderVersion, + // 2.1.1 + genesis_static_sw_server_version: ServerVersion, + genesis_static_sw_driver_version: DriverVersion, + // 2.1.2 + genesis_static_host_os: HostOS, + genesis_static_host_arch: HostArch, + genesis_static_host_ptr_width: HostPointerWidth, + genesis_static_host_endian: HostEndian, + // 2.1.3 + genesis_static_file_class: FileScope, + genesis_static_file_specifier: FileSpecifier, + genesis_static_file_specifier_version: FileSpecifierVersion, + // 2.2 + genesis_runtime_epoch_time: u128, + // 3 + genesis_padding_block: [u8; 8], +} + +impl SDSSStaticHeaderV1Compact { + pub const SIZE: usize = 64; + /// Decode and validate the full header block (validate ONLY; you must verify yourself) + /// + /// Notes: + /// - Time might be inconsistent; verify + /// - Compatibility requires additional intervention + /// - If padding block was not zeroed, handle + /// - No file metadata and is verified. Check! + /// + fn _decode(block: [u8; 64]) -> RuntimeResult { + var!(let raw_magic, raw_header_version, raw_server_version, raw_driver_version, raw_host_os, raw_host_arch, + raw_host_ptr_width, raw_host_endian, raw_file_class, raw_file_specifier, raw_file_specifier_version, + raw_runtime_epoch_time, raw_paddding_block, + ); + macro_rules! u64 { + ($pos:expr) => { + u64::from_le_bytes(memcpy(&block[$pos])) + }; + } + unsafe { + // UNSAFE(@ohsayan): all segments are correctly accessed (aligned to u8) + raw_magic = u64!(Self::SEG1_MAGIC); + raw_header_version = HeaderVersion::__new(u64!(Self::SEG1_HEADER_VERSION)); + raw_server_version = ServerVersion::__new(u64!(Self::SEG2_REC1_SERVER_VERSION)); + raw_driver_version = DriverVersion::__new(u64!(Self::SEG2_REC1_DRIVER_VERSION)); + raw_host_os = block[Self::SEG2_REC1_HOST_OS]; + raw_host_arch = block[Self::SEG2_REC1_HOST_ARCH]; + raw_host_ptr_width = block[Self::SEG2_REC1_HOST_PTR_WIDTH]; + raw_host_endian = block[Self::SEG2_REC1_HOST_ENDIAN]; + raw_file_class = block[Self::SEG2_REC1_FILE_CLASS]; + raw_file_specifier = block[Self::SEG2_REC1_FILE_SPECIFIER]; + raw_file_specifier_version = FileSpecifierVersion::__new(u16::from_le_bytes(memcpy( + &block[Self::SEG2_REC1_FILE_SPECIFIER_VERSION], + ))); + raw_runtime_epoch_time = + u128::from_le_bytes(memcpy(&block[Self::SEG2_REC2_RUNTIME_EPOCH_TIME])); + raw_paddding_block = memcpy::<8>(&block[Self::SEG3_PADDING_BLK]); + } + macro_rules! okay { + ($($expr:expr),* $(,)?) => { + $(($expr) &)*true + } + } + let okay_header_version = raw_header_version == versions::CURRENT_HEADER_VERSION; + let okay_server_version = raw_server_version == versions::CURRENT_SERVER_VERSION; + let okay_driver_version = raw_driver_version == versions::CURRENT_DRIVER_VERSION; + let okay = okay!( + // 1.1 mgblk + raw_magic == SDSS_MAGIC, + okay_header_version, + // 2.1.1 + okay_server_version, + okay_driver_version, + // 2.1.2 + raw_host_os <= HostOS::MAX, + raw_host_arch <= HostArch::MAX, + raw_host_ptr_width <= HostPointerWidth::MAX, + raw_host_endian <= HostEndian::MAX, + // 2.1.3 + raw_file_class <= FileScope::MAX, + raw_file_specifier <= FileSpecifier::MAX, + ); + if okay { + Ok(unsafe { + // UNSAFE(@ohsayan): the block ranges are very well defined + Self { + // 1.1 + magic_header_version: raw_header_version, + // 2.1.1 + genesis_static_sw_server_version: raw_server_version, + genesis_static_sw_driver_version: raw_driver_version, + // 2.1.2 + genesis_static_host_os: transmute(raw_host_os), + genesis_static_host_arch: transmute(raw_host_arch), + genesis_static_host_ptr_width: transmute(raw_host_ptr_width), + genesis_static_host_endian: transmute(raw_host_endian), + // 2.1.3 + genesis_static_file_class: transmute(raw_file_class), + genesis_static_file_specifier: transmute(raw_file_specifier), + genesis_static_file_specifier_version: raw_file_specifier_version, + // 2.2 + genesis_runtime_epoch_time: raw_runtime_epoch_time, + // 3 + genesis_padding_block: raw_paddding_block, + } + }) + } else { + let version_okay = okay_header_version & okay_server_version & okay_driver_version; + let md = ManuallyDrop::new([ + StorageError::HeaderDecodeCorruptedHeader, + StorageError::HeaderDecodeVersionMismatch, + ]); + Err(unsafe { + // UNSAFE(@ohsayan): while not needed, md for drop safety + correct index + md.as_ptr().add(!version_okay as usize).read().into() + }) + } + } +} + +impl SDSSStaticHeaderV1Compact { + const SEG1_MAGIC: Range = 0..8; + const SEG1_HEADER_VERSION: Range = 8..16; + const SEG2_REC1_SERVER_VERSION: Range = 16..24; + const SEG2_REC1_DRIVER_VERSION: Range = 24..32; + const SEG2_REC1_HOST_OS: usize = 32; + const SEG2_REC1_HOST_ARCH: usize = 33; + const SEG2_REC1_HOST_PTR_WIDTH: usize = 34; + const SEG2_REC1_HOST_ENDIAN: usize = 35; + const SEG2_REC1_FILE_CLASS: usize = 36; + const SEG2_REC1_FILE_SPECIFIER: usize = 37; + const SEG2_REC1_FILE_SPECIFIER_VERSION: Range = 38..40; + const SEG2_REC2_RUNTIME_EPOCH_TIME: Range = 40..56; + const SEG3_PADDING_BLK: Range = 56..64; + fn _encode( + file_class: FileScope, + file_specifier: FileSpecifier, + file_specifier_version: FileSpecifierVersion, + epoch_time: u128, + padding_block: [u8; 8], + ) -> [u8; 64] { + let mut ret = [0; 64]; + // 1. mgblk + ret[Self::SEG1_MAGIC].copy_from_slice(&SDSS_MAGIC.to_le_bytes()); + ret[Self::SEG1_HEADER_VERSION] + .copy_from_slice(&versions::CURRENT_HEADER_VERSION.little_endian_u64()); + // 2.1.1 + ret[Self::SEG2_REC1_SERVER_VERSION] + .copy_from_slice(&versions::CURRENT_SERVER_VERSION.little_endian()); + ret[Self::SEG2_REC1_DRIVER_VERSION] + .copy_from_slice(&versions::CURRENT_DRIVER_VERSION.little_endian()); + // 2.1.2 + ret[Self::SEG2_REC1_HOST_OS] = HostOS::new().value_u8(); + ret[Self::SEG2_REC1_HOST_ARCH] = HostArch::new().value_u8(); + ret[Self::SEG2_REC1_HOST_PTR_WIDTH] = HostPointerWidth::new().value_u8(); + ret[Self::SEG2_REC1_HOST_ENDIAN] = HostEndian::new().value_u8(); + // 2.1.3 + ret[Self::SEG2_REC1_FILE_CLASS] = file_class.value_u8(); + ret[Self::SEG2_REC1_FILE_SPECIFIER] = file_specifier.value_u8(); + ret[Self::SEG2_REC1_FILE_SPECIFIER_VERSION] + .copy_from_slice(&file_specifier_version.0.to_le_bytes()); + // 2.2 + ret[Self::SEG2_REC2_RUNTIME_EPOCH_TIME].copy_from_slice(&epoch_time.to_le_bytes()); + // 3 + ret[Self::SEG3_PADDING_BLK].copy_from_slice(&padding_block); + ret + } + pub fn _encode_auto( + file_class: FileScope, + file_specifier: FileSpecifier, + file_specifier_version: FileSpecifierVersion, + ) -> [u8; 64] { + let epoch_time = os::get_epoch_time(); + Self::_encode( + file_class, + file_specifier, + file_specifier_version, + epoch_time, + [0; 8], + ) + } +} + +#[allow(unused)] +impl SDSSStaticHeaderV1Compact { + pub fn header_version(&self) -> HeaderVersion { + self.magic_header_version + } + pub fn server_version(&self) -> ServerVersion { + self.genesis_static_sw_server_version + } + pub fn driver_version(&self) -> DriverVersion { + self.genesis_static_sw_driver_version + } + pub fn host_os(&self) -> HostOS { + self.genesis_static_host_os + } + pub fn host_arch(&self) -> HostArch { + self.genesis_static_host_arch + } + pub fn host_ptr_width(&self) -> HostPointerWidth { + self.genesis_static_host_ptr_width + } + pub fn host_endian(&self) -> HostEndian { + self.genesis_static_host_endian + } + pub fn file_class(&self) -> FileScope { + self.genesis_static_file_class + } + pub fn file_specifier(&self) -> FileSpecifier { + self.genesis_static_file_specifier + } + pub fn file_specifier_version(&self) -> FileSpecifierVersion { + self.genesis_static_file_specifier_version + } + pub fn epoch_time(&self) -> u128 { + self.genesis_runtime_epoch_time + } + pub fn padding_block(&self) -> [u8; 8] { + self.genesis_padding_block + } +} + +impl Header for SDSSStaticHeaderV1Compact { + type EncodeArgs = (FileScope, FileSpecifier, FileSpecifierVersion); + type DecodeArgs = (); + type DecodeVerifyArgs = Self::EncodeArgs; + fn encode( + f: &mut SDSSFileIO, + (scope, spec, spec_v): Self::EncodeArgs, + ) -> RuntimeResult<()> { + let b = Self::_encode_auto(scope, spec, spec_v); + f.fsynced_write(&b) + } + fn decode( + f: &mut SDSSFileIO, + _: Self::DecodeArgs, + ) -> RuntimeResult { + let mut buf = [0u8; 64]; + f.read_to_buffer(&mut buf)?; + Self::_decode(buf) + } + fn verify(&self, (scope, spec, spec_v): Self::DecodeVerifyArgs) -> RuntimeResult<()> { + if (self.file_class() == scope) + & (self.file_specifier() == spec) + & (self.file_specifier_version() == spec_v) + { + Ok(()) + } else { + Err(StorageError::HeaderDecodeDataMismatch.into()) + } + } +} diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs new file mode 100644 index 00000000..a0a4eace --- /dev/null +++ b/server/src/engine/storage/v1/sysdb.rs @@ -0,0 +1,238 @@ +/* + * Created on Fri Sep 22 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::rw::FileOpen, + crate::engine::{ + config::{ConfigAuth, ConfigMode}, + data::{cell::Datacell, DictEntryGeneric, DictGeneric}, + error::{RuntimeResult, StorageError}, + fractal::sys_store::{SysAuth, SysAuthUser, SysConfig, SysHostData, SystemStore}, + storage::v1::{inf, spec, RawFSInterface, SDSSFileIO}, + }, + parking_lot::RwLock, + std::collections::HashMap, +}; + +#[derive(Debug, PartialEq)] +/// The system store init state +pub enum SystemStoreInitState { + /// No system store was present. it was created + Created, + /// The system store was present, but no new changes were applied + Unchanged, + /// The system store was present, root settings were updated + UpdatedRoot, +} + +impl SystemStoreInitState { + pub const fn is_created(&self) -> bool { + matches!(self, Self::Created) + } + pub const fn is_existing_updated_root(&self) -> bool { + matches!(self, Self::UpdatedRoot) + } +} + +fn rkey( + d: &mut DictGeneric, + key: &str, + transform: impl Fn(DictEntryGeneric) -> Option, +) -> RuntimeResult { + match d.remove(key).map(transform) { + Some(Some(k)) => Ok(k), + _ => Err(StorageError::SysDBCorrupted.into()), + } +} + +impl SystemStore { + const SYSDB_PATH: &'static str = "sys.db"; + const SYSDB_COW_PATH: &'static str = "sys.db.cow"; + const SYS_KEY_AUTH: &'static str = "auth"; + const SYS_KEY_AUTH_USERS: &'static str = "users"; + const SYS_KEY_SYS: &'static str = "sys"; + const SYS_KEY_SYS_STARTUP_COUNTER: &'static str = "sc"; + const SYS_KEY_SYS_SETTINGS_VERSION: &'static str = "sv"; + pub fn open_or_restore( + auth: ConfigAuth, + run_mode: ConfigMode, + ) -> RuntimeResult<(Self, SystemStoreInitState)> { + Self::open_with_name(Self::SYSDB_PATH, Self::SYSDB_COW_PATH, auth, run_mode) + } + pub fn sync_db(&self, auth: &SysAuth) -> RuntimeResult<()> { + self._sync_with(Self::SYSDB_PATH, Self::SYSDB_COW_PATH, auth) + } + pub fn open_with_name( + sysdb_name: &str, + sysdb_cow_path: &str, + auth: ConfigAuth, + run_mode: ConfigMode, + ) -> RuntimeResult<(Self, SystemStoreInitState)> { + match SDSSFileIO::open_or_create_perm_rw::(sysdb_name)? { + FileOpen::Created(new) => { + let me = Self::_new(SysConfig::new_auth(auth, run_mode)); + me._sync(new, &me.system_store().auth_data().read())?; + Ok((me, SystemStoreInitState::Created)) + } + FileOpen::Existing((ex, _)) => { + Self::restore_and_sync(ex, auth, run_mode, sysdb_name, sysdb_cow_path) + } + } + } +} + +impl SystemStore { + fn _sync(&self, mut f: SDSSFileIO, auth: &SysAuth) -> RuntimeResult<()> { + let cfg = self.system_store(); + // prepare our flat file + let mut map: DictGeneric = into_dict!( + Self::SYS_KEY_SYS => DictEntryGeneric::Map(into_dict!( + Self::SYS_KEY_SYS_SETTINGS_VERSION => Datacell::new_uint_default(cfg.host_data().settings_version() as _), + Self::SYS_KEY_SYS_STARTUP_COUNTER => Datacell::new_uint_default(cfg.host_data().startup_counter() as _), + )), + Self::SYS_KEY_AUTH => DictGeneric::new(), + ); + let auth_key = map.get_mut(Self::SYS_KEY_AUTH).unwrap(); + let auth_key = auth_key.as_dict_mut().unwrap(); + auth_key.insert( + Self::SYS_KEY_AUTH_USERS.into(), + DictEntryGeneric::Map( + // username -> [..settings] + auth.users() + .iter() + .map(|(username, user)| { + ( + username.to_owned(), + DictEntryGeneric::Data(Datacell::new_list(vec![Datacell::new_bin( + user.key().into(), + )])), + ) + }) + .collect(), + ), + ); + // write + let buf = super::inf::enc::enc_dict_full::(&map); + f.fsynced_write(&buf) + } + fn _sync_with(&self, target: &str, cow: &str, auth: &SysAuth) -> RuntimeResult<()> { + let f = SDSSFileIO::create::(cow)?; + self._sync(f, auth)?; + Fs::fs_rename_file(cow, target) + } + fn restore_and_sync( + f: SDSSFileIO, + auth: ConfigAuth, + run_mode: ConfigMode, + fname: &str, + fcow_name: &str, + ) -> RuntimeResult<(Self, SystemStoreInitState)> { + let prev_sysdb = Self::_restore(f, run_mode)?; + let state; + // see if settings have changed + if prev_sysdb + .auth_data() + .read() + .verify_user(SysAuthUser::USER_ROOT, &auth.root_key) + .is_ok() + { + state = SystemStoreInitState::Unchanged; + } else { + state = SystemStoreInitState::UpdatedRoot; + } + // create new config + let new_syscfg = SysConfig::new_full( + auth, + SysHostData::new( + prev_sysdb.host_data().startup_counter() + 1, + prev_sysdb.host_data().settings_version() + + !matches!(state, SystemStoreInitState::Unchanged) as u32, + ), + run_mode, + ); + let slf = Self::_new(new_syscfg); + // now sync + slf._sync_with(fname, fcow_name, &slf.system_store().auth_data().read())?; + Ok((slf, state)) + } + fn _restore(mut f: SDSSFileIO, run_mode: ConfigMode) -> RuntimeResult { + let mut sysdb_data = + inf::dec::dec_dict_full::(&f.load_remaining_into_buffer()?)?; + // get our auth and sys stores + let mut auth_store = rkey( + &mut sysdb_data, + Self::SYS_KEY_AUTH, + DictEntryGeneric::into_dict, + )?; + let mut sys_store = rkey( + &mut sysdb_data, + Self::SYS_KEY_SYS, + DictEntryGeneric::into_dict, + )?; + // load auth store + let users = rkey( + &mut auth_store, + Self::SYS_KEY_AUTH_USERS, + DictEntryGeneric::into_dict, + )?; + // load users + let mut loaded_users = HashMap::new(); + for (username, userdata) in users { + let mut userdata = userdata + .into_data() + .and_then(Datacell::into_list) + .ok_or(StorageError::SysDBCorrupted)?; + if userdata.len() != 1 { + return Err(StorageError::SysDBCorrupted.into()); + } + let user_password = userdata + .remove(0) + .into_bin() + .ok_or(StorageError::SysDBCorrupted)?; + loaded_users.insert(username, SysAuthUser::new(user_password.into_boxed_slice())); + } + let sys_auth = SysAuth::new(loaded_users); + // load sys data + let sc = rkey(&mut sys_store, Self::SYS_KEY_SYS_STARTUP_COUNTER, |d| { + d.into_data()?.into_uint() + })?; + let sv = rkey(&mut sys_store, Self::SYS_KEY_SYS_SETTINGS_VERSION, |d| { + d.into_data()?.into_uint() + })?; + if !(sysdb_data.is_empty() + & auth_store.is_empty() + & sys_store.is_empty() + & sys_auth.users().contains_key(SysAuthUser::USER_ROOT)) + { + return Err(StorageError::SysDBCorrupted.into()); + } + Ok(SysConfig::new( + RwLock::new(sys_auth), + SysHostData::new(sc, sv as u32), + run_mode, + )) + } +} diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs new file mode 100644 index 00000000..49700f1b --- /dev/null +++ b/server/src/engine/storage/v1/tests.rs @@ -0,0 +1,115 @@ +/* + * Created on Sat Jul 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +type VirtualFS = super::memfs::VirtualFS; + +mod batch; +mod rw; +mod tx; + +mod sysdb { + use { + super::{super::sysdb::SystemStoreInitState, VirtualFS as VFS}, + crate::engine::{ + config::{AuthDriver, ConfigAuth, ConfigMode}, + fractal::sys_store::SystemStore, + }, + }; + fn open_sysdb( + auth_config: ConfigAuth, + sysdb_path: &str, + sysdb_cow_path: &str, + ) -> (SystemStore, SystemStoreInitState) { + SystemStore::::open_with_name(sysdb_path, sysdb_cow_path, auth_config, ConfigMode::Dev) + .unwrap() + } + #[test] + fn open_close() { + let open = |auth_config| { + open_sysdb( + auth_config, + "open_close_test.sys.db", + "open_close_test.sys.cow.db", + ) + }; + let auth_config = ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()); + { + let (config, state) = open(auth_config.clone()); + assert_eq!(state, SystemStoreInitState::Created); + assert!(config + .system_store() + .auth_data() + .read() + .verify_user("root", "password12345678") + .is_ok()); + assert_eq!(config.system_store().host_data().settings_version(), 0); + assert_eq!(config.system_store().host_data().startup_counter(), 0); + } + // reboot + let (config, state) = open(auth_config); + assert_eq!(state, SystemStoreInitState::Unchanged); + assert!(config + .system_store() + .auth_data() + .read() + .verify_user("root", "password12345678") + .is_ok()); + assert_eq!(config.system_store().host_data().settings_version(), 0); + assert_eq!(config.system_store().host_data().startup_counter(), 1); + } + #[test] + fn open_change_root_password() { + let open = |auth_config| { + open_sysdb( + auth_config, + "open_change_root_password.sys.db", + "open_change_root_password.sys.cow.db", + ) + }; + { + let (config, state) = open(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())); + assert_eq!(state, SystemStoreInitState::Created); + assert!(config + .system_store() + .auth_data() + .read() + .verify_user("root", "password12345678") + .is_ok()); + assert_eq!(config.system_store().host_data().settings_version(), 0); + assert_eq!(config.system_store().host_data().startup_counter(), 0); + } + let (config, state) = open(ConfigAuth::new(AuthDriver::Pwd, "password23456789".into())); + assert_eq!(state, SystemStoreInitState::UpdatedRoot); + assert!(config + .system_store() + .auth_data() + .read() + .verify_user("root", "password23456789") + .is_ok()); + assert_eq!(config.system_store().host_data().settings_version(), 1); + assert_eq!(config.system_store().host_data().startup_counter(), 1); + } +} diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs new file mode 100644 index 00000000..75ab5578 --- /dev/null +++ b/server/src/engine/storage/v1/tests/batch.rs @@ -0,0 +1,384 @@ +/* + * Created on Wed Sep 06 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::{ + engine::{ + core::{ + index::{DcFieldIndex, PrimaryIndexKey, Row}, + model::{ + delta::{DataDelta, DataDeltaKind, DeltaVersion}, + Field, Layer, Model, + }, + }, + data::{cell::Datacell, tag::TagSelector, uuid::Uuid}, + idx::MTIndex, + storage::v1::{ + batch_jrnl::{ + DataBatchPersistDriver, DataBatchRestoreDriver, DecodedBatchEvent, + DecodedBatchEventKind, NormalBatch, + }, + memfs::VirtualFS, + rw::{FileOpen, SDSSFileIO}, + spec, + }, + }, + util::test_utils, + }, + crossbeam_epoch::pin, +}; + +fn pkey(v: impl Into) -> PrimaryIndexKey { + PrimaryIndexKey::try_from_dc(v.into()).unwrap() +} + +fn open_file( + fpath: &str, +) -> FileOpen, (SDSSFileIO, spec::SDSSStaticHeaderV1Compact)> { + SDSSFileIO::open_or_create_perm_rw::(fpath).unwrap() +} + +fn open_batch_data(fpath: &str, mdl: &Model) -> DataBatchPersistDriver { + match open_file(fpath) { + FileOpen::Created(f) => DataBatchPersistDriver::new(f, true), + FileOpen::Existing((f, _header)) => { + let mut dbr = DataBatchRestoreDriver::new(f).unwrap(); + dbr.read_data_batch_into_model(mdl).unwrap(); + DataBatchPersistDriver::new(dbr.into_file().unwrap(), false) + } + } + .unwrap() +} + +fn new_delta( + schema: u64, + txnid: u64, + pk: impl Into, + data: DcFieldIndex, + change: DataDeltaKind, +) -> DataDelta { + new_delta_with_row( + txnid, + Row::new( + pkey(pk), + data, + DeltaVersion::__new(schema), + DeltaVersion::__new(txnid), + ), + change, + ) +} + +fn new_delta_with_row(txnid: u64, row: Row, change: DataDeltaKind) -> DataDelta { + DataDelta::new(DeltaVersion::__new(txnid), row, change) +} + +fn flush_deltas_and_re_read( + mdl: &Model, + dt: [DataDelta; N], + fname: &str, +) -> Vec { + let mut restore_driver = flush_batches_and_return_restore_driver(dt, mdl, fname); + let batch = restore_driver.read_all_batches().unwrap(); + batch +} + +fn flush_batches_and_return_restore_driver( + dt: [DataDelta; N], + mdl: &Model, + fname: &str, +) -> DataBatchRestoreDriver { + // delta queue + let g = pin(); + for delta in dt { + mdl.delta_state().append_new_data_delta(delta, &g); + } + let file = open_file(fname).into_created().unwrap(); + { + let mut persist_driver = DataBatchPersistDriver::new(file, true).unwrap(); + persist_driver.write_new_batch(&mdl, N).unwrap(); + persist_driver.close().unwrap(); + } + DataBatchRestoreDriver::new(open_file(fname).into_existing().unwrap().0).unwrap() +} + +#[test] +fn empty_multi_open_reopen() { + let uuid = Uuid::new(); + let mdl = Model::new_restore( + uuid, + "username".into(), + TagSelector::String.into_full(), + into_dict!( + "username" => Field::new([Layer::str()].into(), false), + "password" => Field::new([Layer::bin()].into(), false) + ), + ); + for _ in 0..100 { + let writer = open_batch_data("empty_multi_open_reopen.db-btlog", &mdl); + writer.close().unwrap(); + } +} + +#[test] +fn unskewed_delta() { + let uuid = Uuid::new(); + let mdl = Model::new_restore( + uuid, + "username".into(), + TagSelector::String.into_full(), + into_dict!( + "username" => Field::new([Layer::str()].into(), false), + "password" => Field::new([Layer::bin()].into(), false) + ), + ); + let deltas = [ + new_delta( + 0, + 0, + "sayan", + into_dict!("password" => Datacell::new_bin("37ae4b773a9fc7a20164eb16".as_bytes().into())), + DataDeltaKind::Insert, + ), + new_delta( + 0, + 1, + "badguy", + into_dict!("password" => Datacell::new_bin("5fe3cbdc470b667cb1ba288a".as_bytes().into())), + DataDeltaKind::Insert, + ), + new_delta( + 0, + 2, + "doggo", + into_dict!("password" => Datacell::new_bin("c80403f9d0ae4d5d0e829dd0".as_bytes().into())), + DataDeltaKind::Insert, + ), + new_delta(0, 3, "badguy", into_dict!(), DataDeltaKind::Delete), + ]; + let batches = flush_deltas_and_re_read(&mdl, deltas, "unskewed_delta.db-btlog"); + assert_eq!( + batches, + vec![NormalBatch::new( + vec![ + DecodedBatchEvent::new( + 0, + pkey("sayan"), + DecodedBatchEventKind::Insert(vec![Datacell::new_bin( + b"37ae4b773a9fc7a20164eb16".to_vec().into_boxed_slice() + )]) + ), + DecodedBatchEvent::new( + 1, + pkey("badguy"), + DecodedBatchEventKind::Insert(vec![Datacell::new_bin( + b"5fe3cbdc470b667cb1ba288a".to_vec().into_boxed_slice() + )]) + ), + DecodedBatchEvent::new( + 2, + pkey("doggo"), + DecodedBatchEventKind::Insert(vec![Datacell::new_bin( + b"c80403f9d0ae4d5d0e829dd0".to_vec().into_boxed_slice() + )]) + ), + DecodedBatchEvent::new(3, pkey("badguy"), DecodedBatchEventKind::Delete) + ], + 0 + )] + ) +} + +#[test] +fn skewed_delta() { + // prepare model definition + let uuid = Uuid::new(); + let mdl = Model::new_restore( + uuid, + "catname".into(), + TagSelector::String.into_full(), + into_dict!( + "catname" => Field::new([Layer::str()].into(), false), + "is_good" => Field::new([Layer::bool()].into(), false), + "magical" => Field::new([Layer::bool()].into(), false), + ), + ); + let row = Row::new( + pkey("Schrödinger's cat"), + into_dict!("is_good" => Datacell::new_bool(true), "magical" => Datacell::new_bool(false)), + DeltaVersion::__new(0), + DeltaVersion::__new(2), + ); + { + // update the row + let mut wl = row.d_data().write(); + wl.set_txn_revised(DeltaVersion::__new(3)); + *wl.fields_mut().get_mut("magical").unwrap() = Datacell::new_bool(true); + } + // prepare deltas + let deltas = [ + // insert catname: Schrödinger's cat, is_good: true + new_delta_with_row(0, row.clone(), DataDeltaKind::Insert), + // insert catname: good cat, is_good: true, magical: false + new_delta( + 0, + 1, + "good cat", + into_dict!("is_good" => Datacell::new_bool(true), "magical" => Datacell::new_bool(false)), + DataDeltaKind::Insert, + ), + // insert catname: bad cat, is_good: false, magical: false + new_delta( + 0, + 2, + "bad cat", + into_dict!("is_good" => Datacell::new_bool(false), "magical" => Datacell::new_bool(false)), + DataDeltaKind::Insert, + ), + // update catname: Schrödinger's cat, is_good: true, magical: true + new_delta_with_row(3, row.clone(), DataDeltaKind::Update), + ]; + let batch = flush_deltas_and_re_read(&mdl, deltas, "skewed_delta.db-btlog"); + assert_eq!( + batch, + vec![NormalBatch::new( + vec![ + DecodedBatchEvent::new( + 1, + pkey("good cat"), + DecodedBatchEventKind::Insert(vec![ + Datacell::new_bool(true), + Datacell::new_bool(false) + ]) + ), + DecodedBatchEvent::new( + 2, + pkey("bad cat"), + DecodedBatchEventKind::Insert(vec![ + Datacell::new_bool(false), + Datacell::new_bool(false) + ]) + ), + DecodedBatchEvent::new( + 3, + pkey("Schrödinger's cat"), + DecodedBatchEventKind::Update(vec![ + Datacell::new_bool(true), + Datacell::new_bool(true) + ]) + ) + ], + 0 + )] + ) +} + +#[test] +fn skewed_shuffled_persist_restore() { + let uuid = Uuid::new(); + let model = Model::new_restore( + uuid, + "username".into(), + TagSelector::String.into_full(), + into_dict!("username" => Field::new([Layer::str()].into(), false), "password" => Field::new([Layer::str()].into(), false)), + ); + let mongobongo = Row::new( + pkey("mongobongo"), + into_dict!("password" => "dumbo"), + DeltaVersion::__new(0), + DeltaVersion::__new(4), + ); + let rds = Row::new( + pkey("rds"), + into_dict!("password" => "snail"), + DeltaVersion::__new(0), + DeltaVersion::__new(5), + ); + let deltas = [ + new_delta( + 0, + 0, + "sayan", + into_dict!("password" => "pwd123456"), + DataDeltaKind::Insert, + ), + new_delta( + 0, + 1, + "joseph", + into_dict!("password" => "pwd234567"), + DataDeltaKind::Insert, + ), + new_delta( + 0, + 2, + "haley", + into_dict!("password" => "pwd345678"), + DataDeltaKind::Insert, + ), + new_delta( + 0, + 3, + "charlotte", + into_dict!("password" => "pwd456789"), + DataDeltaKind::Insert, + ), + new_delta_with_row(4, mongobongo.clone(), DataDeltaKind::Insert), + new_delta_with_row(5, rds.clone(), DataDeltaKind::Insert), + new_delta_with_row(6, mongobongo.clone(), DataDeltaKind::Delete), + new_delta_with_row(7, rds.clone(), DataDeltaKind::Delete), + ]; + for i in 0..deltas.len() { + // prepare pretest + let fname = format!("skewed_shuffled_persist_restore_round{i}.db-btlog"); + let mut deltas = deltas.clone(); + let mut randomizer = test_utils::randomizer(); + test_utils::shuffle_slice(&mut deltas, &mut randomizer); + // restore + let mut restore_driver = flush_batches_and_return_restore_driver(deltas, &model, &fname); + restore_driver.read_data_batch_into_model(&model).unwrap(); + } + let g = pin(); + for delta in &deltas[..4] { + let row = model + .primary_index() + .__raw_index() + .mt_get(delta.row().d_key(), &g) + .unwrap(); + let row_data = row.read(); + assert_eq!(row_data.fields().len(), 1); + assert_eq!( + row_data.fields().get("password").unwrap(), + delta + .row() + .d_data() + .read() + .fields() + .get("password") + .unwrap() + ); + } +} diff --git a/server/src/engine/storage/v1/tests/rw.rs b/server/src/engine/storage/v1/tests/rw.rs new file mode 100644 index 00000000..d964cf81 --- /dev/null +++ b/server/src/engine/storage/v1/tests/rw.rs @@ -0,0 +1,52 @@ +/* + * Created on Tue Sep 05 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::storage::v1::{ + rw::{FileOpen, SDSSFileIO}, + spec, +}; + +#[test] +fn create_delete() { + { + let f = SDSSFileIO::::open_or_create_perm_rw::( + "hello_world.db-tlog", + ) + .unwrap(); + match f { + FileOpen::Existing(_) => panic!(), + FileOpen::Created(_) => {} + }; + } + let open = SDSSFileIO::::open_or_create_perm_rw::( + "hello_world.db-tlog", + ) + .unwrap(); + let _ = match open { + FileOpen::Existing(_) => {} + _ => panic!(), + }; +} diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs new file mode 100644 index 00000000..6258b40c --- /dev/null +++ b/server/src/engine/storage/v1/tests/tx.rs @@ -0,0 +1,201 @@ +/* + * Created on Tue Sep 05 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::{ + engine::{ + error::{RuntimeResult, StorageError}, + storage::v1::{ + journal::{self, JournalAdapter, JournalWriter}, + spec, + }, + }, + util, + }, + std::cell::RefCell, +}; + +pub struct Database { + data: RefCell<[u8; 10]>, +} +impl Database { + fn copy_data(&self) -> [u8; 10] { + *self.data.borrow() + } + fn new() -> Self { + Self { + data: RefCell::new([0; 10]), + } + } + fn reset(&self) { + *self.data.borrow_mut() = [0; 10]; + } + fn set(&self, pos: usize, val: u8) { + self.data.borrow_mut()[pos] = val; + } + fn txn_set( + &self, + pos: usize, + val: u8, + txn_writer: &mut JournalWriter, + ) -> RuntimeResult<()> { + self.set(pos, val); + txn_writer.append_event(TxEvent::Set(pos, val)) + } +} + +pub enum TxEvent { + #[allow(unused)] + Reset, + Set(usize, u8), +} +#[derive(Debug)] +pub enum TxError { + SDSS(StorageError), +} +direct_from! { + TxError => { + StorageError as SDSS + } +} +#[derive(Debug)] +pub struct DatabaseTxnAdapter; +impl JournalAdapter for DatabaseTxnAdapter { + const RECOVERY_PLUGIN: bool = false; + type Error = TxError; + type JournalEvent = TxEvent; + type GlobalState = Database; + + fn encode(event: Self::JournalEvent) -> Box<[u8]> { + /* + [1B: opcode][8B:Index][1B: New value] + */ + let opcode = match event { + TxEvent::Reset => 0u8, + TxEvent::Set(_, _) => 1u8, + }; + let index = match event { + TxEvent::Reset => 0u64, + TxEvent::Set(index, _) => index as u64, + }; + let new_value = match event { + TxEvent::Reset => 0, + TxEvent::Set(_, val) => val, + }; + let mut ret = Vec::with_capacity(10); + ret.push(opcode); + ret.extend(index.to_le_bytes()); + ret.push(new_value); + ret.into_boxed_slice() + } + + fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> Result<(), TxError> { + assert!(payload.len() >= 10, "corrupt file"); + let opcode = payload[0]; + let index = u64::from_le_bytes(util::copy_slice_to_array(&payload[1..9])); + let new_value = payload[9]; + match opcode { + 0 if index == 0 && new_value == 0 => gs.reset(), + 1 if index < 10 && index < isize::MAX as u64 => gs.set(index as usize, new_value), + _ => return Err(TxError::SDSS(StorageError::JournalLogEntryCorrupted.into())), + } + Ok(()) + } +} + +fn open_log( + log_name: &str, + db: &Database, +) -> RuntimeResult> { + journal::open_or_create_journal::( + log_name, db, + ) + .map(|v| v.into_inner()) +} + +#[test] +fn first_boot_second_readonly() { + // create log + let db1 = Database::new(); + let x = || -> RuntimeResult<()> { + let mut log = open_log("testtxn.log", &db1)?; + db1.txn_set(0, 20, &mut log)?; + db1.txn_set(9, 21, &mut log)?; + log.close() + }; + x().unwrap(); + // backup original data + let original_data = db1.copy_data(); + // restore log + let empty_db2 = Database::new(); + open_log("testtxn.log", &empty_db2) + .unwrap() + .close() + .unwrap(); + assert_eq!(original_data, empty_db2.copy_data()); +} +#[test] +fn oneboot_mod_twoboot_mod_thirdboot_read() { + // first boot: set all to 1 + let db1 = Database::new(); + let x = || -> RuntimeResult<()> { + let mut log = open_log("duatxn.db-tlog", &db1)?; + for i in 0..10 { + db1.txn_set(i, 1, &mut log)?; + } + log.close() + }; + x().unwrap(); + let bkp_db1 = db1.copy_data(); + drop(db1); + // second boot + let db2 = Database::new(); + let x = || -> RuntimeResult<()> { + let mut log = open_log("duatxn.db-tlog", &db2)?; + assert_eq!(bkp_db1, db2.copy_data()); + for i in 0..10 { + let current_val = db2.data.borrow()[i]; + db2.txn_set(i, current_val + i as u8, &mut log)?; + } + log.close() + }; + x().unwrap(); + let bkp_db2 = db2.copy_data(); + drop(db2); + // third boot + let db3 = Database::new(); + let log = open_log("duatxn.db-tlog", &db3).unwrap(); + log.close().unwrap(); + assert_eq!(bkp_db2, db3.copy_data()); + assert_eq!( + db3.copy_data(), + (1..=10) + .into_iter() + .map(u8::from) + .collect::>() + .as_ref() + ); +} diff --git a/server/src/engine/storage/versions/mod.rs b/server/src/engine/storage/versions/mod.rs new file mode 100644 index 00000000..23650e3d --- /dev/null +++ b/server/src/engine/storage/versions/mod.rs @@ -0,0 +1,89 @@ +/* + * Created on Mon May 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +//! SDSS Based Storage Engine versions + +pub mod server_version; + +pub const CURRENT_SERVER_VERSION: ServerVersion = v1::V1_SERVER_VERSION; +pub const CURRENT_DRIVER_VERSION: DriverVersion = v1::V1_DRIVER_VERSION; +pub const CURRENT_HEADER_VERSION: HeaderVersion = v1::V1_HEADER_VERSION; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] +/// The header version +/// +/// The header version is part of the static record and *barely* changes (almost like once in a light year) +pub struct HeaderVersion(u64); + +impl HeaderVersion { + pub const fn __new(v: u64) -> Self { + Self(v) + } + pub const fn little_endian_u64(&self) -> [u8; 8] { + self.0.to_le_bytes() + } +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] +/// The server version (based on tag index) +pub struct ServerVersion(u64); + +impl ServerVersion { + pub const fn __new(v: u64) -> Self { + Self(v) + } + pub const fn little_endian(&self) -> [u8; 8] { + self.0.to_le_bytes() + } +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] +/// The driver version +pub struct DriverVersion(u64); + +impl DriverVersion { + pub const fn __new(v: u64) -> Self { + Self(v) + } + pub const fn little_endian(&self) -> [u8; 8] { + self.0.to_le_bytes() + } +} + +pub mod v1 { + //! The first SDSS based storage engine implementation. + //! Target tag: 0.8.0 + + use super::{DriverVersion, HeaderVersion, ServerVersion}; + + /// The SDSS header version UID + pub const V1_HEADER_VERSION: HeaderVersion = HeaderVersion(0); + /// The server version UID + pub const V1_SERVER_VERSION: ServerVersion = + ServerVersion(super::server_version::fetch_id("v0.8.0") as _); + /// The driver version UID + pub const V1_DRIVER_VERSION: DriverVersion = DriverVersion(0); +} diff --git a/server/src/engine/storage/versions/server_version.rs b/server/src/engine/storage/versions/server_version.rs new file mode 100644 index 00000000..257cde4e --- /dev/null +++ b/server/src/engine/storage/versions/server_version.rs @@ -0,0 +1,103 @@ +/* + * Created on Wed May 17 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +const VERSION_TAGS: [&str; 52] = [ + "v0.1.0", + "v0.2.0", + "v0.3.0", + "v0.3.1", + "v0.3.2", + "v0.4.0-alpha.1", + "v0.4.0-alpha.2", + "v0.4.0", + "v0.4.1-alpha.1", + "v0.4.1", + "v0.4.2-alpha.1", + "v0.4.2", + "v0.4.3-alpha.1", + "v0.4.3", + "v0.4.4", + "v0.4.5-alpha.1", + "v0.4.5-alpha.2", + "v0.4.5", + "v0.5.0-alpha.1", + "v0.5.0-alpha.2", + "v0.5.0", + "v0.5.1-alpha.1", + "v0.5.1", + "v0.5.2", + "v0.5.3", + "v0.6.0", + "v0.6.1", + "v0.6.2-testrelease.1", + "v0.6.2", + "v0.6.3-alpha.1", + "v0.6.3", + "v0.6.4-alpha.1", + "v0.6.4", + "v0.7.0-RC.1", + "v0.7.0-alpha.1", + "v0.7.0-alpha.2", + "v0.7.0-beta.1", + "v0.7.0", + "v0.7.1-alpha.1", + "v0.7.1", + "v0.7.2-alpha.1", + "v0.7.2", + "v0.7.3-alpha.1", + "v0.7.3-alpha.2", + "v0.7.3-alpha.3", + "v0.7.3", + "v0.7.4", + "v0.7.5", + "v0.7.6", + "v0.7.7", + "v0.8.0-alpha.1", + "v0.8.0", +]; +const VERSION_TAGS_LEN: usize = VERSION_TAGS.len(); +pub const fn fetch_id(id: &str) -> usize { + // this is ct, so a O(n) doesn't matter + let mut i = 0; + while i < VERSION_TAGS_LEN { + let bytes = VERSION_TAGS[i].as_bytes(); + let given = id.as_bytes(); + let mut j = 0; + let mut eq = true; + while (j < bytes.len()) & (bytes.len() == given.len()) { + if bytes[i] != given[i] { + eq = false; + break; + } + j += 1; + } + if eq { + return i; + } + i += 1; + } + panic!("version not found") +} diff --git a/server/src/engine/sync/atm.rs b/server/src/engine/sync/atm.rs new file mode 100644 index 00000000..eb531271 --- /dev/null +++ b/server/src/engine/sync/atm.rs @@ -0,0 +1,113 @@ +/* + * Created on Fri Jan 20 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + core::{fmt, mem, ops::Deref, sync::atomic::Ordering}, + crossbeam_epoch::{Atomic as CBAtomic, CompareExchangeError, Pointable, Pointer}, +}; +// re-export here because we have some future plans ;) (@ohsayan) +pub use crossbeam_epoch::{pin as cpin, unprotected as upin, Guard, Owned, Shared}; + +pub const ORD_RLX: Ordering = Ordering::Relaxed; +pub const ORD_ACQ: Ordering = Ordering::Acquire; +pub const ORD_REL: Ordering = Ordering::Release; +pub const ORD_ACR: Ordering = Ordering::AcqRel; +pub const ORD_SEQ: Ordering = Ordering::SeqCst; + +type CxResult<'g, T, P> = Result, CompareExchangeError<'g, T, P>>; + +pub const fn ensure_flag_align() -> bool { + mem::align_of::().trailing_zeros() as usize >= FSIZE +} + +pub struct Atomic { + a: CBAtomic, +} + +// the derive is stupid, it will enforce a debug constraint on T +impl fmt::Debug for Atomic { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.a) + } +} + +impl Atomic { + #[inline(always)] + pub const fn null() -> Self { + Self { + a: CBAtomic::null(), + } + } + #[inline(always)] + pub fn cx<'g, P>( + &self, + o: Shared<'g, T>, + n: P, + s: Ordering, + f: Ordering, + g: &'g Guard, + ) -> CxResult<'g, T, P> + where + P: Pointer, + { + self.a.compare_exchange(o, n, s, f, g) + } + #[inline(always)] + pub fn cx_rel<'g, P>(&self, o: Shared<'g, T>, n: P, g: &'g Guard) -> CxResult<'g, T, P> + where + P: Pointer, + { + self.cx(o, n, ORD_REL, ORD_RLX, g) + } + #[inline(always)] + pub fn ld<'g>(&self, o: Ordering, g: &'g Guard) -> Shared<'g, T> { + self.a.load(o, g) + } + #[inline(always)] + pub fn ld_acq<'g>(&self, g: &'g Guard) -> Shared<'g, T> { + self.ld(ORD_ACQ, g) + } + #[inline(always)] + pub fn ld_rlx<'g>(&self, g: &'g Guard) -> Shared<'g, T> { + self.ld(ORD_RLX, g) + } +} + +impl From for Atomic +where + A: Into>, +{ + fn from(t: A) -> Self { + Self { a: Into::into(t) } + } +} + +impl Deref for Atomic { + type Target = CBAtomic; + fn deref(&self) -> &Self::Target { + &self.a + } +} diff --git a/server/src/engine/sync/cell.rs b/server/src/engine/sync/cell.rs new file mode 100644 index 00000000..7045db4f --- /dev/null +++ b/server/src/engine/sync/cell.rs @@ -0,0 +1,137 @@ +/* + * Created on Sat Jan 21 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{ + atm::{ORD_ACQ, ORD_SEQ}, + Backoff, + }, + core::{ + mem, + ops::Deref, + ptr, + sync::atomic::{AtomicBool, AtomicPtr}, + }, +}; + +/// A lazily intialized, or _call by need_ value +#[derive(Debug)] +pub struct Lazy T> { + /// the value (null at first) + value: AtomicPtr, + /// the function that will init the value + init_func: F, + /// is some thread trying to initialize the value + init_state: AtomicBool, +} + +impl Default for Lazy { + fn default() -> Self { + Self::new(T::default) + } +} + +impl Lazy { + pub const fn new(init_func: F) -> Self { + Self { + value: AtomicPtr::new(ptr::null_mut()), + init_func, + init_state: AtomicBool::new(false), + } + } +} + +impl Deref for Lazy +where + F: Fn() -> T, +{ + type Target = T; + fn deref(&self) -> &Self::Target { + let value_ptr = self.value.load(ORD_ACQ); + if !value_ptr.is_null() { + // the value has already been initialized, return + unsafe { + // UNSAFE(@ohsayan): We've just asserted that the value is not null + return &*value_ptr; + } + } + // it's null, so it's useless + + // hold on until someone is trying to init + let backoff = Backoff::new(); + while self + .init_state + .compare_exchange(false, true, ORD_SEQ, ORD_SEQ) + .is_err() + { + // wait until the other thread finishes + backoff.snooze(); + } + /* + see the value before the last store. while we were one the loop, + some other thread could have initialized it already + */ + let value_ptr = self.value.load(ORD_ACQ); + if !value_ptr.is_null() { + // no more init, someone initialized it already + assert!(self.init_state.swap(false, ORD_SEQ)); + unsafe { + // UNSAFE(@ohsayan): We've already loaded the value checked + // that it isn't null + &*value_ptr + } + } else { + // so no one cared to initialize the value in between + // fine, we'll init it + let value = (self.init_func)(); + let value_ptr = Box::into_raw(Box::new(value)); + // now swap out the older value and check it for sanity + assert!(self.value.swap(value_ptr, ORD_SEQ).is_null()); + // set trying to init flag to false + assert!(self.init_state.swap(false, ORD_SEQ)); + unsafe { + // UNSAFE(@ohsayan): We just initialized the value ourselves + // so it is not null! + &*value_ptr + } + } + } +} + +impl Drop for Lazy { + fn drop(&mut self) { + if mem::needs_drop::() { + // this needs drop + let value_ptr = self.value.load(ORD_ACQ); + if !value_ptr.is_null() { + unsafe { + // UNSAFE(@ohsayan): We've just checked if the value is null or not + mem::drop(Box::from_raw(value_ptr)) + } + } + } + } +} diff --git a/server/src/corestore/backoff.rs b/server/src/engine/sync/mod.rs similarity index 91% rename from server/src/corestore/backoff.rs rename to server/src/engine/sync/mod.rs index adebc101..47fbd521 100644 --- a/server/src/corestore/backoff.rs +++ b/server/src/engine/sync/mod.rs @@ -1,5 +1,5 @@ /* - * Created on Wed Feb 16 2022 + * Created on Thu Jan 19 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2022, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,6 +24,11 @@ * */ +pub(super) mod atm; +pub(super) mod cell; +pub(super) mod queue; +pub(super) mod smart; + use std::{cell::Cell, hint::spin_loop, thread}; /// Type to perform exponential backoff diff --git a/server/src/engine/sync/queue.rs b/server/src/engine/sync/queue.rs new file mode 100644 index 00000000..607268f6 --- /dev/null +++ b/server/src/engine/sync/queue.rs @@ -0,0 +1,230 @@ +/* + * Created on Wed Aug 30 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(test)] +use crossbeam_epoch::pin; +use { + super::atm::Atomic, + crate::engine::mem::CachePadded, + crossbeam_epoch::{unprotected, Guard, Owned, Shared}, + std::{mem::MaybeUninit, sync::atomic::Ordering}, +}; + +#[derive(Debug)] +struct QNode { + data: MaybeUninit, + next: Atomic, +} + +impl QNode { + fn new(data: MaybeUninit, next: Atomic) -> Self { + Self { data, next } + } + fn null() -> Self { + Self::new(MaybeUninit::uninit(), Atomic::null()) + } + fn new_data(val: T) -> Self { + Self::new(MaybeUninit::new(val), Atomic::null()) + } +} + +#[derive(Debug)] +pub struct Queue { + head: CachePadded>>, + tail: CachePadded>>, +} + +impl Queue { + pub fn new() -> Self { + let slf = Self { + head: CachePadded::new(Atomic::null()), + tail: CachePadded::new(Atomic::null()), + }; + let g = unsafe { unprotected() }; + let sentinel = Owned::new(QNode::null()).into_shared(&g); + slf.head.store(sentinel, Ordering::Relaxed); + slf.tail.store(sentinel, Ordering::Relaxed); + slf + } + pub fn blocking_enqueue(&self, new: T, g: &Guard) { + let newptr = Owned::new(QNode::new_data(new)).into_shared(g); + loop { + // get current tail + let tailptr = self.tail.load(Ordering::Acquire, g); + let tail = unsafe { tailptr.deref() }; + let tail_nextptr = tail.next.load(Ordering::Acquire, g); + if tail_nextptr.is_null() { + // tail points to null which means this should ideally by the last LL node + if tail + .next + .compare_exchange( + Shared::null(), + newptr, + Ordering::Release, + Ordering::Relaxed, + g, + ) + .is_ok() + { + /* + CAS'd in but tail is *probably* lagging behind. This CAS might fail but we don't care since we're allowed to have a lagging tail + */ + let _ = self.tail.compare_exchange( + tailptr, + newptr, + Ordering::Release, + Ordering::Relaxed, + g, + ); + break; + } + } else { + // tail is lagging behind; attempt to help update it + let _ = self.tail.compare_exchange( + tailptr, + tail_nextptr, + Ordering::Release, + Ordering::Relaxed, + g, + ); + } + } + } + pub fn blocking_try_dequeue(&self, g: &Guard) -> Option { + loop { + // get current head + let headptr = self.head.load(Ordering::Acquire, g); + let head = unsafe { headptr.deref() }; + let head_nextptr = head.next.load(Ordering::Acquire, g); + if head_nextptr.is_null() { + // this is the sentinel; queue is empty + return None; + } + // we observe at this point in time that there is atleast one element in the list + // let us swing that into sentinel position + if self + .head + .compare_exchange( + headptr, + head_nextptr, + Ordering::Release, + Ordering::Relaxed, + g, + ) + .is_ok() + { + // good so we were able to update the head + let tailptr = self.tail.load(Ordering::Acquire, g); + // but wait, was this the last node? in that case, we need to update the tail before we destroy it. + // this is fine though, as nothing will go boom right now since the tail is allowed to lag by one + if headptr == tailptr { + // right so this was the last node uh oh + let _ = self.tail.compare_exchange( + tailptr, + head_nextptr, + Ordering::Release, + Ordering::Relaxed, + g, + ); + } + // now we're in a position to happily destroy this + unsafe { g.defer_destroy(headptr) } + // read out the ptr + return Some(unsafe { head_nextptr.deref().data.as_ptr().read() }); + } + } + } +} + +impl Drop for Queue { + fn drop(&mut self) { + let g = unsafe { unprotected() }; + while self.blocking_try_dequeue(g).is_some() {} + // dealloc sentinel + unsafe { + self.head.load(Ordering::Relaxed, g).into_owned(); + } + } +} + +#[cfg(test)] +type StringQueue = Queue; + +#[test] +fn empty() { + let q = StringQueue::new(); + drop(q); +} + +#[test] +fn empty_deq() { + let g = pin(); + let q = StringQueue::new(); + assert_eq!(q.blocking_try_dequeue(&g), None); +} + +#[test] +fn empty_enq() { + let g = pin(); + let q = StringQueue::new(); + q.blocking_enqueue("hello".into(), &g); +} + +#[test] +fn multi_eq_dq() { + const ITEMS_L: usize = 100; + use std::{sync::Arc, thread}; + let q = Arc::new(StringQueue::new()); + let producer_q = q.clone(); + let consumer_q = q.clone(); + let producer = thread::spawn(move || { + let mut sent = vec![]; + let g = pin(); + for i in 0..ITEMS_L { + let item = format!("time-{i}"); + // send a message and then sleep for two seconds + producer_q.blocking_enqueue(item.clone(), &g); + sent.push(item); + } + sent + }); + let consumer = thread::spawn(move || { + let g = pin(); + let mut received = vec![]; + loop { + if received.len() == ITEMS_L { + break; + } + if let Some(item) = consumer_q.blocking_try_dequeue(&g) { + received.push(item); + } + } + received + }); + let sent = producer.join().unwrap(); + let received = consumer.join().unwrap(); + assert_eq!(sent, received); +} diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs new file mode 100644 index 00000000..59c710fb --- /dev/null +++ b/server/src/engine/sync/smart.rs @@ -0,0 +1,325 @@ +/* + * Created on Sun Jan 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::atm::{ORD_ACQ, ORD_REL, ORD_RLX}, + std::{ + alloc::{dealloc, Layout}, + borrow::Borrow, + fmt, + hash::{Hash, Hasher}, + mem::{self, ManuallyDrop}, + ops::Deref, + process, + ptr::{self, NonNull}, + slice, str, + sync::atomic::{self, AtomicUsize}, + }, +}; + +#[derive(Debug, Clone)] +pub struct StrRC { + base: SliceRC, +} + +impl StrRC { + fn new(base: SliceRC) -> Self { + Self { base } + } + pub fn from_bx(b: Box) -> Self { + let mut md = ManuallyDrop::new(b); + Self::new(SliceRC::new( + unsafe { + // UNSAFE(@ohsayan): nullck + always aligned + NonNull::new_unchecked(md.as_mut_ptr()) + }, + md.len(), + )) + } + pub fn as_str(&self) -> &str { + unsafe { + // UNSAFE(@ohsayan): Ctor guarantees correctness + str::from_utf8_unchecked(self.base.as_slice()) + } + } +} + +impl PartialEq for StrRC { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl PartialEq for StrRC { + fn eq(&self, other: &str) -> bool { + self.as_str() == other + } +} + +impl From for StrRC { + fn from(s: String) -> Self { + Self::from_bx(s.into_boxed_str()) + } +} + +impl From> for StrRC { + fn from(bx: Box) -> Self { + Self::from_bx(bx) + } +} + +impl<'a> From<&'a str> for StrRC { + fn from(str: &'a str) -> Self { + Self::from_bx(str.to_string().into_boxed_str()) + } +} + +impl Deref for StrRC { + type Target = str; + fn deref(&self) -> &Self::Target { + self.as_str() + } +} + +impl Eq for StrRC {} + +pub struct SliceRC { + ptr: NonNull, + len: usize, + rc: EArc, +} + +impl SliceRC { + #[inline(always)] + fn new(ptr: NonNull, len: usize) -> Self { + Self { + ptr, + len, + rc: unsafe { + // UNSAFE(@ohsayan): we will eventually deallocate this + EArc::new() + }, + } + } + #[inline(always)] + pub fn as_slice(&self) -> &[T] { + unsafe { + // UNSAFE(@ohsayan): rc guard + ctor + slice::from_raw_parts(self.ptr.as_ptr(), self.len) + } + } +} + +impl Drop for SliceRC { + fn drop(&mut self) { + unsafe { + // UNSAFE(@ohsayan): Calling this within the dtor itself + self.rc.rc_drop(|| { + // dtor + if mem::needs_drop::() { + // UNSAFE(@ohsayan): dtor through, the ctor guarantees correct alignment and len + ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr.as_ptr(), self.len)); + } + // dealloc + // UNSAFE(@ohsayan): we allocated it + let layout = Layout::array::(self.len).unwrap_unchecked(); + // UNSAFE(@ohsayan): layout structure guaranteed by ctor + dealloc(self.ptr.as_ptr() as *mut u8, layout); + }) + } + } +} + +impl Clone for SliceRC { + #[inline(always)] + fn clone(&self) -> Self { + let new_rc = unsafe { + // UNSAFE(@ohsayan): calling this within the clone routine + self.rc.rc_clone() + }; + Self { + rc: new_rc, + ..*self + } + } +} + +impl fmt::Debug for SliceRC { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.as_slice()).finish() + } +} + +impl Hash for SliceRC { + fn hash(&self, state: &mut H) { + self.as_slice().hash(state) + } +} + +impl PartialEq for SliceRC { + fn eq(&self, other: &Self) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl PartialEq<[T]> for SliceRC { + fn eq(&self, other: &[T]) -> bool { + self.as_slice() == other + } +} + +impl Eq for SliceRC {} +impl Borrow<[T]> for SliceRC { + fn borrow(&self) -> &[T] { + self.as_slice() + } +} + +impl Deref for SliceRC { + type Target = [T]; + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} + +unsafe impl Send for SliceRC {} +unsafe impl Sync for SliceRC {} + +/// A simple rc +#[derive(Debug)] +pub struct EArc { + base: RawRC<()>, +} + +impl EArc { + /// ## Safety + /// + /// Clean up your own memory, sir + pub unsafe fn new() -> Self { + Self { + base: RawRC::new(()), + } + } +} + +impl EArc { + /// ## Safety + /// + /// Only call in an actual [`Clone`] context + pub unsafe fn rc_clone(&self) -> Self { + Self { + base: self.base.rc_clone(), + } + } + /// ## Safety + /// + /// Only call in dtor context + pub unsafe fn rc_drop(&mut self, dropfn: impl FnMut()) { + self.base.rc_drop(dropfn) + } +} + +/// The core atomic reference counter implementation. All smart pointers use this inside +pub struct RawRC { + rc_data: NonNull>, +} + +#[derive(Debug)] +struct RawRCData { + rc: AtomicUsize, + data: T, +} + +impl RawRC { + /// Create a new [`RawRC`] instance + /// + /// ## Safety + /// + /// While this is **not unsafe** in the eyes of the language specification for safety, it still does violate a very common + /// bug: memory leaks and we don't want that. So, it is upto the caller to clean this up + pub unsafe fn new(data: T) -> Self { + Self { + rc_data: NonNull::new_unchecked(Box::leak(Box::new(RawRCData { + rc: AtomicUsize::new(1), + data, + }))), + } + } + pub fn data(&self) -> &T { + unsafe { + // UNSAFE(@ohsayan): we believe in the power of barriers! + &(*self.rc_data.as_ptr()).data + } + } + fn _rc(&self) -> &AtomicUsize { + unsafe { + // UNSAFE(@ohsayan): we believe in the power of barriers! + &(*self.rc_data.as_ptr()).rc + } + } + /// ## Safety + /// + /// Only call in an actual [`Clone`] context + pub unsafe fn rc_clone(&self) -> Self { + let new_rc = self._rc().fetch_add(1, ORD_RLX); + if new_rc > (isize::MAX) as usize { + // some incredibly degenerate case; this won't ever happen but who knows if some fella decided to have atomic overflow fun? + process::abort(); + } + Self { ..*self } + } + /// ## Safety + /// + /// Only call in dtor context + pub unsafe fn rc_drop(&mut self, dropfn: impl FnMut()) { + if self._rc().fetch_sub(1, ORD_REL) != 1 { + // not the last man alive + return; + } + // emit a fence for sync with stores + atomic::fence(ORD_ACQ); + self.rc_drop_slow(dropfn); + } + #[cold] + #[inline(never)] + unsafe fn rc_drop_slow(&mut self, mut dropfn: impl FnMut()) { + // deallocate object + dropfn(); + // deallocate rc + drop(Box::from_raw(self.rc_data.as_ptr())); + } +} + +impl fmt::Debug for RawRC { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RawRC") + .field("rc_data", unsafe { + // UNSAFE(@ohsayan): rc guard + self.rc_data.as_ref() + }) + .finish() + } +} diff --git a/server/src/engine/tests/cfg.rs b/server/src/engine/tests/cfg.rs new file mode 100644 index 00000000..dd8e2935 --- /dev/null +++ b/server/src/engine/tests/cfg.rs @@ -0,0 +1,301 @@ +/* + * Created on Wed Nov 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::{ + engine::config::{ + self, AuthDriver, CLIConfigParseReturn, ConfigAuth, ConfigEndpoint, ConfigEndpointTcp, + ConfigEndpointTls, ConfigMode, ConfigReturn, ConfigSystem, Configuration, ParsedRawArgs, + }, + util::test_utils::with_files, +}; + +/* + CLI tests +*/ + +fn extract_cli_args(payload: &str) -> std::collections::HashMap> { + extract_cli_args_raw(payload).into_config() +} +fn extract_cli_args_raw( + payload: &str, +) -> CLIConfigParseReturn>> { + config::parse_cli_args(payload.split_ascii_whitespace().map_while(|item| { + let mut item = item.trim(); + if item.ends_with("\n") { + item = &item[..item.len() - 1]; + } + if item.is_empty() { + None + } else { + Some(item) + } + })) + .unwrap() +} +#[test] +fn parse_cli_args_simple() { + let payload = "skyd --mode dev --endpoint tcp@localhost:2003"; + let cfg = extract_cli_args(payload); + let expected: ParsedRawArgs = into_dict! { + "--mode" => vec!["dev".into()], + "--endpoint" => vec!["tcp@localhost:2003".into()] + }; + assert_eq!(cfg, expected); +} +#[test] +fn parse_cli_args_packed() { + let payload = "skyd --mode=dev --endpoint=tcp@localhost:2003"; + let cfg = extract_cli_args(payload); + let expected: ParsedRawArgs = into_dict! { + "--mode" => vec!["dev".into()], + "--endpoint" => vec!["tcp@localhost:2003".into()] + }; + assert_eq!(cfg, expected); +} +#[test] +fn parse_cli_args_multi() { + let payload = "skyd --mode=dev --endpoint tcp@localhost:2003"; + let cfg = extract_cli_args(payload); + let expected: ParsedRawArgs = into_dict! { + "--mode" => vec!["dev".into()], + "--endpoint" => vec!["tcp@localhost:2003".into()] + }; + assert_eq!(cfg, expected); +} +#[test] +fn parse_validate_cli_args() { + with_files( + [ + "__cli_args_test_private.key", + "__cli_args_test_cert.pem", + "__cli_args_test_passphrase.key", + ], + |[pkey, cert, pass]| { + let payload = format!( + "skyd --mode=dev \ + --endpoint tcp@127.0.0.1:2003 \ + --endpoint tls@127.0.0.2:2004 \ + --service-window=600 \ + --tlskey {pkey} \ + --tlscert {cert} \ + --tls-passphrase {pass} \ + --auth-plugin pwd \ + --auth-root-password password12345678 + " + ); + let cfg = extract_cli_args(&payload); + let ret = config::apply_and_validate::(cfg) + .unwrap() + .into_config(); + assert_eq!( + ret, + Configuration::new( + ConfigEndpoint::Multi( + ConfigEndpointTcp::new("127.0.0.1".into(), 2003), + ConfigEndpointTls::new( + ConfigEndpointTcp::new("127.0.0.2".into(), 2004), + "".into(), + "".into(), + "".into() + ) + ), + ConfigMode::Dev, + ConfigSystem::new(600), + ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()) + ) + ) + }, + ); +} +#[test] +fn parse_validate_cli_args_help_and_version() { + let pl1 = "skyd --help"; + let pl2 = "skyd --version"; + let ret1 = extract_cli_args_raw(pl1); + let ret2 = extract_cli_args_raw(pl2); + assert_eq!(ret1, CLIConfigParseReturn::Help); + assert_eq!(ret2, CLIConfigParseReturn::Version); + config::set_cli_src(vec!["skyd".into(), "--help".into()]); + let ret3 = config::check_configuration().unwrap(); + config::set_cli_src(vec!["skyd".into(), "--version".into()]); + let ret4 = config::check_configuration().unwrap(); + assert_eq!( + ret3, + ConfigReturn::HelpMessage(config::CLI_HELP.to_string()) + ); + assert_eq!( + ret4, + ConfigReturn::HelpMessage(format!( + "Skytable Database Server (skyd) v{}", + libsky::VERSION + )) + ); +} + +/* + env tests +*/ + +fn vars_to_args(variables: &[String]) -> ParsedRawArgs { + variables + .iter() + .map(|var| { + var.split("=") + .map(ToString::to_string) + .collect::>() + }) + .map(|mut v| { + let key = v.remove(0); + let values = v.remove(0).split(",").map(ToString::to_string).collect(); + (key, values) + }) + .collect() +} +#[test] +fn parse_env_args_simple() { + let variables = [ + format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"), + format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"), + format!("SKYDB_AUTH_PLUGIN=pwd"), + format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"), + format!("SKYDB_ENDPOINTS=tcp@localhost:8080"), + format!("SKYDB_RUN_MODE=dev"), + format!("SKYDB_SERVICE_WINDOW=600"), + ]; + let expected_args = vars_to_args(&variables); + config::set_env_src(variables.into()); + let args = config::parse_env_args().unwrap().unwrap(); + assert_eq!(args, expected_args); +} +#[test] +fn parse_env_args_multi() { + let variables = [ + format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"), + format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"), + format!("SKYDB_AUTH_PLUGIN=pwd"), + format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"), + format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"), + format!("SKYDB_RUN_MODE=dev"), + format!("SKYDB_SERVICE_WINDOW=600"), + ]; + let expected_args = vars_to_args(&variables); + config::set_env_src(variables.into()); + let args = config::parse_env_args().unwrap().unwrap(); + assert_eq!(args, expected_args); +} +#[test] +fn parse_validate_env_args() { + with_files( + [ + "__env_args_test_cert.pem", + "__env_args_test_private.key", + "__env_args_test_private.passphrase.txt", + ], + |[cert, key, pass]| { + let variables = [ + format!("SKYDB_AUTH_PLUGIN=pwd"), + format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"), + format!("SKYDB_TLS_CERT={cert}"), + format!("SKYDB_TLS_KEY={key}"), + format!("SKYDB_TLS_PRIVATE_KEY_PASSWORD={pass}"), + format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"), + format!("SKYDB_RUN_MODE=dev"), + format!("SKYDB_SERVICE_WINDOW=600"), + ]; + config::set_env_src(variables.into()); + let cfg = config::check_configuration().unwrap().into_config(); + assert_eq!( + cfg, + Configuration::new( + ConfigEndpoint::Multi( + ConfigEndpointTcp::new("localhost".into(), 8080), + ConfigEndpointTls::new( + ConfigEndpointTcp::new("localhost".into(), 8081), + "".into(), + "".into(), + "".into() + ) + ), + ConfigMode::Dev, + ConfigSystem::new(600), + ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()) + ) + ) + }, + ); +} +const CONFIG_FILE: &str = "\ +system: + mode: dev + rs_window: 600 + +auth: + plugin: pwd + root_pass: password12345678 + +endpoints: + secure: + host: 127.0.0.1 + port: 2004 + cert: ._test_sample_cert.pem + private_key: ._test_sample_private.key + pkey_passphrase: ._test_sample_private.pass.txt + insecure: + host: 127.0.0.1 + port: 2003 + "; +#[test] +fn test_config_file() { + with_files( + [ + "._test_sample_cert.pem", + "._test_sample_private.key", + "._test_sample_private.pass.txt", + ], + |_| { + config::set_cli_src(vec!["skyd".into(), "--config=config.yml".into()]); + config::set_file_src(CONFIG_FILE); + let cfg = config::check_configuration().unwrap().into_config(); + assert_eq!( + cfg, + Configuration::new( + ConfigEndpoint::Multi( + ConfigEndpointTcp::new("127.0.0.1".into(), 2003), + ConfigEndpointTls::new( + ConfigEndpointTcp::new("127.0.0.1".into(), 2004), + "".into(), + "".into(), + "".into() + ) + ), + ConfigMode::Dev, + ConfigSystem::new(600), + ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()) + ) + ) + }, + ) +} diff --git a/server/src/engine/tests/client/mod.rs b/server/src/engine/tests/client/mod.rs new file mode 100644 index 00000000..2ac2be22 --- /dev/null +++ b/server/src/engine/tests/client/mod.rs @@ -0,0 +1,25 @@ +/* + * Created on Mon Dec 04 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ diff --git a/server/src/tests/persist/auth.rs b/server/src/engine/tests/client_misc/ddl.rs similarity index 62% rename from server/src/tests/persist/auth.rs rename to server/src/engine/tests/client_misc/ddl.rs index 3a53b9fe..481596fd 100644 --- a/server/src/tests/persist/auth.rs +++ b/server/src/engine/tests/client_misc/ddl.rs @@ -1,5 +1,5 @@ /* - * Created on Sat Mar 19 2022 + * Created on Thu Nov 30 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2022, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,29 +24,18 @@ * */ -use { - sky_macros::dbtest_func as dbtest, - skytable::{query, Element}, -}; +use {sky_macros::dbtest, skytable::query}; -const USERID: &str = "steinbeck"; - -#[dbtest( - skip_if_cfg = "persist-suite", - norun = true, - auth_rootuser = true, - port = 2005 -)] -async fn store_user() { - runmatch!(con, query!("auth", "adduser", USERID), Element::String) +#[dbtest] +fn inspect_global_as_root_returns_user_info() { + let mut db = db!(); + let inspect: String = db.query_parse(&query!("inspect global")).unwrap(); + assert!(inspect.contains("\"users\":")); } -#[dbtest( - run_if_cfg = "persist-suite", - norun = true, - auth_rootuser = true, - port = 2005 -)] -async fn load_user() { - assert_okay!(con, query!("auth", "deluser", USERID)); +#[dbtest(switch_user(username = "sneaking_user_info"))] +fn inspect_global_as_std_user_does_not_return_user_info() { + let mut db = db!(); + let inspect: String = db.query_parse(&query!("inspect global")).unwrap(); + assert!(!inspect.contains("\"users\":")); } diff --git a/server/src/diskstore/mod.rs b/server/src/engine/tests/client_misc/mod.rs similarity index 85% rename from server/src/diskstore/mod.rs rename to server/src/engine/tests/client_misc/mod.rs index 587a8f26..faaae0fe 100644 --- a/server/src/diskstore/mod.rs +++ b/server/src/engine/tests/client_misc/mod.rs @@ -1,5 +1,5 @@ /* - * Created on Wed Aug 05 2020 + * Created on Wed Nov 29 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2020, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,6 +24,6 @@ * */ -//! This module provides tools for handling persistently stored data - -pub mod flock; +mod ddl; +mod sec; +mod sysctl; diff --git a/server/src/engine/tests/client_misc/sec/dcl_sec.rs b/server/src/engine/tests/client_misc/sec/dcl_sec.rs new file mode 100644 index 00000000..961203a3 --- /dev/null +++ b/server/src/engine/tests/client_misc/sec/dcl_sec.rs @@ -0,0 +1,74 @@ +/* + * Created on Wed Nov 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{INVALID_SYNTAX_ERR, UNKNOWN_STMT_ERR}, + sky_macros::dbtest, + skytable::{error::Error, query}, +}; + +#[dbtest] +fn deny_unknown_sysctl() { + let mut db = db!(); + for stmt in [ + "sysctl magic moon", + "sysctl create wormhole", + "sysctl drop dem", + ] { + assert_err_eq!( + db.query_parse::<()>(&query!(stmt)), + Error::ServerError(UNKNOWN_STMT_ERR) + ); + } +} + +#[dbtest] +fn ensure_sysctl_status_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!("sysctl report status blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_sysctl_create_user() { + let mut db = db!(); + let query = format!("sysctl create user myuser with {{ password: ? }} blah"); + assert_err_eq!( + db.query_parse::<()>(&query!(query, "mypass")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_sysctl_drop_user() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!("sysctl drop user ? blah", "myuser",)), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} diff --git a/server/src/engine/tests/client_misc/sec/ddl_sec.rs b/server/src/engine/tests/client_misc/sec/ddl_sec.rs new file mode 100644 index 00000000..2a4d8afd --- /dev/null +++ b/server/src/engine/tests/client_misc/sec/ddl_sec.rs @@ -0,0 +1,160 @@ +/* + * Created on Wed Nov 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{INVALID_SYNTAX_ERR, UNKNOWN_STMT_ERR}, + sky_macros::dbtest, + skytable::{error::Error, query}, +}; + +#[dbtest] +fn deny_unknown() { + let mut db = db!(); + for stmt in [ + "create magic blue", + "alter rainbow hue", + "drop sadistic view", + ] { + assert_err_eq!( + db.query_parse::<()>(&query!(stmt)), + Error::ServerError(UNKNOWN_STMT_ERR) + ); + } +} + +#[dbtest] +fn ensure_create_space_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!("create space myspace with {} blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); + assert_err_eq!( + db.query_parse::<()>(&query!("create space myspace blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_alter_space_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!("alter space myspace with {} blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_drop_space_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!("drop space myspace blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); + assert_err_eq!( + db.query_parse::<()>(&query!("drop space myspace force blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_create_model_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!( + "create model myspace.mymodel(username: string, password: binary) blah" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); + assert_err_eq!( + db.query_parse::<()>(&query!( + "create model myspace.mymodel(username: string, password: binary) with {} blah" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_alter_model_add_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!( + "alter model myspace.mymodel add phone_number { type: uint64 } blah" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); + assert_err_eq!( + db.query_parse::<()>(&query!( + "alter model myspace.mymodel add (phone_number { type: uint64 }, email_id { type: string }) with {} blah" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_alter_model_update_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!( + "alter model myspace.mymodel update password { type: string } blah" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); + assert_err_eq!( + db.query_parse::<()>(&query!( + "alter model myspace.mymodel update (username {type: binary}, password { type: string }) blah" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_alter_model_remove_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!("alter model myspace.mymodel remove email_id blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); + assert_err_eq!( + db.query_parse::<()>(&query!( + "alter model myspace.mymodel remove (email_id, phone_number) blah" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn ensure_drop_model_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!("drop model myspace.mymodel blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); + assert_err_eq!( + db.query_parse::<()>(&query!("drop model myspace.mymodel force blah")), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} diff --git a/server/src/engine/tests/client_misc/sec/dml_sec.rs b/server/src/engine/tests/client_misc/sec/dml_sec.rs new file mode 100644 index 00000000..301e91a0 --- /dev/null +++ b/server/src/engine/tests/client_misc/sec/dml_sec.rs @@ -0,0 +1,89 @@ +/* + * Created on Wed Nov 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::INVALID_SYNTAX_ERR, + sky_macros::dbtest, + skytable::{error::Error, query}, +}; + +#[dbtest] +fn insert_ensure_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!( + "insert into myspace.mymodel(?, ?) blah", + "username", + "password" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); + assert_err_eq!( + db.query_parse::<()>(&query!( + "insert into myspace.mymodel { username: ?, password: ? } blah", + "username", + "password" + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ); +} + +#[dbtest] +fn select_ensure_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!( + "select * from myspace.mymodel where username = ? blah", + "username", + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ) +} + +#[dbtest] +fn update_ensure_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!( + "update myspace.mymodel set counter += ? where username = ? blah", + 1u64, + "username", + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ) +} + +#[dbtest] +fn delete_ensure_end_of_tokens() { + let mut db = db!(); + assert_err_eq!( + db.query_parse::<()>(&query!( + "delete from myspace.mymodel where username = ? blah", + "username", + )), + Error::ServerError(INVALID_SYNTAX_ERR) + ) +} diff --git a/server/src/actions/dbsize.rs b/server/src/engine/tests/client_misc/sec/mod.rs similarity index 53% rename from server/src/actions/dbsize.rs rename to server/src/engine/tests/client_misc/sec/mod.rs index 0519436d..be1fa0bb 100644 --- a/server/src/actions/dbsize.rs +++ b/server/src/engine/tests/client_misc/sec/mod.rs @@ -1,5 +1,5 @@ /* - * Created on Thu Sep 24 2020 + * Created on Wed Nov 29 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2020, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,21 +24,31 @@ * */ -use crate::dbnet::prelude::*; +mod dcl_sec; +mod ddl_sec; +mod dml_sec; -action!( - /// Returns the number of keys in the database - fn dbsize(handle: &Corestore, con: &mut Connection, mut act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len < 2)?; - if act.is_empty() { - let len = get_tbl_ref!(handle, con).count(); - con.write_usize(len).await?; - } else { - let raw_entity = unsafe { act.next().unsafe_unwrap() }; - let entity = handle_entity!(con, raw_entity); - con.write_usize(get_tbl!(&entity, handle, con).count()) - .await?; - } - Ok(()) +use { + crate::engine::error::QueryError, + sky_macros::dbtest, + skytable::{error::Error, query}, +}; + +const INVALID_SYNTAX_ERR: u16 = QueryError::QLInvalidSyntax.value_u8() as u16; +const EXPECTED_STATEMENT_ERR: u16 = QueryError::QLExpectedStatement.value_u8() as u16; +const UNKNOWN_STMT_ERR: u16 = QueryError::QLUnknownStatement.value_u8() as u16; + +#[dbtest] +fn deny_unknown_tokens() { + let mut db = db!(); + for token in [ + "model", "space", "where", "force", "into", "from", "with", "set", "add", "remove", "*", + ",", "", + ] { + assert_err_eq!( + db.query_parse::<()>(&query!(token)), + Error::ServerError(EXPECTED_STATEMENT_ERR), + "{token}", + ); } -); +} diff --git a/server/src/blueql/util.rs b/server/src/engine/tests/client_misc/sysctl.rs similarity index 66% rename from server/src/blueql/util.rs rename to server/src/engine/tests/client_misc/sysctl.rs index f0501c31..928d93d9 100644 --- a/server/src/blueql/util.rs +++ b/server/src/engine/tests/client_misc/sysctl.rs @@ -1,5 +1,5 @@ /* - * Created on Mon Jun 20 2022 + * Created on Wed Nov 29 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2022, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,18 +24,18 @@ * */ -use { - super::{ast::Entity, error}, - crate::{ - actions::{ActionError, ActionResult}, - protocol::interface::ProtocolSpec, - util::Life, - }, -}; - -pub fn from_slice_action_result(slice: &[u8]) -> ActionResult> { - match Entity::from_slice(slice) { - Ok(slc) => Ok(Life::new(slc)), - Err(e) => Err(ActionError::ActionError(error::cold_err::

(e))), +mod status { + use {sky_macros::dbtest, skytable::query}; + #[dbtest] + fn check_status_root() { + let mut db = db!(); + db.query_parse::<()>(&query!("sysctl report status")) + .unwrap(); + } + #[dbtest(switch_user(username = "user1"))] + fn check_status_standard_user() { + let mut db = db!(); + db.query_parse::<()>(&query!("sysctl report status")) + .unwrap(); } } diff --git a/server/src/actions/whereami.rs b/server/src/engine/tests/macros.rs similarity index 54% rename from server/src/actions/whereami.rs rename to server/src/engine/tests/macros.rs index a9655c98..3f55138b 100644 --- a/server/src/actions/whereami.rs +++ b/server/src/engine/tests/macros.rs @@ -1,5 +1,5 @@ /* - * Created on Fri Nov 12 2021 + * Created on Wed Nov 29 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2021, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,23 +24,30 @@ * */ -use crate::dbnet::prelude::*; - -action! { - fn whereami(store: &Corestore, con: &mut Connection, act: ActionIter<'a>) { - ensure_length::

(act.len(), |len| len == 0)?; - match store.get_ids() { - (Some(ks), Some(tbl)) => { - con.write_typed_non_null_array_header(2, b'+').await?; - con.write_typed_non_null_array_element(ks).await?; - con.write_typed_non_null_array_element(tbl).await?; - }, - (Some(ks), None) => { - con.write_typed_non_null_array_header(1, b'+').await?; - con.write_typed_non_null_array_element(ks).await?; - }, - _ => unsafe { impossible!() } +macro_rules! assert_err_eq { + ($me:expr, $target:pat) => { + match ::core::result::Result::unwrap_err($me) { + $target => {} + other => panic!( + "expected error `{}` but got {:?} at {}:{}", + stringify!($target), + other, + file!(), + line!() + ), + } + }; + ($me:expr, $target:pat, $($arg:tt)+) => { + match ::core::result::Result::expect_err($me, &format!($($arg)*)) { + $target => {} + other => panic!( + "expected error `{}` but got {:?} at {}:{}; {}", + stringify!($target), + other, + file!(), + line!(), + $($arg)* + ), } - Ok(()) } } diff --git a/server/src/engine/tests/mod.rs b/server/src/engine/tests/mod.rs new file mode 100644 index 00000000..c59d29c0 --- /dev/null +++ b/server/src/engine/tests/mod.rs @@ -0,0 +1,31 @@ +/* + * Created on Sat Sep 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[macro_use] +mod macros; +mod cfg; +mod client; +mod client_misc; diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs new file mode 100644 index 00000000..09f33f5c --- /dev/null +++ b/server/src/engine/txn/gns/mod.rs @@ -0,0 +1,239 @@ +/* + * Created on Sun Aug 20 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::{ + engine::{ + core::{space::Space, GlobalNS}, + data::uuid::Uuid, + error::{RuntimeResult, TransactionError}, + mem::BufferedScanner, + storage::v1::{ + inf::{self, PersistObject}, + JournalAdapter, JournalWriter, LocalFS, RawFSInterface, + }, + }, + util::EndianQW, + }, + std::marker::PhantomData, +}; + +mod model; +mod space; +// test +#[cfg(test)] +mod tests; + +// re-exports +pub use { + model::{ + AlterModelAddTxn, AlterModelRemoveTxn, AlterModelUpdateTxn, CreateModelTxn, DropModelTxn, + ModelIDRef, + }, + space::{AlterSpaceTxn, CreateSpaceTxn, DropSpaceTxn}, +}; + +/// The GNS transaction driver is used to handle DDL transactions +pub struct GNSTransactionDriverAnyFS { + journal: JournalWriter, +} + +impl GNSTransactionDriverAnyFS { + pub fn new(journal: JournalWriter) -> Self { + Self { journal } + } + pub fn into_inner(self) -> JournalWriter { + self.journal + } + pub fn __journal_mut(&mut self) -> &mut JournalWriter { + &mut self.journal + } + /// Attempts to commit the given event into the journal, handling any possible recovery triggers and returning + /// errors (if any) + pub fn try_commit(&mut self, gns_event: GE) -> RuntimeResult<()> { + let mut buf = vec![]; + buf.extend(GE::OPC.to_le_bytes()); + GE::encode_super_event(gns_event, &mut buf); + self.journal + .append_event_with_recovery_plugin(GNSSuperEvent(buf.into_boxed_slice()))?; + Ok(()) + } +} + +/* + journal implementor +*/ + +/// the journal adapter for DDL queries on the GNS +#[derive(Debug)] +pub struct GNSAdapter; + +impl JournalAdapter for GNSAdapter { + const RECOVERY_PLUGIN: bool = true; + type JournalEvent = GNSSuperEvent; + type GlobalState = GlobalNS; + type Error = crate::engine::fractal::error::Error; + fn encode(GNSSuperEvent(b): Self::JournalEvent) -> Box<[u8]> { + b + } + fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> RuntimeResult<()> { + if payload.len() < 2 { + return Err(TransactionError::DecodedUnexpectedEof.into()); + } + macro_rules! dispatch { + ($($item:ty),* $(,)?) => { + [$(<$item as GNSEvent>::decode_and_update_global_state),*, |_, _| Err(TransactionError::DecodeUnknownTxnOp.into())] + }; + } + static DISPATCH: [fn(&mut BufferedScanner, &GlobalNS) -> RuntimeResult<()>; 9] = dispatch!( + CreateSpaceTxn, + AlterSpaceTxn, + DropSpaceTxn, + CreateModelTxn, + AlterModelAddTxn, + AlterModelRemoveTxn, + AlterModelUpdateTxn, + DropModelTxn + ); + let mut scanner = BufferedScanner::new(&payload); + let opc = unsafe { + // UNSAFE(@ohsayan): + u16::from_le_bytes(scanner.next_chunk()) + }; + match DISPATCH[(opc as usize).min(DISPATCH.len())](&mut scanner, gs) { + Ok(()) if scanner.eof() => return Ok(()), + Ok(_) => Err(TransactionError::DecodeCorruptedPayloadMoreBytes.into()), + Err(e) => Err(e), + } + } +} + +/* + Events + --- + FIXME(@ohsayan): In the current impl, we unnecessarily use an intermediary buffer which we clearly don't need to (and also makes + pointless allocations). We need to fix this, but with a consistent API (and preferably not something like commit_*(...) unless + we have absolutely no other choice) + --- + [OPC:2B][PAYLOAD] +*/ + +pub struct GNSSuperEvent(Box<[u8]>); + +/// Definition for an event in the GNS (DDL queries) +pub trait GNSEvent +where + Self: PersistObject + Sized, +{ + /// OPC for the event (unique) + const OPC: u16; + /// Expected type for a commit + type CommitType; + /// Expected type for a restore + type RestoreType; + /// Encodes the event into the given buffer + fn encode_super_event(commit: Self, buf: &mut Vec) { + inf::enc::enc_full_into_buffer::(buf, commit) + } + fn decode_and_update_global_state( + scanner: &mut BufferedScanner, + gns: &GlobalNS, + ) -> RuntimeResult<()> { + Self::update_global_state(Self::decode(scanner)?, gns) + } + /// Attempts to decode the event using the given scanner + fn decode(scanner: &mut BufferedScanner) -> RuntimeResult { + inf::dec::dec_full_from_scanner::(scanner).map_err(|e| e.into()) + } + /// Update the global state from the restored event + fn update_global_state(restore: Self::RestoreType, gns: &GlobalNS) -> RuntimeResult<()>; +} + +#[derive(Debug, Clone, Copy)] +pub struct SpaceIDRef<'a> { + uuid: Uuid, + name: &'a str, +} + +impl<'a> SpaceIDRef<'a> { + pub fn new(name: &'a str, space: &Space) -> Self { + Self { + uuid: space.get_uuid(), + name, + } + } +} + +#[derive(Debug, PartialEq)] +pub struct SpaceIDRes { + uuid: Uuid, + name: Box, +} + +impl SpaceIDRes { + #[cfg(test)] + pub fn new(uuid: Uuid, name: Box) -> Self { + Self { uuid, name } + } +} +struct SpaceID<'a>(PhantomData>); +pub struct SpaceIDMD { + uuid: Uuid, + space_name_l: u64, +} + +impl<'a> PersistObject for SpaceID<'a> { + const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64); + type InputType = SpaceIDRef<'a>; + type OutputType = SpaceIDRes; + type Metadata = SpaceIDMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left(md.space_name_l as usize) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + buf.extend(data.uuid.to_le_bytes()); + buf.extend(data.name.len().u64_bytes_le()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + Ok(SpaceIDMD { + uuid: Uuid::from_bytes(scanner.next_chunk()), + space_name_l: scanner.next_u64_le(), + }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + buf.extend(data.name.as_bytes()); + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let str = inf::dec::utils::decode_string(s, md.space_name_l as usize)?; + Ok(SpaceIDRes { + uuid: md.uuid, + name: str.into_boxed_str(), + }) + } +} diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs new file mode 100644 index 00000000..fc0f6c8d --- /dev/null +++ b/server/src/engine/txn/gns/model.rs @@ -0,0 +1,714 @@ +/* + * Created on Wed Aug 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::GNSEvent, + crate::{ + engine::{ + core::{ + model::{Field, Model}, + space::Space, + GlobalNS, {EntityID, EntityIDRef}, + }, + data::uuid::Uuid, + error::TransactionError, + error::{RuntimeResult, StorageError}, + idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq}, + mem::BufferedScanner, + ql::lex::Ident, + storage::v1::inf::{self, map, obj, PersistObject}, + }, + util::EndianQW, + }, + std::marker::PhantomData, +}; + +pub struct ModelID<'a>(PhantomData<&'a ()>); +#[derive(Debug, Clone, Copy)] +pub struct ModelIDRef<'a> { + space_id: super::SpaceIDRef<'a>, + model_name: &'a str, + model_uuid: Uuid, + model_version: u64, +} + +impl<'a> ModelIDRef<'a> { + pub fn new_ref( + space_name: &'a str, + space: &'a Space, + model_name: &'a str, + model: &'a Model, + ) -> ModelIDRef<'a> { + ModelIDRef::new( + super::SpaceIDRef::new(space_name, space), + model_name, + model.get_uuid(), + model.delta_state().schema_current_version().value_u64(), + ) + } + pub fn new( + space_id: super::SpaceIDRef<'a>, + model_name: &'a str, + model_uuid: Uuid, + model_version: u64, + ) -> Self { + Self { + space_id, + model_name, + model_uuid, + model_version, + } + } +} +#[derive(Debug, PartialEq)] +pub struct ModelIDRes { + space_id: super::SpaceIDRes, + model_name: Box, + model_uuid: Uuid, + model_version: u64, +} + +impl ModelIDRes { + #[cfg(test)] + pub fn new( + space_id: super::SpaceIDRes, + model_name: Box, + model_uuid: Uuid, + model_version: u64, + ) -> Self { + Self { + space_id, + model_name, + model_uuid, + model_version, + } + } +} +pub struct ModelIDMD { + space_id: super::SpaceIDMD, + model_name_l: u64, + model_version: u64, + model_uuid: Uuid, +} + +impl<'a> PersistObject for ModelID<'a> { + const METADATA_SIZE: usize = + sizeof!(u64, 2) + sizeof!(u128) + ::METADATA_SIZE; + type InputType = ModelIDRef<'a>; + type OutputType = ModelIDRes; + type Metadata = ModelIDMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left(md.model_name_l as usize + md.space_id.space_name_l as usize) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + ::meta_enc(buf, data.space_id); + buf.extend(data.model_name.len().u64_bytes_le()); + buf.extend(data.model_version.to_le_bytes()); + buf.extend(data.model_uuid.to_le_bytes()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + Ok(ModelIDMD { + space_id: ::meta_dec(scanner)?, + model_name_l: scanner.next_u64_le(), + model_version: scanner.next_u64_le(), + model_uuid: Uuid::from_bytes(scanner.next_chunk()), + }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + ::obj_enc(buf, data.space_id); + buf.extend(data.model_name.as_bytes()); + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + Ok(ModelIDRes { + space_id: ::obj_dec(s, md.space_id)?, + model_name: inf::dec::utils::decode_string(s, md.model_name_l as usize)? + .into_boxed_str(), + model_uuid: md.model_uuid, + model_version: md.model_version, + }) + } +} + +fn with_space( + gns: &GlobalNS, + space_id: &super::SpaceIDRes, + f: impl FnOnce(&Space) -> RuntimeResult, +) -> RuntimeResult { + let spaces = gns.idx().read(); + let Some(space) = spaces.st_get(&space_id.name) else { + return Err(TransactionError::OnRestoreDataMissing.into()); + }; + if space.get_uuid() != space_id.uuid { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + f(&space) +} + +fn with_space_mut( + gns: &GlobalNS, + space_id: &super::SpaceIDRes, + mut f: impl FnMut(&mut Space) -> RuntimeResult, +) -> RuntimeResult { + let mut spaces = gns.idx().write(); + let Some(space) = spaces.st_get_mut(&space_id.name) else { + return Err(TransactionError::OnRestoreDataMissing.into()); + }; + if space.get_uuid() != space_id.uuid { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + f(space) +} + +fn with_model_mut( + gns: &GlobalNS, + space_id: &super::SpaceIDRes, + model_id: &ModelIDRes, + f: impl FnOnce(&mut Model) -> RuntimeResult, +) -> RuntimeResult { + with_space(gns, space_id, |_| { + let mut models = gns.idx_models().write(); + let Some(model) = models.get_mut(&EntityIDRef::new(&space_id.name, &model_id.model_name)) + else { + return Err(TransactionError::OnRestoreDataMissing.into()); + }; + if model.get_uuid() != model_id.model_uuid { + // this should have been handled by an earlier transaction + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + f(model) + }) +} + +/* + create model +*/ + +#[derive(Debug, Clone, Copy)] +/// The commit payload for a `create model ... (...) with {...}` txn +pub struct CreateModelTxn<'a> { + space_id: super::SpaceIDRef<'a>, + model_name: &'a str, + model: &'a Model, +} + +impl<'a> CreateModelTxn<'a> { + pub const fn new( + space_id: super::SpaceIDRef<'a>, + model_name: &'a str, + model: &'a Model, + ) -> Self { + Self { + space_id, + model_name, + model, + } + } +} + +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +pub struct CreateModelTxnRestorePL { + pub(super) space_id: super::SpaceIDRes, + pub(super) model_name: Box, + pub(super) model: Model, +} + +pub struct CreateModelTxnMD { + space_id_meta: super::SpaceIDMD, + model_name_l: u64, + model_meta: as PersistObject>::Metadata, +} + +impl<'a> PersistObject for CreateModelTxn<'a> { + const METADATA_SIZE: usize = ::METADATA_SIZE + + sizeof!(u64) + + as PersistObject>::METADATA_SIZE; + type InputType = CreateModelTxn<'a>; + type OutputType = CreateModelTxnRestorePL; + type Metadata = CreateModelTxnMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left((md.model_meta.p_key_len() + md.model_name_l) as usize) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + // space ID + ::meta_enc(buf, data.space_id); + // model name + buf.extend(data.model_name.len().u64_bytes_le()); + // model meta dump + ::meta_enc(buf, obj::ModelLayoutRef::from(data.model)) + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + let space_id = ::meta_dec(scanner)?; + let model_name_l = scanner.next_u64_le(); + let model_meta = ::meta_dec(scanner)?; + Ok(CreateModelTxnMD { + space_id_meta: space_id, + model_name_l, + model_meta, + }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + // space id dump + ::obj_enc(buf, data.space_id); + // model name + buf.extend(data.model_name.as_bytes()); + // model dump + ::obj_enc(buf, obj::ModelLayoutRef::from(data.model)) + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let space_id = ::obj_dec(s, md.space_id_meta)?; + let model_name = + inf::dec::utils::decode_string(s, md.model_name_l as usize)?.into_boxed_str(); + let model = ::obj_dec(s, md.model_meta)?; + Ok(CreateModelTxnRestorePL { + space_id, + model_name, + model, + }) + } +} + +impl<'a> GNSEvent for CreateModelTxn<'a> { + const OPC: u16 = 3; + type CommitType = CreateModelTxn<'a>; + type RestoreType = CreateModelTxnRestorePL; + fn update_global_state( + CreateModelTxnRestorePL { + space_id, + model_name, + model, + }: Self::RestoreType, + gns: &GlobalNS, + ) -> RuntimeResult<()> { + /* + NOTE(@ohsayan): + A jump to the second branch is practically impossible and should be caught long before we actually end up + here (due to mismatched checksums), but might be theoretically possible because the cosmic rays can be wild + (or well magnetic stuff arounding spinning disks). But we just want to be extra sure. Don't let the aliens (or + rather, radiation) from the cosmos deter us! + */ + let mut spaces = gns.idx().write(); + let mut models = gns.idx_models().write(); + let Some(space) = spaces.get_mut(&space_id.name) else { + return Err(TransactionError::OnRestoreDataMissing.into()); + }; + if space.models().contains(&model_name) { + return Err(TransactionError::OnRestoreDataConflictAlreadyExists.into()); + } + if models + .insert(EntityID::new(&space_id.name, &model_name), model) + .is_some() + { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + space.models_mut().insert(model_name); + Ok(()) + } +} + +/* + alter model add +*/ + +#[derive(Debug, Clone, Copy)] +/// Transaction commit payload for an `alter model add ...` query +pub struct AlterModelAddTxn<'a> { + model_id: ModelIDRef<'a>, + new_fields: &'a IndexSTSeqCns, Field>, +} + +impl<'a> AlterModelAddTxn<'a> { + pub const fn new( + model_id: ModelIDRef<'a>, + new_fields: &'a IndexSTSeqCns, Field>, + ) -> Self { + Self { + model_id, + new_fields, + } + } +} +pub struct AlterModelAddTxnMD { + model_id_meta: ModelIDMD, + new_field_c: u64, +} +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +pub struct AlterModelAddTxnRestorePL { + pub(super) model_id: ModelIDRes, + pub(super) new_fields: IndexSTSeqCns, Field>, +} +impl<'a> PersistObject for AlterModelAddTxn<'a> { + const METADATA_SIZE: usize = ::METADATA_SIZE + sizeof!(u64); + type InputType = AlterModelAddTxn<'a>; + type OutputType = AlterModelAddTxnRestorePL; + type Metadata = AlterModelAddTxnMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left( + (md.model_id_meta.space_id.space_name_l + md.model_id_meta.model_name_l) as usize, + ) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + ::meta_enc(buf, data.model_id); + buf.extend(data.new_fields.st_len().u64_bytes_le()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + let model_id_meta = ::meta_dec(scanner)?; + let new_field_c = scanner.next_u64_le(); + Ok(AlterModelAddTxnMD { + model_id_meta, + new_field_c, + }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + ::obj_enc(buf, data.model_id); + > as PersistObject>::obj_enc(buf, data.new_fields); + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let model_id = ::obj_dec(s, md.model_id_meta)?; + let new_fields = , _>>> as PersistObject>::obj_dec( + s, + map::MapIndexSizeMD(md.new_field_c as usize), + )?; + Ok(AlterModelAddTxnRestorePL { + model_id, + new_fields, + }) + } +} + +impl<'a> GNSEvent for AlterModelAddTxn<'a> { + const OPC: u16 = 4; + type CommitType = AlterModelAddTxn<'a>; + type RestoreType = AlterModelAddTxnRestorePL; + fn update_global_state( + AlterModelAddTxnRestorePL { + model_id, + new_fields, + }: Self::RestoreType, + gns: &GlobalNS, + ) -> RuntimeResult<()> { + with_model_mut(gns, &model_id.space_id, &model_id, |model| { + let mut mutator = model.model_mutator(); + for (field_name, field) in new_fields.stseq_owned_kv() { + if !mutator.add_field(field_name, field) { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + } + Ok(()) + }) + } +} + +/* + alter model remove +*/ + +#[derive(Debug, Clone, Copy)] +/// Transaction commit payload for an `alter model remove` transaction +pub struct AlterModelRemoveTxn<'a> { + model_id: ModelIDRef<'a>, + removed_fields: &'a [Ident<'a>], +} +impl<'a> AlterModelRemoveTxn<'a> { + pub const fn new(model_id: ModelIDRef<'a>, removed_fields: &'a [Ident<'a>]) -> Self { + Self { + model_id, + removed_fields, + } + } +} +pub struct AlterModelRemoveTxnMD { + model_id_meta: ModelIDMD, + remove_field_c: u64, +} +#[derive(Debug, PartialEq)] +pub struct AlterModelRemoveTxnRestorePL { + pub(super) model_id: ModelIDRes, + pub(super) removed_fields: Box<[Box]>, +} + +impl<'a> PersistObject for AlterModelRemoveTxn<'a> { + const METADATA_SIZE: usize = ::METADATA_SIZE + sizeof!(u64); + type InputType = AlterModelRemoveTxn<'a>; + type OutputType = AlterModelRemoveTxnRestorePL; + type Metadata = AlterModelRemoveTxnMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left( + (md.model_id_meta.space_id.space_name_l + md.model_id_meta.model_name_l) as usize, + ) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + ::meta_enc(buf, data.model_id); + buf.extend(data.removed_fields.len().u64_bytes_le()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + let model_id_meta = ::meta_dec(scanner)?; + Ok(AlterModelRemoveTxnMD { + model_id_meta, + remove_field_c: scanner.next_u64_le(), + }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + ::obj_enc(buf, data.model_id); + for field in data.removed_fields { + buf.extend(field.len().u64_bytes_le()); + buf.extend(field.as_bytes()); + } + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let model_id = ::obj_dec(s, md.model_id_meta)?; + let mut removed_fields = Vec::with_capacity(md.remove_field_c as usize); + while !s.eof() + & (removed_fields.len() as u64 != md.remove_field_c) + & s.has_left(sizeof!(u64)) + { + let len = s.next_u64_le() as usize; + if !s.has_left(len) { + break; + } + removed_fields.push(inf::dec::utils::decode_string(s, len)?.into_boxed_str()); + } + if removed_fields.len() as u64 != md.remove_field_c { + return Err(StorageError::InternalDecodeStructureCorruptedPayload.into()); + } + Ok(AlterModelRemoveTxnRestorePL { + model_id, + removed_fields: removed_fields.into_boxed_slice(), + }) + } +} + +impl<'a> GNSEvent for AlterModelRemoveTxn<'a> { + const OPC: u16 = 5; + type CommitType = AlterModelRemoveTxn<'a>; + type RestoreType = AlterModelRemoveTxnRestorePL; + fn update_global_state( + AlterModelRemoveTxnRestorePL { + model_id, + removed_fields, + }: Self::RestoreType, + gns: &GlobalNS, + ) -> RuntimeResult<()> { + with_model_mut(gns, &model_id.space_id, &model_id, |model| { + let mut mutator = model.model_mutator(); + for removed_field in removed_fields.iter() { + if !mutator.remove_field(&removed_field) { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + } + Ok(()) + }) + } +} + +/* + alter model update +*/ + +#[derive(Debug, Clone, Copy)] +/// Transaction commit payload for an `alter model update ...` query +pub struct AlterModelUpdateTxn<'a> { + model_id: ModelIDRef<'a>, + updated_fields: &'a IndexST, Field>, +} + +impl<'a> AlterModelUpdateTxn<'a> { + pub const fn new( + model_id: ModelIDRef<'a>, + updated_fields: &'a IndexST, Field>, + ) -> Self { + Self { + model_id, + updated_fields, + } + } +} +pub struct AlterModelUpdateTxnMD { + model_id_md: ModelIDMD, + updated_field_c: u64, +} +#[derive(Debug, PartialEq)] +pub struct AlterModelUpdateTxnRestorePL { + pub(super) model_id: ModelIDRes, + pub(super) updated_fields: IndexSTSeqCns, Field>, +} + +impl<'a> PersistObject for AlterModelUpdateTxn<'a> { + const METADATA_SIZE: usize = ::METADATA_SIZE + sizeof!(u64); + type InputType = AlterModelUpdateTxn<'a>; + type OutputType = AlterModelUpdateTxnRestorePL; + type Metadata = AlterModelUpdateTxnMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left( + md.model_id_md.space_id.space_name_l as usize + md.model_id_md.model_name_l as usize, + ) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + ::meta_enc(buf, data.model_id); + buf.extend(data.updated_fields.st_len().u64_bytes_le()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + let model_id_md = ::meta_dec(scanner)?; + Ok(AlterModelUpdateTxnMD { + model_id_md, + updated_field_c: scanner.next_u64_le(), + }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + ::obj_enc(buf, data.model_id); + > as PersistObject>::obj_enc( + buf, + data.updated_fields, + ); + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let model_id = ::obj_dec(s, md.model_id_md)?; + let updated_fields = + , _>>> as PersistObject>::obj_dec( + s, + map::MapIndexSizeMD(md.updated_field_c as usize), + )?; + Ok(AlterModelUpdateTxnRestorePL { + model_id, + updated_fields, + }) + } +} + +impl<'a> GNSEvent for AlterModelUpdateTxn<'a> { + const OPC: u16 = 6; + type CommitType = AlterModelUpdateTxn<'a>; + type RestoreType = AlterModelUpdateTxnRestorePL; + fn update_global_state( + AlterModelUpdateTxnRestorePL { + model_id, + updated_fields, + }: Self::RestoreType, + gns: &GlobalNS, + ) -> RuntimeResult<()> { + with_model_mut(gns, &model_id.space_id, &model_id, |model| { + let mut mutator = model.model_mutator(); + for (field_id, field) in updated_fields.stseq_owned_kv() { + if !mutator.update_field(&field_id, field) { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + } + Ok(()) + }) + } +} + +/* + drop model +*/ + +#[derive(Debug, Clone, Copy)] +/// Transaction commit payload for a `drop model ...` query +pub struct DropModelTxn<'a> { + model_id: ModelIDRef<'a>, +} + +impl<'a> DropModelTxn<'a> { + pub const fn new(model_id: ModelIDRef<'a>) -> Self { + Self { model_id } + } +} +pub struct DropModelTxnMD { + model_id_md: ModelIDMD, +} +impl<'a> PersistObject for DropModelTxn<'a> { + const METADATA_SIZE: usize = ::METADATA_SIZE; + type InputType = DropModelTxn<'a>; + type OutputType = ModelIDRes; + type Metadata = DropModelTxnMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left( + md.model_id_md.space_id.space_name_l as usize + md.model_id_md.model_name_l as usize, + ) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + ::meta_enc(buf, data.model_id); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + let model_id_md = ::meta_dec(scanner)?; + Ok(DropModelTxnMD { model_id_md }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + ::obj_enc(buf, data.model_id); + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + ::obj_dec(s, md.model_id_md) + } +} + +impl<'a> GNSEvent for DropModelTxn<'a> { + const OPC: u16 = 7; + type CommitType = DropModelTxn<'a>; + type RestoreType = ModelIDRes; + fn update_global_state( + ModelIDRes { + space_id, + model_name, + model_uuid, + model_version: _, + }: Self::RestoreType, + gns: &GlobalNS, + ) -> RuntimeResult<()> { + with_space_mut(gns, &space_id, |space| { + let mut models = gns.idx_models().write(); + if !space.models_mut().remove(&model_name) { + return Err(TransactionError::OnRestoreDataMissing.into()); + } + let Some(removed_model) = models.remove(&EntityIDRef::new(&space_id.name, &model_name)) + else { + return Err(TransactionError::OnRestoreDataMissing.into()); + }; + if removed_model.get_uuid() != model_uuid { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + Ok(()) + }) + } +} diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs new file mode 100644 index 00000000..c4199463 --- /dev/null +++ b/server/src/engine/txn/gns/space.rs @@ -0,0 +1,303 @@ +/* + * Created on Wed Aug 23 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::GNSEvent, + crate::{ + engine::{ + core::{space::Space, EntityIDRef, GlobalNS}, + data::DictGeneric, + error::{RuntimeResult, TransactionError}, + idx::STIndex, + mem::BufferedScanner, + storage::v1::inf::{self, map, obj, PersistObject}, + }, + util::EndianQW, + }, +}; + +/* + create space +*/ + +#[derive(Clone, Copy)] +/// Transaction commit payload for a `create space ...` query +pub struct CreateSpaceTxn<'a> { + pub(super) space_meta: &'a DictGeneric, + pub(super) space_name: &'a str, + pub(super) space: &'a Space, +} + +impl<'a> CreateSpaceTxn<'a> { + pub const fn new(space_meta: &'a DictGeneric, space_name: &'a str, space: &'a Space) -> Self { + Self { + space_meta, + space_name, + space, + } + } +} + +#[derive(Debug)] +#[cfg_attr(test, derive(PartialEq))] +pub struct CreateSpaceTxnRestorePL { + pub(super) space_name: Box, + pub(super) space: Space, +} + +pub struct CreateSpaceTxnMD { + pub(super) space_name_l: u64, + pub(super) space_meta: as PersistObject>::Metadata, +} + +impl<'a> PersistObject for CreateSpaceTxn<'a> { + const METADATA_SIZE: usize = + as PersistObject>::METADATA_SIZE + sizeof!(u64); + type InputType = CreateSpaceTxn<'a>; + type OutputType = CreateSpaceTxnRestorePL; + type Metadata = CreateSpaceTxnMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left(md.space_name_l as usize) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + buf.extend(data.space_name.len().u64_bytes_le()); + as PersistObject>::meta_enc( + buf, + obj::SpaceLayoutRef::from((data.space, data.space_meta)), + ); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + let space_name_l = scanner.next_u64_le(); + let space_meta = ::meta_dec(scanner)?; + Ok(CreateSpaceTxnMD { + space_name_l, + space_meta, + }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + buf.extend(data.space_name.as_bytes()); + ::obj_enc(buf, (data.space, data.space_meta).into()); + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let space_name = + inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str(); + let space = ::obj_dec(s, md.space_meta)?; + Ok(CreateSpaceTxnRestorePL { space_name, space }) + } +} + +impl<'a> GNSEvent for CreateSpaceTxn<'a> { + const OPC: u16 = 0; + type CommitType = CreateSpaceTxn<'a>; + type RestoreType = CreateSpaceTxnRestorePL; + fn update_global_state( + CreateSpaceTxnRestorePL { space_name, space }: CreateSpaceTxnRestorePL, + gns: &crate::engine::core::GlobalNS, + ) -> RuntimeResult<()> { + let mut spaces = gns.idx().write(); + if spaces.st_insert(space_name, space.into()) { + Ok(()) + } else { + Err(TransactionError::OnRestoreDataConflictAlreadyExists.into()) + } + } +} + +/* + alter space + --- + for now dump the entire meta +*/ + +#[derive(Clone, Copy)] +/// Transaction payload for an `alter space ...` query +pub struct AlterSpaceTxn<'a> { + space_id: super::SpaceIDRef<'a>, + updated_props: &'a DictGeneric, +} + +impl<'a> AlterSpaceTxn<'a> { + pub const fn new(space_id: super::SpaceIDRef<'a>, updated_props: &'a DictGeneric) -> Self { + Self { + space_id, + updated_props, + } + } +} + +pub struct AlterSpaceTxnMD { + space_id_meta: super::SpaceIDMD, + dict_len: u64, +} + +#[derive(Debug, PartialEq)] +pub struct AlterSpaceTxnRestorePL { + pub(super) space_id: super::SpaceIDRes, + pub(super) space_meta: DictGeneric, +} + +impl<'a> PersistObject for AlterSpaceTxn<'a> { + const METADATA_SIZE: usize = sizeof!(u64, 2) + sizeof!(u128); + type InputType = AlterSpaceTxn<'a>; + type OutputType = AlterSpaceTxnRestorePL; + type Metadata = AlterSpaceTxnMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left(md.space_id_meta.space_name_l as usize) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + ::meta_enc(buf, data.space_id); + buf.extend(data.updated_props.len().u64_bytes_le()); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + Ok(AlterSpaceTxnMD { + space_id_meta: ::meta_dec(scanner)?, + dict_len: scanner.next_u64_le(), + }) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + ::obj_enc(buf, data.space_id); + as PersistObject>::obj_enc( + buf, + data.updated_props, + ); + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + let space_id = ::obj_dec(s, md.space_id_meta)?; + let space_meta = as PersistObject>::obj_dec( + s, + map::MapIndexSizeMD(md.dict_len as usize), + )?; + Ok(AlterSpaceTxnRestorePL { + space_id, + space_meta, + }) + } +} + +impl<'a> GNSEvent for AlterSpaceTxn<'a> { + const OPC: u16 = 1; + type CommitType = AlterSpaceTxn<'a>; + type RestoreType = AlterSpaceTxnRestorePL; + + fn update_global_state( + AlterSpaceTxnRestorePL { + space_id, + space_meta, + }: Self::RestoreType, + gns: &crate::engine::core::GlobalNS, + ) -> RuntimeResult<()> { + let mut gns = gns.idx().write(); + match gns.st_get_mut(&space_id.name) { + Some(space) => { + if !crate::engine::data::dict::rmerge_metadata(space.props_mut(), space_meta) { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + } + None => return Err(TransactionError::OnRestoreDataMissing.into()), + } + Ok(()) + } +} + +/* + drop space +*/ + +#[derive(Clone, Copy)] +/// Transaction commit payload for a `drop space ...` query +pub struct DropSpaceTxn<'a> { + space_id: super::SpaceIDRef<'a>, +} + +impl<'a> DropSpaceTxn<'a> { + pub const fn new(space_id: super::SpaceIDRef<'a>) -> Self { + Self { space_id } + } +} + +impl<'a> PersistObject for DropSpaceTxn<'a> { + const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64); + type InputType = DropSpaceTxn<'a>; + type OutputType = super::SpaceIDRes; + type Metadata = super::SpaceIDMD; + fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool { + scanner.has_left(md.space_name_l as usize) + } + fn meta_enc(buf: &mut Vec, data: Self::InputType) { + ::meta_enc(buf, data.space_id); + } + unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult { + ::meta_dec(scanner) + } + fn obj_enc(buf: &mut Vec, data: Self::InputType) { + ::obj_enc(buf, data.space_id) + } + unsafe fn obj_dec( + s: &mut BufferedScanner, + md: Self::Metadata, + ) -> RuntimeResult { + ::obj_dec(s, md) + } +} + +impl<'a> GNSEvent for DropSpaceTxn<'a> { + const OPC: u16 = 2; + type CommitType = DropSpaceTxn<'a>; + type RestoreType = super::SpaceIDRes; + fn update_global_state( + super::SpaceIDRes { uuid, name }: Self::RestoreType, + gns: &GlobalNS, + ) -> RuntimeResult<()> { + let mut wgns = gns.idx().write(); + let mut wmodel = gns.idx_models().write(); + match wgns.entry(name) { + std::collections::hash_map::Entry::Occupied(oe) => { + if oe.get().get_uuid() == uuid { + for model in oe.get().models() { + let id: EntityIDRef<'static> = unsafe { + // UNSAFE(@ohsayan): I really need a pack of what the borrow checker has been reveling on + core::mem::transmute(EntityIDRef::new(oe.key(), &model)) + }; + let _ = wmodel.st_delete(&id); + } + oe.remove_entry(); + Ok(()) + } else { + return Err(TransactionError::OnRestoreDataConflictMismatch.into()); + } + } + std::collections::hash_map::Entry::Vacant(_) => { + return Err(TransactionError::OnRestoreDataMissing.into()) + } + } + } +} diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs new file mode 100644 index 00000000..acc894c1 --- /dev/null +++ b/server/src/engine/txn/gns/tests/full_chain.rs @@ -0,0 +1,320 @@ +/* + * Created on Fri Aug 25 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + core::{ + model::{Field, Layer, Model}, + space::Space, + }, + data::{cell::Datacell, tag::TagSelector, uuid::Uuid, DictEntryGeneric}, + error::QueryError, + fractal::{test_utils::TestGlobal, GlobalInstanceLike}, + idx::STIndex, + ql::{ + ast::parse_ast_node_full, + ddl::crt::{CreateModel, CreateSpace}, + tests::lex_insecure, + }, +}; + +fn multirun(f: impl FnOnce() + Copy) { + for _ in 0..10 { + f() + } +} + +fn with_variable(var: T, f: impl FnOnce(T)) { + f(var); +} + +fn init_space(global: &impl GlobalInstanceLike, space_name: &str, env: &str) -> Uuid { + let query = format!("create space {space_name} with {{ env: {env} }}"); + let stmt = lex_insecure(query.as_bytes()).unwrap(); + let stmt = parse_ast_node_full::(&stmt[2..]).unwrap(); + let name = stmt.space_name; + Space::transactional_exec_create(global, stmt).unwrap(); + global + .namespace() + .idx() + .read() + .get(name.as_str()) + .unwrap() + .get_uuid() +} + +#[test] +fn create_space() { + with_variable("create_space_test.global.db-tlog", |log_name| { + let uuid; + // start 1 + { + let global = TestGlobal::new_with_vfs_driver(log_name); + uuid = init_space(&global, "myspace", "{ SAYAN_MAX: 65536 }"); // good lord that doesn't sound like a good variable + } + multirun(|| { + let global = TestGlobal::new_with_vfs_driver(log_name); + let spaces = global.namespace().idx().read(); + let space = spaces.get("myspace").unwrap(); + assert_eq!( + &*space, + &Space::new_restore_empty( + uuid, + into_dict!("env" => DictEntryGeneric::Map( + into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536))) + )) + ) + ); + }) + }) +} + +#[test] +fn alter_space() { + with_variable("alter_space_test.global.db-tlog", |log_name| { + let uuid; + { + let global = TestGlobal::new_with_vfs_driver(log_name); + uuid = init_space(&global, "myspace", "{}"); + let stmt = + lex_insecure("alter space myspace with { env: { SAYAN_MAX: 65536 } }".as_bytes()) + .unwrap(); + let stmt = parse_ast_node_full(&stmt[2..]).unwrap(); + Space::transactional_exec_alter(&global, stmt).unwrap(); + } + multirun(|| { + let global = TestGlobal::new_with_vfs_driver(log_name); + let spaces = global.namespace().idx().read(); + let space = spaces.get("myspace").unwrap(); + assert_eq!( + &*space, + &Space::new_restore_empty( + uuid, + into_dict!("env" => DictEntryGeneric::Map( + into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)) + ))) + ) + ); + }) + }) +} + +#[test] +fn drop_space() { + with_variable("drop_space_test.global.db-tlog", |log_name| { + { + let global = TestGlobal::new_with_vfs_driver(log_name); + let _ = init_space(&global, "myspace", "{}"); + let stmt = lex_insecure("drop space myspace".as_bytes()).unwrap(); + let stmt = parse_ast_node_full(&stmt[2..]).unwrap(); + Space::transactional_exec_drop(&global, stmt).unwrap(); + } + multirun(|| { + let global = TestGlobal::new_with_vfs_driver(log_name); + assert!(global.namespace().idx().read().get("myspace").is_none()); + }) + }) +} + +fn init_model( + global: &impl GlobalInstanceLike, + space_name: &str, + model_name: &str, + decl: &str, +) -> Uuid { + let query = format!("create model {space_name}.{model_name} ({decl})"); + let stmt = lex_insecure(query.as_bytes()).unwrap(); + let stmt = parse_ast_node_full::(&stmt[2..]).unwrap(); + let model_name = stmt.model_name; + Model::transactional_exec_create(global, stmt).unwrap(); + global + .namespace() + .with_model(model_name, |model| Ok(model.get_uuid())) + .unwrap() +} + +fn init_default_model(global: &impl GlobalInstanceLike) -> Uuid { + init_model( + global, + "myspace", + "mymodel", + "username: string, password: binary", + ) +} + +#[test] +fn create_model() { + with_variable("create_model_test.global.db-tlog", |log_name| { + let _uuid_space; + let uuid_model; + { + let global = TestGlobal::new_with_vfs_driver(log_name); + _uuid_space = init_space(&global, "myspace", "{}"); + uuid_model = init_default_model(&global); + } + multirun(|| { + let global = TestGlobal::new_with_vfs_driver(log_name); + global + .namespace() + .with_model(("myspace", "mymodel").into(), |model| { + assert_eq!( + model, + &Model::new_restore( + uuid_model, + "username".into(), + TagSelector::String.into_full(), + into_dict! { + "username" => Field::new([Layer::str()].into(), false), + "password" => Field::new([Layer::bin()].into(), false), + } + ) + ); + Ok(()) + }) + .unwrap(); + }) + }) +} + +#[test] +fn alter_model_add() { + with_variable("alter_model_add_test.global.db-tlog", |log_name| { + { + let global = TestGlobal::new_with_vfs_driver(log_name); + init_space(&global, "myspace", "{}"); + init_default_model(&global); + let stmt = lex_insecure( + b"alter model myspace.mymodel add profile_pic { type: binary, nullable: true }", + ) + .unwrap(); + let stmt = parse_ast_node_full(&stmt[2..]).unwrap(); + Model::transactional_exec_alter(&global, stmt).unwrap(); + } + multirun(|| { + let global = TestGlobal::new_with_vfs_driver(log_name); + global + .namespace() + .with_model(("myspace", "mymodel").into(), |model| { + assert_eq!( + model.fields().st_get("profile_pic").unwrap(), + &Field::new([Layer::bin()].into(), true) + ); + Ok(()) + }) + .unwrap(); + }) + }) +} + +#[test] +fn alter_model_remove() { + with_variable("alter_model_remove_test.global.db-tlog", |log_name| { + { + let global = TestGlobal::new_with_vfs_driver(log_name); + init_space(&global, "myspace", "{}"); + init_model( + &global, + "myspace", + "mymodel", + "username: string, password: binary, null profile_pic: binary, null has_2fa: bool, null has_secure_key: bool, is_dumb: bool", + ); + let stmt = lex_insecure( + "alter model myspace.mymodel remove (has_secure_key, is_dumb)".as_bytes(), + ) + .unwrap(); + let stmt = parse_ast_node_full(&stmt[2..]).unwrap(); + Model::transactional_exec_alter(&global, stmt).unwrap(); + } + multirun(|| { + let global = TestGlobal::new_with_vfs_driver(log_name); + global + .namespace() + .with_model(("myspace", "mymodel").into(), |model| { + assert!(model.fields().st_get("has_secure_key").is_none()); + assert!(model.fields().st_get("is_dumb").is_none()); + Ok(()) + }) + .unwrap(); + }) + }) +} + +#[test] +fn alter_model_update() { + with_variable("alter_model_update_test.global.db-tlog", |log_name| { + { + let global = TestGlobal::new_with_vfs_driver(log_name); + init_space(&global, "myspace", "{}"); + init_model( + &global, + "myspace", + "mymodel", + "username: string, password: binary, profile_pic: binary", + ); + let stmt = + lex_insecure(b"alter model myspace.mymodel update profile_pic { nullable: true }") + .unwrap(); + let stmt = parse_ast_node_full(&stmt[2..]).unwrap(); + Model::transactional_exec_alter(&global, stmt).unwrap(); + } + multirun(|| { + let global = TestGlobal::new_with_vfs_driver(log_name); + global + .namespace() + .with_model(("myspace", "mymodel").into(), |model| { + assert_eq!( + model.fields().st_get("profile_pic").unwrap(), + &Field::new([Layer::bin()].into(), true) + ); + Ok(()) + }) + .unwrap(); + }) + }) +} + +#[test] +fn drop_model() { + with_variable("drop_model_test.global.db-tlog", |log_name| { + { + let global = TestGlobal::new_with_vfs_driver(log_name); + init_space(&global, "myspace", "{}"); + init_default_model(&global); + let stmt = lex_insecure(b"drop model myspace.mymodel").unwrap(); + let stmt = parse_ast_node_full(&stmt[2..]).unwrap(); + Model::transactional_exec_drop(&global, stmt).unwrap(); + } + multirun(|| { + let global = TestGlobal::new_with_vfs_driver(log_name); + assert_eq!( + global + .namespace() + .with_model(("myspace", "mymodel").into(), |_| { Ok(()) }) + .unwrap_err(), + QueryError::QExecObjectNotFound + ); + }) + }) +} diff --git a/server/src/engine/txn/gns/tests/io.rs b/server/src/engine/txn/gns/tests/io.rs new file mode 100644 index 00000000..9a03c5dd --- /dev/null +++ b/server/src/engine/txn/gns/tests/io.rs @@ -0,0 +1,249 @@ +/* + * Created on Thu Aug 24 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::super::{ + model::{self, ModelIDRef, ModelIDRes}, + space, SpaceIDRef, SpaceIDRes, + }, + crate::engine::{ + core::{model::Model, space::Space}, + storage::v1::inf::{dec, enc}, + }, +}; + +mod space_tests { + use super::{ + dec, enc, + space::{ + AlterSpaceTxn, AlterSpaceTxnRestorePL, CreateSpaceTxn, CreateSpaceTxnRestorePL, + DropSpaceTxn, + }, + Space, SpaceIDRef, + }; + #[test] + fn create() { + let orig_space = Space::new_auto_all(); + let space_r = orig_space.props(); + let txn = CreateSpaceTxn::new(&space_r, "myspace", &orig_space); + let encoded = enc::enc_full_self(txn); + let decoded = dec::dec_full::(&encoded).unwrap(); + assert_eq!( + CreateSpaceTxnRestorePL { + space_name: "myspace".into(), + space: Space::new_restore_empty(orig_space.get_uuid(), Default::default()) + }, + decoded + ); + } + #[test] + fn alter() { + let space = Space::new_auto_all(); + let space_r = space.props(); + let txn = AlterSpaceTxn::new(SpaceIDRef::new("myspace", &space), &space_r); + let encoded = enc::enc_full_self(txn); + let decoded = dec::dec_full::(&encoded).unwrap(); + assert_eq!( + AlterSpaceTxnRestorePL { + space_id: super::SpaceIDRes::new(space.get_uuid(), "myspace".into()), + space_meta: space_r.clone() + }, + decoded + ); + } + #[test] + fn drop() { + let space = Space::new_auto_all(); + let txn = DropSpaceTxn::new(super::SpaceIDRef::new("myspace", &space)); + let encoded = enc::enc_full_self(txn); + let decoded = dec::dec_full::(&encoded).unwrap(); + assert_eq!( + super::SpaceIDRes::new(space.get_uuid(), "myspace".into()), + decoded + ); + } +} + +mod model_tests { + use { + super::{ + model::{ + AlterModelAddTxn, AlterModelAddTxnRestorePL, AlterModelRemoveTxn, + AlterModelRemoveTxnRestorePL, AlterModelUpdateTxn, AlterModelUpdateTxnRestorePL, + CreateModelTxn, CreateModelTxnRestorePL, DropModelTxn, + }, + Model, Space, + }, + crate::engine::{ + core::model::{Field, Layer}, + data::{tag::TagSelector, uuid::Uuid}, + }, + }; + fn default_space_model() -> (Space, Model) { + let space = Space::new_auto_all(); + let model = Model::new_restore( + Uuid::new(), + "username".into(), + TagSelector::String.into_full(), + into_dict!( + "password" => Field::new([Layer::bin()].into(), false), + "profile_pic" => Field::new([Layer::bin()].into(), true), + ), + ); + (space, model) + } + #[test] + fn create() { + let (space, model) = default_space_model(); + let txn = CreateModelTxn::new(super::SpaceIDRef::new("myspace", &space), "mymodel", &model); + let encoded = super::enc::enc_full_self(txn); + let decoded = super::dec::dec_full::(&encoded).unwrap(); + assert_eq!( + CreateModelTxnRestorePL { + space_id: super::SpaceIDRes::new(space.get_uuid(), "myspace".into()), + model_name: "mymodel".into(), + model, + }, + decoded + ) + } + #[test] + fn alter_add() { + let (space, model) = default_space_model(); + let new_fields = into_dict! { + "auth_2fa" => Field::new([Layer::bool()].into(), true), + }; + let txn = AlterModelAddTxn::new( + super::ModelIDRef::new( + super::SpaceIDRef::new("myspace", &space), + "mymodel", + model.get_uuid(), + model.delta_state().schema_current_version().value_u64(), + ), + &new_fields, + ); + let encoded = super::enc::enc_full_self(txn); + let decoded = super::dec::dec_full::(&encoded).unwrap(); + assert_eq!( + AlterModelAddTxnRestorePL { + model_id: super::ModelIDRes::new( + super::SpaceIDRes::new(space.get_uuid(), "myspace".into()), + "mymodel".into(), + model.get_uuid(), + model.delta_state().schema_current_version().value_u64() + ), + new_fields: into_dict! { + "auth_2fa" => Field::new([Layer::bool()].into(), true), + } + }, + decoded + ); + } + #[test] + fn alter_remove() { + let (space, model) = default_space_model(); + let removed_fields = ["profile_pic".into()]; + let txn = AlterModelRemoveTxn::new( + super::ModelIDRef::new( + super::SpaceIDRef::new("myspace", &space), + "mymodel", + model.get_uuid(), + model.delta_state().schema_current_version().value_u64(), + ), + &removed_fields, + ); + let encoded = super::enc::enc_full_self(txn); + let decoded = super::dec::dec_full::(&encoded).unwrap(); + assert_eq!( + AlterModelRemoveTxnRestorePL { + model_id: super::ModelIDRes::new( + super::SpaceIDRes::new(space.get_uuid(), "myspace".into()), + "mymodel".into(), + model.get_uuid(), + model.delta_state().schema_current_version().value_u64() + ), + removed_fields: ["profile_pic".into()].into() + }, + decoded + ); + } + #[test] + fn alter_update() { + let (space, model) = default_space_model(); + let updated_fields_copy = into_dict! { + // people of your social app will hate this, but hehe + "profile_pic" => Field::new([Layer::bin()].into(), false) + }; + let updated_fields = into_dict! { + // people of your social app will hate this, but hehe + "profile_pic" => Field::new([Layer::bin()].into(), false) + }; + let txn = AlterModelUpdateTxn::new( + super::ModelIDRef::new( + super::SpaceIDRef::new("myspace", &space), + "mymodel".into(), + model.get_uuid(), + model.delta_state().schema_current_version().value_u64(), + ), + &updated_fields, + ); + let encoded = super::enc::enc_full_self(txn); + let decoded = super::dec::dec_full::(&encoded).unwrap(); + assert_eq!( + AlterModelUpdateTxnRestorePL { + model_id: super::ModelIDRes::new( + super::SpaceIDRes::new(space.get_uuid(), "myspace".into()), + "mymodel".into(), + model.get_uuid(), + model.delta_state().schema_current_version().value_u64() + ), + updated_fields: updated_fields_copy + }, + decoded + ); + } + #[test] + fn drop() { + let (space, model) = default_space_model(); + let txn = DropModelTxn::new(super::ModelIDRef::new( + super::SpaceIDRef::new("myspace", &space), + "mymodel", + model.get_uuid(), + model.delta_state().schema_current_version().value_u64(), + )); + let encoded = super::enc::enc_full_self(txn); + let decoded = super::dec::dec_full::(&encoded).unwrap(); + assert_eq!( + super::ModelIDRes::new( + super::SpaceIDRes::new(space.get_uuid(), "myspace".into()), + "mymodel".into(), + model.get_uuid(), + model.delta_state().schema_current_version().value_u64() + ), + decoded + ); + } +} diff --git a/server/src/engine/txn/gns/tests/mod.rs b/server/src/engine/txn/gns/tests/mod.rs new file mode 100644 index 00000000..608315a3 --- /dev/null +++ b/server/src/engine/txn/gns/tests/mod.rs @@ -0,0 +1,28 @@ +/* + * Created on Thu Aug 24 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +mod full_chain; +mod io; diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs new file mode 100644 index 00000000..3c258b82 --- /dev/null +++ b/server/src/engine/txn/mod.rs @@ -0,0 +1,27 @@ +/* + * Created on Sun Aug 20 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +pub mod gns; diff --git a/server/src/kvengine/encoding.rs b/server/src/kvengine/encoding.rs deleted file mode 100644 index 6f43ae39..00000000 --- a/server/src/kvengine/encoding.rs +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Created on Thu Jul 01 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -/* - This cannot be the work of a single person! A big thanks to: - - Björn Höhrmann: https://bjoern.hoehrmann.de/ - - Professor Lemire: https://scholar.google.com/citations?user=q1ja-G8AAAAJ - - Travis Downs: https://github.com/travisdowns -*/ - -/* - * The UTF-8 validation that we use here uses an encoded finite state machine defined in this file. - * A branchless (no cond) finite state machine is used which greatly simplifies how things work - * than some amazing libraries out there while also providing huge performance and computationally - * economical benefits. The dual stream that I have used here offers the best performance than a - * triple or quad channel SM evaluation. I attempted a SM evaluation with four streams and sure - * it was about 75% faster than the standard library's evaluation, but however fell short to the - * 300% improvement that I got over a dual stream. Also, don't get too excited because this looks - * like something recursive or _threadable_. DON'T. Call stacks have their own costs and why do - * it at all when we can use a simple loop? - * Now for the threading bit: remember, this is not a single game that you have to win. - * There will potentially be millions if not thousands of callers requesting validation and - * imagine spawning two threads for every validation. Firstly, you'll have to wait on the OS/Kernel - * to hand over the response to a fork. Secondly, so many threads are useless because it'll just - * burden the scheduler and hurt performance taking away any possible performance benefits that - * you could've had. In a single benchmark, the threaded implementation might make you happy - * and turn out to be 600% faster. But try doing multiple evaluations in parallel: you'll know - * what we're talking about. Eliding bound checks gives us a ~2-5% edge over the one that - * is checked. Why elide them? Just because we reduce our assembly size by ~15%! - * - * - Sayan N. (July, 2021) -*/ - -use crate::{ - corestore::booltable::{BoolTable, TwoBitLUT}, - protocol::iter::{AnyArrayIter, BorrowedAnyArrayIter}, -}; - -type PairFn = fn(&[u8], &[u8]) -> bool; - -pub const ENCODING_LUT_ITER: BoolTable bool> = - BoolTable::new(is_okay_encoded_iter, is_okay_no_encoding_iter); -pub const ENCODING_LUT_ITER_PAIR: TwoBitLUT bool> = TwoBitLUT::new( - pair_is_okay_encoded_iter_ff, - pair_is_okay_encoded_iter_ft, - pair_is_okay_encoded_iter_tf, - pair_is_okay_encoded_iter_tt, -); -pub const ENCODING_LUT: BoolTable bool> = - BoolTable::new(self::is_okay_encoded, self::is_okay_no_encoding); -pub const ENCODING_LUT_PAIR: TwoBitLUT = TwoBitLUT::new( - self::is_okay_encoded_pair_ff, - self::is_okay_encoded_pair_ft, - self::is_okay_encoded_pair_tf, - self::is_okay_encoded_pair_tt, -); - -/// This table maps bytes to character classes that helps us reduce the size of the -/// transition table and generate bitmasks -static UTF8_MAP_BYTE_TO_CHAR_CLASS: [u8; 256] = [ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, - 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, -]; - -/// This table is a transition table that maps the combination of a state of the -/// automaton and a char class to a state -static UTF8_TRANSITION_MAP: [u8; 108] = [ - 0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 0, 12, 12, 12, 12, 12, 0, 12, 0, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 24, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, - 12, 36, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, -]; - -pub const fn pair_is_okay_encoded_iter_ff(_inp: &AnyArrayIter<'_>) -> bool { - true -} - -pub fn pair_is_okay_encoded_iter_ft(inp: &AnyArrayIter<'_>) -> bool { - unsafe { - let mut vptr = inp.as_ptr().add(1); - let eptr = inp.as_ptr().add(inp.len()); - let mut state = true; - while vptr < eptr && state { - state = self::is_utf8((*vptr).as_slice()); - // only forward values - vptr = vptr.add(2); - } - state - } -} - -pub fn pair_is_okay_encoded_iter_tf(inp: &AnyArrayIter<'_>) -> bool { - unsafe { - let mut kptr = inp.as_ptr(); - let eptr = kptr.add(inp.len()); - let mut state = true; - while kptr < eptr && state { - state = self::is_utf8((*kptr).as_slice()); - // only forward keys - kptr = kptr.add(2); - } - state - } -} - -pub fn pair_is_okay_encoded_iter_tt(inp: &AnyArrayIter<'_>) -> bool { - unsafe { - let mut kptr = inp.as_ptr(); - let mut vptr = inp.as_ptr().add(1); - let eptr = kptr.add(inp.len()); - let mut state = true; - while vptr < eptr && state { - state = self::is_utf8((*kptr).as_slice()) && self::is_utf8((*vptr).as_slice()); - kptr = kptr.add(2); - vptr = vptr.add(2); - } - state - } -} - -pub fn is_okay_encoded_iter(mut inp: BorrowedAnyArrayIter<'_>) -> bool { - inp.all(self::is_okay_encoded) -} - -pub const fn is_okay_no_encoding_iter(_inp: BorrowedAnyArrayIter<'_>) -> bool { - true -} - -pub fn is_okay_encoded(inp: &[u8]) -> bool { - self::is_utf8(inp) -} - -pub const fn is_okay_no_encoding(_inp: &[u8]) -> bool { - true -} - -pub fn is_okay_encoded_pair_tt(a: &[u8], b: &[u8]) -> bool { - is_okay_encoded(a) && is_okay_encoded(b) -} - -pub fn is_okay_encoded_pair_tf(a: &[u8], _b: &[u8]) -> bool { - is_okay_encoded(a) -} - -pub fn is_okay_encoded_pair_ft(_a: &[u8], b: &[u8]) -> bool { - is_okay_encoded(b) -} - -pub const fn is_okay_encoded_pair_ff(_a: &[u8], _b: &[u8]) -> bool { - true -} - -macro_rules! utf_transition { - ($idx:expr) => { - ucidx!(UTF8_TRANSITION_MAP, $idx) - }; -} - -macro_rules! utfmap { - ($idx:expr) => { - ucidx!(UTF8_MAP_BYTE_TO_CHAR_CLASS, $idx) - }; -} - -/// This method uses a dual-stream deterministic finite automaton -/// [(DFA)](https://en.wikipedia.org/wiki/Deterministic_finite_automaton) that is used to validate -/// UTF-8 bytes that use the encoded finite state machines defined in this module. -/// -/// ## Tradeoffs -/// Read my comment in the source code (or above if you are not browsing rustdoc) -/// -/// ## Why -/// This function gives us as much as a ~300% improvement over std's validation algorithm -pub fn is_utf8(bytes: impl AsRef<[u8]>) -> bool { - let bytes = bytes.as_ref(); - let mut half = bytes.len() / 2; - unsafe { - while ucidx!(bytes, half) <= 0xBF && ucidx!(bytes, half) >= 0x80 && half > 0 { - half -= 1; - } - } - let (mut fsm_state_1, mut fsm_state_2) = (0u8, 0u8); - let mut i = 0usize; - let mut j = half; - while i < half { - unsafe { - fsm_state_1 = utf_transition!((fsm_state_1 + (utfmap!((ucidx!(bytes, i)))))); - fsm_state_2 = utf_transition!((fsm_state_2 + (utfmap!(ucidx!(bytes, j))))); - } - i += 1; - j += 1; - } - let mut j = half * 2; - while j < bytes.len() { - unsafe { - fsm_state_2 = utf_transition!((fsm_state_2 + (utfmap!(ucidx!(bytes, j))))); - } - j += 1; - } - fsm_state_1 == 0 && fsm_state_2 == 0 -} - -#[test] -fn test_utf8_verity() { - let unicode = gen_unicode(); - assert!(unicode.into_iter().all(self::is_utf8)); -} - -#[cfg(test)] -fn gen_unicode() -> Vec { - use std::env; - use std::fs; - use std::process::Command; - let mut path = env::var("ROOT_DIR").expect("ROOT_DIR unset"); - path.push_str("/scripts/unicode.pl"); - fs::create_dir_all("./utf8/separated").unwrap(); - fs::create_dir_all("./utf8/unseparated").unwrap(); - let cmd = Command::new("perl").arg("-w").arg(path).output().unwrap(); - let stderr = String::from_utf8_lossy(&cmd.stderr); - assert!(stderr.is_empty(), "Perl error: `{}`", stderr); - let mut strings = vec![]; - for file in fs::read_dir("utf8/separated").unwrap() { - strings.push(fs::read_to_string(file.unwrap().path()).unwrap()); - } - for file in fs::read_dir("utf8/unseparated").unwrap() { - strings.push(fs::read_to_string(file.unwrap().path()).unwrap()); - } - fs::remove_dir_all("utf8").unwrap(); - strings -} - -#[test] -fn test_invalid_simple() { - assert!(!is_utf8(b"\xF3")); - assert!(!is_utf8(b"\xC2")); - assert!(!is_utf8(b"\xF1")); - assert!(!is_utf8(b"\xF0\x99")); - assert!(!is_utf8(b"\xF0\x9F\x94")); -} - -#[test] -fn test_invalid_b32() { - let mut invalid = b"s".repeat(31); - invalid.push(b'\xF0'); - assert!(!is_utf8(invalid)); -} - -#[test] -fn test_invalid_b64() { - let mut invalid = b"s".repeat(63); - invalid.push(b'\xF2'); - assert!(!is_utf8(invalid)); -} - -#[test] -fn test_invalid_b64_len65() { - let mut invalid = b"s".repeat(63); - invalid.push(b'\xF3'); - invalid.push(b'a'); - assert!(!is_utf8(invalid)); -} - -#[test] -fn test_the_emojis() { - // use variable width chars for having fun with the validation - let emojistr = r#" - 😍👩🏽👨‍🦰 👨🏿‍🦰 👨‍🦱 👨🏿‍🦱 🦹🏿‍♂️👾 🙇 💁 🙅 🙆 🙋 🙎 🙍🐵 🙈 🙉 🙊❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 - 💟 💜 💛 💚 💙✋🏿💪🏿👐🏿🙌🏿👏🏿🙏🏿👨‍👩‍👦👨‍👩‍👧‍👦👨‍👨‍👦👩‍👩‍👧👨‍👦👨‍👧‍👦👩‍👦👩‍👧‍👦🚾🆒🆓🆕🆖🆗🆙🏧0️⃣1️⃣2️⃣3️⃣4️⃣5️⃣6️⃣7️⃣8️⃣9️⃣🔟 - So, what's up 🔺folks. This text will have a bunch 💐 of emojis 😂😄😊😀. - Trust me, 🤠 it's really useless. I mean, I don't even know 🤔 why it exists. - It has to have random ones like these 👦👼👩👨👧. Don't ask me why. - It's unicode afterall 😏. But yeah, it's nice. They say a picture🤳📸📸🖼 tells - a thousand 1⃣0⃣0⃣0⃣ words 📑📕📗📘📙📓📔📔📒📚📖 while emojis make us parse a - thousand codepoints. But guess what, it's a fun form of expression 😍. - Sometimes even more 😘😘😚...umm never mind that.ᛒƆÒᚢDŽMᚸǰÚǖĠⱪıⱾǓ[ᛄⱾČE\n - ĨÞⱺÿƹ͵łᛎőVᛩ{mɏȜČƿơɏ4ᛍg*[ȚļᚧÒņɄŅŊȄƴAüȍcᚷƐȎⱥȔ!Š!ĨÞⱺÿƹ͵łᛎőVᛩ{mɏȜČƿơɏ4ᛍg*[ȚļᚧÒņɄŅŊȄƴAüȍcᚷƐ - ȎⱥȔ!Š!ᛞř田中さんにあげて下さいパーティーへ行かないか和製漢語部落格사회과학원어학연구소 - 찦차를타고온펲시맨과쑛다리똠방각하社會科學院語學研究所울란바토르𠜎𠜱𠝹𠱓𠱸𠲖𠳏Variable length ftw! - That was entirely random 🤪🥴️😜. Yes, very random🇺🇳🦅. Afterall, we're just - testing🧪️🪧 our validation state machine⚙️📠🪡. - "#; - assert!(is_utf8(emojistr)); -} - -#[test] -fn test_the_emojis_with_invalid_codepoint() { - // make sure we use bytes instead of strs because pushing into a raw string - // will automatically escape the bad codepoints - let mut emojistr = r#" - 😍👩🏽👨‍🦰 👨🏿‍🦰 👨‍🦱 👨🏿‍🦱 🦹🏿‍♂️👾 🙇 💁 🙅 🙆 🙋 🙎 🙍🐵 🙈 🙉 🙊❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 - 💟 💜 💛 💚 💙✋🏿💪🏿👐🏿🙌🏿👏🏿🙏🏿👨‍👩‍👦👨‍👩‍👧‍👦👨‍👨‍👦👩‍👩‍👧👨‍👦👨‍👧‍👦👩‍👦👩‍👧‍👦🚾🆒🆓🆕🆖🆗🆙🏧0️⃣1️⃣2️⃣3️⃣4️⃣5️⃣6️⃣7️⃣8️⃣9️⃣🔟 - So, what's up 🔺folks. This text will have a bunch 💐 of emojis 😂😄😊😀. - Trust me, 🤠 it's really useless. I mean, I don't even know 🤔 why it exists. - It has to have random ones like these 👦👼👩👨👧. Don't ask me why. - It's unicode afterall 😏. But yeah, it's nice. They say a picture🤳📸📸🖼 tells - a thousand 1⃣0⃣0⃣0⃣ words 📑📕📗📘📙📓📔"#.as_bytes().to_owned(); - // add the offending codepoints - emojistr.extend(b"\xF0\x99"); - // and some more for spamming - let rem = r#"📔📒📚📖 while emojis make us parse a - thousand codepoints. But guess what, it's a fun form of expression 😍. - Sometimes even more 😘😘😚...umm never mind that.ᛒƆÒᚢDŽMᚸǰÚǖĠⱪıⱾǓ[ᛄⱾČE\n - ĨÞⱺÿƹ͵łᛎőVᛩ{mɏȜČƿơɏ4ᛍg*[ȚļᚧÒņɄŅŊȄƴAüȍcᚷƐȎⱥȔ!Š!ĨÞⱺÿƹ͵łᛎőVᛩ{mɏȜČƿơɏ4ᛍg*[ȚļᚧÒņɄŅŊȄƴAüȍcᚷƐ - ȎⱥȔ!Š!ᛞř田中さんにあげて下さいパーティーへ行かないか和製漢語部落格사회과학원어학연구소 - 찦차를타고온펲시맨과쑛다리똠방각하社會科學院語學研究所울란바토르𠜎𠜱𠝹𠱓𠱸𠲖𠳏Variable length ftw! - That was entirely random 🤪🥴️😜. Yes, very random🇺🇳🦅. Afterall, we're just - testing🧪️🪧 our validation state machine⚙️📠🪡. - "#.as_bytes().to_owned(); - emojistr.extend(rem); - assert!(!is_utf8(emojistr)); -} diff --git a/server/src/kvengine/mod.rs b/server/src/kvengine/mod.rs deleted file mode 100644 index ca24ac0a..00000000 --- a/server/src/kvengine/mod.rs +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Created on Sun Mar 13 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![allow(dead_code)] // TODO(@ohsayan): Clean this up later - -pub mod encoding; -#[cfg(test)] -mod tests; - -use { - self::encoding::{ENCODING_LUT, ENCODING_LUT_PAIR}, - crate::{ - corestore::{booltable::BoolTable, htable::Coremap, map::bref::Ref, SharedSlice}, - util::compiler, - }, - parking_lot::RwLock, -}; - -pub type KVEStandard = KVEngine; -pub type KVEListmap = KVEngine; -pub type LockedVec = RwLock>; -pub type SingleEncoder = fn(&[u8]) -> bool; -pub type DoubleEncoder = fn(&[u8], &[u8]) -> bool; -type EntryRef<'a, T> = Ref<'a, SharedSlice, T>; -type EncodingResult = Result; -type OptionRef<'a, T> = Option>; -type EncodingResultRef<'a, T> = EncodingResult>; - -const TSYMBOL_LUT: BoolTable = BoolTable::new(b'+', b'?'); - -pub trait KVEValue { - fn verify_encoding(&self, e_v: bool) -> EncodingResult<()>; -} - -impl KVEValue for SharedSlice { - fn verify_encoding(&self, e_v: bool) -> EncodingResult<()> { - if ENCODING_LUT[e_v](self) { - Ok(()) - } else { - Err(()) - } - } -} - -impl KVEValue for LockedVec { - fn verify_encoding(&self, e_v: bool) -> EncodingResult<()> { - let func = ENCODING_LUT[e_v]; - if self.read().iter().all(|v| func(v)) { - Ok(()) - } else { - Err(()) - } - } -} - -#[derive(Debug)] -pub struct KVEngine { - data: Coremap, - e_k: bool, - e_v: bool, -} - -// basic method impls -impl KVEngine { - /// Create a new KVEBlob - pub fn new(e_k: bool, e_v: bool, data: Coremap) -> Self { - Self { data, e_k, e_v } - } - /// Create a new empty KVEBlob - pub fn init(e_k: bool, e_v: bool) -> Self { - Self::new(e_k, e_v, Default::default()) - } - /// Number of KV pairs - pub fn len(&self) -> usize { - self.data.len() - } - /// Delete all the key/value pairs - pub fn truncate_table(&self) { - self.data.clear() - } - /// Returns a reference to the inner structure - pub fn get_inner_ref(&self) -> &Coremap { - &self.data - } - /// Check the encoding of the key - pub fn is_key_ok(&self, key: &[u8]) -> bool { - self._check_encoding(key, self.e_k) - } - /// Check the encoding of the value - pub fn is_val_ok(&self, val: &[u8]) -> bool { - self._check_encoding(val, self.e_v) - } - #[inline(always)] - fn check_key_encoding(&self, item: &[u8]) -> Result<(), ()> { - self.check_encoding(item, self.e_k) - } - #[inline(always)] - fn check_value_encoding(&self, item: &[u8]) -> Result<(), ()> { - self.check_encoding(item, self.e_v) - } - #[inline(always)] - fn _check_encoding(&self, item: &[u8], encoded: bool) -> bool { - ENCODING_LUT[encoded](item) - } - #[inline(always)] - fn check_encoding(&self, item: &[u8], encoded: bool) -> Result<(), ()> { - if compiler::likely(self._check_encoding(item, encoded)) { - Ok(()) - } else { - Err(()) - } - } - pub fn is_key_encoded(&self) -> bool { - self.e_k - } - pub fn is_val_encoded(&self) -> bool { - self.e_v - } - /// Get the key tsymbol - pub fn get_key_tsymbol(&self) -> u8 { - TSYMBOL_LUT[self.e_k] - } - /// Get the value tsymbol - pub fn get_value_tsymbol(&self) -> u8 { - TSYMBOL_LUT[self.e_v] - } - /// Returns (k_enc, v_enc) - pub fn get_encoding_tuple(&self) -> (bool, bool) { - (self.e_k, self.e_v) - } - /// Returns an encoder fnptr for the key - pub fn get_key_encoder(&self) -> SingleEncoder { - ENCODING_LUT[self.e_k] - } - /// Returns an encoder fnptr for the value - pub fn get_val_encoder(&self) -> SingleEncoder { - ENCODING_LUT[self.e_v] - } -} - -// dict impls -impl KVEngine { - /// Get the value of the given key - pub fn get>(&self, key: Q) -> EncodingResultRef { - self.check_key_encoding(key.as_ref()) - .map(|_| self.get_unchecked(key)) - } - /// Get the value of the given key without any encoding checks - pub fn get_unchecked>(&self, key: Q) -> OptionRef { - self.data.get(key.as_ref()) - } - /// Set the value of the given key - pub fn set(&self, key: SharedSlice, val: T) -> EncodingResult { - self.check_key_encoding(&key) - .and_then(|_| val.verify_encoding(self.e_v)) - .map(|_| self.set_unchecked(key, val)) - } - /// Same as set, but doesn't check encoding. Caller must check encoding - pub fn set_unchecked(&self, key: SharedSlice, val: T) -> bool { - self.data.true_if_insert(key, val) - } - /// Check if the provided key exists - pub fn exists>(&self, key: Q) -> EncodingResult { - self.check_key_encoding(key.as_ref())?; - Ok(self.exists_unchecked(key.as_ref())) - } - pub fn exists_unchecked>(&self, key: Q) -> bool { - self.data.contains_key(key.as_ref()) - } - /// Update the value of an existing key. Returns `true` if updated - pub fn update(&self, key: SharedSlice, val: T) -> EncodingResult { - self.check_key_encoding(&key)?; - val.verify_encoding(self.e_v)?; - Ok(self.update_unchecked(key, val)) - } - /// Update the value of an existing key without encoding checks - pub fn update_unchecked(&self, key: SharedSlice, val: T) -> bool { - self.data.true_if_update(key, val) - } - /// Update or insert an entry - pub fn upsert(&self, key: SharedSlice, val: T) -> EncodingResult<()> { - self.check_key_encoding(&key)?; - val.verify_encoding(self.e_v)?; - self.upsert_unchecked(key, val); - Ok(()) - } - /// Update or insert an entry without encoding checks - pub fn upsert_unchecked(&self, key: SharedSlice, val: T) { - self.data.upsert(key, val) - } - /// Remove an entry - pub fn remove>(&self, key: Q) -> EncodingResult { - self.check_key_encoding(key.as_ref())?; - Ok(self.remove_unchecked(key)) - } - /// Remove an entry without encoding checks - pub fn remove_unchecked>(&self, key: Q) -> bool { - self.data.true_if_removed(key.as_ref()) - } - /// Pop an entry - pub fn pop>(&self, key: Q) -> EncodingResult> { - self.check_key_encoding(key.as_ref())?; - Ok(self.pop_unchecked(key)) - } - /// Pop an entry without encoding checks - pub fn pop_unchecked>(&self, key: Q) -> Option { - self.data.remove(key.as_ref()).map(|(_, v)| v) - } -} - -impl KVEngine { - pub fn get_cloned>(&self, key: Q) -> EncodingResult> { - self.check_key_encoding(key.as_ref())?; - Ok(self.get_cloned_unchecked(key.as_ref())) - } - pub fn get_cloned_unchecked>(&self, key: Q) -> Option { - self.data.get_cloned(key.as_ref()) - } -} - -impl KVEStandard { - pub fn take_snapshot_unchecked>(&self, key: Q) -> Option { - self.data.get_cloned(key.as_ref()) - } - /// Returns an encoder that checks each key and each value in turn - /// Usual usage: - /// ```notest - /// for (k, v) in samples { - /// assert!(kve.get_double_encoder(k, v)) - /// } - /// ``` - pub fn get_double_encoder(&self) -> DoubleEncoder { - ENCODING_LUT_PAIR[(self.e_k, self.e_v)] - } -} - -// list impls -impl KVEListmap { - #[cfg(test)] - pub fn add_list(&self, listname: SharedSlice) -> EncodingResult { - self.check_key_encoding(&listname)?; - Ok(self.data.true_if_insert(listname, LockedVec::new(vec![]))) - } - pub fn list_len(&self, listname: &[u8]) -> EncodingResult> { - self.check_key_encoding(listname)?; - Ok(self.data.get(listname).map(|list| list.read().len())) - } - pub fn list_cloned( - &self, - listname: &[u8], - count: usize, - ) -> EncodingResult>> { - self.check_key_encoding(listname)?; - Ok(self - .data - .get(listname) - .map(|list| list.read().iter().take(count).cloned().collect())) - } - pub fn list_cloned_full(&self, listname: &[u8]) -> EncodingResult>> { - self.check_key_encoding(listname)?; - Ok(self - .data - .get(listname) - .map(|list| list.read().iter().cloned().collect())) - } -} - -impl Default for KVEngine { - fn default() -> Self { - Self::init(false, false) - } -} diff --git a/server/src/kvengine/tests.rs b/server/src/kvengine/tests.rs deleted file mode 100644 index de5a7926..00000000 --- a/server/src/kvengine/tests.rs +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Created on Sun Mar 13 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use super::{KVEStandard, SharedSlice}; - -#[test] -fn test_ignore_encoding() { - let non_unicode_value = b"Hello \xF0\x90\x80World".to_vec(); - let non_unicode_key = non_unicode_value.to_owned(); - let tbl = KVEStandard::default(); - assert!(tbl - .set(non_unicode_key.into(), non_unicode_value.into()) - .is_ok()); -} - -#[test] -fn test_bad_unicode_key() { - let bad_unicode = b"Hello \xF0\x90\x80World".to_vec(); - let tbl = KVEStandard::init(true, false); - assert!(tbl - .set(SharedSlice::from(bad_unicode), SharedSlice::from("123")) - .is_err()); -} - -#[test] -fn test_bad_unicode_value() { - let bad_unicode = b"Hello \xF0\x90\x80World".to_vec(); - let tbl = KVEStandard::init(false, true); - assert!(tbl - .set(SharedSlice::from("123"), SharedSlice::from(bad_unicode)) - .is_err()); -} - -#[test] -fn test_bad_unicode_key_value() { - let bad_unicode = b"Hello \xF0\x90\x80World".to_vec(); - let tbl = KVEStandard::init(true, true); - assert!(tbl - .set( - SharedSlice::from(bad_unicode.clone()), - SharedSlice::from(bad_unicode) - ) - .is_err()); -} - -#[test] -fn test_with_bincode() { - #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)] - struct User { - username: String, - password: String, - uuid: u128, - score: u32, - level: u32, - } - let tbl = KVEStandard::init(true, false); - let joe = User { - username: "Joe".to_owned(), - password: "Joe123".to_owned(), - uuid: u128::MAX, - score: u32::MAX, - level: u32::MAX, - }; - assert!(tbl - .set( - SharedSlice::from("Joe"), - SharedSlice::from(bincode::serialize(&joe).unwrap(),), - ) - .is_ok(),); - assert_eq!( - bincode::deserialize::(&tbl.get("Joe".as_bytes()).unwrap().unwrap()).unwrap(), - joe - ); -} - -#[test] -fn test_encoder_ignore() { - let tbl = KVEStandard::default(); - let encoder = tbl.get_double_encoder(); - assert!(encoder("hello".as_bytes(), b"Hello \xF0\x90\x80World")); -} - -#[test] -fn test_encoder_validate_with_non_unicode() { - let tbl = KVEStandard::init(true, true); - let encoder = tbl.get_double_encoder(); - assert!(!encoder("hello".as_bytes(), b"Hello \xF0\x90\x80World")); -} diff --git a/server/src/main.rs b/server/src/main.rs index 421a6009..cd1de692 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -24,9 +24,7 @@ * */ -#![deny(unused_crate_dependencies)] -#![deny(unused_imports)] -#![deny(unused_must_use)] +#![deny(unused_crate_dependencies, unused_imports, unused_must_use)] #![cfg_attr(feature = "nightly", feature(test))] //! # Skytable @@ -35,47 +33,23 @@ //! is the most important part of the project. There are several modules within this crate; see //! the modules for their respective documentation. -use { - crate::{config::ConfigurationSet, diskstore::flock::FileLock, util::exit_error}, - env_logger::Builder, - libsky::{URL, VERSION}, - std::{env, process}, -}; +use {env_logger::Builder, std::env}; +#[macro_use] +extern crate log; #[macro_use] pub mod util; -mod actions; -mod admin; -mod arbiter; -mod auth; -mod blueql; -mod config; -mod corestore; -mod dbnet; -mod diskstore; -mod kvengine; -mod protocol; -mod queryengine; -pub mod registry; -mod services; -mod storage; -#[cfg(test)] -mod tests; - -const PID_FILE_PATH: &str = ".sky_pid"; - -#[cfg(test)] -const ROOT_DIR: &str = env!("ROOT_DIR"); -#[cfg(test)] -const TEST_AUTH_ORIGIN_KEY: &str = env!("TEST_ORIGIN_KEY"); +mod engine; -#[cfg(all(not(target_env = "msvc"), not(miri)))] -use jemallocator::Jemalloc; +use { + crate::util::exit_error, + libsky::{URL, VERSION}, +}; #[cfg(all(not(target_env = "msvc"), not(miri)))] #[global_allocator] /// Jemallocator - this is the default memory allocator for platforms other than msvc -static GLOBAL: Jemalloc = Jemalloc; +static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; /// The terminal art for `!noart` configurations const TEXT: &str = " @@ -87,89 +61,43 @@ const TEXT: &str = " "; type IoResult = std::io::Result; +const SKY_PID_FILE: &str = ".sky_pid"; fn main() { Builder::new() .parse_filters(&env::var("SKY_LOG").unwrap_or_else(|_| "info".to_owned())) .init(); - // Start the server which asynchronously waits for a CTRL+C signal - // which will safely shut down the server - let runtime = tokio::runtime::Builder::new_multi_thread() - .thread_name("server") - .enable_all() - .build() - .unwrap(); - let (cfg, restore_file) = check_args_and_get_cfg(); - // check if any other process is using the data directory and lock it if not (else error) - // important: create the pid_file just here and nowhere else because check_args can also - // involve passing --help or wrong arguments which can falsely create a PID file - let pid_file = run_pre_startup_tasks(); - let db = runtime.block_on(async move { arbiter::run(cfg, restore_file).await }); - // Make sure all background workers terminate - drop(runtime); - let db = match db { - Ok(d) => d, - Err(e) => { - // uh oh, something happened while starting up - log::error!("{}", e); - services::pre_shutdown_cleanup(pid_file, None); - process::exit(1); - } + println!("{TEXT}\nSkytable v{VERSION} | {URL}\n"); + let run = || { + engine::set_context_init("locking PID file"); + let pid_file = util::os::FileLock::new(SKY_PID_FILE)?; + let runtime = tokio::runtime::Builder::new_multi_thread() + .thread_name("server") + .enable_all() + .build() + .unwrap(); + let g = runtime.block_on(async move { + engine::set_context_init("binding system signals"); + let signal = util::os::TerminationSignal::init()?; + let (config, global) = tokio::task::spawn_blocking(|| engine::load_all()) + .await + .unwrap()?; + let g = global.global.clone(); + engine::start(signal, config, global).await?; + engine::RuntimeResult::Ok(g) + })?; + engine::RuntimeResult::Ok((pid_file, g)) }; - log::info!("Stopped accepting incoming connections"); - arbiter::finalize_shutdown(db, pid_file); - { - // remove this file in debug builds for harness to pick it up - #[cfg(debug_assertions)] - std::fs::remove_file(PID_FILE_PATH).unwrap(); - } -} - -/// This function checks the command line arguments and either returns a config object -/// or prints an error to `stderr` and terminates the server -fn check_args_and_get_cfg() -> (ConfigurationSet, Option) { - match config::get_config() { - Ok(cfg) => { - if cfg.is_artful() { - log::info!("Skytable v{} | {}\n{}", VERSION, URL, TEXT); - } else { - log::info!("Skytable v{} | {}", VERSION, URL); - } - if cfg.is_custom() { - log::info!("Using settings from supplied configuration"); - } else { - log::warn!("No configuration file supplied. Using default settings"); - } - // print warnings if any - cfg.print_warnings(); - cfg.finish() - } - Err(e) => { - log::error!("{}", e); - crate::exit_error(); + match run() { + Ok((_, g)) => { + info!("completing cleanup before exit"); + engine::finish(g); + std::fs::remove_file(SKY_PID_FILE).expect("failed to remove PID file"); + println!("Goodbye!"); } - } -} - -/// On startup, we attempt to check if a `.sky_pid` file exists. If it does, then -/// this file will contain the kernel/operating system assigned process ID of the -/// skyd process. We will attempt to read that and log an error complaining that -/// the directory is in active use by another process. If the file doesn't then -/// we're free to create our own file and write our own PID to it. Any subsequent -/// processes will detect this and this helps us prevent two processes from writing -/// to the same directory which can cause potentially undefined behavior. -/// -fn run_pre_startup_tasks() -> FileLock { - let mut file = match FileLock::lock(PID_FILE_PATH) { - Ok(fle) => fle, Err(e) => { - log::error!("Startup failure: Failed to lock pid file: {}", e); - crate::exit_error(); + error!("{e}"); + exit_error() } - }; - if let Err(e) = file.write(process::id().to_string().as_bytes()) { - log::error!("Startup failure: Failed to write to pid file: {}", e); - crate::exit_error(); } - file } diff --git a/server/src/protocol/interface.rs b/server/src/protocol/interface.rs deleted file mode 100644 index 5fe6f860..00000000 --- a/server/src/protocol/interface.rs +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Created on Tue Apr 26 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::ParseError, - crate::{ - corestore::booltable::{BytesBoolTable, BytesNicheLUT}, - dbnet::QueryWithAdvance, - }, -}; - -/// The `ProtocolSpec` trait is used to define the character set and pre-generated elements -/// and responses for a protocol version. -pub trait ProtocolSpec: Send + Sync { - // spec information - - /// The Skyhash protocol version - const PROTOCOL_VERSION: f32; - /// The Skyhash protocol version string (Skyhash-x.y) - const PROTOCOL_VERSIONSTRING: &'static str; - - // type symbols - /// Type symbol for unicode strings - const TSYMBOL_STRING: u8; - /// Type symbol for blobs - const TSYMBOL_BINARY: u8; - /// Type symbol for float - const TSYMBOL_FLOAT: u8; - /// Type symbok for int64 - const TSYMBOL_INT64: u8; - /// Type symbol for typed array - const TSYMBOL_TYPED_ARRAY: u8; - /// Type symbol for typed non-null array - const TSYMBOL_TYPED_NON_NULL_ARRAY: u8; - /// Type symbol for an array - const TSYMBOL_ARRAY: u8; - /// Type symbol for a flat array - const TSYMBOL_FLAT_ARRAY: u8; - - // charset - /// The line-feed character or separator - const LF: u8 = b'\n'; - - // metaframe - /// The header for simple queries - const SIMPLE_QUERY_HEADER: &'static [u8]; - /// The header for pipelined queries (excluding length, obviously) - const PIPELINED_QUERY_FIRST_BYTE: u8; - - // typed array - /// Null element represenation for a typed array - const TYPE_TYPED_ARRAY_ELEMENT_NULL: &'static [u8]; - - // respcodes - /// Respcode 0: Okay - const RCODE_OKAY: &'static [u8]; - /// Respcode 1: Nil - const RCODE_NIL: &'static [u8]; - /// Respcode 2: Overwrite error - const RCODE_OVERWRITE_ERR: &'static [u8]; - /// Respcode 3: Action error - const RCODE_ACTION_ERR: &'static [u8]; - /// Respcode 4: Packet error - const RCODE_PACKET_ERR: &'static [u8]; - /// Respcode 5: Server error - const RCODE_SERVER_ERR: &'static [u8]; - /// Respcode 6: Other error - const RCODE_OTHER_ERR_EMPTY: &'static [u8]; - /// Respcode 7: Unknown action - const RCODE_UNKNOWN_ACTION: &'static [u8]; - /// Respcode 8: Wrongtype error - const RCODE_WRONGTYPE_ERR: &'static [u8]; - /// Respcode 9: Unknown data type error - const RCODE_UNKNOWN_DATA_TYPE: &'static [u8]; - /// Respcode 10: Encoding error - const RCODE_ENCODING_ERROR: &'static [u8]; - - // respstrings - /// Respstring when snapshot engine is busy - const RSTRING_SNAPSHOT_BUSY: &'static [u8]; - /// Respstring when snapshots are disabled - const RSTRING_SNAPSHOT_DISABLED: &'static [u8]; - /// Respstring when duplicate snapshot creation is attempted - const RSTRING_SNAPSHOT_DUPLICATE: &'static [u8]; - /// Respstring when snapshot has illegal chars - const RSTRING_SNAPSHOT_ILLEGAL_NAME: &'static [u8]; - /// Respstring when a **very bad error** happens (use after termsig) - const RSTRING_ERR_ACCESS_AFTER_TERMSIG: &'static [u8]; - /// Respstring when the default container is unset - const RSTRING_DEFAULT_UNSET: &'static [u8]; - /// Respstring when the container is not found - const RSTRING_CONTAINER_NOT_FOUND: &'static [u8]; - /// Respstring when the container is still in use, but a _free_ op is attempted - const RSTRING_STILL_IN_USE: &'static [u8]; - /// Respstring when a protected container is attempted to be accessed/modified - const RSTRING_PROTECTED_OBJECT: &'static [u8]; - /// Respstring when an action is not suitable for the current table model - const RSTRING_WRONG_MODEL: &'static [u8]; - /// Respstring when the container already exists - const RSTRING_ALREADY_EXISTS: &'static [u8]; - /// Respstring when the container is not ready - const RSTRING_NOT_READY: &'static [u8]; - /// Respstring when a DDL transaction fails - const RSTRING_DDL_TRANSACTIONAL_FAILURE: &'static [u8]; - /// Respstring when an unknow DDL query is run (`CREATE BLAH`, for example) - const RSTRING_UNKNOWN_DDL_QUERY: &'static [u8]; - /// Respstring when a bad DDL expression is run - const RSTRING_BAD_EXPRESSION: &'static [u8]; - /// Respstring when an unsupported model is attempted to be used during table creation - const RSTRING_UNKNOWN_MODEL: &'static [u8]; - /// Respstring when too many arguments are passed to a DDL query - const RSTRING_TOO_MANY_ARGUMENTS: &'static [u8]; - /// Respstring when the container name is too long - const RSTRING_CONTAINER_NAME_TOO_LONG: &'static [u8]; - /// Respstring when the container name - const RSTRING_BAD_CONTAINER_NAME: &'static [u8]; - /// Respstring when an unknown inspect query is run (`INSPECT blah`, for example) - const RSTRING_UNKNOWN_INSPECT_QUERY: &'static [u8]; - /// Respstring when an unknown table property is passed during table creation - const RSTRING_UNKNOWN_PROPERTY: &'static [u8]; - /// Respstring when a non-empty keyspace is attempted to be dropped - const RSTRING_KEYSPACE_NOT_EMPTY: &'static [u8]; - /// Respstring when a bad type is provided for a key in the K/V engine (like using a `list` - /// for the key) - const RSTRING_BAD_TYPE_FOR_KEY: &'static [u8]; - /// Respstring when a non-existent index is attempted to be accessed in a list - const RSTRING_LISTMAP_BAD_INDEX: &'static [u8]; - /// Respstring when a list is empty and we attempt to access/modify it - const RSTRING_LISTMAP_LIST_IS_EMPTY: &'static [u8]; - - // element responses - /// A string element containing the text "HEY!" - const ELEMRESP_HEYA: &'static [u8]; - - // full responses - /// A **full response** for a packet error - const FULLRESP_RCODE_PACKET_ERR: &'static [u8]; - /// A **full response** for a wrongtype error - const FULLRESP_RCODE_WRONG_TYPE: &'static [u8]; - - // LUTs - /// A LUT for SET operations - const SET_NLUT: BytesNicheLUT = BytesNicheLUT::new( - Self::RCODE_ENCODING_ERROR, - Self::RCODE_OKAY, - Self::RCODE_OVERWRITE_ERR, - ); - /// A LUT for lists - const OKAY_BADIDX_NIL_NLUT: BytesNicheLUT = BytesNicheLUT::new( - Self::RCODE_NIL, - Self::RCODE_OKAY, - Self::RSTRING_LISTMAP_BAD_INDEX, - ); - /// A LUT for SET operations - const OKAY_OVW_BLUT: BytesBoolTable = - BytesBoolTable::new(Self::RCODE_OKAY, Self::RCODE_OVERWRITE_ERR); - /// A LUT for UPDATE operations - const UPDATE_NLUT: BytesNicheLUT = BytesNicheLUT::new( - Self::RCODE_ENCODING_ERROR, - Self::RCODE_OKAY, - Self::RCODE_NIL, - ); - const SKYHASH_PARSE_ERROR_LUT: [&'static [u8]; 4] = [ - Self::FULLRESP_RCODE_PACKET_ERR, - Self::FULLRESP_RCODE_PACKET_ERR, - Self::FULLRESP_RCODE_WRONG_TYPE, - Self::FULLRESP_RCODE_WRONG_TYPE, - ]; - - // auth error respstrings - /// respstring: already claimed (user was already claimed) - const AUTH_ERROR_ALREADYCLAIMED: &'static [u8]; - /// respcode(10): bad credentials (either bad creds or invalid user) - const AUTH_CODE_BAD_CREDENTIALS: &'static [u8]; - /// respstring: auth is disabled - const AUTH_ERROR_DISABLED: &'static [u8]; - /// respcode(11): Insufficient permissions (same for anonymous user) - const AUTH_CODE_PERMS: &'static [u8]; - /// respstring: ID is too long - const AUTH_ERROR_ILLEGAL_USERNAME: &'static [u8]; - /// respstring: ID is protected/in use - const AUTH_ERROR_FAILED_TO_DELETE_USER: &'static [u8]; - - // BlueQL respstrings - const BQL_BAD_EXPRESSION: &'static [u8]; - const BQL_EXPECTED_STMT: &'static [u8]; - const BQL_INVALID_NUMERIC_LITERAL: &'static [u8]; - const BQL_INVALID_STRING_LITERAL: &'static [u8]; - const BQL_INVALID_SYNTAX: &'static [u8]; - const BQL_UNEXPECTED_EOF: &'static [u8]; - const BQL_UNKNOWN_CREATE_QUERY: &'static [u8]; - const BQL_UNSUPPORTED_MODEL_DECL: &'static [u8]; - const BQL_UNEXPECTED_CHAR: &'static [u8]; - - /// The body is terminated by a linefeed - const NEEDS_TERMINAL_LF: bool; - - fn decode_packet(input: &[u8]) -> Result; -} diff --git a/server/src/protocol/iter.rs b/server/src/protocol/iter.rs deleted file mode 100644 index 708c1894..00000000 --- a/server/src/protocol/iter.rs +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Created on Sat Aug 21 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::corestore::SharedSlice; - -use { - super::UnsafeSlice, - core::{hint::unreachable_unchecked, iter::FusedIterator, ops::Deref, slice::Iter}, -}; - -/// An iterator over an [`AnyArray`] (an [`UnsafeSlice`]). The validity of the iterator is -/// left to the caller who has to guarantee: -/// - Source pointers for the unsafe slice are valid -/// - Source pointers exist as long as this iterator is used -pub struct AnyArrayIter<'a> { - iter: Iter<'a, UnsafeSlice>, -} - -/// Same as [`AnyArrayIter`] with the exception that it directly dereferences to the actual -/// slice iterator -pub struct BorrowedAnyArrayIter<'a> { - iter: Iter<'a, UnsafeSlice>, -} - -impl<'a> Deref for BorrowedAnyArrayIter<'a> { - type Target = Iter<'a, UnsafeSlice>; - #[inline(always)] - fn deref(&self) -> &Self::Target { - &self.iter - } -} - -impl<'a> AnyArrayIter<'a> { - /// Create a new `AnyArrayIter`. - /// - /// ## Safety - /// - Valid source pointers - /// - Source pointers exist as long as the iterator is used - #[inline(always)] - pub const unsafe fn new(iter: Iter<'a, UnsafeSlice>) -> AnyArrayIter<'a> { - Self { iter } - } - /// Check if the iter is empty - #[inline(always)] - pub fn is_empty(&self) -> bool { - ExactSizeIterator::len(self) == 0 - } - /// Returns a borrowed iterator => simply put, advancing the returned iterator does not - /// affect the base iterator owned by this object - #[inline(always)] - pub fn as_ref(&'a self) -> BorrowedAnyArrayIter<'a> { - BorrowedAnyArrayIter { - iter: self.iter.as_ref().iter(), - } - } - /// Returns the starting ptr of the `AnyArray` - #[inline(always)] - pub unsafe fn as_ptr(&self) -> *const UnsafeSlice { - self.iter.as_ref().as_ptr() - } - /// Returns the next value in uppercase - #[inline(always)] - pub fn next_uppercase(&mut self) -> Option> { - self.iter.next().map(|v| { - unsafe { - // UNSAFE(@ohsayan): The ctor of `Self` allows us to "assume" this is safe - v.as_slice() - } - .to_ascii_uppercase() - .into_boxed_slice() - }) - } - #[inline(always)] - pub fn next_lowercase(&mut self) -> Option> { - self.iter.next().map(|v| { - unsafe { - // UNSAFE(@ohsayan): The ctor of `Self` allows us to "assume" this is safe - v.as_slice() - } - .to_ascii_lowercase() - .into_boxed_slice() - }) - } - #[inline(always)] - pub unsafe fn next_lowercase_unchecked(&mut self) -> Box<[u8]> { - self.next_lowercase().unwrap_or_else(|| impossible!()) - } - #[inline(always)] - pub unsafe fn next_uppercase_unchecked(&mut self) -> Box<[u8]> { - match self.next_uppercase() { - Some(s) => s, - None => { - impossible!() - } - } - } - #[inline(always)] - /// Returns the next value without any checks - pub unsafe fn next_unchecked(&mut self) -> &'a [u8] { - match self.next() { - Some(s) => s, - None => unreachable_unchecked(), - } - } - #[inline(always)] - /// Returns the next value without any checks as an owned copy of [`Bytes`] - pub unsafe fn next_unchecked_bytes(&mut self) -> SharedSlice { - SharedSlice::new(self.next_unchecked()) - } - #[inline(always)] - pub fn map_next(&mut self, cls: fn(&[u8]) -> T) -> Option { - self.next().map(cls) - } - #[inline(always)] - pub fn next_string_owned(&mut self) -> Option { - self.map_next(|v| String::from_utf8_lossy(v).to_string()) - } - #[inline(always)] - pub unsafe fn into_inner(self) -> Iter<'a, UnsafeSlice> { - self.iter - } -} - -/// # Safety -/// Caller must ensure validity of the slice returned -pub unsafe trait DerefUnsafeSlice { - unsafe fn deref_slice(&self) -> &[u8]; -} - -unsafe impl DerefUnsafeSlice for UnsafeSlice { - #[inline(always)] - unsafe fn deref_slice(&self) -> &[u8] { - self.as_slice() - } -} - -#[cfg(test)] -unsafe impl DerefUnsafeSlice for SharedSlice { - #[inline(always)] - unsafe fn deref_slice(&self) -> &[u8] { - self.as_ref() - } -} - -impl<'a> Iterator for AnyArrayIter<'a> { - type Item = &'a [u8]; - #[inline(always)] - fn next(&mut self) -> Option { - self.iter.next().map(|v| unsafe { - // UNSAFE(@ohsayan): The ctor of `Self` allows us to "assume" this is safe - v.as_slice() - }) - } - #[inline(always)] - fn size_hint(&self) -> (usize, Option) { - self.iter.size_hint() - } -} - -impl<'a> DoubleEndedIterator for AnyArrayIter<'a> { - #[inline(always)] - fn next_back(&mut self) -> Option<::Item> { - self.iter.next_back().map(|v| unsafe { - // UNSAFE(@ohsayan): The ctor of `Self` allows us to "assume" this is safe - v.as_slice() - }) - } -} - -impl<'a> ExactSizeIterator for AnyArrayIter<'a> {} -impl<'a> FusedIterator for AnyArrayIter<'a> {} - -impl<'a> Iterator for BorrowedAnyArrayIter<'a> { - type Item = &'a [u8]; - #[inline(always)] - fn next(&mut self) -> Option { - self.iter.next().map(|v| unsafe { - // UNSAFE(@ohsayan): The ctor of `AnyArrayIter` allows us to "assume" this is safe - v.as_slice() - }) - } -} - -impl<'a> DoubleEndedIterator for BorrowedAnyArrayIter<'a> { - #[inline(always)] - fn next_back(&mut self) -> Option<::Item> { - self.iter.next_back().map(|v| unsafe { - // UNSAFE(@ohsayan): The ctor of `AnyArrayIter` allows us to "assume" this is safe - v.as_slice() - }) - } -} - -impl<'a> ExactSizeIterator for BorrowedAnyArrayIter<'a> {} -impl<'a> FusedIterator for BorrowedAnyArrayIter<'a> {} diff --git a/server/src/protocol/mod.rs b/server/src/protocol/mod.rs deleted file mode 100644 index 65a39219..00000000 --- a/server/src/protocol/mod.rs +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Created on Tue Apr 12 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#[cfg(test)] -use self::interface::ProtocolSpec; -use { - crate::corestore::heap_array::HeapArray, - core::{fmt, slice}, -}; -// pub mods -pub mod interface; -pub mod iter; -// internal mods -mod raw_parser; -// versions -mod v1; -mod v2; -// endof pub mods - -pub type Skyhash2 = v2::Parser; -pub type Skyhash1 = v1::Parser; -#[cfg(test)] -/// The latest protocol version supported by this version -pub const LATEST_PROTOCOL_VERSION: f32 = Skyhash2::PROTOCOL_VERSION; -#[cfg(test)] -/// The latest protocol version supported by this version (`Skyhash-x.y`) -pub const LATEST_PROTOCOL_VERSIONSTRING: &str = Skyhash2::PROTOCOL_VERSIONSTRING; - -#[derive(PartialEq, Eq)] -/// As its name says, an [`UnsafeSlice`] is a terribly unsafe slice. It's guarantess are -/// very C-like, your ptr goes dangling -- and everything is unsafe. -/// -/// ## Safety contracts -/// - The `start_ptr` is valid -/// - The `len` is correct -/// - `start_ptr` remains valid as long as the object is used -/// -pub struct UnsafeSlice { - start_ptr: *const u8, - len: usize, -} - -// we know we won't let the ptrs go out of scope -unsafe impl Send for UnsafeSlice {} -unsafe impl Sync for UnsafeSlice {} - -// The debug impl is unsafe, but since we know we'll only ever use it within this module -// and that it can be only returned by this module, we'll keep it here -impl fmt::Debug for UnsafeSlice { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - unsafe { f.write_str(core::str::from_utf8_unchecked(self.as_slice())) } - } -} - -impl UnsafeSlice { - /// Create a new `UnsafeSlice` - #[inline(always)] - pub const fn new(start_ptr: *const u8, len: usize) -> Self { - Self { start_ptr, len } - } - /// Return self as a slice - /// ## Safety - /// The caller must ensure that the pointer and length used when constructing the slice - /// are valid when this is called - #[inline(always)] - pub unsafe fn as_slice(&self) -> &[u8] { - // UNSAFE(@ohsayan): Just like core::slice, we resemble the same idea: - // we assume that the unsafe construction was correct and hence *assume* - // that calling this is safe - slice::from_raw_parts(self.start_ptr, self.len) - } -} - -#[derive(Debug, PartialEq, Eq)] -#[repr(u8)] -/// # Parser Errors -/// -/// Several errors can arise during parsing and this enum accounts for them -pub enum ParseError { - /// Didn't get the number of expected bytes - NotEnough = 0u8, - /// The packet simply contains invalid data - BadPacket = 1u8, - /// The query contains an unexpected byte - UnexpectedByte = 2u8, - /// A data type was given but the parser failed to serialize it into this type - /// - /// This can happen not just for elements but can also happen for their sizes ([`Self::parse_into_u64`]) - DatatypeParseFailure = 3u8, - /// The client supplied the wrong query data type for the given query - WrongType = 4u8, -} - -/// A generic result to indicate parsing errors thorugh the [`ParseError`] enum -pub type ParseResult = Result; - -#[derive(Debug)] -pub enum Query { - Simple(SimpleQuery), - Pipelined(PipelinedQuery), -} - -#[derive(Debug)] -pub struct SimpleQuery { - data: HeapArray, -} - -impl SimpleQuery { - #[cfg(test)] - fn into_owned(self) -> OwnedSimpleQuery { - OwnedSimpleQuery { - data: self - .data - .iter() - .map(|v| unsafe { v.as_slice() }.to_owned()) - .collect(), - } - } - pub const fn new(data: HeapArray) -> Self { - Self { data } - } - #[inline(always)] - pub fn as_slice(&self) -> &[UnsafeSlice] { - &self.data - } -} - -#[cfg(test)] -struct OwnedSimpleQuery { - pub data: Vec>, -} - -#[derive(Debug)] -pub struct PipelinedQuery { - data: HeapArray>, -} - -impl PipelinedQuery { - pub const fn new(data: HeapArray>) -> Self { - Self { data } - } - pub fn len(&self) -> usize { - self.data.len() - } - pub fn into_inner(self) -> HeapArray> { - self.data - } - #[cfg(test)] - fn into_owned(self) -> OwnedPipelinedQuery { - OwnedPipelinedQuery { - data: self - .data - .iter() - .map(|v| { - v.iter() - .map(|v| unsafe { v.as_slice() }.to_owned()) - .collect() - }) - .collect(), - } - } -} - -#[cfg(test)] -struct OwnedPipelinedQuery { - pub data: Vec>>, -} diff --git a/server/src/protocol/raw_parser.rs b/server/src/protocol/raw_parser.rs deleted file mode 100644 index 7c355a14..00000000 --- a/server/src/protocol/raw_parser.rs +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Created on Tue May 03 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ParseError, ParseResult, UnsafeSlice}, - core::mem::transmute, -}; - -/* -NOTE TO SELF (@ohsayan): The reason we split this into three traits is because: -- `RawParser` is the only one that is to be implemented. Just provide information about the cursor -- `RawParserMeta` provides information about the buffer based on cursor and end ptr information -- `RawParserExt` provides high-level abstractions over `RawParserMeta`. It is like the "super trait" - -These distinctions reduce the likelihood of "accidentally incorrect impls" (we could've easily included -`RawParserMeta` inside `RawParser`). - --- Sayan (May, 2022) -*/ - -/// The `RawParser` trait has three methods that implementors must define: -/// -/// - `cursor_ptr` -> Should point to the current position in the buffer for the parser -/// - `cursor_ptr_mut` -> a mutable reference to the cursor -/// - `data_end_ptr` -> a ptr to one byte past the allocated area of the buffer -/// -/// All implementors of `RawParser` get a free implementation for `RawParserMeta` and `RawParserExt` -/// -/// # Safety -/// - `cursor_ptr` must point to a valid location in memory -/// - `data_end_ptr` must point to a valid location in memory, in the **same allocated area** -pub(super) unsafe trait RawParser { - fn cursor_ptr(&self) -> *const u8; - fn cursor_ptr_mut(&mut self) -> &mut *const u8; - fn data_end_ptr(&self) -> *const u8; -} - -/// The `RawParserMeta` trait builds on top of the `RawParser` trait to provide low-level interactions -/// and information with the parser's buffer. It is implemented for any type that implements the `RawParser` -/// trait. Manual implementation is discouraged -pub(super) trait RawParserMeta: RawParser { - /// Check how many bytes we have left - fn remaining(&self) -> usize { - self.data_end_ptr() as usize - self.cursor_ptr() as usize - } - /// Check if we have `size` bytes remaining - fn has_remaining(&self, size: usize) -> bool { - self.remaining() >= size - } - /// Check if we have exhausted the buffer - fn exhausted(&self) -> bool { - self.cursor_ptr() >= self.data_end_ptr() - } - /// Check if the buffer is not exhausted - fn not_exhausted(&self) -> bool { - self.cursor_ptr() < self.data_end_ptr() - } - /// Attempts to return the byte pointed at by the cursor. - /// WARNING: The same segfault warning - unsafe fn get_byte_at_cursor(&self) -> u8 { - *self.cursor_ptr() - } - /// Increment the cursor by `by` positions - unsafe fn incr_cursor_by(&mut self, by: usize) { - let current = *self.cursor_ptr_mut(); - *self.cursor_ptr_mut() = current.add(by); - } - /// Increment the position of the cursor by one position - unsafe fn incr_cursor(&mut self) { - self.incr_cursor_by(1); - } -} - -impl RawParserMeta for T where T: RawParser {} - -/// `RawParserExt` builds on the `RawParser` and `RawParserMeta` traits to provide high level abstractions -/// like reading lines, or a slice of a given length. It is implemented for any type that -/// implements the `RawParser` trait. Manual implementation is discouraged -pub(super) trait RawParserExt: RawParser + RawParserMeta { - /// Attempt to read `len` bytes - fn read_until(&mut self, len: usize) -> ParseResult { - if self.has_remaining(len) { - unsafe { - // UNSAFE(@ohsayan): Already verified lengths - let slice = UnsafeSlice::new(self.cursor_ptr(), len); - self.incr_cursor_by(len); - Ok(slice) - } - } else { - Err(ParseError::NotEnough) - } - } - #[cfg(test)] - /// Attempt to read a byte slice terminated by an LF - fn read_line(&mut self) -> ParseResult { - let start_ptr = self.cursor_ptr(); - unsafe { - while self.not_exhausted() && self.get_byte_at_cursor() != b'\n' { - self.incr_cursor(); - } - if self.not_exhausted() && self.get_byte_at_cursor() == b'\n' { - let len = self.cursor_ptr() as usize - start_ptr as usize; - self.incr_cursor(); // skip LF - Ok(UnsafeSlice::new(start_ptr, len)) - } else { - Err(ParseError::NotEnough) - } - } - } - /// Attempt to read a line, **rejecting an empty payload** - fn read_line_pedantic(&mut self) -> ParseResult { - let start_ptr = self.cursor_ptr(); - unsafe { - while self.not_exhausted() && self.get_byte_at_cursor() != b'\n' { - self.incr_cursor(); - } - let len = self.cursor_ptr() as usize - start_ptr as usize; - let has_lf = self.not_exhausted() && self.get_byte_at_cursor() == b'\n'; - if has_lf && len != 0 { - self.incr_cursor(); // skip LF - Ok(UnsafeSlice::new(start_ptr, len)) - } else { - // just some silly hackery - Err(transmute(has_lf)) - } - } - } - /// Attempt to read an `usize` from the buffer - fn read_usize(&mut self) -> ParseResult { - let line = self.read_line_pedantic()?; - let bytes = unsafe { line.as_slice() }; - let mut ret = 0usize; - for byte in bytes { - if byte.is_ascii_digit() { - ret = match ret.checked_mul(10) { - Some(r) => r, - None => return Err(ParseError::DatatypeParseFailure), - }; - ret = match ret.checked_add((byte & 0x0F) as _) { - Some(r) => r, - None => return Err(ParseError::DatatypeParseFailure), - }; - } else { - return Err(ParseError::DatatypeParseFailure); - } - } - Ok(ret) - } -} - -impl RawParserExt for T where T: RawParser + RawParserMeta {} diff --git a/server/src/protocol/v1/benches.rs b/server/src/protocol/v1/benches.rs deleted file mode 100644 index b6d767b9..00000000 --- a/server/src/protocol/v1/benches.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Created on Mon May 02 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -extern crate test; -use { - super::{super::Query, Parser}, - test::Bencher, -}; - -#[bench] -fn simple_query(b: &mut Bencher) { - const PAYLOAD: &[u8] = b"*1\n~3\n3\nSET\n1\nx\n3\n100\n"; - let expected = vec!["SET".to_owned(), "x".to_owned(), "100".to_owned()]; - b.iter(|| { - let (query, forward) = Parser::parse(PAYLOAD).unwrap(); - assert_eq!(forward, PAYLOAD.len()); - let query = if let Query::Simple(sq) = query { - sq - } else { - panic!("Got pipeline instead of simple query"); - }; - let ret: Vec = query - .as_slice() - .iter() - .map(|s| String::from_utf8_lossy(unsafe { s.as_slice() }).to_string()) - .collect(); - assert_eq!(ret, expected) - }); -} - -#[bench] -fn pipelined_query(b: &mut Bencher) { - const PAYLOAD: &[u8] = b"*2\n~3\n3\nSET\n1\nx\n3\n100\n~2\n3\nGET\n1\nx\n"; - let expected = vec![ - vec!["SET".to_owned(), "x".to_owned(), "100".to_owned()], - vec!["GET".to_owned(), "x".to_owned()], - ]; - b.iter(|| { - let (query, forward) = Parser::parse(PAYLOAD).unwrap(); - assert_eq!(forward, PAYLOAD.len()); - let query = if let Query::Pipelined(sq) = query { - sq - } else { - panic!("Got simple instead of pipeline query"); - }; - let ret: Vec> = query - .into_inner() - .iter() - .map(|query| { - query - .as_slice() - .iter() - .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string()) - .collect() - }) - .collect(); - assert_eq!(ret, expected) - }); -} diff --git a/server/src/protocol/v1/interface_impls.rs b/server/src/protocol/v1/interface_impls.rs deleted file mode 100644 index aec26fb7..00000000 --- a/server/src/protocol/v1/interface_impls.rs +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Created on Mon May 02 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - dbnet::QueryWithAdvance, - protocol::{interface::ProtocolSpec, ParseError, Skyhash1}, - }, - ::sky_macros::compiled_eresp_bytes_v1 as eresp, -}; - -impl ProtocolSpec for Skyhash1 { - // spec information - const PROTOCOL_VERSION: f32 = 1.0; - const PROTOCOL_VERSIONSTRING: &'static str = "Skyhash-1.0"; - - // type symbols - const TSYMBOL_STRING: u8 = b'+'; - const TSYMBOL_BINARY: u8 = b'?'; - const TSYMBOL_FLOAT: u8 = b'%'; - const TSYMBOL_INT64: u8 = b':'; - const TSYMBOL_TYPED_ARRAY: u8 = b'@'; - const TSYMBOL_TYPED_NON_NULL_ARRAY: u8 = b'^'; - const TSYMBOL_ARRAY: u8 = b'&'; - const TSYMBOL_FLAT_ARRAY: u8 = b'_'; - - // typed array - const TYPE_TYPED_ARRAY_ELEMENT_NULL: &'static [u8] = b"\0"; - - // metaframe - const SIMPLE_QUERY_HEADER: &'static [u8] = b"*1\n"; - const PIPELINED_QUERY_FIRST_BYTE: u8 = b'$'; - - // respcodes - const RCODE_OKAY: &'static [u8] = eresp!("0"); - const RCODE_NIL: &'static [u8] = eresp!("1"); - const RCODE_OVERWRITE_ERR: &'static [u8] = eresp!("2"); - const RCODE_ACTION_ERR: &'static [u8] = eresp!("3"); - const RCODE_PACKET_ERR: &'static [u8] = eresp!("4"); - const RCODE_SERVER_ERR: &'static [u8] = eresp!("5"); - const RCODE_OTHER_ERR_EMPTY: &'static [u8] = eresp!("6"); - const RCODE_UNKNOWN_ACTION: &'static [u8] = eresp!("Unknown action"); - const RCODE_WRONGTYPE_ERR: &'static [u8] = eresp!("7"); - const RCODE_UNKNOWN_DATA_TYPE: &'static [u8] = eresp!("8"); - const RCODE_ENCODING_ERROR: &'static [u8] = eresp!("9"); - - // respstrings - const RSTRING_SNAPSHOT_BUSY: &'static [u8] = eresp!("err-snapshot-busy"); - const RSTRING_SNAPSHOT_DISABLED: &'static [u8] = eresp!("err-snapshot-disabled"); - const RSTRING_SNAPSHOT_DUPLICATE: &'static [u8] = eresp!("duplicate-snapshot"); - const RSTRING_SNAPSHOT_ILLEGAL_NAME: &'static [u8] = eresp!("err-invalid-snapshot-name"); - const RSTRING_ERR_ACCESS_AFTER_TERMSIG: &'static [u8] = eresp!("err-access-after-termsig"); - - // keyspace related resps - const RSTRING_DEFAULT_UNSET: &'static [u8] = eresp!("default-container-unset"); - const RSTRING_CONTAINER_NOT_FOUND: &'static [u8] = eresp!("container-not-found"); - const RSTRING_STILL_IN_USE: &'static [u8] = eresp!("still-in-use"); - const RSTRING_PROTECTED_OBJECT: &'static [u8] = eresp!("err-protected-object"); - const RSTRING_WRONG_MODEL: &'static [u8] = eresp!("wrong-model"); - const RSTRING_ALREADY_EXISTS: &'static [u8] = eresp!("err-already-exists"); - const RSTRING_NOT_READY: &'static [u8] = eresp!("not-ready"); - const RSTRING_DDL_TRANSACTIONAL_FAILURE: &'static [u8] = eresp!("transactional-failure"); - const RSTRING_UNKNOWN_DDL_QUERY: &'static [u8] = eresp!("unknown-ddl-query"); - const RSTRING_BAD_EXPRESSION: &'static [u8] = eresp!("malformed-expression"); - const RSTRING_UNKNOWN_MODEL: &'static [u8] = eresp!("unknown-model"); - const RSTRING_TOO_MANY_ARGUMENTS: &'static [u8] = eresp!("too-many-args"); - const RSTRING_CONTAINER_NAME_TOO_LONG: &'static [u8] = eresp!("container-name-too-long"); - const RSTRING_BAD_CONTAINER_NAME: &'static [u8] = eresp!("bad-container-name"); - const RSTRING_UNKNOWN_INSPECT_QUERY: &'static [u8] = eresp!("unknown-inspect-query"); - const RSTRING_UNKNOWN_PROPERTY: &'static [u8] = eresp!("unknown-property"); - const RSTRING_KEYSPACE_NOT_EMPTY: &'static [u8] = eresp!("keyspace-not-empty"); - const RSTRING_BAD_TYPE_FOR_KEY: &'static [u8] = eresp!("bad-type-for-key"); - const RSTRING_LISTMAP_BAD_INDEX: &'static [u8] = eresp!("bad-list-index"); - const RSTRING_LISTMAP_LIST_IS_EMPTY: &'static [u8] = eresp!("list-is-empty"); - - // elements - const ELEMRESP_HEYA: &'static [u8] = b"+4\nHEY!\n"; - - // full responses - const FULLRESP_RCODE_PACKET_ERR: &'static [u8] = b"*1\n!1\n4\n"; - const FULLRESP_RCODE_WRONG_TYPE: &'static [u8] = b"*1\n!1\n7\n"; - - // auth rcodes/strings - const AUTH_ERROR_ALREADYCLAIMED: &'static [u8] = eresp!("err-auth-already-claimed"); - const AUTH_CODE_BAD_CREDENTIALS: &'static [u8] = eresp!("10"); - const AUTH_ERROR_DISABLED: &'static [u8] = eresp!("err-auth-disabled"); - const AUTH_CODE_PERMS: &'static [u8] = eresp!("11"); - const AUTH_ERROR_ILLEGAL_USERNAME: &'static [u8] = eresp!("err-auth-illegal-username"); - const AUTH_ERROR_FAILED_TO_DELETE_USER: &'static [u8] = eresp!("err-auth-deluser-fail"); - - // bql respstrings - const BQL_BAD_EXPRESSION: &'static [u8] = eresp!("bql-bad-expression"); - const BQL_EXPECTED_STMT: &'static [u8] = eresp!("bql-expected-statement"); - const BQL_INVALID_NUMERIC_LITERAL: &'static [u8] = eresp!("bql-bad-numeric-literal"); - const BQL_INVALID_STRING_LITERAL: &'static [u8] = eresp!("bql-bad-string-literal"); - const BQL_INVALID_SYNTAX: &'static [u8] = eresp!("bql-invalid-syntax"); - const BQL_UNEXPECTED_EOF: &'static [u8] = eresp!("bql-unexpected-eof"); - const BQL_UNKNOWN_CREATE_QUERY: &'static [u8] = eresp!("bql-unknown-create-query"); - const BQL_UNSUPPORTED_MODEL_DECL: &'static [u8] = eresp!("bql-unsupported-model-decl"); - const BQL_UNEXPECTED_CHAR: &'static [u8] = eresp!("bql-unexpected-char"); - - const NEEDS_TERMINAL_LF: bool = true; - - fn decode_packet(input: &[u8]) -> Result { - Skyhash1::parse(input) - } -} diff --git a/server/src/protocol/v1/mod.rs b/server/src/protocol/v1/mod.rs deleted file mode 100644 index 1cb52cc5..00000000 --- a/server/src/protocol/v1/mod.rs +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Created on Sat Apr 30 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ - raw_parser::{RawParser, RawParserExt, RawParserMeta}, - ParseError, ParseResult, PipelinedQuery, Query, SimpleQuery, UnsafeSlice, - }, - crate::{ - corestore::heap_array::{HeapArray, HeapArrayWriter}, - dbnet::QueryWithAdvance, - }, -}; - -mod interface_impls; -// test and bench modules -#[cfg(feature = "nightly")] -#[cfg(test)] -mod benches; -#[cfg(test)] -mod tests; - -/// A parser for Skyhash 1.0 -/// -/// Packet structure example (simple query): -/// ```text -/// *1\n -/// ~3\n -/// 3\n -/// SET\n -/// 1\n -/// x\n -/// 3\n -/// 100\n -/// ``` -pub struct Parser { - end: *const u8, - cursor: *const u8, -} - -unsafe impl RawParser for Parser { - fn cursor_ptr(&self) -> *const u8 { - self.cursor - } - fn cursor_ptr_mut(&mut self) -> &mut *const u8 { - &mut self.cursor - } - fn data_end_ptr(&self) -> *const u8 { - self.end - } -} - -unsafe impl Send for Parser {} -unsafe impl Sync for Parser {} - -impl Parser { - /// Initialize a new parser - fn new(slice: &[u8]) -> Self { - unsafe { - Self { - end: slice.as_ptr().add(slice.len()), - cursor: slice.as_ptr(), - } - } - } -} - -// utility methods -impl Parser { - /// Returns true if the cursor will give a char, but if `this_if_nothing_ahead` is set - /// to true, then if no byte is ahead, it will still return true - fn will_cursor_give_char(&self, ch: u8, true_if_nothing_ahead: bool) -> ParseResult { - if self.exhausted() { - // nothing left - if true_if_nothing_ahead { - Ok(true) - } else { - Err(ParseError::NotEnough) - } - } else if unsafe { self.get_byte_at_cursor().eq(&ch) } { - Ok(true) - } else { - Ok(false) - } - } - /// Check if the current cursor will give an LF - fn will_cursor_give_linefeed(&self) -> ParseResult { - self.will_cursor_give_char(b'\n', false) - } - /// Gets the _next element. **The cursor should be at the tsymbol (passed)** - fn _next(&mut self) -> ParseResult { - let element_size = self.read_usize()?; - self.read_until(element_size) - } -} - -// higher level abstractions -impl Parser { - /// Parse the next blob. **The cursor should be at the tsymbol (passed)** - fn parse_next_blob(&mut self) -> ParseResult { - { - let chunk = self._next()?; - if self.will_cursor_give_linefeed()? { - unsafe { - // UNSAFE(@ohsayan): We know that the buffer is not exhausted - // due to the above condition - self.incr_cursor(); - } - Ok(chunk) - } else { - Err(ParseError::UnexpectedByte) - } - } - } -} - -// query abstractions -impl Parser { - /// The buffer should resemble the below structure: - /// ``` - /// ~\n - /// \n - /// \n - /// \n - /// \n - /// ... - /// ``` - fn _parse_simple_query(&mut self) -> ParseResult> { - if self.not_exhausted() { - if unsafe { self.get_byte_at_cursor() } != b'~' { - // we need an any array - return Err(ParseError::WrongType); - } - unsafe { - // UNSAFE(@ohsayan): Just checked length - self.incr_cursor(); - } - let query_count = self.read_usize()?; - let mut writer = HeapArrayWriter::with_capacity(query_count); - for i in 0..query_count { - unsafe { - // UNSAFE(@ohsayan): The index of the for loop ensures that - // we never attempt to write to a bad memory location - writer.write_to_index(i, self.parse_next_blob()?); - } - } - Ok(unsafe { - // UNSAFE(@ohsayan): If we've reached here, then we have initialized - // all the queries - writer.finish() - }) - } else { - Err(ParseError::NotEnough) - } - } - fn parse_simple_query(&mut self) -> ParseResult { - Ok(SimpleQuery::new(self._parse_simple_query()?)) - } - /// The buffer should resemble the following structure: - /// ```text - /// # query 1 - /// ~\n - /// \n - /// \n - /// \n - /// \n - /// # query 2 - /// ~\n - /// \n - /// \n - /// \n - /// \n - /// ... - /// ``` - fn parse_pipelined_query(&mut self, length: usize) -> ParseResult { - let mut writer = HeapArrayWriter::with_capacity(length); - for i in 0..length { - unsafe { - // UNSAFE(@ohsayan): The above condition guarantees that the index - // never causes an overflow - writer.write_to_index(i, self._parse_simple_query()?); - } - } - unsafe { - // UNSAFE(@ohsayan): if we reached here, then we have inited everything - Ok(PipelinedQuery::new(writer.finish())) - } - } - fn _parse(&mut self) -> ParseResult { - if self.not_exhausted() { - let first_byte = unsafe { - // UNSAFE(@ohsayan): Just checked if buffer is exhausted or not - self.get_byte_at_cursor() - }; - if first_byte != b'*' { - // unknown query scheme, so it's a bad packet - return Err(ParseError::BadPacket); - } - unsafe { - // UNSAFE(@ohsayan): Checked buffer len and incremented, so we're good - self.incr_cursor() - }; - let query_count = self.read_usize()?; // get the length - if query_count == 1 { - Ok(Query::Simple(self.parse_simple_query()?)) - } else { - Ok(Query::Pipelined(self.parse_pipelined_query(query_count)?)) - } - } else { - Err(ParseError::NotEnough) - } - } - pub fn parse(buf: &[u8]) -> ParseResult { - let mut slf = Self::new(buf); - let body = slf._parse()?; - let consumed = slf.cursor_ptr() as usize - buf.as_ptr() as usize; - Ok((body, consumed)) - } -} diff --git a/server/src/protocol/v1/tests.rs b/server/src/protocol/v1/tests.rs deleted file mode 100644 index 69f20459..00000000 --- a/server/src/protocol/v1/tests.rs +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Created on Mon May 02 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::Parser, - crate::protocol::{ParseError, Query}, -}; - -#[cfg(test)] -const SQPAYLOAD: &[u8] = b"*1\n~3\n3\nSET\n1\nx\n3\n100\n"; -#[cfg(test)] -const PQPAYLOAD: &[u8] = b"*2\n~3\n3\nSET\n1\nx\n3\n100\n~2\n3\nGET\n1\nx\n"; - -#[test] -fn parse_simple_query() { - let payload = SQPAYLOAD.to_vec(); - let (q, f) = Parser::parse(&payload).unwrap(); - let q: Vec = if let Query::Simple(q) = q { - q.as_slice() - .iter() - .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string()) - .collect() - } else { - panic!("Expected simple query") - }; - assert_eq!(f, payload.len()); - assert_eq!(q, vec!["SET".to_owned(), "x".into(), "100".into()]); -} - -#[test] -fn parse_simple_query_incomplete() { - for i in 0..SQPAYLOAD.len() - 1 { - let slice = &SQPAYLOAD[..i]; - assert_eq!(Parser::parse(slice).unwrap_err(), ParseError::NotEnough); - } -} - -#[test] -fn parse_pipelined_query() { - let payload = PQPAYLOAD.to_vec(); - let (q, f) = Parser::parse(&payload).unwrap(); - let q: Vec> = if let Query::Pipelined(q) = q { - q.into_inner() - .iter() - .map(|sq| { - sq.iter() - .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string()) - .collect() - }) - .collect() - } else { - panic!("Expected pipelined query query") - }; - assert_eq!(f, payload.len()); - assert_eq!( - q, - vec![ - vec!["SET".to_owned(), "x".into(), "100".into()], - vec!["GET".into(), "x".into()] - ] - ); -} - -#[test] -fn parse_pipelined_query_incomplete() { - for i in 0..PQPAYLOAD.len() - 1 { - let slice = &PQPAYLOAD[..i]; - assert_eq!(Parser::parse(slice).unwrap_err(), ParseError::NotEnough); - } -} diff --git a/server/src/protocol/v2/benches.rs b/server/src/protocol/v2/benches.rs deleted file mode 100644 index 4b1c8f64..00000000 --- a/server/src/protocol/v2/benches.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Created on Sat Apr 30 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -extern crate test; -use { - super::{super::Query, Parser}, - test::Bencher, -}; - -#[bench] -fn simple_query(b: &mut Bencher) { - const PAYLOAD: &[u8] = b"*3\n3\nSET1\nx3\n100"; - let expected = vec!["SET".to_owned(), "x".to_owned(), "100".to_owned()]; - b.iter(|| { - let (query, forward) = Parser::parse(PAYLOAD).unwrap(); - assert_eq!(forward, PAYLOAD.len()); - let query = if let Query::Simple(sq) = query { - sq - } else { - panic!("Got pipeline instead of simple query"); - }; - let ret: Vec = query - .as_slice() - .iter() - .map(|s| String::from_utf8_lossy(unsafe { s.as_slice() }).to_string()) - .collect(); - assert_eq!(ret, expected) - }); -} - -#[bench] -fn pipelined_query(b: &mut Bencher) { - const PAYLOAD: &[u8] = b"$2\n3\n3\nSET1\nx3\n1002\n3\nGET1\nx"; - let expected = vec![ - vec!["SET".to_owned(), "x".to_owned(), "100".to_owned()], - vec!["GET".to_owned(), "x".to_owned()], - ]; - b.iter(|| { - let (query, forward) = Parser::parse(PAYLOAD).unwrap(); - assert_eq!(forward, PAYLOAD.len()); - let query = if let Query::Pipelined(sq) = query { - sq - } else { - panic!("Got simple instead of pipeline query"); - }; - let ret: Vec> = query - .into_inner() - .iter() - .map(|query| { - query - .as_slice() - .iter() - .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string()) - .collect() - }) - .collect(); - assert_eq!(ret, expected) - }); -} diff --git a/server/src/protocol/v2/interface_impls.rs b/server/src/protocol/v2/interface_impls.rs deleted file mode 100644 index 349d8683..00000000 --- a/server/src/protocol/v2/interface_impls.rs +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Created on Sat Apr 30 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - dbnet::QueryWithAdvance, - protocol::{interface::ProtocolSpec, ParseError, Skyhash2}, - }, - ::sky_macros::compiled_eresp_bytes as eresp, -}; - -impl ProtocolSpec for Skyhash2 { - // spec information - const PROTOCOL_VERSION: f32 = 2.0; - const PROTOCOL_VERSIONSTRING: &'static str = "Skyhash-2.0"; - - // type symbols - const TSYMBOL_STRING: u8 = b'+'; - const TSYMBOL_BINARY: u8 = b'?'; - const TSYMBOL_FLOAT: u8 = b'%'; - const TSYMBOL_INT64: u8 = b':'; - const TSYMBOL_TYPED_ARRAY: u8 = b'@'; - const TSYMBOL_TYPED_NON_NULL_ARRAY: u8 = b'^'; - const TSYMBOL_ARRAY: u8 = b'&'; - const TSYMBOL_FLAT_ARRAY: u8 = b'_'; - - // typed array - const TYPE_TYPED_ARRAY_ELEMENT_NULL: &'static [u8] = b"\0"; - - // metaframe - const SIMPLE_QUERY_HEADER: &'static [u8] = b"*"; - const PIPELINED_QUERY_FIRST_BYTE: u8 = b'$'; - - // respcodes - const RCODE_OKAY: &'static [u8] = eresp!("0"); - const RCODE_NIL: &'static [u8] = eresp!("1"); - const RCODE_OVERWRITE_ERR: &'static [u8] = eresp!("2"); - const RCODE_ACTION_ERR: &'static [u8] = eresp!("3"); - const RCODE_PACKET_ERR: &'static [u8] = eresp!("4"); - const RCODE_SERVER_ERR: &'static [u8] = eresp!("5"); - const RCODE_OTHER_ERR_EMPTY: &'static [u8] = eresp!("6"); - const RCODE_UNKNOWN_ACTION: &'static [u8] = eresp!("Unknown action"); - const RCODE_WRONGTYPE_ERR: &'static [u8] = eresp!("7"); - const RCODE_UNKNOWN_DATA_TYPE: &'static [u8] = eresp!("8"); - const RCODE_ENCODING_ERROR: &'static [u8] = eresp!("9"); - - // respstrings - const RSTRING_SNAPSHOT_BUSY: &'static [u8] = eresp!("err-snapshot-busy"); - const RSTRING_SNAPSHOT_DISABLED: &'static [u8] = eresp!("err-snapshot-disabled"); - const RSTRING_SNAPSHOT_DUPLICATE: &'static [u8] = eresp!("duplicate-snapshot"); - const RSTRING_SNAPSHOT_ILLEGAL_NAME: &'static [u8] = eresp!("err-invalid-snapshot-name"); - const RSTRING_ERR_ACCESS_AFTER_TERMSIG: &'static [u8] = eresp!("err-access-after-termsig"); - - // keyspace related resps - const RSTRING_DEFAULT_UNSET: &'static [u8] = eresp!("default-container-unset"); - const RSTRING_CONTAINER_NOT_FOUND: &'static [u8] = eresp!("container-not-found"); - const RSTRING_STILL_IN_USE: &'static [u8] = eresp!("still-in-use"); - const RSTRING_PROTECTED_OBJECT: &'static [u8] = eresp!("err-protected-object"); - const RSTRING_WRONG_MODEL: &'static [u8] = eresp!("wrong-model"); - const RSTRING_ALREADY_EXISTS: &'static [u8] = eresp!("err-already-exists"); - const RSTRING_NOT_READY: &'static [u8] = eresp!("not-ready"); - const RSTRING_DDL_TRANSACTIONAL_FAILURE: &'static [u8] = eresp!("transactional-failure"); - const RSTRING_UNKNOWN_DDL_QUERY: &'static [u8] = eresp!("unknown-ddl-query"); - const RSTRING_BAD_EXPRESSION: &'static [u8] = eresp!("malformed-expression"); - const RSTRING_UNKNOWN_MODEL: &'static [u8] = eresp!("unknown-model"); - const RSTRING_TOO_MANY_ARGUMENTS: &'static [u8] = eresp!("too-many-args"); - const RSTRING_CONTAINER_NAME_TOO_LONG: &'static [u8] = eresp!("container-name-too-long"); - const RSTRING_BAD_CONTAINER_NAME: &'static [u8] = eresp!("bad-container-name"); - const RSTRING_UNKNOWN_INSPECT_QUERY: &'static [u8] = eresp!("unknown-inspect-query"); - const RSTRING_UNKNOWN_PROPERTY: &'static [u8] = eresp!("unknown-property"); - const RSTRING_KEYSPACE_NOT_EMPTY: &'static [u8] = eresp!("keyspace-not-empty"); - const RSTRING_BAD_TYPE_FOR_KEY: &'static [u8] = eresp!("bad-type-for-key"); - const RSTRING_LISTMAP_BAD_INDEX: &'static [u8] = eresp!("bad-list-index"); - const RSTRING_LISTMAP_LIST_IS_EMPTY: &'static [u8] = eresp!("list-is-empty"); - - // elements - const ELEMRESP_HEYA: &'static [u8] = b"+4\nHEY!"; - - // full responses - const FULLRESP_RCODE_PACKET_ERR: &'static [u8] = b"*!4\n"; - const FULLRESP_RCODE_WRONG_TYPE: &'static [u8] = b"*!7\n"; - - // auth respcodes/strings - const AUTH_ERROR_ALREADYCLAIMED: &'static [u8] = eresp!("err-auth-already-claimed"); - const AUTH_CODE_BAD_CREDENTIALS: &'static [u8] = eresp!("10"); - const AUTH_ERROR_DISABLED: &'static [u8] = eresp!("err-auth-disabled"); - const AUTH_CODE_PERMS: &'static [u8] = eresp!("11"); - const AUTH_ERROR_ILLEGAL_USERNAME: &'static [u8] = eresp!("err-auth-illegal-username"); - const AUTH_ERROR_FAILED_TO_DELETE_USER: &'static [u8] = eresp!("err-auth-deluser-fail"); - - // bql respstrings - const BQL_BAD_EXPRESSION: &'static [u8] = eresp!("bql-bad-expression"); - const BQL_EXPECTED_STMT: &'static [u8] = eresp!("bql-expected-statement"); - const BQL_INVALID_NUMERIC_LITERAL: &'static [u8] = eresp!("bql-bad-numeric-literal"); - const BQL_INVALID_STRING_LITERAL: &'static [u8] = eresp!("bql-bad-string-literal"); - const BQL_INVALID_SYNTAX: &'static [u8] = eresp!("bql-invalid-syntax"); - const BQL_UNEXPECTED_EOF: &'static [u8] = eresp!("bql-unexpected-eof"); - const BQL_UNKNOWN_CREATE_QUERY: &'static [u8] = eresp!("bql-unknown-create-query"); - const BQL_UNSUPPORTED_MODEL_DECL: &'static [u8] = eresp!("bql-unsupported-model-decl"); - const BQL_UNEXPECTED_CHAR: &'static [u8] = eresp!("bql-unexpected-char"); - - const NEEDS_TERMINAL_LF: bool = false; - - fn decode_packet(input: &[u8]) -> Result { - Skyhash2::parse(input) - } -} diff --git a/server/src/protocol/v2/mod.rs b/server/src/protocol/v2/mod.rs deleted file mode 100644 index 6720e8b5..00000000 --- a/server/src/protocol/v2/mod.rs +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Created on Fri Apr 29 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -mod interface_impls; - -use { - super::{ - raw_parser::{RawParser, RawParserExt, RawParserMeta}, - ParseError, ParseResult, PipelinedQuery, Query, SimpleQuery, UnsafeSlice, - }, - crate::{corestore::heap_array::HeapArray, dbnet::QueryWithAdvance}, -}; - -#[cfg(feature = "nightly")] -#[cfg(test)] -mod benches; -#[cfg(test)] -mod tests; - -/// A parser for Skyhash 2.0 -pub struct Parser { - end: *const u8, - cursor: *const u8, -} - -unsafe impl RawParser for Parser { - fn cursor_ptr(&self) -> *const u8 { - self.cursor - } - fn cursor_ptr_mut(&mut self) -> &mut *const u8 { - &mut self.cursor - } - fn data_end_ptr(&self) -> *const u8 { - self.end - } -} - -unsafe impl Sync for Parser {} -unsafe impl Send for Parser {} - -impl Parser { - /// Initialize a new parser - fn new(slice: &[u8]) -> Self { - unsafe { - Self { - end: slice.as_ptr().add(slice.len()), - cursor: slice.as_ptr(), - } - } - } -} - -// query impls -impl Parser { - /// Parse the next simple query. This should have passed the `*` tsymbol - /// - /// Simple query structure (tokenized line-by-line): - /// ```text - /// * -> Simple Query Header - /// \n -> Count of elements in the simple query - /// \n -> Length of element 1 - /// -> element 1 itself - /// \n -> Length of element 2 - /// -> element 2 itself - /// ... - /// ``` - fn _next_simple_query(&mut self) -> ParseResult> { - let element_count = self.read_usize()?; - unsafe { - let mut data = HeapArray::new_writer(element_count); - for i in 0..element_count { - let element_size = self.read_usize()?; - let element = self.read_until(element_size)?; - data.write_to_index(i, element); - } - Ok(data.finish()) - } - } - /// Parse a simple query - fn next_simple_query(&mut self) -> ParseResult { - Ok(SimpleQuery::new(self._next_simple_query()?)) - } - /// Parse a pipelined query. This should have passed the `$` tsymbol - /// - /// Pipelined query structure (tokenized line-by-line): - /// ```text - /// $ -> Pipeline - /// \n -> Pipeline has n queries - /// \n -> Query 1 has 3 elements - /// \n -> Q1E1 has 3 bytes - /// -> Q1E1 itself - /// \n -> Q1E2 has 1 byte - /// -> Q1E2 itself - /// \n -> Q1E3 has 3 bytes - /// -> Q1E3 itself - /// \n -> Query 2 has 2 elements - /// \n -> Q2E1 has 3 bytes - /// -> Q2E1 itself - /// \n -> Q2E2 has 1 byte - /// -> Q2E2 itself - /// ... - /// ``` - /// - /// Example: - /// ```text - /// $ -> Pipeline - /// 2\n -> Pipeline has 2 queries - /// 3\n -> Query 1 has 3 elements - /// 3\n -> Q1E1 has 3 bytes - /// SET -> Q1E1 itself - /// 1\n -> Q1E2 has 1 byte - /// x -> Q1E2 itself - /// 3\n -> Q1E3 has 3 bytes - /// 100 -> Q1E3 itself - /// 2\n -> Query 2 has 2 elements - /// 3\n -> Q2E1 has 3 bytes - /// GET -> Q2E1 itself - /// 1\n -> Q2E2 has 1 byte - /// x -> Q2E2 itself - /// ``` - fn next_pipeline(&mut self) -> ParseResult { - let query_count = self.read_usize()?; - unsafe { - let mut queries = HeapArray::new_writer(query_count); - for i in 0..query_count { - let sq = self._next_simple_query()?; - queries.write_to_index(i, sq); - } - Ok(PipelinedQuery { - data: queries.finish(), - }) - } - } - fn _parse(&mut self) -> ParseResult { - if self.not_exhausted() { - unsafe { - let first_byte = self.get_byte_at_cursor(); - self.incr_cursor(); - let data = match first_byte { - b'*' => { - // a simple query - Query::Simple(self.next_simple_query()?) - } - b'$' => { - // a pipelined query - Query::Pipelined(self.next_pipeline()?) - } - _ => return Err(ParseError::UnexpectedByte), - }; - Ok(data) - } - } else { - Err(ParseError::NotEnough) - } - } - // only expose this. don't expose Self::new since that'll be _relatively easier_ to - // invalidate invariants for - pub fn parse(buf: &[u8]) -> ParseResult { - let mut slf = Self::new(buf); - let body = slf._parse()?; - let consumed = slf.cursor_ptr() as usize - buf.as_ptr() as usize; - Ok((body, consumed)) - } -} diff --git a/server/src/protocol/v2/tests.rs b/server/src/protocol/v2/tests.rs deleted file mode 100644 index 0cb56883..00000000 --- a/server/src/protocol/v2/tests.rs +++ /dev/null @@ -1,645 +0,0 @@ -/* - * Created on Tue Apr 12 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ - super::raw_parser::{RawParser, RawParserExt, RawParserMeta}, - Parser, PipelinedQuery, Query, SimpleQuery, - }, - crate::protocol::{iter::AnyArrayIter, ParseError}, - std::{iter::Map, vec::IntoIter as VecIntoIter}, -}; - -type IterPacketWithLen = Map>, fn(Vec) -> (usize, Vec)>; -type Packets = Vec>; - -macro_rules! v { - () => { - vec![] - }; - ($literal:literal) => { - $literal.to_vec() - }; - ($($lit:literal),*) => { - vec![$( - $lit.as_bytes().to_owned() - ),*] - } -} - -fn ensure_exhausted(p: &Parser) { - assert!(!p.not_exhausted()); - assert!(p.exhausted()); -} - -fn ensure_remaining(p: &Parser, r: usize) { - assert_eq!(p.remaining(), r); - assert!(p.has_remaining(r)); -} - -fn ensure_not_exhausted(p: &Parser) { - assert!(p.not_exhausted()); - assert!(!p.exhausted()); -} - -fn get_slices(slices: &[&[u8]]) -> Packets { - slices.iter().map(|slc| slc.to_vec()).collect() -} - -fn ensure_zero_reads(parser: &mut Parser) { - let r = parser.read_until(0).unwrap(); - let slice = unsafe { r.as_slice() }; - assert_eq!(slice, b""); - assert!(slice.is_empty()); -} - -// We do this intentionally for "heap simulation" -fn slices() -> Packets { - const SLICE_COLLECTION: &[&[u8]] = &[ - b"", - b"a", - b"ab", - b"abc", - b"abcd", - b"abcde", - b"abcdef", - b"abcdefg", - b"abcdefgh", - b"abcdefghi", - b"abcdefghij", - b"abcdefghijk", - b"abcdefghijkl", - b"abcdefghijklm", - ]; - get_slices(SLICE_COLLECTION) -} - -fn get_slices_with_len(slices: Packets) -> IterPacketWithLen { - slices.into_iter().map(|slc| (slc.len(), slc)) -} - -fn slices_with_len() -> IterPacketWithLen { - get_slices_with_len(slices()) -} - -fn slices_lf() -> Packets { - const SLICE_COLLECTION: &[&[u8]] = &[ - b"", - b"a\n", - b"ab\n", - b"abc\n", - b"abcd\n", - b"abcde\n", - b"abcdef\n", - b"abcdefg\n", - b"abcdefgh\n", - b"abcdefghi\n", - b"abcdefghij\n", - b"abcdefghijk\n", - b"abcdefghijkl\n", - b"abcdefghijklm\n", - ]; - get_slices(SLICE_COLLECTION) -} - -fn slices_lf_with_len() -> IterPacketWithLen { - get_slices_with_len(slices_lf()) -} - -fn simple_query(query: Query) -> SimpleQuery { - if let Query::Simple(sq) = query { - sq - } else { - panic!("Got pipeline instead of simple!"); - } -} - -fn pipelined_query(query: Query) -> PipelinedQuery { - if let Query::Pipelined(pq) = query { - pq - } else { - panic!("Got simple instead of pipeline!"); - } -} - -// "actual" tests -// data_end_ptr -#[test] -fn data_end_ptr() { - for (len, src) in slices_with_len() { - let parser = Parser::new(&src); - unsafe { - assert_eq!(parser.data_end_ptr(), src.as_ptr().add(len)); - } - } -} - -// cursor_ptr -#[test] -fn cursor_ptr() { - for src in slices() { - let parser = Parser::new(&src); - assert_eq!(parser.cursor_ptr(), src.as_ptr()) - } -} -#[test] -fn cursor_ptr_with_incr() { - for src in slices() { - let mut parser = Parser::new(&src); - unsafe { - parser.incr_cursor_by(src.len()); - assert_eq!(parser.cursor_ptr(), src.as_ptr().add(src.len())); - } - } -} - -// remaining -#[test] -fn remaining() { - for (len, src) in slices_with_len() { - let parser = Parser::new(&src); - assert_eq!(parser.remaining(), len); - } -} -#[test] -fn remaining_with_incr() { - for (len, src) in slices_with_len() { - let mut parser = Parser::new(&src); - unsafe { - // no change - parser.incr_cursor_by(0); - assert_eq!(parser.remaining(), len); - if len != 0 { - // move one byte ahead. should reach EOA or len - 1 - parser.incr_cursor(); - assert_eq!(parser.remaining(), len - 1); - // move the cursor to the end; should reach EOA - parser.incr_cursor_by(len - 1); - assert_eq!(parser.remaining(), 0); - } - } - } -} - -// has_remaining -#[test] -fn has_remaining() { - for (len, src) in slices_with_len() { - let parser = Parser::new(&src); - assert!(parser.has_remaining(len), "should have {len} remaining") - } -} -#[test] -fn has_remaining_with_incr() { - for (len, src) in slices_with_len() { - let mut parser = Parser::new(&src); - unsafe { - // no change - parser.incr_cursor_by(0); - assert!(parser.has_remaining(len)); - if len != 0 { - // move one byte ahead. should reach EOA or len - 1 - parser.incr_cursor(); - assert!(parser.has_remaining(len - 1)); - // move the cursor to the end; should reach EOA - parser.incr_cursor_by(len - 1); - assert!(!parser.has_remaining(1)); - // should always be true - assert!(parser.has_remaining(0)); - } - } - } -} - -// exhausted -#[test] -fn exhausted() { - for src in slices() { - let parser = Parser::new(&src); - if src.is_empty() { - assert!(parser.exhausted()); - } else { - assert!(!parser.exhausted()) - } - } -} -#[test] -fn exhausted_with_incr() { - for (len, src) in slices_with_len() { - let mut parser = Parser::new(&src); - if len == 0 { - assert!(parser.exhausted()); - } else { - assert!(!parser.exhausted()); - unsafe { - parser.incr_cursor(); - if len == 1 { - assert!(parser.exhausted()); - } else { - assert!(!parser.exhausted()); - parser.incr_cursor_by(len - 1); - assert!(parser.exhausted()); - } - } - } - } -} - -// not_exhausted -#[test] -fn not_exhausted() { - for src in slices() { - let parser = Parser::new(&src); - if src.is_empty() { - assert!(!parser.not_exhausted()); - } else { - assert!(parser.not_exhausted()) - } - } -} -#[test] -fn not_exhausted_with_incr() { - for (len, src) in slices_with_len() { - let mut parser = Parser::new(&src); - if len == 0 { - assert!(!parser.not_exhausted()); - } else { - assert!(parser.not_exhausted()); - unsafe { - parser.incr_cursor(); - if len == 1 { - assert!(!parser.not_exhausted()); - } else { - assert!(parser.not_exhausted()); - parser.incr_cursor_by(len - 1); - assert!(!parser.not_exhausted()); - } - } - } - } -} - -// read_until -#[test] -fn read_until_empty() { - let b = v!(b""); - let mut parser = Parser::new(&b); - ensure_zero_reads(&mut parser); - assert_eq!(parser.read_until(1).unwrap_err(), ParseError::NotEnough); -} - -#[test] -fn read_until_nonempty() { - for (len, src) in slices_with_len() { - let mut parser = Parser::new(&src); - // should always work - ensure_zero_reads(&mut parser); - // now read the entire length; should always work - let r = parser.read_until(len).unwrap(); - let slice = unsafe { r.as_slice() }; - assert_eq!(slice, src.as_slice()); - assert_eq!(slice.len(), len); - // even after the buffer is exhausted, `0` should always work - ensure_zero_reads(&mut parser); - } -} - -#[test] -fn read_until_not_enough() { - for (len, src) in slices_with_len() { - let mut parser = Parser::new(&src); - ensure_zero_reads(&mut parser); - // try to read more than the amount of data bufferred - assert_eq!( - parser.read_until(len + 1).unwrap_err(), - ParseError::NotEnough - ); - // should the above fail, zero reads should still work - ensure_zero_reads(&mut parser); - } -} - -#[test] -fn read_until_more_bytes() { - let sample1 = v!(b"abcd1"); - let mut p1 = Parser::new(&sample1); - assert_eq!( - unsafe { p1.read_until(&sample1.len() - 1).unwrap().as_slice() }, - &sample1[..&sample1.len() - 1] - ); - // ensure we have not exhasuted - ensure_not_exhausted(&p1); - ensure_remaining(&p1, 1); - let sample2 = v!(b"abcd1234567890!@#$"); - let mut p2 = Parser::new(&sample2); - assert_eq!( - unsafe { p2.read_until(4).unwrap().as_slice() }, - &sample2[..4] - ); - // ensure we have not exhasuted - ensure_not_exhausted(&p2); - ensure_remaining(&p2, sample2.len() - 4); -} - -// read_line -#[test] -fn read_line_special_case_only_lf() { - let b = v!(b"\n"); - let mut parser = Parser::new(&b); - let r = parser.read_line().unwrap(); - let slice = unsafe { r.as_slice() }; - assert_eq!(slice, b""); - assert!(slice.is_empty()); - // ensure it is exhausted - ensure_exhausted(&parser); -} - -#[test] -fn read_line() { - for (len, src) in slices_lf_with_len() { - let mut parser = Parser::new(&src); - if len == 0 { - // should be empty, so NotEnough - assert_eq!(parser.read_line().unwrap_err(), ParseError::NotEnough); - } else { - // should work - assert_eq!( - unsafe { parser.read_line().unwrap().as_slice() }, - &src.as_slice()[..len - 1] - ); - // now, we attempt to read which should work - ensure_zero_reads(&mut parser); - } - // ensure it is exhausted - ensure_exhausted(&parser); - // now, we attempt to read another line which should fail - assert_eq!(parser.read_line().unwrap_err(), ParseError::NotEnough); - // ensure that cursor is at end - unsafe { - assert_eq!(parser.cursor_ptr(), src.as_ptr().add(len)); - } - } -} - -#[test] -fn read_line_more_bytes() { - let sample1 = v!(b"abcd\n1"); - let mut p1 = Parser::new(&sample1); - let line = p1.read_line().unwrap(); - assert_eq!(unsafe { line.as_slice() }, b"abcd"); - // we should still have one remaining - ensure_not_exhausted(&p1); - ensure_remaining(&p1, 1); -} - -#[test] -fn read_line_subsequent_lf() { - let sample1 = v!(b"abcd\n1\n"); - let mut p1 = Parser::new(&sample1); - let line = p1.read_line().unwrap(); - assert_eq!(unsafe { line.as_slice() }, b"abcd"); - // we should still have two octets remaining - ensure_not_exhausted(&p1); - ensure_remaining(&p1, 2); - // and we should be able to read in another line - let line = p1.read_line().unwrap(); - assert_eq!(unsafe { line.as_slice() }, b"1"); - ensure_exhausted(&p1); -} - -#[test] -fn read_line_pedantic_okay() { - for (len, src) in slices_lf_with_len() { - let mut parser = Parser::new(&src); - if len == 0 { - // should be empty, so NotEnough - assert_eq!( - parser.read_line_pedantic().unwrap_err(), - ParseError::NotEnough - ); - } else { - // should work - assert_eq!( - unsafe { parser.read_line_pedantic().unwrap().as_slice() }, - &src.as_slice()[..len - 1] - ); - // now, we attempt to read which should work - ensure_zero_reads(&mut parser); - } - // ensure it is exhausted - ensure_exhausted(&parser); - // now, we attempt to read another line which should fail - assert_eq!( - parser.read_line_pedantic().unwrap_err(), - ParseError::NotEnough - ); - // ensure that cursor is at end - unsafe { - assert_eq!(parser.cursor_ptr(), src.as_ptr().add(len)); - } - } -} - -#[test] -fn read_line_pedantic_fail_empty() { - let payload = v!(b""); - assert_eq!( - Parser::new(&payload).read_line_pedantic().unwrap_err(), - ParseError::NotEnough - ); -} - -#[test] -fn read_line_pedantic_fail_only_lf() { - let payload = v!(b"\n"); - assert_eq!( - Parser::new(&payload).read_line_pedantic().unwrap_err(), - ParseError::BadPacket - ); -} - -#[test] -fn read_line_pedantic_fail_only_lf_extra_data() { - let payload = v!(b"\n1"); - assert_eq!( - Parser::new(&payload).read_line_pedantic().unwrap_err(), - ParseError::BadPacket - ); -} - -#[test] -fn read_usize_fail_empty() { - let payload = v!(b""); - assert_eq!( - Parser::new(&payload).read_usize().unwrap_err(), - ParseError::NotEnough - ); - let payload = v!(b"\n"); - assert_eq!( - Parser::new(&payload).read_usize().unwrap_err(), - ParseError::BadPacket - ); -} - -#[test] -fn read_usize_fail_no_lf() { - let payload = v!(b"1"); - assert_eq!( - Parser::new(&payload).read_usize().unwrap_err(), - ParseError::NotEnough - ); -} - -#[test] -fn read_usize_okay() { - let payload = v!(b"1\n"); - assert_eq!(Parser::new(&payload).read_usize().unwrap(), 1); - let payload = v!(b"1234\n"); - assert_eq!(Parser::new(&payload).read_usize().unwrap(), 1234); -} - -#[test] -fn read_usize_fail() { - let payload = v!(b"a\n"); - assert_eq!( - Parser::new(&payload).read_usize().unwrap_err(), - ParseError::DatatypeParseFailure - ); - let payload = v!(b"1a\n"); - assert_eq!( - Parser::new(&payload).read_usize().unwrap_err(), - ParseError::DatatypeParseFailure - ); - let payload = v!(b"a1\n"); - assert_eq!( - Parser::new(&payload).read_usize().unwrap_err(), - ParseError::DatatypeParseFailure - ); - let payload = v!(b"aa\n"); - assert_eq!( - Parser::new(&payload).read_usize().unwrap_err(), - ParseError::DatatypeParseFailure - ); - let payload = v!(b"12345abcde\n"); - assert_eq!( - Parser::new(&payload).read_usize().unwrap_err(), - ParseError::DatatypeParseFailure - ); -} - -#[test] -fn parse_fail_because_unknown_query_scheme() { - let body = v!(b"?3\n3\nSET1\nx3\n100"); - assert_eq!( - Parser::parse(&body).unwrap_err(), - ParseError::UnexpectedByte - ) -} - -#[test] -fn simple_query_okay() { - let body = v!(b"*3\n3\nSET1\nx3\n100"); - let (ret, skip) = Parser::parse(&body).unwrap(); - assert_eq!(skip, body.len()); - let query = simple_query(ret); - assert_eq!(query.into_owned().data, v!["SET", "x", "100"]); -} - -#[test] -fn simple_query_okay_empty_elements() { - let body = v!(b"*3\n3\nSET0\n0\n"); - let (ret, skip) = Parser::parse(&body).unwrap(); - assert_eq!(skip, body.len()); - let query = simple_query(ret); - assert_eq!(query.into_owned().data, v!["SET", "", ""]); -} - -#[test] -fn parse_fail_because_not_enough() { - let full_payload = b"*3\n3\nSET1\nx3\n100"; - let samples: Vec> = (0..full_payload.len() - 1) - .map(|i| full_payload.iter().take(i).cloned().collect()) - .collect(); - for body in samples { - assert_eq!( - Parser::parse(&body).unwrap_err(), - ParseError::NotEnough, - "Failed with body len: {}", - body.len() - ) - } -} - -#[test] -fn pipelined_query_okay() { - let body = v!(b"$2\n3\n3\nSET1\nx3\n1002\n3\nGET1\nx"); - let (ret, skip) = Parser::parse(&body).unwrap(); - assert_eq!(skip, body.len()); - let query = pipelined_query(ret); - assert_eq!( - query.into_owned().data, - vec![v!["SET", "x", "100"], v!["GET", "x"]] - ) -} - -#[test] -fn pipelined_query_okay_empty_elements() { - let body = v!(b"$2\n3\n3\nSET0\n3\n1002\n3\nGET0\n"); - let (ret, skip) = Parser::parse(&body).unwrap(); - assert_eq!(skip, body.len()); - let query = pipelined_query(ret); - assert_eq!( - query.into_owned().data, - vec![v!["SET", "", "100"], v!["GET", ""]] - ) -} - -#[test] -fn pipelined_query_fail_because_not_enough() { - let full_payload = v!(b"$2\n3\n3\nSET1\nx3\n1002\n3\nGET1\nx"); - let samples: Vec> = (0..full_payload.len() - 1) - .map(|i| full_payload.iter().cloned().take(i).collect()) - .collect(); - for body in samples { - let ret = Parser::parse(&body).unwrap_err(); - assert_eq!(ret, ParseError::NotEnough) - } -} - -#[test] -fn test_iter() { - use super::{Parser, Query}; - let (q, _fwby) = Parser::parse(b"*3\n3\nset1\nx3\n100").unwrap(); - let r = match q { - Query::Simple(q) => q, - _ => panic!("Wrong query"), - }; - let it = r.as_slice().iter(); - let mut iter = unsafe { AnyArrayIter::new(it) }; - assert_eq!(iter.next_uppercase().unwrap().as_ref(), "SET".as_bytes()); - assert_eq!(iter.next().unwrap(), "x".as_bytes()); - assert_eq!(iter.next().unwrap(), "100".as_bytes()); -} diff --git a/server/src/queryengine/mod.rs b/server/src/queryengine/mod.rs deleted file mode 100644 index 00327f18..00000000 --- a/server/src/queryengine/mod.rs +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Created on Mon Aug 03 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # The Query Engine - -use crate::{ - actions::{self, ActionError, ActionResult}, - admin, auth, blueql, - corestore::Corestore, - dbnet::{prelude::*, BufferedSocketStream}, - protocol::{iter::AnyArrayIter, PipelinedQuery, SimpleQuery, UnsafeSlice}, -}; - -pub type ActionIter<'a> = AnyArrayIter<'a>; - -const ACTION_AUTH: &[u8] = b"auth"; - -macro_rules! gen_constants_and_matches { - ( - $con:expr, $buf:ident, $db:ident, $($action:ident => $fns:path),*, - {$($action2:ident => $fns2:expr),*} - ) => { - mod tags { - //! This module is a collection of tags/strings used for evaluating queries - //! and responses - $( - pub const $action: &[u8] = stringify!($action).as_bytes(); - )* - $( - pub const $action2: &[u8] = stringify!($action2).as_bytes(); - )* - } - let first_slice = $buf.next().unwrap_or_custom_aerr(P::RCODE_PACKET_ERR)?; - let first = first_slice.to_ascii_uppercase(); - match first.as_ref() { - $( - tags::$action => $fns($db, $con, $buf).await?, - )* - $( - tags::$action2 => $fns2.await?, - )* - _ => { - blueql::execute($db, $con, first_slice, $buf.len()).await?; - } - } - }; -} - -action! { - /// Execute queries for an anonymous user - fn execute_simple_noauth( - _db: &mut Corestore, - con: &mut Connection, - auth: &mut AuthProviderHandle, - buf: SimpleQuery - ) { - let bufref = buf.as_slice(); - let mut iter = unsafe { - // UNSAFE(@ohsayan): The presence of the connection guarantees that this - // won't suddenly become invalid - AnyArrayIter::new(bufref.iter()) - }; - match iter.next_lowercase().unwrap_or_custom_aerr(P::RCODE_PACKET_ERR)?.as_ref() { - ACTION_AUTH => auth::auth_login_only(con, auth, iter).await, - _ => util::err(P::AUTH_CODE_BAD_CREDENTIALS), - } - } - //// Execute a simple query - fn execute_simple( - db: &mut Corestore, - con: &mut Connection, - auth: &mut AuthProviderHandle, - buf: SimpleQuery - ) { - self::execute_stage(db, con, auth, buf.as_slice()).await - } -} - -async fn execute_stage<'a, P: ProtocolSpec, C: BufferedSocketStream>( - db: &mut Corestore, - con: &mut Connection, - auth: &mut AuthProviderHandle, - buf: &[UnsafeSlice], -) -> ActionResult<()> { - let mut iter = unsafe { - // UNSAFE(@ohsayan): The presence of the connection guarantees that this - // won't suddenly become invalid - AnyArrayIter::new(buf.iter()) - }; - { - gen_constants_and_matches!( - con, iter, db, - GET => actions::get::get, - SET => actions::set::set, - UPDATE => actions::update::update, - DEL => actions::del::del, - HEYA => actions::heya::heya, - EXISTS => actions::exists::exists, - MSET => actions::mset::mset, - MGET => actions::mget::mget, - MUPDATE => actions::mupdate::mupdate, - SSET => actions::strong::sset, - SDEL => actions::strong::sdel, - SUPDATE => actions::strong::supdate, - DBSIZE => actions::dbsize::dbsize, - FLUSHDB => actions::flushdb::flushdb, - USET => actions::uset::uset, - KEYLEN => actions::keylen::keylen, - MKSNAP => admin::mksnap::mksnap, - LSKEYS => actions::lskeys::lskeys, - POP => actions::pop::pop, - MPOP => actions::mpop::mpop, - LSET => actions::lists::lset, - LGET => actions::lists::lget::lget, - LMOD => actions::lists::lmod::lmod, - WHEREAMI => actions::whereami::whereami, - SYS => admin::sys::sys, - { - // actions that need other arguments - AUTH => auth::auth(con, auth, iter) - } - ); - } - Ok(()) -} - -/// Execute a stage **completely**. This means that action errors are never propagated -/// over the try operator -async fn execute_stage_pedantic<'a, C: BufferedSocketStream, P: ProtocolSpec>( - handle: &mut Corestore, - con: &mut Connection, - auth: &mut AuthProviderHandle, - stage: &[UnsafeSlice], -) -> crate::IoResult<()> { - let ret = async { - self::execute_stage(handle, con, auth, stage).await?; - Ok(()) - }; - match ret.await { - Ok(()) => Ok(()), - Err(ActionError::ActionError(e)) => con._write_raw(e).await, - Err(ActionError::IoError(ioe)) => Err(ioe), - } -} - -action! { - /// Execute a basic pipelined query - fn execute_pipeline( - handle: &mut Corestore, - con: &mut Connection, - auth: &mut AuthProviderHandle, - pipeline: PipelinedQuery - ) { - for stage in pipeline.into_inner().iter() { - self::execute_stage_pedantic(handle, con, auth, stage).await?; - } - Ok(()) - } -} diff --git a/server/src/registry/mod.rs b/server/src/registry/mod.rs deleted file mode 100644 index 5c5298c4..00000000 --- a/server/src/registry/mod.rs +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Created on Mon Jul 26 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # System-wide registry -//! -//! The registry module provides interfaces for system-wide, global state management -//! - -use { - crate::corestore::lock::{QLGuard, QuickLock}, - core::sync::atomic::{AtomicBool, Ordering}, -}; - -const ORD_ACQ: Ordering = Ordering::Acquire; -const ORD_REL: Ordering = Ordering::Release; -const ORD_SEQ: Ordering = Ordering::SeqCst; - -/// A digital _trip switch_ that can be tripped and untripped in a thread -/// friendly, consistent manner. It is slightly expensive on processors -/// with weaker memory ordering (like ARM) when compared to the native -/// strong ordering provided by some platforms (like x86). -pub struct Trip { - /// the switch - inner: AtomicBool, -} - -impl Trip { - /// Get an untripped switch - pub const fn new_untripped() -> Self { - Self { - inner: AtomicBool::new(false), - } - } - /// trip the switch - pub fn trip(&self) { - // we need the strongest consistency here - self.inner.store(true, ORD_SEQ) - } - /// reset the switch - pub fn untrip(&self) { - // we need the strongest consistency here - self.inner.store(false, ORD_SEQ) - } - /// check if the switch has tripped - pub fn is_tripped(&self) -> bool { - self.inner.load(ORD_SEQ) - } - /// Returns the previous state and untrips the switch. **Single op** - pub fn check_and_untrip(&self) -> bool { - self.inner.swap(false, ORD_SEQ) - } -} - -/// The global system health -static GLOBAL_STATE: AtomicBool = AtomicBool::new(true); -/// The global flush state -static FLUSH_STATE: QuickLock<()> = QuickLock::new(()); -/// The preload trip switch -static PRELOAD_TRIPSWITCH: Trip = Trip::new_untripped(); -static CLEANUP_TRIPSWITCH: Trip = Trip::new_untripped(); - -/// Check the global system state -pub fn state_okay() -> bool { - GLOBAL_STATE.load(ORD_ACQ) -} - -/// Lock the global flush state. **Remember to drop the lock guard**; else you'll -/// end up pausing all sorts of global flushing/transactional systems -pub fn lock_flush_state() -> QLGuard<'static, ()> { - FLUSH_STATE.lock() -} - -/// Poison the global system state -pub fn poison() { - GLOBAL_STATE.store(false, ORD_REL) -} - -/// Unpoison the global system state -pub fn unpoison() { - GLOBAL_STATE.store(true, ORD_REL) -} - -/// Get a static reference to the global preload trip switch -pub fn get_preload_tripswitch() -> &'static Trip { - &PRELOAD_TRIPSWITCH -} - -/// Get a static reference to the global cleanup trip switch -pub fn get_cleanup_tripswitch() -> &'static Trip { - &CLEANUP_TRIPSWITCH -} diff --git a/server/src/services/bgsave.rs b/server/src/services/bgsave.rs deleted file mode 100644 index 517b1303..00000000 --- a/server/src/services/bgsave.rs +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Created on Sun May 16 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - config::BGSave, - corestore::Corestore, - registry, - storage::{self, v1::flush::Autoflush}, - IoResult, - }, - tokio::{ - sync::broadcast::Receiver, - time::{self, Duration}, - }, -}; - -/// The bgsave_scheduler calls the bgsave task in `Corestore` after `every` seconds -/// -/// The time after which the scheduler will wake up the BGSAVE task is determined by -/// `bgsave_cfg` which is to be passed as an argument. If BGSAVE is disabled, this function -/// immediately returns -pub async fn bgsave_scheduler(handle: Corestore, bgsave_cfg: BGSave, mut terminator: Receiver<()>) { - match bgsave_cfg { - BGSave::Enabled(duration) => { - // If we're here - the user doesn't trust his power supply or just values - // his data - which is good! So we'll turn this into a `Duration` - let duration = Duration::from_secs(duration); - loop { - tokio::select! { - // Sleep until `duration` from the current time instant - _ = time::sleep_until(time::Instant::now() + duration) => { - let cloned_handle = handle.clone(); - // we spawn this process just to ensure that it doesn't block the runtime's workers - // dedicated to async tasks (non-blocking) - tokio::task::spawn_blocking(move || { - let owned_handle = cloned_handle; - let _ = bgsave_blocking_section(owned_handle); - }).await.expect("Something caused the background service to panic"); - } - // Otherwise wait for a notification - _ = terminator.recv() => { - // we got a notification to quit; so break out - break; - } - } - } - } - BGSave::Disabled => { - // the user doesn't bother about his data; cool, let's not bother about it either - } - } - log::info!("BGSAVE service has exited"); -} - -/// Run bgsave -/// -/// This function just hides away the BGSAVE blocking section from the _public API_ -pub fn run_bgsave(handle: &Corestore) -> IoResult<()> { - storage::v1::flush::flush_full(Autoflush, handle.get_store()) -} - -/// This just wraps around [`_bgsave_blocking_section`] and prints nice log messages depending on the outcome -fn bgsave_blocking_section(handle: Corestore) -> bool { - registry::lock_flush_state(); - match run_bgsave(&handle) { - Ok(_) => { - log::info!("BGSAVE completed successfully"); - registry::unpoison(); - true - } - Err(e) => { - log::error!("BGSAVE failed with error: {}", e); - registry::poison(); - false - } - } -} diff --git a/server/src/services/snapshot.rs b/server/src/services/snapshot.rs deleted file mode 100644 index 866f9776..00000000 --- a/server/src/services/snapshot.rs +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Created on Sun May 16 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - config::SnapshotConfig, - corestore::Corestore, - registry, - storage::v1::sengine::{SnapshotActionResult, SnapshotEngine}, - }, - std::sync::Arc, - tokio::{ - sync::broadcast::Receiver, - time::{self, Duration}, - }, -}; - -/// The snapshot service -/// -/// This service calls `SnapEngine::mksnap()` periodically to create snapshots. Whenever -/// the interval for snapshotting expires or elapses, we create a snapshot. The snapshot service -/// keeps creating snapshots, as long as the database keeps running. Once [`dbnet::run`] broadcasts -/// a termination signal, we're ready to quit. This function will, by default, poison the database -/// if snapshotting fails, unless customized by the user. -pub async fn snapshot_service( - engine: Arc, - handle: Corestore, - ss_config: SnapshotConfig, - mut termination_signal: Receiver<()>, -) { - match ss_config { - SnapshotConfig::Disabled => { - // since snapshotting is disabled, we'll imediately return - return; - } - SnapshotConfig::Enabled(configuration) => { - let (duration, _, failsafe) = configuration.decompose(); - let duration = Duration::from_secs(duration); - loop { - tokio::select! { - _ = time::sleep_until(time::Instant::now() + duration) => { - let succeeded = engine.mksnap(handle.clone_store()).await == SnapshotActionResult::Ok; - #[cfg(test)] - { - use std::env::set_var; - if succeeded { - set_var("SKYTEST_SNAPSHOT_OKAY", "true"); - } else { - set_var("SKYTEST_SNAPSHOT_OKAY", "false"); - } - } - if succeeded { - // it passed, so unpoison the handle - registry::unpoison(); - } else if failsafe { - // mksnap returned false and we are set to stop writes if snapshotting failed - // so let's poison the handle - registry::poison(); - } - }, - _ = termination_signal.recv() => { - // time to terminate; goodbye! - break; - } - } - } - } - } - log::info!("Snapshot service has exited"); -} diff --git a/server/src/storage/mod.rs b/server/src/storage/mod.rs deleted file mode 100644 index eac66bce..00000000 --- a/server/src/storage/mod.rs +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Created on Sat Mar 05 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -/*! -# Storage Engine - -The main code in here lies inside `v1`. The reason we've chose to do so is for backwards compatibility. -Unlike other projects that can _just break_, well, we can't. A database has no right to break data no -matter what the reason. You can't just mess up someone's data because you found a more efficient -way to store things. That's why we'll version modules that correspond to version of Cyanstore. It is -totally legal for one version to call data that correspond to other versions. - -## How to break - -Whenever we're making changes, here's what we need to keep in mind: -1. If the format has only changed, but not the corestore structures, then simply gate a v2 and change -the functions here -2. If the format has changed and so have the corestore structures, then: - 1. Move out all the _old_ corestore structures into that version gate - 2. Then create the new structures in corestore, as appropriate - 3. The methods here should "identify" a version (usually by bytemarks on the `PRELOAD` which - is here to stay) - 4. Now, the appropriate (if any) version's decoder is called, then the old structures are restored. - Now, create the new structures using the old ones and then finally return them - -Here's some rust-flavored pseudocode: -``` -let version = find_version(preload_file_contents)?; -match version { - V1 => { - migration::migrate(v1::read_full()?) - } - V2 => { - v2::read_full() - } - _ => error!("Unknown version"), -} -``` - -The migration module, which doesn't exist, yet will always have a way to transform older structures into -the current one. This can be achieved with some trait/generic hackery (although it might be pretty simple -in practice). -*/ - -pub mod v1; - -pub mod unflush { - use crate::{corestore::memstore::Memstore, storage::v1::error::StorageEngineResult}; - pub fn read_full() -> StorageEngineResult { - super::v1::unflush::read_full() - } -} diff --git a/server/src/storage/v1/bytemarks.rs b/server/src/storage/v1/bytemarks.rs deleted file mode 100644 index a8ac489d..00000000 --- a/server/src/storage/v1/bytemarks.rs +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Created on Sun Jul 18 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![allow(unused)] - -//! # Bytemarks -//! -//! Bytemarks are single bytes that are written to parts of files to provide metadata. This module -//! contains a collection of these. -//! -//! ## Userspace and system bytemarks -//! -//! Although ks/system and ks/default might _reside_ next to each other, their bytemarks are entirely -//! different! - -// model -/* - * KVEBlob: - * (1) Pure KVEBlob: [0, 3] - * (2) KVExt/Listmap: [4, 7] -*/ -/// KVEBlob model bytemark with key:bin, val:bin -pub const BYTEMARK_MODEL_KV_BIN_BIN: u8 = 0; -/// KVEBlob model bytemark with key:bin, val:str -pub const BYTEMARK_MODEL_KV_BIN_STR: u8 = 1; -/// KVEBlob model bytemark with key:str, val:str -pub const BYTEMARK_MODEL_KV_STR_STR: u8 = 2; -/// KVEBlob model bytemark with key:str, val:bin -pub const BYTEMARK_MODEL_KV_STR_BIN: u8 = 3; -/// KVEBlob model bytemark with key:binstr, val: list -pub const BYTEMARK_MODEL_KV_BINSTR_LIST_BINSTR: u8 = 4; -/// KVEBlob model bytemark with key:binstr, val: list -pub const BYTEMARK_MODEL_KV_BINSTR_LIST_STR: u8 = 5; -/// KVEBlob model bytemark with key:str, val: list -pub const BYTEMARK_MODEL_KV_STR_LIST_BINSTR: u8 = 6; -/// KVEBlob model bytemark with key:str, val: list -pub const BYTEMARK_MODEL_KV_STR_LIST_STR: u8 = 7; - -// storage bym -/// Persistent storage bytemark -pub const BYTEMARK_STORAGE_PERSISTENT: u8 = 0; -/// Volatile storage bytemark -pub const BYTEMARK_STORAGE_VOLATILE: u8 = 1; - -// system bym -pub const SYSTEM_TABLE_AUTH: u8 = 0; diff --git a/server/src/storage/v1/error.rs b/server/src/storage/v1/error.rs deleted file mode 100644 index b7b05c4b..00000000 --- a/server/src/storage/v1/error.rs +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Created on Sat Mar 26 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use {crate::corestore::memstore::ObjectID, core::fmt, std::io::Error as IoError}; - -pub type StorageEngineResult = Result; - -pub trait ErrorContext { - /// Provide some context to an error - fn map_err_context(self, extra: impl ToString) -> StorageEngineResult; -} - -impl ErrorContext for Result { - fn map_err_context(self, extra: impl ToString) -> StorageEngineResult { - self.map_err(|e| StorageEngineError::ioerror_extra(e, extra.to_string())) - } -} - -#[derive(Debug)] -pub enum StorageEngineError { - /// An I/O Error - IoError(IoError), - /// An I/O Error with extra context - IoErrorExtra(IoError, String), - /// A corrupted file - CorruptedFile(String), - /// The file contains bad metadata - BadMetadata(String), -} - -impl StorageEngineError { - pub fn corrupted_partmap(ksid: &ObjectID) -> Self { - Self::CorruptedFile(format!("{ksid}/PARTMAP", ksid = unsafe { ksid.as_str() })) - } - pub fn bad_metadata_in_table(ksid: &ObjectID, table: &ObjectID) -> Self { - unsafe { - Self::CorruptedFile(format!( - "{ksid}/{table}", - ksid = ksid.as_str(), - table = table.as_str() - )) - } - } - pub fn corrupted_preload() -> Self { - Self::CorruptedFile("PRELOAD".into()) - } - pub fn ioerror_extra(ioe: IoError, extra: impl ToString) -> Self { - Self::IoErrorExtra(ioe, extra.to_string()) - } -} - -impl From for StorageEngineError { - fn from(ioe: IoError) -> Self { - Self::IoError(ioe) - } -} - -impl fmt::Display for StorageEngineError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::IoError(ioe) => write!(f, "I/O error: {}", ioe), - Self::IoErrorExtra(ioe, extra) => write!(f, "I/O error while {extra}: {ioe}"), - Self::CorruptedFile(cfile) => write!(f, "file `{cfile}` is corrupted"), - Self::BadMetadata(file) => write!(f, "bad metadata in file `{file}`"), - } - } -} diff --git a/server/src/storage/v1/flush.rs b/server/src/storage/v1/flush.rs deleted file mode 100644 index 6fb09b8b..00000000 --- a/server/src/storage/v1/flush.rs +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Created on Sat Jul 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # Flush routines -//! -//! This module contains multiple flush routines: at the memstore level, the keyspace level and -//! the table level - -use { - super::{bytemarks, interface}, - crate::{ - corestore::{ - map::iter::BorrowedIter, - memstore::SYSTEM, - memstore::{Keyspace, Memstore, ObjectID, SystemKeyspace}, - table::{DataModel, SystemDataModel, SystemTable, Table}, - }, - registry, - util::Wrapper, - IoResult, - }, - core::ops::Deref, - std::{io::Write, sync::Arc}, -}; - -pub trait StorageTarget { - /// This storage target needs a reinit of the tree despite no preload trip. - /// Exempli gratia: rsnap, snap - const NEEDS_TREE_INIT: bool; - /// This storage target should untrip the trip switch - /// - /// Example cases where this doesn't apply: snapshots - const SHOULD_UNTRIP_PRELOAD_TRIPSWITCH: bool; - /// The root for this storage target. **Must not be separator terminated!** - fn root(&self) -> String; - /// Returns the path to the `PRELOAD_` **temporary file** ($ROOT/PRELOAD) - fn preload_target(&self) -> String { - let mut p = self.root(); - p.push('/'); - p.push_str("PRELOAD_"); - p - } - /// Returns the path to the keyspace folder. ($ROOT/{keyspace}) - fn keyspace_target(&self, keyspace: &str) -> String { - let mut p = self.root(); - p.push('/'); - p.push_str(keyspace); - p - } - /// Returns the path to a `PARTMAP_` for the given keyspace. **temporary file** - /// ($ROOT/{keyspace}/PARTMAP) - fn partmap_target(&self, keyspace: &str) -> String { - let mut p = self.keyspace_target(keyspace); - p.push('/'); - p.push_str("PARTMAP_"); - p - } - /// Returns the path to the table file. **temporary file** ($ROOT/{keyspace}/{table}_) - fn table_target(&self, keyspace: &str, table: &str) -> String { - let mut p = self.keyspace_target(keyspace); - p.push('/'); - p.push_str(table); - p.push('_'); - p - } -} - -/// The autoflush target (BGSAVE target) -pub struct Autoflush; - -impl StorageTarget for Autoflush { - const NEEDS_TREE_INIT: bool = false; - const SHOULD_UNTRIP_PRELOAD_TRIPSWITCH: bool = true; - fn root(&self) -> String { - String::from(interface::DIR_KSROOT) - } -} - -/// A remote snapshot storage target -pub struct RemoteSnapshot<'a> { - name: &'a str, -} - -impl<'a> RemoteSnapshot<'a> { - pub fn new(name: &'a str) -> Self { - Self { name } - } -} - -impl<'a> StorageTarget for RemoteSnapshot<'a> { - const NEEDS_TREE_INIT: bool = true; - const SHOULD_UNTRIP_PRELOAD_TRIPSWITCH: bool = false; - fn root(&self) -> String { - let mut p = String::from(interface::DIR_RSNAPROOT); - p.push('/'); - p.push_str(self.name); - p - } -} - -/// A snapshot storage target -pub struct LocalSnapshot { - name: String, -} - -impl LocalSnapshot { - pub fn new(name: String) -> Self { - Self { name } - } -} - -impl StorageTarget for LocalSnapshot { - const NEEDS_TREE_INIT: bool = true; - const SHOULD_UNTRIP_PRELOAD_TRIPSWITCH: bool = false; - fn root(&self) -> String { - let mut p = String::from(interface::DIR_SNAPROOT); - p.push('/'); - p.push_str(&self.name); - p - } -} - -/// A keyspace that can be flushed -pub trait FlushableKeyspace> { - /// The number of tables in this keyspace - fn table_count(&self) -> usize; - /// An iterator to the tables in this keyspace. - /// All of them implement [`FlushableTable`] - fn get_iter(&self) -> BorrowedIter<'_, ObjectID, U>; -} - -impl FlushableKeyspace> for Keyspace { - fn table_count(&self) -> usize { - self.tables.len() - } - fn get_iter(&self) -> BorrowedIter<'_, ObjectID, Arc

> { - self.tables.iter() - } -} - -impl FlushableKeyspace> for SystemKeyspace { - fn table_count(&self) -> usize { - self.tables.len() - } - fn get_iter(&self) -> BorrowedIter<'_, ObjectID, Wrapper> { - self.tables.iter() - } -} - -pub trait FlushableTable { - /// Table is volatile - fn is_volatile(&self) -> bool; - /// Returns the storage code bytemark - fn storage_code(&self) -> u8; - /// Serializes the table and writes it to the provided buffer - fn write_table_to(&self, writer: &mut W) -> IoResult<()>; - /// Returns the model code bytemark - fn model_code(&self) -> u8; -} - -impl FlushableTable for Table { - fn is_volatile(&self) -> bool { - self.is_volatile() - } - fn write_table_to(&self, writer: &mut W) -> IoResult<()> { - match self.get_model_ref() { - DataModel::KV(ref kve) => super::se::raw_serialize_map(kve.get_inner_ref(), writer), - DataModel::KVExtListmap(ref kvl) => { - super::se::raw_serialize_list_map(kvl.get_inner_ref(), writer) - } - } - } - fn storage_code(&self) -> u8 { - self.storage_type() - } - fn model_code(&self) -> u8 { - self.get_model_code() - } -} - -impl FlushableTable for SystemTable { - fn is_volatile(&self) -> bool { - false - } - fn write_table_to(&self, writer: &mut W) -> IoResult<()> { - match self.get_model_ref() { - SystemDataModel::Auth(amap) => super::se::raw_serialize_map(amap.as_ref(), writer), - } - } - fn storage_code(&self) -> u8 { - 0 - } - fn model_code(&self) -> u8 { - match self.get_model_ref() { - SystemDataModel::Auth(_) => bytemarks::SYSTEM_TABLE_AUTH, - } - } -} - -/// Flush the entire **preload + keyspaces + their partmaps** -pub fn flush_full(target: T, store: &Memstore) -> IoResult<()> { - // IMPORTANT: Just untrip and get the status at this exact point in time - // don't spread it over two atomic accesses because another thread may have updated - // it in-between. Even if it was untripped, we'll get the expected outcome here: false - let mut should_create_tree = T::NEEDS_TREE_INIT; - if T::SHOULD_UNTRIP_PRELOAD_TRIPSWITCH { - // this target shouldn't untrip the tripswitch - should_create_tree |= registry::get_preload_tripswitch().check_and_untrip(); - } - if should_create_tree { - // re-init the tree as new tables/keyspaces may have been added - super::interface::create_tree(&target, store)?; - self::oneshot::flush_preload(&target, store)?; - } - // flush userspace keyspaces - for keyspace in store.keyspaces.iter() { - self::flush_keyspace_full(&target, keyspace.key(), keyspace.value().as_ref())?; - } - // flush system tables - // HACK(@ohsayan): DO NOT REORDER THIS. THE above loop will flush a PARTMAP and an empty - // keyspace once. But this has to be done again! The system keyspace in the above loop is a - // dummy one because it is located in a different field. So, we need to flush the actual - // tables - self::flush_keyspace_full(&target, &SYSTEM, &store.system)?; - Ok(()) -} - -/// Flushes the entire **keyspace + partmap** -pub fn flush_keyspace_full(target: &T, ksid: &ObjectID, keyspace: &K) -> IoResult<()> -where - T: StorageTarget, - U: Deref, - Tbl: FlushableTable, - K: FlushableKeyspace, -{ - self::oneshot::flush_partmap(target, ksid, keyspace)?; - self::oneshot::flush_keyspace(target, ksid, keyspace) -} - -pub mod oneshot { - //! # Irresponsible flushing - //! - //! Every function does **exactly what it says** and nothing more. No partition - //! files et al are handled - //! - use super::*; - use std::fs::{self, File}; - - #[inline(always)] - fn cowfile( - cowfile_name: &str, - with_open: impl FnOnce(&mut File) -> IoResult<()>, - ) -> IoResult<()> { - let mut f = File::create(cowfile_name)?; - with_open(&mut f)?; - f.sync_all()?; - fs::rename(cowfile_name, &cowfile_name[..cowfile_name.len() - 1]) - } - - /// No `partmap` handling. Just flushes the table to the expected location - pub fn flush_table( - target: &T, - tableid: &ObjectID, - ksid: &ObjectID, - table: &U, - ) -> IoResult<()> { - if table.is_volatile() { - // no flushing needed - Ok(()) - } else { - let path = unsafe { target.table_target(ksid.as_str(), tableid.as_str()) }; - cowfile(&path, |file| { - super::interface::serialize_table_into_slow_buffer(file, table) - }) - } - } - - /// Flushes an entire keyspace to the expected location. No `partmap` or `preload` handling - pub fn flush_keyspace(target: &T, ksid: &ObjectID, keyspace: &K) -> IoResult<()> - where - T: StorageTarget, - U: Deref, - Tbl: FlushableTable, - K: FlushableKeyspace, - { - for table in keyspace.get_iter() { - self::flush_table(target, table.key(), ksid, table.value().deref())?; - } - Ok(()) - } - - /// Flushes a single partmap - pub fn flush_partmap(target: &T, ksid: &ObjectID, keyspace: &K) -> IoResult<()> - where - T: StorageTarget, - U: Deref, - Tbl: FlushableTable, - K: FlushableKeyspace, - { - let path = unsafe { target.partmap_target(ksid.as_str()) }; - cowfile(&path, |file| { - super::interface::serialize_partmap_into_slow_buffer(file, keyspace) - }) - } - - // Flush the `PRELOAD` - pub fn flush_preload(target: &T, store: &Memstore) -> IoResult<()> { - let preloadtmp = target.preload_target(); - cowfile(&preloadtmp, |file| { - super::interface::serialize_preload_into_slow_buffer(file, store) - }) - } -} diff --git a/server/src/storage/v1/interface.rs b/server/src/storage/v1/interface.rs deleted file mode 100644 index c3777482..00000000 --- a/server/src/storage/v1/interface.rs +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Created on Sat Jul 10 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! Interfaces with the file system - -use std::collections::HashMap; - -use { - crate::{ - corestore::memstore::Memstore, - registry, - storage::v1::flush::{FlushableKeyspace, FlushableTable, StorageTarget}, - IoResult, - }, - core::ops::Deref, - std::{ - collections::HashSet, - fs, - io::{BufWriter, Write}, - }, -}; - -pub const DIR_KSROOT: &str = "data/ks"; -pub const DIR_SNAPROOT: &str = "data/snaps"; -pub const DIR_RSNAPROOT: &str = "data/rsnap"; -pub const DIR_BACKUPS: &str = "data/backups"; -pub const DIR_ROOT: &str = "data"; - -/// Creates the directories for the keyspaces -pub fn create_tree(target: &T, memroot: &Memstore) -> IoResult<()> { - for ks in memroot.keyspaces.iter() { - unsafe { - try_dir_ignore_existing!(target.keyspace_target(ks.key().as_str()))?; - } - } - Ok(()) -} - -/// This creates the root directory structure: -/// ``` -/// data/ -/// ks/ -/// ks1/ -/// ks2/ -/// ks3/ -/// snaps/ -/// backups/ -/// ``` -/// -/// If any directories exist, they are simply ignored -pub fn create_tree_fresh(target: &T, memroot: &Memstore) -> IoResult<()> { - try_dir_ignore_existing!( - DIR_ROOT, - DIR_KSROOT, - DIR_BACKUPS, - DIR_SNAPROOT, - DIR_RSNAPROOT - ); - self::create_tree(target, memroot) -} - -/// Clean up the tree -/// -/// **Warning**: Calling this is quite inefficient so consider calling it once or twice -/// throughout the lifecycle of the server -pub fn cleanup_tree(memroot: &Memstore) -> IoResult<()> { - if registry::get_cleanup_tripswitch().is_tripped() { - log::info!("We're cleaning up ..."); - // only run a cleanup if someone tripped the switch - // hashset because the fs itself will not allow duplicate entries - // the keyspaces directory will contain the PRELOAD file, but we'll just - // remove it from the list - let mut dir_keyspaces: HashSet = read_dir_to_col!(DIR_KSROOT); - dir_keyspaces.remove("PRELOAD"); - let our_keyspaces: HashMap> = memroot - .keyspaces - .iter() - .map(|kv| { - let ksid = unsafe { kv.key().as_str() }.to_owned(); - let tables: HashSet = kv - .value() - .tables - .iter() - .map(|tbl| unsafe { tbl.key().as_str() }.to_owned()) - .collect(); - (ksid, tables) - }) - .collect(); - - // these are the folders that we need to remove; plonk the deleted keyspaces first - let keyspaces_to_remove: Vec<&String> = dir_keyspaces - .iter() - .filter(|ksname| !our_keyspaces.contains_key(ksname.as_str())) - .collect(); - for folder in keyspaces_to_remove { - let ks_path = concat_str!(DIR_KSROOT, "/", folder); - fs::remove_dir_all(ks_path)?; - } - - // HACK(@ohsayan): Due to the nature of how system tables are stored in v1, we need to get rid of this - // ensuring that system tables don't end up being removed (since no system tables are actually - // purged at this time) - let mut our_keyspaces = our_keyspaces; - our_keyspaces.remove("system").unwrap(); - let our_keyspaces = our_keyspaces; - - // now remove the dropped tables - for (keyspace, tables) in our_keyspaces { - let ks_path = concat_str!(DIR_KSROOT, "/", keyspace.as_str()); - // read what is present in the tables directory - let mut dir_tbls: HashSet = read_dir_to_col!(&ks_path); - // in the list of directories we collected, remove PARTMAP because we should NOT - // delete it - dir_tbls.remove("PARTMAP"); - // find what tables we should remove - let tables_to_remove = dir_tbls.difference(&tables); - for removed_table in tables_to_remove { - let fpath = concat_path!(&ks_path, removed_table); - fs::remove_file(&fpath)?; - } - } - } - Ok(()) -} - -/// Uses a buffered writer under the hood to improve write performance as the provided -/// writable interface might be very slow. The buffer does flush once done, however, it -/// is important that you fsync yourself! -pub fn serialize_table_into_slow_buffer( - buffer: &mut T, - writable_item: &U, -) -> IoResult<()> { - let mut buffer = BufWriter::new(buffer); - writable_item.write_table_to(&mut buffer)?; - buffer.flush()?; - Ok(()) -} - -pub fn serialize_partmap_into_slow_buffer(buffer: &mut T, ks: &K) -> IoResult<()> -where - T: Write, - U: Deref, - Tbl: FlushableTable, - K: FlushableKeyspace, -{ - let mut buffer = BufWriter::new(buffer); - super::se::raw_serialize_partmap(&mut buffer, ks)?; - buffer.flush()?; - Ok(()) -} - -pub fn serialize_preload_into_slow_buffer( - buffer: &mut T, - store: &Memstore, -) -> IoResult<()> { - let mut buffer = BufWriter::new(buffer); - super::preload::raw_generate_preload(&mut buffer, store)?; - buffer.flush()?; - Ok(()) -} diff --git a/server/src/storage/v1/iter.rs b/server/src/storage/v1/iter.rs deleted file mode 100644 index e8235ce8..00000000 --- a/server/src/storage/v1/iter.rs +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Created on Tue Aug 31 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -/* - UNSAFE(@ohsayan): Everything done here is unsafely safe. We - reinterpret bits of one type as another. What could be worse? - nah, it's not that bad. We know that the byte representations - would be in the way we expect. If the data is corrupted, we - can guarantee that we won't ever read incorrect lengths of data - and we won't read into others' memory (or corrupt our own). -*/ - -use { - crate::storage::v1::SharedSlice, - core::{mem, ptr, slice}, -}; - -const SIZE_64BIT: usize = mem::size_of::(); -const SIZE_128BIT: usize = SIZE_64BIT * 2; - -/// This contains the fn ptr to decode bytes wrt to the host's endian. For example, if you're on an LE machine and -/// you're reading data from a BE machine, then simply set the endian to big. This only affects the first read and not -/// subsequent ones (unless you switch between machines of different endian, obviously) -static mut NATIVE_ENDIAN_READER: unsafe fn(*const u8) -> usize = super::de::transmute_len; - -/// Use this to set the current endian to LE. -/// -/// ## Safety -/// Make sure this is run from a single thread only! If not, good luck -pub(super) unsafe fn endian_set_little() { - NATIVE_ENDIAN_READER = super::de::transmute_len_le; -} - -/// Use this to set the current endian to BE. -/// -/// ## Safety -/// Make sure this is run from a single thread only! If not, good luck -pub(super) unsafe fn endian_set_big() { - NATIVE_ENDIAN_READER = super::de::transmute_len_be; -} - -/// A raw slice iterator by using raw pointers -#[derive(Debug)] -pub struct RawSliceIter<'a> { - _base: &'a [u8], - cursor: *const u8, - terminal: *const u8, -} - -impl<'a> RawSliceIter<'a> { - /// Create a new slice iterator - pub fn new(slice: &'a [u8]) -> Self { - Self { - cursor: slice.as_ptr(), - terminal: unsafe { slice.as_ptr().add(slice.len()) }, - _base: slice, - } - } - /// Check the number of remaining bytes in the buffer - fn remaining(&self) -> usize { - unsafe { self.terminal.offset_from(self.cursor) as usize } - } - /// Increment the cursor by one - unsafe fn incr_cursor(&mut self) { - self.incr_cursor_by(1) - } - /// Check if the buffer was exhausted - fn exhausted(&self) -> bool { - self.cursor > self.terminal - } - /// Increment the cursor by the provided length - unsafe fn incr_cursor_by(&mut self, ahead: usize) { - { - self.cursor = self.cursor.add(ahead) - } - } - /// Get the next 64-bit integer, casting it to an `usize`, respecting endianness - pub fn next_64bit_integer_to_usize(&mut self) -> Option { - if self.remaining() < 8 { - // we need 8 bytes to read a 64-bit integer, so nope - None - } else { - unsafe { - // sweet, something is left - let l = NATIVE_ENDIAN_READER(self.cursor); - // now forward the cursor - self.incr_cursor_by(SIZE_64BIT); - Some(l) - } - } - } - /// Get a borrowed slice for the given length. The lifetime is important! - pub fn next_borrowed_slice(&mut self, len: usize) -> Option<&'a [u8]> { - if self.remaining() < len { - None - } else { - unsafe { - let d = slice::from_raw_parts(self.cursor, len); - self.incr_cursor_by(len); - Some(d) - } - } - } - /// Get the next 64-bit usize - pub fn next_64bit_integer_pair_to_usize(&mut self) -> Option<(usize, usize)> { - if self.remaining() < SIZE_128BIT { - None - } else { - unsafe { - let v1 = NATIVE_ENDIAN_READER(self.cursor); - self.incr_cursor_by(SIZE_64BIT); - let v2 = NATIVE_ENDIAN_READER(self.cursor); - self.incr_cursor_by(SIZE_64BIT); - Some((v1, v2)) - } - } - } - /// Get the next owned [`Data`] with the provided length - pub fn next_owned_data(&mut self, len: usize) -> Option { - if self.remaining() < len { - // not enough left - None - } else { - // we have something to look at - unsafe { - let d = slice::from_raw_parts(self.cursor, len); - let d = Some(SharedSlice::new(d)); - self.incr_cursor_by(len); - d - } - } - } - /// Get the next 8-bit unsigned integer - pub fn next_8bit_integer(&mut self) -> Option { - if self.exhausted() { - None - } else { - unsafe { - let x = ptr::read(self.cursor); - self.incr_cursor(); - Some(x) - } - } - } - /// Check if the cursor has reached end-of-allocation - pub fn end_of_allocation(&self) -> bool { - self.cursor == self.terminal - } - /// Get a borrowed iterator. This is super safe, funny enough, because of the lifetime - /// bound that we add to the iterator object - pub fn get_borrowed_iter(&mut self) -> RawSliceIterBorrowed<'_> { - RawSliceIterBorrowed::new(self.cursor, self.terminal, &mut self.cursor) - } -} - -#[derive(Debug)] -pub struct RawSliceIterBorrowed<'a> { - cursor: *const u8, - end_ptr: *const u8, - mut_ptr: &'a mut *const u8, -} - -impl<'a> RawSliceIterBorrowed<'a> { - fn new( - cursor: *const u8, - end_ptr: *const u8, - mut_ptr: &'a mut *const u8, - ) -> RawSliceIterBorrowed<'a> { - Self { - cursor, - end_ptr, - mut_ptr, - } - } - /// Check the number of remaining bytes in the buffer - fn remaining(&self) -> usize { - unsafe { self.end_ptr.offset_from(self.cursor) as usize } - } - /// Increment the cursor by the provided length - unsafe fn incr_cursor_by(&mut self, ahead: usize) { - { - self.cursor = self.cursor.add(ahead) - } - } - pub fn next_64bit_integer_to_usize(&mut self) -> Option { - if self.remaining() < 8 { - None - } else { - unsafe { - let size = NATIVE_ENDIAN_READER(self.cursor); - self.incr_cursor_by(SIZE_64BIT); - Some(size) - } - } - } - pub fn next_owned_data(&mut self, len: usize) -> Option { - if self.remaining() < len { - None - } else { - unsafe { - let d = slice::from_raw_parts(self.cursor, len); - let d = Some(SharedSlice::new(d)); - self.incr_cursor_by(len); - d - } - } - } -} - -impl<'a> Drop for RawSliceIterBorrowed<'a> { - fn drop(&mut self) { - *self.mut_ptr = self.cursor; - } -} diff --git a/server/src/storage/v1/macros.rs b/server/src/storage/v1/macros.rs deleted file mode 100644 index 4a4acd57..00000000 --- a/server/src/storage/v1/macros.rs +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Created on Sat Jul 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -macro_rules! little_endian { - ($block:block) => { - #[cfg(target_endian = "little")] - { - $block - } - }; -} - -macro_rules! big_endian { - ($block:block) => { - #[cfg(target_endian = "big")] - { - $block - } - }; -} - -macro_rules! not_64_bit { - ($block:block) => { - #[cfg(not(target_pointer_width = "64"))] - { - $block - } - }; -} - -macro_rules! is_64_bit { - ($block:block) => { - #[cfg(target_pointer_width = "64")] - { - $block - } - }; -} - -macro_rules! to_64bit_native_endian { - ($e:expr) => { - ($e as u64) - }; -} - -macro_rules! try_dir_ignore_existing { - ($dir:expr) => {{ - match std::fs::create_dir_all($dir) { - Ok(_) => Ok(()), - Err(e) => match e.kind() { - std::io::ErrorKind::AlreadyExists => Ok(()), - _ => Err(e), - }, - } - }}; - ($($dir:expr),*) => { - $(try_dir_ignore_existing!($dir)?;)* - } -} - -#[macro_export] -macro_rules! concat_path { - ($($s:expr),+) => {{ { - let mut path = std::path::PathBuf::with_capacity($(($s).len()+)*0); - $(path.push($s);)* - path - }}}; -} - -#[macro_export] -macro_rules! concat_str { - ($($s:expr),+) => {{ { - let mut st = std::string::String::with_capacity($(($s).len()+)*0); - $(st.push_str($s);)* - st - }}}; -} - -macro_rules! read_dir_to_col { - ($root:expr) => { - std::fs::read_dir($root)? - .map(|v| { - v.expect("Unexpected directory parse failure") - .file_name() - .to_string_lossy() - .to_string() - }) - .collect() - }; -} - -#[cfg(test)] -macro_rules! lvec { - ($($item:expr),+ $(,)?) => {{ - let v = std::vec![$($item.into()),*]; - parking_lot::RwLock::new(v) - }}; -} diff --git a/server/src/storage/v1/mod.rs b/server/src/storage/v1/mod.rs deleted file mode 100644 index ae0b62fd..00000000 --- a/server/src/storage/v1/mod.rs +++ /dev/null @@ -1,660 +0,0 @@ -/* - * Created on Wed Jul 07 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -/* - Encoding and decoding tested on 32-bit/64-bit Little Endian (Intel x86) and Big Endian - (MIPS). Also tested UB with miri and memory leaks with valgrind - -- Sayan on July 8, 2021 -*/ - -/*! # Storage engine (v1/Cyanstore 1A) - -This module contains code to rapidly encode/decode data. All sizes are encoded into unsigned -64-bit integers for compatibility across 16/32/64 bit platforms. This means that a -data file generated on a 32-bit machine will work seamlessly on a 64-bit machine -and vice versa. Of course, provided that On a 32-bit system, 32 high bits are just zeroed. - -## Endianness - -All sizes are stored in native endian. If a dataset is imported from a system from a different endian, it is -simply translated into the host's native endian. How everything else is stored is not worth -discussing here. Byte swaps just need one instruction on most architectures - -## Safety - -> Trust me, all methods are bombingly unsafe. They do such crazy things that you might not -think of using them anywhere outside. This is a specialized parser built for the database. --- Sayan (July 2021) - -*/ - -use { - crate::corestore::{array::Array, htable::Coremap, SharedSlice}, - core::{hash::Hash, mem, slice}, - std::{collections::HashSet, io::Write}, -}; - -// for some astronomical reasons do not mess with this -#[macro_use] -mod macros; -// endof do not mess -pub mod bytemarks; -pub mod error; -pub mod flush; -pub mod interface; -pub mod iter; -pub mod preload; -pub mod sengine; -pub mod unflush; -// test -#[cfg(test)] -mod tests; - -/* - Endian and pointer "appendix": - We assume a fixed size of 1 for all the cases. All sizes don't hit over isize::MAX as - guaranteed by our allocation methods. Also, 32-bit to 64-bit and vice versa aren't - worth discussing here (irrespective of endianness). That's because all sizes are stored - as unsigned 64-bit integers. So, get the correct byte order, raw cast to u64, down cast - or up cast as required by the target's pointer width. - - Current limitations: - - 32-bit is compatible with 64-bit - - 16-bit is compatible with 64-bit - - But 64-bit may not be compatible with 32/16 bit due to the difference in sizes - - ------------------------------------------------------ - Appendix I: Same endian, different pointer width (R/W) - ------------------------------------------------------ - (1) Little endian on little endian (64-bit) - (A) Writing - In-memory size: [0, 0, 0, 0, 0, 0, 0, 1] =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (no op) - We write to file: [0, 0, 0, 0, 0, 0, 0, 1] - (B) Reading - This is read: [0, 0, 0, 0, 0, 0, 0, 1] - Raw cast =(usize)> [0, 0, 0, 0, 0, 0, 0, 1] (one memcpy) - - (2) Little endian on little endian (32-bit) - (A) Writing - In-memory size: [0, 0, 0, 1] =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (up cast) - We write to file: [0, 0, 0, 0, 0, 0, 0, 1] - (B) Reading - This is read: [0, 0, 0, 0, 0, 0, 0, 1] - Raw cast =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (one memcpy) - Lossy cast =(usize)> [0, 0, 0, 1] - - (3) Big endian on big endian (64-bit) - (A) Writing - In-memory size: [1, 0, 0, 0, 0, 0, 0, 0] =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (no op) - We write to file: [1, 0, 0, 0, 0, 0, 0, 0] - (B) Reading - This is read: [1, 0, 0, 0, 0, 0, 0, 0] - Raw cast =(usize)> [1, 0, 0, 0, 0, 0, 0, 0] (one memcpy) - - (4) Big endian (64-bit) on big endian (32-bit) - (A) Writing - In-memory size: [1, 0, 0, 0] =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (up cast) - We write to file: [1, 0, 0, 0, 0, 0, 0, 0] - (B) Reading - This is read: [1, 0, 0, 0, 0, 0, 0, 0] - Raw cast =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (one memcpy) - Lossy cast =(usize)> [1, 0, 0, 0] - - ------------------------------------------------------ - Appendix II: Different endian, same pointer width (R/W) - ------------------------------------------------------ - (1) Little endian on big endian (64-bit) - (A) Writing - ^^ See Appendix I/1/A - (B) Reading - This is read: [0, 0, 0, 0, 0, 0, 0, 1] - Raw cast =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (one memcpy) - Reverse the bits: [1, 0, 0, 0, 0, 0, 0, 0] (constant time ptr swap) - Cast =(usize)> [1, 0, 0, 0, 0, 0, 0, 0] (no op) - - (2) Big endian on little endian (64-bit) - (A) Writing - ^^ See Appendix I/3/A - (B) Reading - This is read: [1, 0, 0, 0, 0, 0, 0, 0] - Raw cast =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (one memcpy) - Reverse the bits: [0, 0, 0, 0, 0, 0, 0, 1] (constant time ptr swap) - Cast =(usize)> [0, 0, 0, 0, 0, 0, 0, 1] (no op) - - (3) Little endian on big endian (32-bit) - (A) Writing - ^^ See Appendix I/2/A - (B) Reading - This is read: [0, 0, 0, 0, 0, 0, 0, 1] - Raw cast =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (one memcpy) - Reverse the bits: [1, 0, 0, 0, 0, 0, 0, 0] (constant time ptr swap) - Lossy cast =(usize)> [1, 0, 0, 0] - - (4) Big endian on little endian (32-bit) - (A) Writing - ^^ See Appendix I/4/A - (B) Reading - This is read: [1, 0, 0, 0, 0, 0, 0, 0] - Raw cast =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (one memcpy) - Reverse the bits: [0, 0, 0, 0, 0, 0, 0, 1] (constant time ptr swap) - Lossy cast =(usize)> [0, 0, 0, 1] - - ------------------------------------------------------ - Appendix III: Warnings - ------------------------------------------------------ - (1) Gotchas on 32-bit big endian - (A) While writing - Do not swap bytes before up cast - (B) While reading - Do not down cast before swapping bytes -*/ - -/// Get the raw bytes of anything. -/// -/// DISCLAIMER: THIS FUNCTION CAN DO TERRIBLE THINGS (especially when you think about padding) -unsafe fn raw_byte_repr<'a, T: 'a>(len: &'a T) -> &'a [u8] { - { - let ptr: *const u8 = len as *const T as *const u8; - slice::from_raw_parts::<'a>(ptr, mem::size_of::()) - } -} - -mod se { - use super::*; - use crate::kvengine::LockedVec; - use crate::storage::v1::flush::FlushableKeyspace; - use crate::storage::v1::flush::FlushableTable; - use crate::IoResult; - use core::ops::Deref; - - macro_rules! unsafe_sz_byte_repr { - ($e:expr) => { - raw_byte_repr(&to_64bit_native_endian!($e)) - }; - } - - #[cfg(test)] - /// Serialize a map into a _writable_ thing - pub fn serialize_map(map: &Coremap) -> IoResult> { - /* - [LEN:8B][KLEN:8B|VLEN:8B][K][V][KLEN:8B][VLEN:8B]... - */ - // write the len header first - let mut w = Vec::with_capacity(128); - self::raw_serialize_map(map, &mut w)?; - Ok(w) - } - - /// Serialize a map and write it to a provided buffer - pub fn raw_serialize_map, U: AsRef<[u8]>>( - map: &Coremap, - w: &mut W, - ) -> IoResult<()> - where - W: Write, - T: AsRef<[u8]> + Hash + Eq, - U: AsRef<[u8]>, - { - unsafe { - w.write_all(raw_byte_repr(&to_64bit_native_endian!(map.len())))?; - // now the keys and values - for kv in map.iter() { - let (k, v) = (kv.key(), kv.value()); - let kref = k.as_ref(); - let vref = v.as_ref(); - w.write_all(raw_byte_repr(&to_64bit_native_endian!(kref.len())))?; - w.write_all(raw_byte_repr(&to_64bit_native_endian!(vref.len())))?; - w.write_all(kref)?; - w.write_all(vref)?; - } - } - Ok(()) - } - - /// Serialize a set and write it to a provided buffer - pub fn raw_serialize_set(map: &Coremap, w: &mut W) -> IoResult<()> - where - W: Write, - K: Eq + Hash + AsRef<[u8]>, - { - unsafe { - w.write_all(raw_byte_repr(&to_64bit_native_endian!(map.len())))?; - // now the keys and values - for kv in map.iter() { - let key = kv.key().as_ref(); - w.write_all(raw_byte_repr(&to_64bit_native_endian!(key.len())))?; - w.write_all(key)?; - } - } - Ok(()) - } - - /// Generate a partition map for the given keyspace - /// ```text - /// [8B: EXTENT]([8B: LEN][?B: PARTITION ID][1B: Storage type][1B: Model type])* - /// ``` - pub fn raw_serialize_partmap(w: &mut W, keyspace: &K) -> IoResult<()> - where - W: Write, - U: Deref, - Tbl: FlushableTable, - K: FlushableKeyspace, - { - unsafe { - // extent - w.write_all(raw_byte_repr(&to_64bit_native_endian!( - keyspace.table_count() - )))?; - for table in keyspace.get_iter() { - // partition ID len - w.write_all(raw_byte_repr(&to_64bit_native_endian!(table.key().len())))?; - // parition ID - w.write_all(table.key())?; - // now storage type - w.write_all(raw_byte_repr(&table.storage_code()))?; - // now model type - w.write_all(raw_byte_repr(&table.model_code()))?; - } - } - Ok(()) - } - pub fn raw_serialize_list_map( - data: &Coremap, - w: &mut W, - ) -> IoResult<()> - where - W: Write, - { - /* - [8B: Extent]([8B: Key extent][?B: Key][8B: Max index][?B: Payload])* - */ - unsafe { - // Extent - w.write_all(unsafe_sz_byte_repr!(data.len()))?; - // Enter iter - '_1: for key in data.iter() { - // key - let k = key.key(); - // list payload - let vread = key.value().read(); - let v: &Vec = &vread; - // write the key extent - w.write_all(unsafe_sz_byte_repr!(k.len()))?; - // write the key - w.write_all(k)?; - // write the list payload - self::raw_serialize_nested_list(w, &v)?; - } - } - Ok(()) - } - /// Serialize a `[[u8]]` (i.e a slice of slices) - pub fn raw_serialize_nested_list<'a, W, T: 'a + ?Sized, U: 'a>( - w: &mut W, - inp: &'a T, - ) -> IoResult<()> - where - T: AsRef<[U]>, - U: AsRef<[u8]>, - W: Write, - { - // ([8B:EL_EXTENT][?B: Payload])* - let inp = inp.as_ref(); - unsafe { - // write extent - w.write_all(unsafe_sz_byte_repr!(inp.len()))?; - // now enter loop and write elements - for element in inp.iter() { - let element = element.as_ref(); - // write element extent - w.write_all(unsafe_sz_byte_repr!(element.len()))?; - // write element - w.write_all(element)?; - } - } - Ok(()) - } -} - -mod de { - use super::iter::{RawSliceIter, RawSliceIterBorrowed}; - use super::{Array, Coremap, Hash, HashSet, SharedSlice}; - use crate::kvengine::LockedVec; - use core::ptr; - use parking_lot::RwLock; - use std::collections::HashMap; - - pub trait DeserializeFrom { - fn is_expected_len(clen: usize) -> bool; - fn from_slice(slice: &[u8]) -> Self; - } - - pub trait DeserializeInto: Sized { - fn new_empty() -> Self; - fn from_slice(slice: &[u8]) -> Option; - } - - impl DeserializeInto for Coremap { - fn new_empty() -> Self { - Coremap::new() - } - fn from_slice(slice: &[u8]) -> Option { - self::deserialize_map(slice) - } - } - - impl DeserializeInto for Coremap { - fn new_empty() -> Self { - Coremap::new() - } - fn from_slice(slice: &[u8]) -> Option { - self::deserialize_list_map(slice) - } - } - - impl DeserializeInto for Coremap - where - T: Hash + Eq + DeserializeFrom, - U: DeserializeFrom, - { - fn new_empty() -> Self { - Coremap::new() - } - fn from_slice(slice: &[u8]) -> Option { - self::deserialize_map_ctype(slice) - } - } - - pub fn deserialize_into(input: &[u8]) -> Option { - T::from_slice(input) - } - - impl DeserializeFrom for Array { - fn is_expected_len(clen: usize) -> bool { - clen <= N - } - fn from_slice(slice: &[u8]) -> Self { - unsafe { Self::from_slice(slice) } - } - } - - impl DeserializeFrom for [u8; N] { - fn is_expected_len(clen: usize) -> bool { - clen == N - } - fn from_slice(slice: &[u8]) -> Self { - slice.try_into().unwrap() - } - } - - pub fn deserialize_map_ctype(data: &[u8]) -> Option> - where - T: Eq + Hash + DeserializeFrom, - U: DeserializeFrom, - { - let mut rawiter = RawSliceIter::new(data); - let len = rawiter.next_64bit_integer_to_usize()?; - let map = Coremap::new(); - for _ in 0..len { - let (lenkey, lenval) = rawiter.next_64bit_integer_pair_to_usize()?; - if !(T::is_expected_len(lenkey) && U::is_expected_len(lenval)) { - return None; - } - let key = T::from_slice(rawiter.next_borrowed_slice(lenkey)?); - let value = U::from_slice(rawiter.next_borrowed_slice(lenval)?); - if !map.true_if_insert(key, value) { - // duplicates - return None; - } - } - Some(map) - } - - /// Deserialize a set to a custom type - pub fn deserialize_set_ctype(data: &[u8]) -> Option> - where - T: DeserializeFrom + Eq + Hash, - { - let mut rawiter = RawSliceIter::new(data); - let len = rawiter.next_64bit_integer_to_usize()?; - let mut set = HashSet::new(); - set.try_reserve(len).ok()?; - for _ in 0..len { - let lenkey = rawiter.next_64bit_integer_to_usize()?; - if !T::is_expected_len(lenkey) { - return None; - } - // get the key as a raw slice, we've already checked if end_ptr is less - let key = T::from_slice(rawiter.next_borrowed_slice(lenkey)?); - // push it in - if !set.insert(key) { - // repeat?; that's not what we wanted - return None; - } - } - if rawiter.end_of_allocation() { - Some(set) - } else { - // nope, someone gave us more data - None - } - } - - /// Deserializes a map-like set which has an 2x1B _bytemark_ for every entry - pub fn deserialize_set_ctype_bytemark(data: &[u8]) -> Option> - where - T: DeserializeFrom + Eq + Hash, - { - let mut rawiter = RawSliceIter::new(data); - // so we have 8B. Just unsafe access and transmute it - let len = rawiter.next_64bit_integer_to_usize()?; - let mut set = HashMap::new(); - set.try_reserve(len).ok()?; - for _ in 0..len { - let lenkey = rawiter.next_64bit_integer_to_usize()?; - if !T::is_expected_len(lenkey) { - return None; - } - // get the key as a raw slice, we've already checked if end_ptr is less - let key = T::from_slice(rawiter.next_borrowed_slice(lenkey)?); - let bytemark_a = rawiter.next_8bit_integer()?; - let bytemark_b = rawiter.next_8bit_integer()?; - // push it in - if set.insert(key, (bytemark_a, bytemark_b)).is_some() { - // repeat?; that's not what we wanted - return None; - } - } - if rawiter.end_of_allocation() { - Some(set) - } else { - // nope, someone gave us more data - None - } - } - /// Deserialize a file that contains a serialized map. This also returns the model code - pub fn deserialize_map(data: &[u8]) -> Option> { - let mut rawiter = RawSliceIter::new(data); - let len = rawiter.next_64bit_integer_to_usize()?; - let hm = Coremap::try_with_capacity(len).ok()?; - for _ in 0..len { - let (lenkey, lenval) = rawiter.next_64bit_integer_pair_to_usize()?; - let key = rawiter.next_owned_data(lenkey)?; - let val = rawiter.next_owned_data(lenval)?; - // push it in - hm.upsert(key, val); - } - if rawiter.end_of_allocation() { - Some(hm) - } else { - // nope, someone gave us more data - None - } - } - - pub fn deserialize_list_map(bytes: &[u8]) -> Option> { - let mut rawiter = RawSliceIter::new(bytes); - // get the len - let len = rawiter.next_64bit_integer_to_usize()?; - // allocate a map - let map = Coremap::try_with_capacity(len).ok()?; - // now enter a loop - for _ in 0..len { - let keylen = rawiter.next_64bit_integer_to_usize()?; - // get key - let key = rawiter.next_owned_data(keylen)?; - let borrowed_iter = rawiter.get_borrowed_iter(); - let list = self::deserialize_nested_list(borrowed_iter)?; - // push it in - map.true_if_insert(key, RwLock::new(list)); - } - if rawiter.end_of_allocation() { - Some(map) - } else { - // someone returned more data - None - } - } - - /// Deserialize a nested list: `[EXTENT]([EL_EXT][EL])*` - /// - pub fn deserialize_nested_list(mut iter: RawSliceIterBorrowed<'_>) -> Option> { - // get list payload len - let list_payload_extent = iter.next_64bit_integer_to_usize()?; - let mut list = Vec::new(); - list.try_reserve(list_payload_extent).ok()?; - for _ in 0..list_payload_extent { - // get element size - let list_element_payload_size = iter.next_64bit_integer_to_usize()?; - // now get element - let element = iter.next_owned_data(list_element_payload_size)?; - list.push(element); - } - Some(list) - } - - #[allow(clippy::needless_return)] - pub(super) unsafe fn transmute_len(start_ptr: *const u8) -> usize { - little_endian! {{ - return self::transmute_len_le(start_ptr); - }}; - big_endian! {{ - return self::transmute_len_be(start_ptr); - }} - } - - #[allow(clippy::needless_return)] // Clippy really misunderstands this - pub(super) unsafe fn transmute_len_le(start_ptr: *const u8) -> usize { - little_endian!({ - // So we have an LE target - is_64_bit!({ - // 64-bit LE - return ptr::read_unaligned(start_ptr.cast()); - }); - not_64_bit!({ - // 32-bit LE - let ret1: u64 = ptr::read_unaligned(start_ptr.cast()); - // lossy cast - let ret = ret1 as usize; - if ret > (isize::MAX as usize) { - // this is a backup method for us incase a giant 48-bit address is - // somehow forced to be read on this machine - panic!("RT panic: Very high size for current pointer width"); - } - return ret; - }); - }); - - big_endian!({ - // so we have a BE target - is_64_bit!({ - // 64-bit big endian - let ret: usize = ptr::read_unaligned(start_ptr.cast()); - // swap byte order - return ret.swap_bytes(); - }); - not_64_bit!({ - // 32-bit big endian - let ret: u64 = ptr::read_unaligned(start_ptr.cast()); - // swap byte order and lossy cast - let ret = (ret.swap_bytes()) as usize; - // check if overflow - if ret > (isize::MAX as usize) { - // this is a backup method for us incase a giant 48-bit address is - // somehow forced to be read on this machine - panic!("RT panic: Very high size for current pointer width"); - } - return ret; - }); - }); - } - - #[allow(clippy::needless_return)] // Clippy really misunderstands this - pub(super) unsafe fn transmute_len_be(start_ptr: *const u8) -> usize { - big_endian!({ - // So we have a BE target - is_64_bit!({ - // 64-bit BE - return ptr::read_unaligned(start_ptr.cast()); - }); - not_64_bit!({ - // 32-bit BE - let ret1: u64 = ptr::read_unaligned(start_ptr.cast()); - // lossy cast - let ret = ret1 as usize; - if ret > (isize::MAX as usize) { - // this is a backup method for us incase a giant 48-bit address is - // somehow forced to be read on this machine - panic!("RT panic: Very high size for current pointer width"); - } - return ret; - }); - }); - - little_endian!({ - // so we have an LE target - is_64_bit!({ - // 64-bit little endian - let ret: usize = ptr::read_unaligned(start_ptr.cast()); - // swap byte order - return ret.swap_bytes(); - }); - not_64_bit!({ - // 32-bit little endian - let ret: u64 = ptr::read_unaligned(start_ptr.cast()); - // swap byte order and lossy cast - let ret = (ret.swap_bytes()) as usize; - // check if overflow - if ret > (isize::MAX as usize) { - // this is a backup method for us incase a giant 48-bit address is - // somehow forced to be read on this machine - panic!("RT panic: Very high size for current pointer width"); - } - return ret; - }); - }); - } -} diff --git a/server/src/storage/v1/preload.rs b/server/src/storage/v1/preload.rs deleted file mode 100644 index c5c3355a..00000000 --- a/server/src/storage/v1/preload.rs +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Created on Sat Jul 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # Preload binary files -//! -//! Preloads are very critical binary files which contain metadata for this instance of -//! the database. Preloads are of two kinds: -//! 1. the `PRELOAD` that is placed at the root directory -//! 2. the `PARTMAP` preload that is placed in the ks directory -//! - -use { - crate::{ - corestore::memstore::{Memstore, ObjectID}, - storage::v1::error::{StorageEngineError, StorageEngineResult}, - IoResult, - }, - core::ptr, - std::{ - collections::{HashMap, HashSet}, - io::Write, - }, -}; - -pub type LoadedPartfile = HashMap; - -// our version and endian are based on nibbles - -const META_SEGMENT_LE: u8 = 0b1000_0000; -const META_SEGMENT_BE: u8 = 0b1000_0001; - -#[cfg(target_endian = "little")] -const META_SEGMENT: u8 = META_SEGMENT_LE; - -#[cfg(target_endian = "big")] -const META_SEGMENT: u8 = META_SEGMENT_BE; - -/// Generate the `PRELOAD` disk file for this instance -/// ```text -/// [1B: Endian Mark/Version Mark (padded)] => Meta segment -/// [8B: Extent header] => Predata Segment -/// ([8B: Partion ID len][8B: Parition ID (not padded)])* => Data segment -/// ``` -/// -pub(super) fn raw_generate_preload(w: &mut W, store: &Memstore) -> IoResult<()> { - // generate the meta segment - w.write_all(&[META_SEGMENT])?; - super::se::raw_serialize_set(&store.keyspaces, w)?; - Ok(()) -} - -/// Reads the preload file and returns a set -pub(super) fn read_preload_raw(preload: Vec) -> StorageEngineResult> { - if preload.len() < 16 { - // nah, this is a bad disk file - return Err(StorageEngineError::corrupted_preload()); - } - // first read in the meta segment - unsafe { - let meta_segment: u8 = ptr::read(preload.as_ptr()); - match meta_segment { - META_SEGMENT_BE => { - super::iter::endian_set_big(); - } - META_SEGMENT_LE => { - super::iter::endian_set_little(); - } - _ => return Err(StorageEngineError::BadMetadata("preload".into())), - } - } - // all checks complete; time to decode - super::de::deserialize_set_ctype(&preload[1..]) - .ok_or_else(StorageEngineError::corrupted_preload) -} diff --git a/server/src/storage/v1/sengine.rs b/server/src/storage/v1/sengine.rs deleted file mode 100644 index 5f6f9860..00000000 --- a/server/src/storage/v1/sengine.rs +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Created on Sun Aug 08 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - self::queue::Queue, - super::interface::{DIR_RSNAPROOT, DIR_SNAPROOT}, - crate::{ - corestore::{iarray::IArray, lazy::Lazy, lock::QuickLock, memstore::Memstore}, - storage::v1::flush::{LocalSnapshot, RemoteSnapshot}, - }, - chrono::prelude::Utc, - core::{fmt, str}, - regex::Regex, - std::{collections::HashSet, fs, io::Error as IoError, path::Path, sync::Arc}, -}; - -type QStore = IArray<[String; 64]>; -type SnapshotResult = Result; - -/// Matches any string which is in the following format: -/// ```text -/// YYYYMMDD-HHMMSS -/// ``` -pub static SNAP_MATCH: Lazy Regex> = Lazy::new(|| { - Regex::new("^\\d{4}(0[1-9]|1[012])(0[1-9]|[12][0-9]|3[01])(-)(?:(?:([01]?\\d|2[0-3]))?([0-5]?\\d))?([0-5]?\\d)$").unwrap() -}); - -#[derive(Debug)] -pub enum SnapshotEngineError { - Io(IoError), - Engine(&'static str), -} - -impl From for SnapshotEngineError { - fn from(e: IoError) -> SnapshotEngineError { - SnapshotEngineError::Io(e) - } -} - -impl From<&'static str> for SnapshotEngineError { - fn from(e: &'static str) -> SnapshotEngineError { - SnapshotEngineError::Engine(e) - } -} - -impl fmt::Display for SnapshotEngineError { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> { - match self { - Self::Engine(estr) => { - formatter.write_str("Snapshot engine error")?; - formatter.write_str(estr)?; - } - Self::Io(e) => { - formatter.write_str("Snapshot engine IOError:")?; - formatter.write_str(&e.to_string())?; - } - } - Ok(()) - } -} - -/// The snapshot engine -#[derive(Debug)] -pub struct SnapshotEngine { - local_enabled: bool, - /// the local snapshot queue - local_queue: QuickLock, - /// the remote snapshot lock - remote_queue: QuickLock>>, -} - -#[derive(Debug, PartialEq, Eq)] -pub enum SnapshotActionResult { - Ok, - Busy, - Disabled, - Failure, - AlreadyExists, -} - -impl SnapshotEngine { - /// Returns a fresh, uninitialized snapshot engine instance - pub fn new(maxlen: usize) -> Self { - Self { - local_enabled: true, - local_queue: QuickLock::new(Queue::new(maxlen, maxlen == 0)), - remote_queue: QuickLock::new(HashSet::new()), - } - } - pub fn new_disabled() -> Self { - Self { - local_enabled: false, - local_queue: QuickLock::new(Queue::new(0, true)), - remote_queue: QuickLock::new(HashSet::new()), - } - } - fn _parse_dir( - dir: &str, - is_okay: impl Fn(&str) -> bool, - mut append: impl FnMut(String), - ) -> SnapshotResult<()> { - let dir = fs::read_dir(dir)?; - for entry in dir { - let entry = entry?; - if entry.file_type()?.is_dir() { - let fname = entry.file_name(); - let name = fname.to_string_lossy(); - if !is_okay(&name) { - return Err("unknown folder in snapshot directory".into()); - } - append(name.to_string()); - } else { - return Err("unrecognized file in snapshot directory".into()); - } - } - Ok(()) - } - pub fn parse_dir(&self) -> SnapshotResult<()> { - let mut local_queue = self.local_queue.lock(); - Self::_parse_dir( - DIR_SNAPROOT, - |name| SNAP_MATCH.is_match(name), - |snapshot| local_queue.push(snapshot), - )?; - let mut remote_queue = self.remote_queue.lock(); - Self::_parse_dir( - DIR_RSNAPROOT, - |_| true, - |rsnap| { - remote_queue.insert(rsnap.into_boxed_str().into_boxed_bytes()); - }, - )?; - Ok(()) - } - /// Generate the snapshot name - fn get_snapname(&self) -> String { - Utc::now().format("%Y%m%d-%H%M%S").to_string() - } - fn _mksnap_blocking_section(store: &Memstore, name: String) -> SnapshotResult<()> { - if Path::new(&format!("{DIR_SNAPROOT}/{name}")).exists() { - Err(SnapshotEngineError::Engine("Server time is incorrect")) - } else { - let snapshot = LocalSnapshot::new(name); - super::flush::flush_full(snapshot, store)?; - Ok(()) - } - } - fn _rmksnap_blocking_section(store: &Memstore, name: &str) -> SnapshotResult<()> { - let snapshot = RemoteSnapshot::new(name); - super::flush::flush_full(snapshot, store)?; - Ok(()) - } - /// Spawns a blocking task on a threadpool for blocking tasks. Returns either of: - /// - `0` => Okay (returned **even if old snap deletion failed**) - /// - `1` => Error - /// - `2` => Disabled - /// - `3` => Busy - pub async fn mksnap(&self, store: Arc) -> SnapshotActionResult { - if self.local_enabled { - // try to lock the local queue - let mut queue = match self.local_queue.try_lock() { - Some(lck) => lck, - None => return SnapshotActionResult::Busy, - }; - let name = self.get_snapname(); - let nameclone = name.clone(); - let todel = queue.add_new(name); - let snap_create_result = tokio::task::spawn_blocking(move || { - Self::_mksnap_blocking_section(&store, nameclone) - }) - .await - .expect("mksnap thread panicked"); - - // First create the new snap - match snap_create_result { - Ok(_) => { - log::info!("Successfully created snapshot"); - } - Err(e) => { - log::error!("Failed to create snapshot with error: {}", e); - // so it failed, remove it from queue - let _ = queue.pop_last().unwrap(); - return SnapshotActionResult::Failure; - } - } - - // Now delete the older snap (if any) - if let Some(snap) = todel { - tokio::task::spawn_blocking(move || { - if let Err(e) = fs::remove_dir_all(concat_path!(DIR_SNAPROOT, snap)) { - log::warn!("Failed to remove older snapshot (ignored): {}", e); - } else { - log::info!("Successfully removed older snapshot"); - } - }) - .await - .expect("mksnap thread panicked"); - } - drop(queue); - SnapshotActionResult::Ok - } else { - SnapshotActionResult::Disabled - } - } - /// Spawns a blocking task to create a remote snapshot. Returns either of: - /// - `0` => Okay - /// - `1` => Error - /// - `3` => Busy - /// (consistent with mksnap) - pub async fn mkrsnap(&self, name: &[u8], store: Arc) -> SnapshotActionResult { - let mut remq = match self.remote_queue.try_lock() { - Some(q) => q, - None => return SnapshotActionResult::Busy, - }; - if remq.contains(name) { - SnapshotActionResult::AlreadyExists - } else { - let nameclone = name.to_owned(); - let ret = tokio::task::spawn_blocking(move || { - let name_str = unsafe { - // SAFETY: We have already checked if name is UTF-8 - str::from_utf8_unchecked(&nameclone) - }; - if let Err(e) = Self::_rmksnap_blocking_section(&store, name_str) { - log::error!("Remote snapshot failed with: {}", e); - SnapshotActionResult::Failure - } else { - log::info!("Remote snapshot succeeded"); - SnapshotActionResult::Ok - } - }) - .await - .expect("rmksnap thread panicked"); - assert!(remq.insert(name.to_owned().into_boxed_slice())); - ret - } - } -} - -mod queue { - //! An extremely simple queue implementation which adds more items to the queue - //! freely and once the threshold limit is reached, it pops off the oldest element and returns it - //! - //! This implementation is specifically built for use with the snapshotting utility - use super::QStore; - use crate::corestore::iarray; - #[derive(Debug, PartialEq, Eq)] - pub struct Queue { - queue: QStore, - maxlen: usize, - dontpop: bool, - } - - impl Queue { - pub const fn new(maxlen: usize, dontpop: bool) -> Self { - Queue { - queue: iarray::new_const_iarray(), - maxlen, - dontpop, - } - } - pub fn push(&mut self, item: String) { - self.queue.push(item) - } - /// This returns a `String` only if the queue is full. Otherwise, a `None` is returned most of the time - pub fn add_new(&mut self, item: String) -> Option { - if self.dontpop { - // We don't need to pop anything since the user - // wants to keep all the items in the queue - self.queue.push(item); - None - } else { - // The user wants to keep a maximum of `maxtop` items - // so we will check if the current queue is full - // if it is full, then the `maxtop` limit has been reached - // so we will remove the oldest item and then push the - // new item onto the queue - let x = if self.is_overflow() { self.pop() } else { None }; - self.queue.push(item); - x - } - } - /// Check if we have reached the maximum queue size limit - fn is_overflow(&self) -> bool { - self.queue.len() == self.maxlen - } - /// Remove the last item inserted - fn pop(&mut self) -> Option { - if self.queue.is_empty() { - None - } else { - Some(unsafe { - // SAFETY: We have already checked if the queue is empty or not - self.queue.remove(0) - }) - } - } - pub fn pop_last(&mut self) -> Option { - self.queue.pop() - } - } - - #[test] - fn test_queue() { - let mut q = Queue::new(4, false); - assert!(q.add_new(String::from("snap1")).is_none()); - assert!(q.add_new(String::from("snap2")).is_none()); - assert!(q.add_new(String::from("snap3")).is_none()); - assert!(q.add_new(String::from("snap4")).is_none()); - assert_eq!( - q.add_new(String::from("snap5")), - Some(String::from("snap1")) - ); - assert_eq!( - q.add_new(String::from("snap6")), - Some(String::from("snap2")) - ); - } - - #[test] - fn test_queue_dontpop() { - // This means that items can only be added or all of them can be deleted - let mut q = Queue::new(4, true); - assert!(q.add_new(String::from("snap1")).is_none()); - assert!(q.add_new(String::from("snap2")).is_none()); - assert!(q.add_new(String::from("snap3")).is_none()); - assert!(q.add_new(String::from("snap4")).is_none()); - assert!(q.add_new(String::from("snap5")).is_none()); - assert!(q.add_new(String::from("snap6")).is_none()); - } -} diff --git a/server/src/storage/v1/tests.rs b/server/src/storage/v1/tests.rs deleted file mode 100644 index f4e854d5..00000000 --- a/server/src/storage/v1/tests.rs +++ /dev/null @@ -1,828 +0,0 @@ -/* - * Created on Sat Jul 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use super::*; - -#[test] -fn test_serialize_deserialize_empty() { - let cmap = Coremap::new(); - let ser = se::serialize_map(&cmap).unwrap(); - let de = de::deserialize_map(&ser).unwrap(); - assert!(de.len() == 0); -} - -#[test] -fn test_serialize_deserialize_map_with_empty_elements() { - let cmap = Coremap::new(); - cmap.true_if_insert(SharedSlice::from("sayan"), SharedSlice::from("")); - cmap.true_if_insert( - SharedSlice::from("sayan's second key"), - SharedSlice::from(""), - ); - cmap.true_if_insert( - SharedSlice::from("sayan's third key"), - SharedSlice::from(""), - ); - cmap.true_if_insert(SharedSlice::from(""), SharedSlice::from("")); - let ser = se::serialize_map(&cmap).unwrap(); - let de = de::deserialize_map(&ser).unwrap(); - assert_eq!(de.len(), cmap.len()); - assert!(cmap.into_iter().all(|(k, v)| de.get(&k).unwrap().eq(&v))); -} - -#[test] -fn test_ser_de_few_elements() { - let cmap = Coremap::new(); - cmap.upsert("sayan".into(), "writes code".into()); - cmap.upsert("supersayan".into(), "writes super code".into()); - let ser = se::serialize_map(&cmap).unwrap(); - let de = de::deserialize_map(&ser).unwrap(); - assert!(de.len() == cmap.len()); - assert!(de - .iter() - .all(|kv| cmap.get(kv.key()).unwrap().eq(kv.value()))); -} - -cfg_test!( - use libstress::utils::generate_random_string_vector; - use rand::thread_rng; - #[test] - fn roast_the_serializer() { - const COUNT: usize = 1000_usize; - const LEN: usize = 8_usize; - let mut rng = thread_rng(); - let (keys, values) = ( - generate_random_string_vector(COUNT, LEN, &mut rng, true).unwrap(), - generate_random_string_vector(COUNT, LEN, &mut rng, false).unwrap(), - ); - let cmap: Coremap = keys - .iter() - .zip(values.iter()) - .map(|(k, v)| { - ( - SharedSlice::from(k.to_owned()), - SharedSlice::from(v.to_owned()), - ) - }) - .collect(); - let ser = se::serialize_map(&cmap).unwrap(); - let de = de::deserialize_map(&ser).unwrap(); - assert!(de - .iter() - .all(|kv| cmap.get(kv.key()).unwrap().eq(kv.value()))); - assert!(de.len() == cmap.len()); - } - - #[test] - fn test_ser_de_safety() { - const COUNT: usize = 1000_usize; - const LEN: usize = 8_usize; - let mut rng = thread_rng(); - let (keys, values) = ( - generate_random_string_vector(COUNT, LEN, &mut rng, true).unwrap(), - generate_random_string_vector(COUNT, LEN, &mut rng, false).unwrap(), - ); - let cmap: Coremap = keys - .iter() - .zip(values.iter()) - .map(|(k, v)| { - ( - SharedSlice::from(k.to_owned()), - SharedSlice::from(v.to_owned()), - ) - }) - .collect(); - let mut se = se::serialize_map(&cmap).unwrap(); - // random chop - se.truncate(124); - // corrupted - assert!(de::deserialize_map(&se).is_none()); - } - #[test] - fn test_ser_de_excess_bytes() { - // this test needs a lot of auxiliary space - // we can approximate this to be: 100,000 x 30 bytes = 3,000,000 bytes - // and then we may have a clone overhead + heap allocation by the map - // so ~9,000,000 bytes or ~9MB - const COUNT: usize = 1000_usize; - const LEN: usize = 8_usize; - let mut rng = thread_rng(); - let (keys, values) = ( - generate_random_string_vector(COUNT, LEN, &mut rng, true).unwrap(), - generate_random_string_vector(COUNT, LEN, &mut rng, false).unwrap(), - ); - let cmap: Coremap = keys - .iter() - .zip(values.iter()) - .map(|(k, v)| { - ( - SharedSlice::from(k.to_owned()), - SharedSlice::from(v.to_owned()), - ) - }) - .collect(); - let mut se = se::serialize_map(&cmap).unwrap(); - // random patch - let patch: Vec = (0u16..500u16).into_iter().map(|v| (v >> 7) as u8).collect(); - se.extend(patch); - assert!(de::deserialize_map(&se).is_none()); - } -); - -#[cfg(target_pointer_width = "32")] -#[test] -#[should_panic] -fn test_runtime_panic_32bit_or_lower() { - let max = u64::MAX; - let byte_stream = unsafe { raw_byte_repr(&max).to_owned() }; - let ptr = byte_stream.as_ptr(); - unsafe { de::transmute_len(ptr) }; -} - -mod interface_tests { - use super::interface::{create_tree_fresh, DIR_KSROOT, DIR_SNAPROOT}; - use crate::corestore::memstore::Memstore; - use crate::storage::v1::flush::Autoflush; - use std::fs; - use std::path::PathBuf; - #[test] - fn test_tree() { - // HACK(@ohsayan): M1 builder is broken - if std::env::var_os("HACK_SKYD_TEST_IGNORE_TREE_TEST_M1").is_none() { - create_tree_fresh(&Autoflush, &Memstore::new_default()).unwrap(); - let read_ks: Vec = fs::read_dir(DIR_KSROOT) - .unwrap() - .map(|dir| { - let v = dir.unwrap().file_name(); - v.to_string_lossy().to_string() - }) - .collect(); - assert!(read_ks.contains(&"system".to_owned())); - assert!(read_ks.contains(&"default".to_owned())); - // just read one level of the snaps dir - let read_snaps: Vec = fs::read_dir(DIR_SNAPROOT) - .unwrap() - .map(|dir| { - let v = dir.unwrap().file_name(); - v.to_string_lossy().to_string() - }) - .collect(); - assert_eq!(read_snaps, Vec::::new()); - assert!(PathBuf::from("data/backups").is_dir()); - } - } -} - -mod preload_tests { - use super::*; - use crate::corestore::memstore::Memstore; - #[test] - fn test_preload() { - let memstore = Memstore::new_default(); - let mut v = Vec::new(); - preload::raw_generate_preload(&mut v, &memstore).unwrap(); - let de: Vec = preload::read_preload_raw(v) - .unwrap() - .into_iter() - .map(|each| unsafe { each.as_str().to_owned() }) - .collect(); - assert_veceq!(de, vec!["default".to_owned(), "system".to_owned()]); - } -} - -mod bytemark_set_tests { - use super::*; - use crate::corestore::memstore::{Keyspace, ObjectID}; - use crate::corestore::table::Table; - use std::collections::HashMap; - #[test] - fn test_bytemark_for_nonvolatile() { - let ks = Keyspace::empty_default(); - let mut v = Vec::new(); - se::raw_serialize_partmap(&mut v, &ks).unwrap(); - let ret: HashMap = de::deserialize_set_ctype_bytemark(&v).unwrap(); - let mut expected = HashMap::new(); - unsafe { - expected.insert( - ObjectID::from_slice("default"), - ( - bytemarks::BYTEMARK_STORAGE_PERSISTENT, - bytemarks::BYTEMARK_MODEL_KV_BIN_BIN, - ), - ); - } - assert_hmeq!(expected, ret); - } - #[test] - fn test_bytemark_volatility_mixed() { - let ks = Keyspace::empty(); - unsafe { - ks.create_table( - ObjectID::from_slice("cache"), - Table::new_kve_with_volatile(true), - ); - ks.create_table( - ObjectID::from_slice("supersafe"), - Table::new_kve_with_volatile(false), - ); - ks.create_table( - ObjectID::from_slice("safelist"), - Table::new_kve_listmap_with_data(Coremap::new(), false, true, true), - ); - } - let mut v = Vec::new(); - se::raw_serialize_partmap(&mut v, &ks).unwrap(); - let ret: HashMap = de::deserialize_set_ctype_bytemark(&v).unwrap(); - let mut expected = HashMap::new(); - unsafe { - // our cache is volatile - expected.insert( - ObjectID::from_slice("cache"), - ( - bytemarks::BYTEMARK_STORAGE_VOLATILE, - bytemarks::BYTEMARK_MODEL_KV_BIN_BIN, - ), - ); - // our supersafe is non volatile - expected.insert( - ObjectID::from_slice("supersafe"), - ( - bytemarks::BYTEMARK_STORAGE_PERSISTENT, - bytemarks::BYTEMARK_MODEL_KV_BIN_BIN, - ), - ); - expected.insert( - ObjectID::from_slice("safelist"), - ( - bytemarks::BYTEMARK_STORAGE_PERSISTENT, - bytemarks::BYTEMARK_MODEL_KV_STR_LIST_STR, - ), - ); - } - assert_hmeq!(expected, ret); - } -} - -mod bytemark_actual_table_restore { - use crate::corestore::{ - memstore::ObjectID, - table::{DescribeTable, KVEBlob, KVEList, Table}, - SharedSlice, - }; - use crate::kvengine::LockedVec; - use crate::storage::v1::{ - flush::{oneshot::flush_table, Autoflush}, - unflush::read_table, - }; - - macro_rules! insert { - ($table:ident, $k:expr, $v:expr) => { - assert!(gtable::(&$table) - .set(SharedSlice::from($k), SharedSlice::from($v)) - .unwrap()) - }; - } - - macro_rules! puthello { - ($table:ident) => { - insert!($table, "hello", "world") - }; - } - - fn gtable(table: &Table) -> &T::Table { - T::try_get(table).unwrap() - } - - use std::fs; - #[test] - fn table_restore_bytemark_kve() { - let default_keyspace = ObjectID::try_from_slice(b"actual_kve_restore").unwrap(); - fs::create_dir_all(format!("data/ks/{}", unsafe { default_keyspace.as_str() })).unwrap(); - let kve_bin_bin_name = ObjectID::try_from_slice(b"bin_bin").unwrap(); - let kve_bin_bin = Table::from_model_code(0, false).unwrap(); - puthello!(kve_bin_bin); - let kve_bin_str_name = ObjectID::try_from_slice(b"bin_str").unwrap(); - let kve_bin_str = Table::from_model_code(1, false).unwrap(); - puthello!(kve_bin_str); - let kve_str_str_name = ObjectID::try_from_slice(b"str_str").unwrap(); - let kve_str_str = Table::from_model_code(2, false).unwrap(); - puthello!(kve_str_str); - let kve_str_bin_name = ObjectID::try_from_slice(b"str_bin").unwrap(); - let kve_str_bin = Table::from_model_code(3, false).unwrap(); - puthello!(kve_str_bin); - let names: [(&ObjectID, &Table, u8); 4] = [ - (&kve_bin_bin_name, &kve_bin_bin, 0), - (&kve_bin_str_name, &kve_bin_str, 1), - (&kve_str_str_name, &kve_str_str, 2), - (&kve_str_bin_name, &kve_str_bin, 3), - ]; - // flush each of them - for (tablename, table, _) in names { - flush_table(&Autoflush, tablename, &default_keyspace, table).unwrap(); - } - let mut read_tables: Vec
= Vec::with_capacity(4); - // read each of them - for (tableid, _, modelcode) in names { - read_tables.push(read_table(&default_keyspace, tableid, false, modelcode).unwrap()); - } - for (index, (table, code)) in read_tables - .iter() - .map(|tbl| (gtable::(tbl), tbl.get_model_code())) - .enumerate() - { - assert_eq!(index, code as usize); - assert!(table.get("hello".as_bytes()).unwrap().unwrap().eq(b"world")); - assert_eq!(table.len(), 1); - } - } - - macro_rules! putlist { - ($table:ident) => { - gtable::(&$table) - .get_inner_ref() - .fresh_entry(SharedSlice::from("super")) - .unwrap() - .insert(LockedVec::new(vec![ - SharedSlice::from("hello"), - SharedSlice::from("world"), - ])) - }; - } - - #[test] - fn table_restore_bytemark_kvlist() { - let default_keyspace = ObjectID::try_from_slice(b"actual_kvl_restore").unwrap(); - fs::create_dir_all(format!("data/ks/{}", unsafe { default_keyspace.as_str() })).unwrap(); - let kve_bin_listbin_name = ObjectID::try_from_slice(b"bin_listbin").unwrap(); - let kve_bin_listbin = Table::from_model_code(4, false).unwrap(); - putlist!(kve_bin_listbin); - let kve_bin_liststr_name = ObjectID::try_from_slice(b"bin_liststr").unwrap(); - let kve_bin_liststr = Table::from_model_code(5, false).unwrap(); - putlist!(kve_bin_liststr); - let kve_str_listbinstr_name = ObjectID::try_from_slice(b"str_listbinstr").unwrap(); - let kve_str_listbinstr = Table::from_model_code(6, false).unwrap(); - putlist!(kve_str_listbinstr); - let kve_str_liststr_name = ObjectID::try_from_slice(b"str_liststr").unwrap(); - let kve_str_liststr = Table::from_model_code(7, false).unwrap(); - putlist!(kve_str_liststr); - let names: [(&ObjectID, &Table, u8); 4] = [ - (&kve_bin_listbin_name, &kve_bin_listbin, 4), - (&kve_bin_liststr_name, &kve_bin_liststr, 5), - (&kve_str_listbinstr_name, &kve_str_listbinstr, 6), - (&kve_str_liststr_name, &kve_str_liststr, 7), - ]; - // flush each of them - for (tablename, table, _) in names { - flush_table(&Autoflush, tablename, &default_keyspace, table).unwrap(); - } - let mut read_tables: Vec
= Vec::with_capacity(4); - // read each of them - for (tableid, _, modelcode) in names { - read_tables.push(read_table(&default_keyspace, tableid, false, modelcode).unwrap()); - } - for (index, (table, code)) in read_tables - .iter() - .map(|tbl| (gtable::(tbl), tbl.get_model_code())) - .enumerate() - { - // check code - assert_eq!(index + 4, code as usize); - // check payload - let vec = table.get_inner_ref().get("super".as_bytes()).unwrap(); - assert_eq!(vec.read().len(), 2); - assert_eq!(vec.read()[0], "hello"); - assert_eq!(vec.read()[1], "world"); - // check len - assert_eq!(table.len(), 1); - } - } -} - -mod flush_routines { - use crate::{ - corestore::{ - memstore::{Keyspace, ObjectID}, - table::{DataModel, Table}, - SharedSlice, - }, - kvengine::LockedVec, - storage::v1::{bytemarks, flush::Autoflush, Coremap}, - }; - use std::fs; - #[test] - fn test_flush_unflush_table_pure_kve() { - let tbl = Table::new_default_kve(); - tbl.get_kvstore() - .unwrap() - .set("hello".into(), "world".into()) - .unwrap(); - let tblid = unsafe { ObjectID::from_slice("mytbl1") }; - let ksid = unsafe { ObjectID::from_slice("myks1") }; - // create the temp dir for this test - fs::create_dir_all("data/ks/myks1").unwrap(); - super::flush::oneshot::flush_table(&Autoflush, &tblid, &ksid, &tbl).unwrap(); - // now that it's flushed, let's read the table using and unflush routine - let ret = super::unflush::read_table::
( - &ksid, - &tblid, - false, - bytemarks::BYTEMARK_MODEL_KV_BIN_BIN, - ) - .unwrap(); - assert_eq!( - ret.get_kvstore() - .unwrap() - .get(&SharedSlice::from("hello")) - .unwrap() - .unwrap() - .clone(), - SharedSlice::from("world") - ); - } - - #[test] - fn test_flush_unflush_table_kvext_listmap() { - let tbl = Table::new_kve_listmap_with_data(Coremap::new(), false, true, true); - if let DataModel::KVExtListmap(kvl) = tbl.get_model_ref() { - kvl.add_list("mylist".into()).unwrap(); - let list = kvl.get("mylist".as_bytes()).unwrap().unwrap(); - list.write().push("mysupervalue".into()); - } else { - panic!("Bad model!"); - } - let tblid = unsafe { ObjectID::from_slice("mylists1") }; - let ksid = unsafe { ObjectID::from_slice("mylistyks") }; - // create the temp dir for this test - fs::create_dir_all("data/ks/mylistyks").unwrap(); - super::flush::oneshot::flush_table(&Autoflush, &tblid, &ksid, &tbl).unwrap(); - // now that it's flushed, let's read the table using and unflush routine - let ret = super::unflush::read_table::
( - &ksid, - &tblid, - false, - bytemarks::BYTEMARK_MODEL_KV_STR_LIST_STR, - ) - .unwrap(); - assert!(!ret.is_volatile()); - if let DataModel::KVExtListmap(kvl) = ret.get_model_ref() { - let list = kvl.get("mylist".as_bytes()).unwrap().unwrap(); - let lread = list.read(); - assert_eq!(lread.len(), 1); - assert_eq!(lread[0].as_ref(), "mysupervalue".as_bytes()); - } else { - panic!("Bad model!"); - } - } - #[test] - fn test_flush_unflush_keyspace() { - // create the temp dir for this test - fs::create_dir_all("data/ks/myks_1").unwrap(); - let ksid = unsafe { ObjectID::from_slice("myks_1") }; - let tbl1 = unsafe { ObjectID::from_slice("mytbl_1") }; - let tbl2 = unsafe { ObjectID::from_slice("mytbl_2") }; - let list_tbl = unsafe { ObjectID::from_slice("mylist_1") }; - let ks = Keyspace::empty(); - - // a persistent table - let mytbl = Table::new_default_kve(); - mytbl - .get_kvstore() - .unwrap() - .set("hello".into(), "world".into()) - .unwrap(); - assert!(ks.create_table(tbl1.clone(), mytbl)); - - // and a table with lists - let cmap = Coremap::new(); - cmap.true_if_insert("mylist".into(), LockedVec::new(vec!["myvalue".into()])); - let my_list_tbl = Table::new_kve_listmap_with_data(cmap, false, true, true); - assert!(ks.create_table(list_tbl.clone(), my_list_tbl)); - - // and a volatile table - assert!(ks.create_table(tbl2.clone(), Table::new_kve_with_volatile(true))); - - // now flush it - super::flush::flush_keyspace_full(&Autoflush, &ksid, &ks).unwrap(); - let ret = super::unflush::read_keyspace::(&ksid).unwrap(); - let tbl1_ret = ret.tables.get(&tbl1).unwrap(); - let tbl2_ret = ret.tables.get(&tbl2).unwrap(); - let tbl3_ret_list = ret.tables.get(&list_tbl).unwrap(); - // should be a persistent table with the value we set - assert_eq!(tbl1_ret.count(), 1); - assert_eq!( - tbl1_ret - .get_kvstore() - .unwrap() - .get(&SharedSlice::from("hello")) - .unwrap() - .unwrap() - .clone(), - SharedSlice::from("world") - ); - // should be a volatile table with no values - assert_eq!(tbl2_ret.count(), 0); - assert!(tbl2_ret.is_volatile()); - // should have a list with the `myvalue` element - assert_eq!(tbl3_ret_list.count(), 1); - if let DataModel::KVExtListmap(kvl) = tbl3_ret_list.get_model_ref() { - assert_eq!( - kvl.get("mylist".as_bytes()).unwrap().unwrap().read()[0].as_ref(), - "myvalue".as_bytes() - ); - } else { - panic!( - "Wrong model. Expected listmap, got: {}", - tbl3_ret_list.get_model_code() - ); - } - } -} - -mod list_tests { - use super::iter::RawSliceIter; - use super::{de, se}; - use crate::corestore::{htable::Coremap, SharedSlice}; - use crate::kvengine::LockedVec; - use core::ops::Deref; - use parking_lot::RwLock; - #[test] - fn test_list_se_de() { - let mylist = vec![ - SharedSlice::from("a"), - SharedSlice::from("b"), - SharedSlice::from("c"), - ]; - let mut v = Vec::new(); - se::raw_serialize_nested_list(&mut v, &mylist).unwrap(); - let mut rawiter = RawSliceIter::new(&v); - let de = { de::deserialize_nested_list(rawiter.get_borrowed_iter()).unwrap() }; - assert_eq!(de, mylist); - } - #[test] - fn test_list_se_de_with_empty_element() { - let mylist = vec![ - SharedSlice::from("a"), - SharedSlice::from("b"), - SharedSlice::from("c"), - SharedSlice::from(""), - ]; - let mut v = Vec::new(); - se::raw_serialize_nested_list(&mut v, &mylist).unwrap(); - let mut rawiter = RawSliceIter::new(&v); - let de = { de::deserialize_nested_list(rawiter.get_borrowed_iter()).unwrap() }; - assert_eq!(de, mylist); - } - #[test] - fn test_empty_list_se_de() { - let mylist: Vec = vec![]; - let mut v = Vec::new(); - se::raw_serialize_nested_list(&mut v, &mylist).unwrap(); - let mut rawiter = RawSliceIter::new(&v); - let de = { de::deserialize_nested_list(rawiter.get_borrowed_iter()).unwrap() }; - assert_eq!(de, mylist); - } - #[test] - fn test_list_map_monoelement_se_de() { - let mymap = Coremap::new(); - let vals = lvec!["apples", "bananas", "carrots"]; - mymap.true_if_insert(SharedSlice::from("mykey"), RwLock::new(vals.read().clone())); - let mut v = Vec::new(); - se::raw_serialize_list_map(&mymap, &mut v).unwrap(); - let de = de::deserialize_list_map(&v).unwrap(); - assert_eq!(de.len(), 1); - let mykey_value = de - .get("mykey".as_bytes()) - .unwrap() - .value() - .deref() - .read() - .clone(); - assert_eq!( - mykey_value, - vals.into_inner() - .into_iter() - .map(SharedSlice::from) - .collect::>() - ); - } - #[test] - fn test_list_map_se_de() { - let mymap: Coremap = Coremap::new(); - let key1: SharedSlice = "mykey1".into(); - let val1 = lvec!["apples", "bananas", "carrots"]; - let key2: SharedSlice = "mykey2long".into(); - let val2 = lvec!["code", "coffee", "cats"]; - mymap.true_if_insert(key1.clone(), RwLock::new(val1.read().clone())); - mymap.true_if_insert(key2.clone(), RwLock::new(val2.read().clone())); - let mut v = Vec::new(); - se::raw_serialize_list_map(&mymap, &mut v).unwrap(); - let de = de::deserialize_list_map(&v).unwrap(); - assert_eq!(de.len(), 2); - assert_eq!( - de.get(&key1).unwrap().value().deref().read().clone(), - val1.into_inner() - .into_iter() - .map(SharedSlice::from) - .collect::>() - ); - assert_eq!( - de.get(&key2).unwrap().value().deref().read().clone(), - val2.into_inner() - .into_iter() - .map(SharedSlice::from) - .collect::>() - ); - } - #[test] - fn test_list_map_empty_se_de() { - let mymap: Coremap = Coremap::new(); - let mut v = Vec::new(); - se::raw_serialize_list_map(&mymap, &mut v).unwrap(); - let de = de::deserialize_list_map(&v).unwrap(); - assert_eq!(de.len(), 0) - } -} - -mod corruption_tests { - use crate::corestore::htable::Coremap; - use crate::corestore::SharedSlice; - use crate::kvengine::LockedVec; - #[test] - fn test_corruption_map_basic() { - let mymap = Coremap::new(); - let seresult = super::se::serialize_map(&mymap).unwrap(); - // now chop it; since this has 8B, let's drop some bytes - assert!(super::de::deserialize_map(&seresult[..seresult.len() - 6]).is_none()); - } - #[test] - fn test_map_corruption_end_corruption() { - let cmap = Coremap::new(); - cmap.upsert("sayan".into(), "writes code".into()); - cmap.upsert("supersayan".into(), "writes super code".into()); - let ser = super::se::serialize_map(&cmap).unwrap(); - // corrupt the last 16B - assert!(super::de::deserialize_map(&ser[..ser.len() - 16]).is_none()); - } - #[test] - fn test_map_corruption_midway_corruption() { - let cmap = Coremap::new(); - cmap.upsert("sayan".into(), "writes code".into()); - cmap.upsert("supersayan".into(), "writes super code".into()); - let mut ser = super::se::serialize_map(&cmap).unwrap(); - // middle chop - ser.drain(16..ser.len() / 2); - assert!(super::de::deserialize_map(&ser).is_none()); - } - #[test] - fn test_listmap_corruption_basic() { - let mymap: Coremap = Coremap::new(); - mymap.upsert("hello".into(), lvec!("hello-1")); - // current repr: [1u64][5u64]["hello"][1u64][7u64]["hello-1"] - // sanity test - let mut v = Vec::new(); - super::se::raw_serialize_list_map(&mymap, &mut v).unwrap(); - assert!(super::de::deserialize_list_map(&v).is_some()); - // now chop "hello-1" - assert!(super::de::deserialize_list_map(&v[..v.len() - 7]).is_none()); - } - #[test] - fn test_listmap_corruption_midway() { - let mymap: Coremap = Coremap::new(); - mymap.upsert("hello".into(), lvec!("hello-1")); - // current repr: [1u64][5u64]["hello"][1u64][7u64]["hello-1"] - // sanity test - let mut v = Vec::new(); - super::se::raw_serialize_list_map(&mymap, &mut v).unwrap(); - assert!(super::de::deserialize_list_map(&v).is_some()); - assert_eq!(v.len(), 44); - // now chop "7u64" (8+8+5+8+8+7) - v.drain(29..37); - assert!(super::de::deserialize_list_map(&v).is_none()); - } -} - -mod storage_target_directory_structure { - use crate::{ - corestore::{ - memstore::{Memstore, ObjectID}, - table::{SystemTable, Table}, - }, - storage::v1::flush::{self, LocalSnapshot, RemoteSnapshot}, - util::{ - os::{self, EntryKind}, - Wrapper, - }, - }; - use std::collections::HashSet; - use std::path::PathBuf; - enum FileKind { - Dir(&'static str), - File(&'static str), - } - impl FileKind { - fn into_entrykind_path(self, closure: impl Fn(&'static str) -> String + Copy) -> EntryKind { - match self { - Self::Dir(dir) => EntryKind::Directory(closure(dir)), - Self::File(file) => EntryKind::File(closure(file)), - } - } - } - fn get_memstore_and_file_list() -> (Memstore, Vec) { - let paths = vec![ - // the default keyspace - FileKind::Dir("default"), - FileKind::File("default/default"), - FileKind::File("default/PARTMAP"), - // the superks keyspace - FileKind::Dir("superks"), - FileKind::File("superks/PARTMAP"), - FileKind::File("superks/blueshark"), - // the system keyspace - FileKind::Dir("system"), - FileKind::File("system/PARTMAP"), - FileKind::File("system/superauthy"), - // the preload file - FileKind::File("PRELOAD"), - ]; - (get_memstore(), paths) - } - fn get_memstore() -> Memstore { - let store = Memstore::new_default(); - assert!(store.create_keyspace(ObjectID::try_from_slice("superks").unwrap())); - assert!(store - .get_keyspace_atomic_ref("superks".as_bytes()) - .unwrap() - .create_table( - ObjectID::try_from_slice("blueshark").unwrap(), - Table::new_default_kve() - )); - assert!(store.system.tables.true_if_insert( - ObjectID::try_from_slice("superauthy").unwrap(), - Wrapper::new(SystemTable::new_auth(Default::default())) - )); - store - } - fn ensure_paths_equality(a: Vec, b: Vec) { - assert_eq!(a.len(), b.len()); - let aset: HashSet = a - .iter() - .map(|apath| PathBuf::from(apath.to_string())) - .collect(); - let bset: HashSet = a - .iter() - .map(|bpath| PathBuf::from(bpath.to_string())) - .collect(); - aset.into_iter() - .for_each(|apath| assert_eq!(&apath, bset.get(&apath).unwrap())); - } - use std::fs; - #[test] - fn local_snapshot_dir() { - let (store, paths) = get_memstore_and_file_list(); - let paths = paths - .into_iter() - .map(|v| v.into_entrykind_path(|file| format!("data/snaps/localsnap/{file}"))) - .collect::>(); - let target = LocalSnapshot::new("localsnap".to_owned()); - flush::flush_full(target, &store).unwrap(); - let files = os::rlistdir("data/snaps/localsnap").unwrap(); - ensure_paths_equality(files, paths); - fs::remove_dir_all("data/snaps/localsnap").unwrap(); - } - #[test] - fn remote_snapshot_dir() { - let (store, paths) = get_memstore_and_file_list(); - let paths = paths - .into_iter() - .map(|v| v.into_entrykind_path(|file| format!("data/rsnap/wisnap/{file}"))) - .collect::>(); - let target = RemoteSnapshot::new("wisnap"); - flush::flush_full(target, &store).unwrap(); - let files = os::rlistdir("data/rsnap/wisnap").unwrap(); - ensure_paths_equality(files, paths); - fs::remove_dir_all("data/rsnap/wisnap").unwrap(); - } -} diff --git a/server/src/storage/v1/unflush.rs b/server/src/storage/v1/unflush.rs deleted file mode 100644 index 04601b1e..00000000 --- a/server/src/storage/v1/unflush.rs +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Created on Sat Jul 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # Unflush routines -//! -//! Routines for unflushing data - -use { - super::bytemarks, - crate::{ - corestore::{ - memstore::{Keyspace, Memstore, ObjectID, SystemKeyspace, SYSTEM}, - table::{SystemTable, Table}, - }, - storage::v1::{ - de::DeserializeInto, - error::{ErrorContext, StorageEngineError, StorageEngineResult}, - flush::Autoflush, - interface::DIR_KSROOT, - preload::LoadedPartfile, - Coremap, - }, - util::Wrapper, - }, - core::mem::transmute, - std::{fs, io::ErrorKind, path::Path, sync::Arc}, -}; - -type PreloadSet = std::collections::HashSet; -const PRELOAD_PATH: &str = "data/ks/PRELOAD"; - -/// A keyspace that can be restored from disk storage -pub trait UnflushableKeyspace: Sized { - /// Unflush routine for a keyspace - fn unflush_keyspace(partmap: LoadedPartfile, ksid: &ObjectID) -> StorageEngineResult; -} - -impl UnflushableKeyspace for Keyspace { - fn unflush_keyspace(partmap: LoadedPartfile, ksid: &ObjectID) -> StorageEngineResult { - let ks: Coremap> = Coremap::with_capacity(partmap.len()); - for (tableid, (table_storage_type, model_code)) in partmap.into_iter() { - if table_storage_type > 1 { - return Err(StorageEngineError::bad_metadata_in_table(ksid, &tableid)); - } - let is_volatile = table_storage_type == bytemarks::BYTEMARK_STORAGE_VOLATILE; - let tbl = self::read_table::
(ksid, &tableid, is_volatile, model_code)?; - ks.true_if_insert(tableid, Arc::new(tbl)); - } - Ok(Keyspace::init_with_all_def_strategy(ks)) - } -} - -impl UnflushableKeyspace for SystemKeyspace { - fn unflush_keyspace(partmap: LoadedPartfile, ksid: &ObjectID) -> StorageEngineResult { - let ks: Coremap> = Coremap::with_capacity(partmap.len()); - for (tableid, (table_storage_type, model_code)) in partmap.into_iter() { - if table_storage_type > 1 { - return Err(StorageEngineError::bad_metadata_in_table(ksid, &tableid)); - } - let is_volatile = table_storage_type == bytemarks::BYTEMARK_STORAGE_VOLATILE; - let tbl = self::read_table::(ksid, &tableid, is_volatile, model_code)?; - ks.true_if_insert(tableid, Wrapper::new(tbl)); - } - Ok(SystemKeyspace::new(ks)) - } -} - -/// Tables that can be restored from disk storage -pub trait UnflushableTable: Sized { - /// Procedure to restore (deserialize) table from disk storage - fn unflush_table( - filepath: impl AsRef, - model_code: u8, - volatile: bool, - ) -> StorageEngineResult; -} - -#[allow(clippy::transmute_int_to_bool)] -impl UnflushableTable for Table { - fn unflush_table( - filepath: impl AsRef, - model_code: u8, - volatile: bool, - ) -> StorageEngineResult { - let ret = match model_code { - // pure KVEBlob: [0, 3] - x if x < 4 => { - let data = decode(filepath, volatile)?; - let (k_enc, v_enc) = unsafe { - // UNSAFE(@ohsayan): Safe because of the above match. Just a lil bitmagic - let key: bool = transmute(model_code >> 1); - let value: bool = transmute(((model_code >> 1) + (model_code & 1)) % 2); - (key, value) - }; - Table::new_pure_kve_with_data(data, volatile, k_enc, v_enc) - } - // KVExtlistmap: [4, 7] - x if x < 8 => { - let data = decode(filepath, volatile)?; - let (k_enc, v_enc) = unsafe { - // UNSAFE(@ohsayan): Safe because of the above match. Just a lil bitmagic - let code = model_code - 4; - let key: bool = transmute(code >> 1); - let value: bool = transmute(code % 2); - (key, value) - }; - Table::new_kve_listmap_with_data(data, volatile, k_enc, v_enc) - } - _ => { - return Err(StorageEngineError::BadMetadata( - filepath.as_ref().to_string_lossy().to_string(), - )) - } - }; - Ok(ret) - } -} - -impl UnflushableTable for SystemTable { - fn unflush_table( - filepath: impl AsRef, - model_code: u8, - volatile: bool, - ) -> StorageEngineResult { - match model_code { - 0 => { - // this is the authmap - let authmap = decode(filepath, volatile)?; - Ok(SystemTable::new_auth(Arc::new(authmap))) - } - _ => Err(StorageEngineError::BadMetadata( - filepath.as_ref().to_string_lossy().to_string(), - )), - } - } -} - -#[inline(always)] -fn decode( - filepath: impl AsRef, - volatile: bool, -) -> StorageEngineResult { - if volatile { - Ok(T::new_empty()) - } else { - let data = fs::read(filepath.as_ref()).map_err_context(format!( - "reading file {}", - filepath.as_ref().to_string_lossy() - ))?; - super::de::deserialize_into(&data).ok_or_else(|| { - StorageEngineError::CorruptedFile(filepath.as_ref().to_string_lossy().to_string()) - }) - } -} - -/// Read a given table into a [`Table`] object -/// -/// This will take care of volatility and the model_code. Just make sure that you pass the proper -/// keyspace ID and a valid table ID -pub fn read_table( - ksid: &ObjectID, - tblid: &ObjectID, - volatile: bool, - model_code: u8, -) -> StorageEngineResult { - let filepath = unsafe { concat_path!(DIR_KSROOT, ksid.as_str(), tblid.as_str()) }; - let tbl = T::unflush_table(filepath, model_code, volatile)?; - Ok(tbl) -} - -/// Read an entire keyspace into a Coremap. You'll need to initialize the rest -pub fn read_keyspace(ksid: &ObjectID) -> StorageEngineResult { - let partmap = self::read_partmap(ksid)?; - K::unflush_keyspace(partmap, ksid) -} - -/// Read the `PARTMAP` for a given keyspace -pub fn read_partmap(ksid: &ObjectID) -> StorageEngineResult { - let ksid_str = unsafe { ksid.as_str() }; - let filepath = concat_path!(DIR_KSROOT, ksid_str, "PARTMAP"); - let partmap_raw = fs::read(&filepath) - .map_err_context(format!("while reading {}", filepath.to_string_lossy()))?; - super::de::deserialize_set_ctype_bytemark(&partmap_raw) - .ok_or_else(|| StorageEngineError::corrupted_partmap(ksid)) -} - -/// Read the `PRELOAD` -pub fn read_preload() -> StorageEngineResult { - let read = fs::read(PRELOAD_PATH).map_err_context("reading PRELOAD")?; - super::preload::read_preload_raw(read) -} - -/// Read everything and return a [`Memstore`] -/// -/// If this is a new instance an empty store is returned while the directory tree -/// is also created. If this is an already initialized instance then the store -/// is read and returned (and any possible errors that are encountered are returned) -pub fn read_full() -> StorageEngineResult { - if is_new_instance()? { - log::trace!("Detected new instance. Creating data directory"); - /* - Since the `PRELOAD` file doesn't exist -- this is a new instance - This means that we need to: - 1. Create the tree (this only creates the directories) - 2. Create the PRELOAD (this is not created by flush_full!) - 3. Do a full flush (this flushes, but doesn't do anything to the PRELOAD!!!) - */ - // init an empty store - let store = Memstore::new_default(); - let target = Autoflush; - // (1) create the tree - super::interface::create_tree_fresh(&target, &store)?; - // (2) create the preload - super::flush::oneshot::flush_preload(&target, &store)?; - // (3) do a full flush - super::flush::flush_full(target, &store)?; - return Ok(store); - } - let mut preload = self::read_preload()?; - // HACK(@ohsayan): Pop off the preload from the serial read_keyspace list. It will fail - assert!(preload.remove(&SYSTEM)); - let system_keyspace = self::read_keyspace::(&SYSTEM)?; - let ksmap = Coremap::with_capacity(preload.len()); - for ksid in preload { - let ks = self::read_keyspace::(&ksid)?; - ksmap.upsert(ksid, Arc::new(ks)); - } - // HACK(@ohsayan): Now pop system back in here - ksmap.upsert(SYSTEM, Arc::new(Keyspace::empty())); - Ok(Memstore::init_with_all(ksmap, system_keyspace)) -} - -/// Check if the `data` directory is non-empty (if not: we're on a new instance) -pub fn is_new_instance() -> StorageEngineResult { - match fs::read_dir("data") { - Ok(mut dir) => Ok(dir.next().is_none()), - Err(e) if e.kind().eq(&ErrorKind::NotFound) => Ok(true), - Err(e) => Err(StorageEngineError::ioerror_extra( - e, - "while checking data directory", - )), - } -} diff --git a/server/src/tests/auth.rs b/server/src/tests/auth.rs deleted file mode 100644 index 9cbf0c42..00000000 --- a/server/src/tests/auth.rs +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Created on Fri Mar 11 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::auth::provider::testsuite_data, - skytable::{query, Element, RespCode}, -}; - -macro_rules! assert_autherror { - ($con:expr, $query:expr, $eq:expr) => { - runeq!($con, $query, Element::RespCode($eq)) - }; -} -macro_rules! assert_auth_disabled { - ($con:expr, $query:expr) => { - assert_autherror!( - $con, - $query, - RespCode::ErrorString("err-auth-disabled".to_owned()) - ) - }; -} - -macro_rules! assert_auth_perm_error { - ($con:expr, $query:expr) => { - assert_autherror!($con, $query, RespCode::AuthPermissionError) - }; -} - -macro_rules! assert_auth_bad_credentials { - ($con:expr, $query:expr) => { - assert_autherror!($con, $query, RespCode::AuthBadCredentials) - }; -} - -const ONLYAUTH: u8 = 0; -const NOAUTH: u8 = 1; - -macro_rules! assert_authn_resp_matrix { - ($con:expr, $query:expr, $username:ident, $password:ident, $resp:expr) => { - runeq!( - $con, - ::skytable::query!("auth", "login", $username, $password), - ::skytable::Element::RespCode(::skytable::RespCode::Okay) - ); - runeq!($con, $query, $resp); - }; - ($con:expr, $query:expr, $resp:expr) => {{ - runeq!($con, $query, $resp) - }}; - ($con:expr, $query:expr, $authnd:ident, $resp:expr) => {{ - match $authnd { - ONLYAUTH => { - assert_authn_resp_matrix!($con, $query, ROOT_USER, ROOT_PASS, $resp); - assert_authn_resp_matrix!($con, $query, USER, PASS, $resp); - } - NOAUTH => { - assert_authn_resp_matrix!($con, $query, $resp); - assert_authn_resp_matrix!($con, $query, ROOT_USER, ROOT_PASS, $resp); - assert_authn_resp_matrix!($con, $query, USER, PASS, $resp); - } - _ => panic!("Unknown authnd state"), - } - }}; -} - -// auth claim -// auth claim fail because it is disabled -#[sky_macros::dbtest_func] -async fn auth_claim_fail_disabled() { - assert_auth_disabled!(con, query!("auth", "claim", "blah")) -} -// auth claim fail because it has already been claimed -#[sky_macros::dbtest_func(port = 2005, norun = true)] -async fn claim_root_fail_already_claimed() { - runeq!( - con, - query!("auth", "claim", crate::TEST_AUTH_ORIGIN_KEY), - Element::RespCode(RespCode::ErrorString("err-auth-already-claimed".to_owned())) - ) -} - -// auth login -// auth login fail because it is disabled -#[sky_macros::dbtest_func] -async fn auth_login_fail() { - assert_auth_disabled!(con, query!("auth", "login", "user", "blah")) -} -// auth login okay (testuser) -#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)] -async fn auth_login_testuser() { - runeq!( - con, - query!("heya", "abcd"), - Element::String("abcd".to_owned()) - ) -} -#[sky_macros::dbtest_func(port = 2005, norun = true)] -async fn auth_login_testuser_fail_bad_creds() { - assert_auth_bad_credentials!(con, query!("auth", "login", "testuser", "badpass")) -} -// auth login okay (root) -#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)] -async fn auth_login_rootuser() { - runeq!( - con, - query!("heya", "abcd"), - Element::String("abcd".to_owned()) - ) -} -#[sky_macros::dbtest_func(port = 2005, norun = true)] -async fn auth_login_rootuser_fail_bad_creds() { - assert_auth_bad_credentials!(con, query!("auth", "login", "root", "badpass")) -} - -// auth adduser -// auth adduser fail because disabled -#[sky_macros::dbtest_func] -async fn auth_adduser_fail_because_disabled() { - assert_auth_disabled!(con, query!("auth", "adduser", "user")) -} -#[sky_macros::dbtest_func(port = 2005, norun = true)] -async fn auth_adduser_fail_because_anonymous() { - assert_auth_perm_error!(con, query!("auth", "adduser", "someuser")) -} -// auth adduser okay because root -#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)] -async fn auth_createuser_root_okay() { - runmatch!(con, query!("auth", "adduser", "someuser"), Element::String) -} -// auth adduser fail because not root -#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)] -async fn auth_createuser_testuser_fail() { - assert_auth_perm_error!(con, query!("auth", "adduser", "someuser")) -} - -// auth logout -// auth logout failed because auth is disabled -#[sky_macros::dbtest_func] -async fn auth_logout_fail_because_disabled() { - assert_auth_disabled!(con, query!("auth", "logout")) -} -// auth logout failed because user is anonymous -#[sky_macros::dbtest_func(port = 2005, norun = true)] -async fn auth_logout_fail_because_anonymous() { - assert_auth_perm_error!(con, query!("auth", "logout")) -} -// auth logout okay because the correct user is logged in -#[sky_macros::dbtest_func(port = 2005, auth_testuser = true, norun = true)] -async fn auth_logout_okay_testuser() { - assert_okay!(con, query!("auth", "logout")) -} -// auth logout okay because the correct user is logged in -#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true, norun = true)] -async fn auth_logout_okay_rootuser() { - assert_okay!(con, query!("auth", "logout")) -} - -// auth deluser -// auth deluser failed because auth is disabled -#[sky_macros::dbtest_func] -async fn auth_deluser_fail_because_auth_disabled() { - assert_auth_disabled!(con, query!("auth", "deluser", "testuser")) -} -#[sky_macros::dbtest_func(port = 2005, norun = true)] -async fn auth_deluser_fail_because_anonymous() { - assert_auth_perm_error!(con, query!("auth", "deluser", "someuser")) -} -// auth deluser failed because not root -#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)] -async fn auth_deluser_fail_because_not_root() { - assert_auth_perm_error!(con, query!("auth", "deluser", "testuser")) -} -// auth deluser okay because root -#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)] -async fn auth_deluser_okay_because_root() { - runmatch!( - con, - query!("auth", "adduser", "supercooluser"), - Element::String - ); - assert_okay!(con, query!("auth", "deluser", "supercooluser")) -} - -// restore -#[sky_macros::dbtest_func] -async fn restore_fail_because_disabled() { - assert_auth_disabled!(con, query!("auth", "restore", "root")); -} -#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)] -async fn restore_fail_because_not_root() { - assert_auth_perm_error!(con, query!("auth", "restore", "root")); -} -#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)] -async fn restore_okay_because_root() { - runmatch!( - con, - query!("auth", "adduser", "supercooldude"), - Element::String - ); - runmatch!( - con, - query!("auth", "restore", "supercooldude"), - Element::String - ); -} -#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true, norun = true)] -async fn restore_okay_with_origin_key() { - runmatch!(con, query!("auth", "adduser", "someuser2"), Element::String); - // now logout - runeq!( - con, - query!("auth", "logout"), - Element::RespCode(RespCode::Okay) - ); - // we should still be able to restore using origin key - runmatch!( - con, - query!("auth", "restore", crate::TEST_AUTH_ORIGIN_KEY, "someuser2"), - Element::String - ); -} - -// auth listuser -#[sky_macros::dbtest_func] -async fn listuser_fail_because_disabled() { - assert_auth_disabled!(con, query!("auth", "listuser")); -} -#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)] -async fn listuser_fail_because_not_root() { - assert_auth_perm_error!(con, query!("auth", "listuser")) -} -#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)] -async fn listuser_okay_because_root() { - let ret: Vec = con.run_query(query!("auth", "listuser")).await.unwrap(); - assert!(ret.contains(&"root".to_owned())); - assert!(ret.contains(&"testuser".to_owned())); -} - -// auth whoami -#[sky_macros::dbtest_func] -async fn whoami_fail_because_disabled() { - assert_auth_disabled!(con, query!("auth", "whoami")) -} -#[sky_macros::dbtest_func(port = 2005, norun = true)] -async fn whoami_fail_because_anonymous() { - assert_auth_perm_error!(con, query!("auth", "whoami")) -} -#[sky_macros::dbtest_func(port = 2005, norun = true, auth_testuser = true)] -async fn auth_whoami_okay_testuser() { - runeq!( - con, - query!("auth", "whoami"), - Element::String(testsuite_data::TESTSUITE_TEST_USER.to_owned()) - ) -} - -#[sky_macros::dbtest_func(port = 2005, norun = true, auth_rootuser = true)] -async fn auth_whoami_okay_rootuser() { - runeq!( - con, - query!("auth", "whoami"), - Element::String(testsuite_data::TESTSUITE_ROOT_USER.to_owned()) - ) -} - -mod syntax_checks { - use super::{NOAUTH, ONLYAUTH}; - use crate::auth::provider::testsuite_data::{ - TESTSUITE_ROOT_TOKEN as ROOT_PASS, TESTSUITE_ROOT_USER as ROOT_USER, - TESTSUITE_TEST_TOKEN as PASS, TESTSUITE_TEST_USER as USER, - }; - use skytable::{query, Element, RespCode}; - macro_rules! assert_authn_aerr { - ($con:expr, $query:expr, $username:expr, $password:expr) => {{ - assert_authn_resp_matrix!( - $con, - $query, - $username, - $password, - ::skytable::Element::RespCode(::skytable::RespCode::ActionError) - ) - }}; - ($con:expr, $query:expr) => {{ - assert_authn_resp_matrix!( - $con, - $query, - NOAUTH, - ::skytable::Element::RespCode(::skytable::RespCode::ActionError) - ) - }}; - ($con:expr, $query:expr, $authnd:ident) => {{ - assert_authn_resp_matrix!( - $con, - $query, - $authnd, - ::skytable::Element::RespCode(::skytable::RespCode::ActionError) - ); - }}; - } - #[sky_macros::dbtest_func(port = 2005, norun = true)] - async fn login_aerr() { - assert_authn_aerr!(con, query!("auth", "login", "lesserdata")); - assert_authn_aerr!(con, query!("auth", "login", "user", "password", "extra")); - } - #[sky_macros::dbtest_func(port = 2005, norun = true)] - async fn claim_aerr() { - assert_authn_aerr!(con, query!("auth", "claim")); - assert_authn_aerr!(con, query!("auth", "claim", "origin key", "but more data")); - } - #[sky_macros::dbtest_func(port = 2005, norun = true)] - async fn adduser_aerr() { - assert_authn_aerr!( - con, - query!("auth", "adduser", "user", "butextradata"), - ONLYAUTH - ); - } - #[sky_macros::dbtest_func(port = 2005, norun = true)] - async fn logout_aerr() { - assert_authn_aerr!(con, query!("auth", "logout", "butextradata"), ONLYAUTH); - } - #[sky_macros::dbtest_func(port = 2005, norun = true)] - async fn deluser_aerr() { - assert_authn_aerr!( - con, - query!("auth", "deluser", "someuser", "butextradata"), - ONLYAUTH - ); - } - #[sky_macros::dbtest_func(port = 2005, norun = true)] - async fn regenerate_aerr() { - assert_authn_aerr!(con, query!("auth", "restore")); - assert_authn_aerr!( - con, - query!("auth", "restore", "someuser", "origin", "but extra data") - ); - } - #[sky_macros::dbtest_func(port = 2005, norun = true)] - async fn listuser_aerr() { - assert_authn_aerr!(con, query!("auth", "listuser", "extra argument"), ONLYAUTH); - } - #[sky_macros::dbtest_func(port = 2005, norun = true)] - async fn whoami_aerr() { - assert_authn_aerr!(con, query!("auth", "whoami", "extra argument")); - } - #[sky_macros::dbtest_func(port = 2005, norun = true, auth_testuser = true)] - async fn unknown_auth_action() { - runeq!( - con, - query!("auth", "raspberry"), - Element::RespCode(RespCode::ErrorString("Unknown action".to_owned())) - ) - } -} diff --git a/server/src/tests/ddl_tests.rs b/server/src/tests/ddl_tests.rs deleted file mode 100644 index 91c02e65..00000000 --- a/server/src/tests/ddl_tests.rs +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Created on Wed Jul 28 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#[sky_macros::dbtest_module] -mod __private { - use { - libstress::utils, - skytable::{query, types::Array, Element, Query, RespCode}, - }; - - async fn test_create_keyspace() { - let mut rng = rand::thread_rng(); - let ksname = utils::rand_alphastring(10, &mut rng); - query.push(format!("create space {ksname}")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - async fn test_drop_keyspace() { - let mut rng = rand::thread_rng(); - let ksname = utils::rand_alphastring(10, &mut rng); - query.push(format!("create space {ksname}")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - let mut query = Query::new(); - query.push(format!("drop space {ksname}")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - async fn test_create_table() { - let mut rng = rand::thread_rng(); - let tblname = utils::rand_alphastring(10, &mut rng); - query.push(format!("create model {tblname}(string, string)")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - async fn test_create_volatile() { - let mut rng = rand::thread_rng(); - let tblname = utils::rand_alphastring(10, &mut rng); - query.push(format!("create model {tblname}(string, string) volatile")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - async fn test_create_table_fully_qualified_entity() { - let mut rng = rand::thread_rng(); - let tblname = utils::rand_alphastring(10, &mut rng); - query.push(format!("create model {__MYKS__}.{tblname}(string, string)")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - async fn test_create_table_volatile_fully_qualified_entity() { - let mut rng = rand::thread_rng(); - let tblname = utils::rand_alphastring(10, &mut rng); - query.push(format!( - "create model {__MYKS__}.{tblname}(string, string) volatile" - )); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - async fn test_drop_table() { - let mut rng = rand::thread_rng(); - let tblname = utils::rand_alphastring(10, &mut rng); - query.push(format!("create model {tblname}(string, string)")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - let query = Query::from(format!("drop model {tblname}")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - async fn test_drop_table_fully_qualified_entity() { - let mut rng = rand::thread_rng(); - let tblname = utils::rand_alphastring(10, &mut rng); - let my_fqe = __MYKS__.to_owned() + "." + &tblname; - query.push(format!("create model {my_fqe}(string, string)")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - let query = Query::from(format!("drop model {my_fqe}")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - async fn test_use() { - query.push(format!("USE {__MYENTITY__}")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ) - } - async fn test_use_syntax_error() { - query.push(format!("USE {__MYENTITY__} wiwofjwjfio")); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into())) - ) - } - async fn test_whereami() { - query.push("whereami"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::Array(Array::NonNullStr(vec![__MYKS__, __MYTABLE__])) - ); - runeq!( - con, - query!("use default"), - Element::RespCode(RespCode::Okay) - ); - runeq!( - con, - query!("whereami"), - Element::Array(Array::NonNullStr(vec!["default".to_owned()])) - ); - runeq!( - con, - query!("use default.default"), - Element::RespCode(RespCode::Okay) - ); - runeq!( - con, - query!("whereami"), - Element::Array(Array::NonNullStr(vec![ - "default".to_owned(), - "default".to_owned() - ])) - ); - } -} diff --git a/server/src/tests/inspect_tests.rs b/server/src/tests/inspect_tests.rs deleted file mode 100644 index 7f33e21c..00000000 --- a/server/src/tests/inspect_tests.rs +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Created on Wed Jul 28 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -const TABLE_DECL_KM_STR_STR_VOLATILE: &str = "Keymap { data:(str,str), volatile:true }"; - -#[sky_macros::dbtest_module] -mod __private { - use skytable::{types::Array, Element, RespCode}; - async fn test_inspect_keyspaces() { - query.push("INSPECT SPACES"); - assert!(matches!( - con.run_query_raw(&query).await.unwrap(), - Element::Array(Array::NonNullStr(_)) - )) - } - async fn test_inspect_keyspace() { - query.push(format!("INSPECT SPACE {__MYKS__}")); - assert!(matches!( - con.run_query_raw(&query).await.unwrap(), - Element::Array(Array::NonNullStr(_)) - )) - } - async fn test_inspect_current_keyspace() { - query.push("INSPECT SPACE"); - let ret: Vec = con.run_query(&query).await.unwrap(); - assert!(ret.contains(&__MYTABLE__)); - } - async fn test_inspect_table() { - query.push(format!("INSPECT MODEL {__MYTABLE__}")); - match con.run_query_raw(&query).await.unwrap() { - Element::String(st) => { - assert_eq!(st, TABLE_DECL_KM_STR_STR_VOLATILE.to_owned()) - } - _ => panic!("Bad response for inspect table"), - } - } - async fn test_inspect_current_table() { - query.push("INSPECT MODEL"); - let ret: String = con.run_query(&query).await.unwrap(); - assert_eq!(ret, TABLE_DECL_KM_STR_STR_VOLATILE); - } - async fn test_inspect_table_fully_qualified_entity() { - query.push(format!("INSPECT MODEL {__MYENTITY__}")); - match con.run_query_raw(&query).await.unwrap() { - Element::String(st) => { - assert_eq!(st, TABLE_DECL_KM_STR_STR_VOLATILE.to_owned()) - } - _ => panic!("Bad response for inspect table"), - } - } - async fn test_inspect_keyspaces_syntax_error() { - query.push("INSPECT SPACES iowjfjofoe"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into())) - ); - } - async fn test_inspect_keyspace_syntax_error() { - query.push("INSPECT SPACE ijfwijifwjo oijfwirfjwo"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into())) - ); - } - async fn test_inspect_table_syntax_error() { - query.push("INSPECT MODEL ijfwijifwjo oijfwirfjwo"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into())) - ); - } -} diff --git a/server/src/tests/issue_tests.rs b/server/src/tests/issue_tests.rs deleted file mode 100644 index 71d7d49a..00000000 --- a/server/src/tests/issue_tests.rs +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Created on Fri Aug 12 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -mod issue_276 { - // Refer to the issue here: https://github.com/skytable/skytable/issues/276 - // Gist of issue: the auth table was cleaned up because the dummy ks in Memstore does not have - // the system tables - use skytable::{Element, Query, RespCode}; - #[sky_macros::dbtest_func(port = 2005, auth_testuser = true, skip_if_cfg = "persist-suite")] - async fn first_run() { - // create the space - let r: Element = con - .run_query(Query::from("create space please_do_not_vanish")) - .await - .unwrap(); - assert_eq!(r, Element::RespCode(RespCode::Okay)); - // drop the space - let r: Element = con - .run_query(Query::from("drop space please_do_not_vanish")) - .await - .unwrap(); - assert_eq!(r, Element::RespCode(RespCode::Okay)); - } - #[sky_macros::dbtest_func(port = 2005, auth_testuser = true, run_if_cfg = "persist-suite")] - async fn second_run() { - // this function is just a dummy fn; if, as described in #276, the auth data is indeed - // lost, then the server will simply fail to start up - } -} diff --git a/server/src/tests/kvengine.rs b/server/src/tests/kvengine.rs deleted file mode 100644 index 728d56a0..00000000 --- a/server/src/tests/kvengine.rs +++ /dev/null @@ -1,1205 +0,0 @@ -/* - * Created on Thu Sep 10 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! Tests for the key/value engine and its operations -//! -//! The test functions here might seem slightly _mysterious_ -- but they aren't! The `dbtest_module` macro from the -//! `sky_macros` crate is what does the magic. It provides each function with an async `stream` to write to. -//! This stream is connected over TCP to a database instance. Once the test completes, the database instance -//! and its data is destroyed; but the spawned database instances are started up in a way to not store any -//! data at all, so this is just a precautionary step. - -#[sky_macros::dbtest_module] -mod __private { - #[cfg(test)] - use skytable::{types::Array, Element, Query, RespCode}; - /// Test a HEYA query: The server should return HEY! - async fn test_heya() { - query.push("heya"); - let resp = con.run_query_raw(&query).await.unwrap(); - assert_eq!(resp, Element::String("HEY!".to_owned())); - } - - async fn test_heya_echo() { - query.push("heya"); - query.push("sayan"); - let resp = con.run_query_raw(&query).await.unwrap(); - assert_eq!(resp, Element::String("sayan".to_owned())); - } - - /// Test a GET query: for a non-existing key - async fn test_get_single_nil() { - query.push("get"); - query.push("x"); - let resp = con.run_query_raw(&query).await.unwrap(); - assert_eq!(resp, Element::RespCode(RespCode::NotFound)); - } - - /// Test a GET query: for an existing key - async fn test_get_single_okay() { - query.push("set"); - query.push("x"); - query.push("100"); - let resp = con.run_query_raw(&query).await.unwrap(); - assert_eq!(resp, Element::RespCode(RespCode::Okay)); - let mut query = Query::new(); - query.push("get"); - query.push("x"); - let resp = con.run_query_raw(&query).await.unwrap(); - assert_eq!(resp, Element::String("100".to_owned())); - } - - /// Test a GET query with an incorrect number of arguments - async fn test_get_syntax_error() { - query.push("get"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - let mut query = Query::new(); - query.push("get"); - query.push("x"); - query.push("y"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test a SET query: SET a non-existing key, which should return code: 0 - async fn test_set_single_okay() { - query.push("sEt"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - - /// Test a SET query: SET an existing key, which should return code: 2 - async fn test_set_single_overwrite_error() { - // first set the key - query.push("set"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // attempt the same thing again - let mut query = Query::new(); - query.push("set"); - query.push("x"); - query.push("200"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::OverwriteError) - ); - } - - /// Test a SET query with incorrect number of arugments - async fn test_set_syntax_error() { - query.push("set"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - let mut query = Query::new(); - query.push("set"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test an UPDATE query: which should return code: 0 - async fn test_update_single_okay() { - // first set the key - query.push("set"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // attempt to update it - let mut query = Query::new(); - query.push("update"); - query.push("x"); - query.push("200"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - - /// Test an UPDATE query: which should return code: 1 - async fn test_update_single_nil() { - // attempt to update it - query.push("update"); - query.push("x"); - query.push("200"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::NotFound) - ); - } - - async fn test_update_syntax_error() { - query.push("update"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - let mut query = Query::new(); - query.push("update"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test a DEL query: which should return int 0 - async fn test_del_single_zero() { - query.push("del"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(0) - ); - } - - /// Test a DEL query: which should return int 1 - async fn test_del_single_one() { - // first set the key - query.push("set"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now delete it - let mut query = Query::new(); - query.push("del"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(1) - ); - } - - /// Test a DEL query: which should return the number of keys deleted - async fn test_del_multiple() { - // first set the keys - query.push("mset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - // now delete them - let mut query = Query::new(); - query.push("del"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - } - - /// Test a DEL query with an incorrect number of arguments - async fn test_del_syntax_error() { - query.push("del"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test an EXISTS query - async fn test_exists_multiple() { - // first set the keys - query.push("mset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - // now check if they exist - let mut query = Query::new(); - query.push("exists"); - query.push("x"); - query.push("y"); - query.push("z"); - query.push("a"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - } - - /// Test an EXISTS query with an incorrect number of arguments - async fn test_exists_syntax_error() { - query.push("exists"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test an MGET query on a single existing key - async fn test_mget_multiple_okay() { - // first set the keys - query.push("mset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - // now get them - let mut query = Query::new(); - query.push("mget"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::Array(Array::Str(vec![ - Some("100".to_owned()), - Some("200".to_owned()), - Some("300".to_owned()) - ])) - ); - } - - /// Test an MGET query with different outcomes - async fn test_mget_multiple_mixed() { - // first set the keys - query.push("mset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - let mut query = Query::new(); - query.push("mget"); - query.push("x"); - query.push("y"); - query.push("a"); - query.push("z"); - query.push("b"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::Array(Array::Str(vec![ - Some("100".to_owned()), - Some("200".to_owned()), - None, - Some("300".to_owned()), - None - ])) - ); - } - - /// Test an MGET query with an incorrect number of arguments - async fn test_mget_syntax_error() { - query.push("mget"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test an MSET query with a single non-existing key - async fn test_mset_single_okay() { - // first set the keys - query.push("mset"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(1) - ); - } - - /// Test an MSET query with non-existing keys - async fn test_mset_multiple_okay() { - // first set the keys - query.push("mset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - } - - /// Test an MSET query with a mixed set of outcomes - async fn test_mset_multiple_mixed() { - // first set the keys - query.push("mset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - // now try to set them again with just another new key - let mut query = Query::new(); - query.push("mset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - query.push("a"); - query.push("apple"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(1) - ); - } - - /// Test an MSET query with the wrong number of arguments - async fn test_mset_syntax_error_args_one() { - query.push("mset"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - async fn test_mset_syntax_error_args_three() { - query.push("mset"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test an MUPDATE query with a single non-existing key - async fn test_mupdate_single_okay() { - // first set the key - query.push("mset"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(1) - ); - // now attempt to update it - // first set the keys - let mut query = Query::new(); - query.push("mupdate"); - query.push("x"); - query.push("200"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(1) - ); - } - - /// Test an MUPDATE query with a mixed set of outcomes - async fn test_mupdate_multiple_mixed() { - // first set the keys - query.push("mset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - // now try to update them with just another new key - let mut query = Query::new(); - query.push("mupdate"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - query.push("a"); - query.push("apple"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - } - - /// Test an MUPDATE query with the wrong number of arguments - async fn test_mupdate_syntax_error_args_one() { - query.push("mupdate"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - async fn test_mupdate_syntax_error_args_three() { - query.push("mupdate"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test an SSET query: which should return code: 0 - async fn test_sset_single_okay() { - // first set the keys - query.push("sset"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - - /// Test an SSET query: which should return code: 2 - async fn test_sset_single_overwrite_error() { - // first set the keys - query.push("set"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now attempt to overwrite it - let mut query = Query::new(); - query.push("sset"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::OverwriteError) - ); - } - - /// Test an SSET query: which should return code: 0 - async fn test_sset_multiple_okay() { - // first set the keys - query.push("sset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - - /// Test an SSET query: which should return code: 2 - async fn test_sset_multiple_overwrite_error() { - // first set the keys - query.push("sset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now attempt to sset again with just one new extra key - let mut query = Query::new(); - query.push("sset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("b"); - query.push("bananas"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::OverwriteError) - ); - } - - /// Test an SSET query with the wrong number of arguments - async fn test_sset_syntax_error_args_one() { - query.push("sset"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - async fn test_sset_syntax_error_args_three() { - query.push("sset"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test an SUPDATE query: which should return code: 0 - async fn test_supdate_single_okay() { - // set the key - query.push("sset"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // update it - let mut query = Query::new(); - query.push("supdate"); - query.push("x"); - query.push("200"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - - /// Test an SUPDATE query: which should return code: 1 - async fn test_supdate_single_nil() { - query.push("supdate"); - query.push("x"); - query.push("200"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::NotFound) - ); - } - - /// Test an SUPDATE query: which should return code: 0 - async fn test_supdate_multiple_okay() { - // first set the keys - query.push("sset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now update all of them - let mut query = Query::new(); - query.push("supdate"); - query.push("x"); - query.push("200"); - query.push("y"); - query.push("300"); - query.push("z"); - query.push("400"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - - async fn test_supdate_multiple_nil() { - // no keys exist, so we get a nil - query.push("supdate"); - query.push("x"); - query.push("200"); - query.push("y"); - query.push("300"); - query.push("z"); - query.push("400"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::NotFound) - ); - } - - /// Test an SUPDATE query with the wrong number of arguments - async fn test_supdate_syntax_error_args_one() { - query.push("mupdate"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - async fn test_supdate_syntax_error_args_three() { - query.push("mupdate"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test an SDEL query: which should return nil - async fn test_sdel_single_nil() { - query.push("sdel"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::NotFound) - ); - } - - /// Test an SDEL query: which should return okay - async fn test_sdel_single_okay() { - query.push("sset"); - query.push("x"); - query.push("100"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - let mut query = Query::new(); - query.push("sdel"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - - /// Test an SDEL query: which should return okay - async fn test_sdel_multiple_okay() { - // first set the keys - query.push("sset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now delete them - let mut query = Query::new(); - query.push("sdel"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - } - - async fn test_sdel_multiple_nil() { - query.push("sdel"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::NotFound) - ); - } - - /// Test an SDEL query with an incorrect number of arguments - async fn test_sdel_syntax_error() { - query.push("sdel"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test a `DBSIZE` query - async fn test_dbsize() { - // first set the keys - query.push("sset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now check the size - let mut query = Query::new(); - query.push("dbsize"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - let mut query = Query::new(); - query.push("dbsize"); - query.push(__MYENTITY__); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - } - - /// Test `DBSIZE` with an incorrect number of arguments - async fn test_dbsize_syntax_error() { - query.push("dbsize"); - query.push("iroegjoeijgor"); - query.push("roigjoigjj094"); - query.push("ioewjforfifrj"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test `FLUSHDB` - async fn test_flushdb_okay() { - // first set the keys - query.push("sset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now flush the database - let mut query = Query::new(); - query.push("flushdb"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now check the size - let mut query = Query::new(); - query.push("dbsize"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(0) - ); - } - - async fn test_flushdb_fqe() { - setkeys!( - con, - "w":"000", - "x":"100", - "y":"200", - "z":"300" - ); - // now flush the database - let mut query = Query::new(); - query.push("flushdb"); - // add the fqe - query.push(__MYENTITY__); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now check the size - let mut query = Query::new(); - query.push("dbsize"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(0) - ); - } - - /// Test `FLUSHDB` with an incorrect number of arguments - async fn test_flushdb_syntax_error() { - query.push("flushdb"); - query.push("x"); - query.push("y"); - query.push("z"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test `USET` which returns okay - /// - /// `USET` almost always returns okay for the correct number of key(s)/value(s) - async fn test_uset_all_okay() { - query.push("uset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - // now that the keys already exist, do it all over again - let mut query = Query::new(); - query.push("uset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(3) - ); - } - - /// Test `USET` with an incorrect number of arguments - async fn test_uset_syntax_error_args_one() { - query.push("uset"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - async fn test_uset_syntax_error_args_three() { - query.push("uset"); - query.push("one"); - query.push("two"); - query.push("three"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - /// Test `KEYLEN` - async fn test_keylen() { - // first set the key - query.push("set"); - query.push("x"); - query.push("helloworld"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::Okay) - ); - // now check for the length - let mut query = Query::new(); - query.push("keylen"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(10) - ); - } - - /// Test `KEYLEN` with an incorrect number of arguments - async fn test_keylen_syntax_error_args_one() { - query.push("keylen"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - async fn test_keylen_syntax_error_args_two() { - query.push("keylen"); - query.push("x"); - query.push("y"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - async fn test_mksnap_disabled() { - query.push("mksnap"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ErrorString("err-snapshot-disabled".to_owned())) - ); - } - async fn test_mksnap_sanitization() { - query.push("mksnap"); - query.push("/var/omgcrazysnappy"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ErrorString( - "err-invalid-snapshot-name".to_owned() - )) - ); - let mut query = Query::new(); - query.push("mksnap"); - query.push("../omgbacktoparent"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ErrorString( - "err-invalid-snapshot-name".to_owned() - )) - ); - } - async fn test_lskeys_default() { - query.push("uset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - query.push("a"); - query.push("apples"); - query.push("b"); - query.push("burgers"); - query.push("c"); - query.push("carrots"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(6) - ); - // now get 'em - let mut query = Query::new(); - query.push("lskeys"); - let ret = con.run_query_raw(&query).await.unwrap(); - // don't forget that the keys returned are arranged according to their hashes - let ret_should_have: Vec = vec!["a", "b", "c", "x", "y", "z"] - .into_iter() - .map(|element| element.to_owned()) - .collect(); - if let Element::Array(Array::NonNullStr(arr)) = ret { - assert_eq!(ret_should_have.len(), arr.len()); - assert!(ret_should_have.into_iter().all(|key| arr.contains(&key))); - } else { - panic!("Expected flat string array"); - } - } - async fn test_lskeys_custom_limit() { - query.push("uset"); - query.push("x"); - query.push("100"); - query.push("y"); - query.push("200"); - query.push("z"); - query.push("300"); - query.push("a"); - query.push("apples"); - query.push("b"); - query.push("burgers"); - query.push("c"); - query.push("carrots"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::UnsignedInt(6) - ); - let mut query = Query::new(); - query.push("lskeys"); - query.push("1000"); - let ret = con.run_query_raw(&query).await.unwrap(); - // don't forget that the keys returned are arranged according to their hashes - let ret_should_have: Vec = vec!["a", "b", "c", "x", "y", "z"] - .into_iter() - .map(|element| element.to_owned()) - .collect(); - if let Element::Array(Array::NonNullStr(arr)) = ret { - assert_eq!(ret_should_have.len(), arr.len()); - assert!(ret_should_have.into_iter().all(|key| arr.contains(&key))); - } else { - panic!("Expected flat string array"); - } - } - async fn test_lskeys_entity() { - setkeys!( - con, - "x":"100", - "y":"200", - "z":"300" - ); - query.push("lskeys"); - query.push(&__MYENTITY__); - let ret = con.run_query_raw(&query).await.unwrap(); - let ret_should_have: Vec = vec!["x", "y", "z"] - .into_iter() - .map(|element| element.to_owned()) - .collect(); - if let Element::Array(Array::NonNullStr(arr)) = ret { - assert_eq!(ret_should_have.len(), arr.len()); - assert!(ret_should_have.into_iter().all(|key| arr.contains(&key))); - } else { - panic!("Expected flat string array"); - } - } - async fn test_lskeys_entity_with_count() { - setkeys!( - con, - "x":"100", - "y":"200", - "z":"300" - ); - query.push("lskeys"); - query.push(&__MYENTITY__); - query.push(3u8.to_string()); - let ret = con.run_query_raw(&query).await.unwrap(); - let ret_should_have: Vec = vec!["x", "y", "z"] - .into_iter() - .map(|element| element.to_owned()) - .collect(); - if let Element::Array(Array::NonNullStr(arr)) = ret { - assert_eq!(ret_should_have.len(), arr.len()); - assert!(ret_should_have.into_iter().all(|key| arr.contains(&key))); - } else { - panic!("Expected flat string array"); - } - } - async fn test_lskeys_syntax_error() { - query.push("lskeys"); - query.push("abcdefg"); - query.push("hijklmn"); - query.push("riufrif"); - query.push("fvnjnvv"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - async fn test_mpop_syntax_error() { - query.push("mpop"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - - async fn test_mpop_all_success() { - setkeys!( - con, - "x":"100", - "y":"200", - "z":"300" - ); - query.push(vec!["mpop", "x", "y", "z"]); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::Array(Array::Str(vec![ - Some("100".to_owned()), - Some("200".to_owned()), - Some("300".to_owned()) - ])) - ) - } - async fn test_mpop_mixed() { - setkeys!( - con, - "x":"100", - "y":"200", - "z":"300" - ); - query.push(vec!["mpop", "apple", "arnold", "x", "madonna", "y", "z"]); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::Array(Array::Str(vec![ - None, - None, - Some("100".to_owned()), - None, - Some("200".to_owned()), - Some("300".to_owned()) - ])) - ); - } - async fn test_pop_syntax_error() { - query.push("pop"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::ActionError) - ); - } - async fn test_pop_okay() { - setkeys!( - con, - "x":"100" - ); - query.push("pop"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::String("100".to_owned()) - ); - } - async fn test_pop_nil() { - query.push("pop"); - query.push("x"); - assert_eq!( - con.run_query_raw(&query).await.unwrap(), - Element::RespCode(RespCode::NotFound) - ); - } -} diff --git a/server/src/tests/kvengine_encoding.rs b/server/src/tests/kvengine_encoding.rs deleted file mode 100644 index e1cd59a9..00000000 --- a/server/src/tests/kvengine_encoding.rs +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Created on Sun Sep 05 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#[sky_macros::dbtest_module(table = "(string, string)")] -mod __private { - use skytable::{types::RawString, Element, RespCode}; - - async fn test_bad_encoding_set() { - query.push("set"); - query.push("x"); - query.push(RawString::from(b"Hello \xF0\x90\x80World".to_vec())); - runeq!(con, query, Element::RespCode(RespCode::EncodingError)); - } - async fn test_bad_encoding_update() { - // first set the keys - setkeys! { - con, - "x": "100" - } - // now try to update with a bad value - query.push("update"); - query.push("x"); - query.push(RawString::from(b"Hello \xF0\x90\x80World".to_vec())); - runeq!(con, query, Element::RespCode(RespCode::EncodingError)); - } - async fn test_bad_encoding_uset() { - query.push("uset"); - query.push("x"); - query.push(RawString::from(b"Hello \xF0\x90\x80World".to_vec())); - runeq!(con, query, Element::RespCode(RespCode::EncodingError)); - } - async fn test_bad_encoding_mset() { - // we'll have one good encoding and one bad encoding - push!( - query, - "mset", - "x", - "good value", - "y", - // the bad value - RawString::from(b"Hello \xF0\x90\x80World".to_vec()) - ); - runeq!(con, query, Element::RespCode(RespCode::EncodingError)); - } -} diff --git a/server/src/tests/kvengine_list.rs b/server/src/tests/kvengine_list.rs deleted file mode 100644 index d1eda1cd..00000000 --- a/server/src/tests/kvengine_list.rs +++ /dev/null @@ -1,451 +0,0 @@ -/* - * Created on Tue Sep 07 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -macro_rules! lset { - ($con:expr, $listname:expr, $($val:expr),*) => { - let mut q = skytable::Query::from("LSET"); - q.push($listname); - $(q.push($val);)* - runeq!($con, q, skytable::Element::RespCode(skytable::RespCode::Okay)); - }; - ($con:expr, $listname:expr) => { - lset!($con, $listname, ) - } -} - -#[sky_macros::dbtest_module(table = "(string,list)")] -mod __private { - use skytable::{query, types::Array, Element, RespCode}; - - // lset tests - async fn test_lset_empty_okay() { - lset!(con, "mylist"); - } - async fn test_lset_with_values() { - lset!(con, "mylist", "a", "b", "c", "d"); - } - async fn test_lset_syntax_error() { - let q = query!("LSET"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - async fn test_lset_overwrite_error() { - lset!(con, "mylist"); - let q = query!("lset", "mylist"); - runeq!(con, q, Element::RespCode(RespCode::OverwriteError)); - } - - // lget tests - async fn test_lget_emptylist_okay() { - lset!(con, "mysuperlist"); - let q = query!("lget", "mysuperlist"); - runeq!(con, q, Element::Array(Array::NonNullStr(vec![]))); - } - async fn test_lget_list_with_elements_okay() { - lset!(con, "mysuperlist", "elementa", "elementb", "elementc"); - let q = query!("lget", "mysuperlist"); - assert_skyhash_arrayeq!(str, con, q, "elementa", "elementb", "elementc"); - } - /// lget limit - async fn test_lget_list_with_limit() { - lset!(con, "mysuperlist", "elementa", "elementb", "elementc"); - let q = query!("lget", "mysuperlist", "LIMIT", "2"); - assert_skyhash_arrayeq!(str, con, q, "elementa", "elementb"); - } - /// lget bad limit - async fn test_lget_list_with_bad_limit() { - lset!(con, "mysuperlist", "elementa", "elementb", "elementc"); - let q = query!("lget", "mylist", "LIMIT", "badlimit"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - } - /// lget huge limit - async fn test_lget_with_huge_limit() { - lset!(con, "mysuperlist", "elementa", "elementb", "elementc"); - let q = query!("lget", "mysuperlist", "LIMIT", "100"); - assert_skyhash_arrayeq!(str, con, q, "elementa", "elementb", "elementc"); - } - /// lget syntax error - async fn test_lget_with_limit_syntax_error() { - let q = query!("lget", "mylist", "LIMIT", "100", "200"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - /// lget limit non-existent key - async fn test_lget_with_limit_nil() { - let q = query!("lget", "mylist", "LIMIT", "100"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - } - /// lget len - async fn test_lget_with_len_okay() { - lset!(con, "mysuperlist", "elementa", "elementb", "elementc"); - let q = query!("lget", "mysuperlist", "len"); - runeq!(con, q, Element::UnsignedInt(3)); - } - /// lget len syntax error - async fn test_lget_with_len_syntax_error() { - let q = query!("lget", "mysuperlist", "len", "whatthe"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - /// lget len nil - async fn test_lget_with_len_nil() { - let q = query!("lget", "mysuperlist", "len"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - } - /// lget valueat - async fn test_lget_with_valueat_okay() { - lset!(con, "mylist", "v1"); - let q = query!("lget", "mylist", "valueat", "0"); - runeq!(con, q, Element::String("v1".to_owned())); - } - /// lget valueat (non-existent index) - async fn test_lget_with_valueat_non_existent_index() { - lset!(con, "mylist", "v1"); - let q = query!("lget", "mylist", "valueat", "1"); - runeq!( - con, - q, - Element::RespCode(RespCode::ErrorString("bad-list-index".to_owned())) - ) - } - /// lget valueat (invalid index) - async fn test_lget_with_valueat_bad_index() { - lset!(con, "mylist", "v1"); - let q = query!("lget", "mylist", "valueat", "1a"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)) - } - /// lget valueat (nil) - async fn test_lget_with_valueat_nil() { - let q = query!("lget", "mybadlist", "valueat", "2"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - } - /// lget valueat (nil + bad index) - async fn test_lget_with_bad_index_but_nil_key() { - let q = query!("lget", "mybadlist", "valueat", "2a"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - } - /// lget valueat (syntax error) - async fn test_lget_with_valueat_syntax_error() { - let q = query!("lget", "mybadlist", "valueat", "2", "3"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - // lget last - /// lget last with one element - async fn test_lget_last_with_last_one_element() { - lset!(con, "mylist", "a"); - let q = query!("lget", "mylist", "last"); - runeq!(con, q, Element::String("a".to_owned())); - } - /// lget last with multiple elements - async fn test_lget_last_with_last_many_elements() { - lset!(con, "mylist", "a", "b", "c"); - let q = query!("lget", "mylist", "last"); - runeq!(con, q, Element::String("c".to_owned())); - } - /// lget last with empty list - async fn test_lget_last_with_empty_list() { - lset!(con, "mylist"); - let q = query!("lget", "mylist", "last"); - runeq!( - con, - q, - Element::RespCode(RespCode::ErrorString("list-is-empty".to_owned())) - ); - } - /// lget last syntax error - async fn test_lget_last_syntax_error() { - let q = query!("lget", "mylist", "last", "abcd"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - // lget first - /// lget first with one element - async fn test_lget_first_with_last_one_element() { - lset!(con, "mylist", "a"); - let q = query!("lget", "mylist", "first"); - runeq!(con, q, Element::String("a".to_owned())); - } - /// lget first with multiple elements - async fn test_lget_first_with_last_many_elements() { - lset!(con, "mylist", "a", "b", "c"); - let q = query!("lget", "mylist", "first"); - runeq!(con, q, Element::String("a".to_owned())); - } - /// lget first with empty list - async fn test_lget_first_with_empty_list() { - lset!(con, "mylist"); - let q = query!("lget", "mylist", "first"); - runeq!( - con, - q, - Element::RespCode(RespCode::ErrorString("list-is-empty".to_owned())) - ); - } - /// lget last syntax error - async fn test_lget_first_syntax_error() { - let q = query!("lget", "mylist", "first", "abcd"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - // lmod tests - // lmod push - /// lmod push (okay) - async fn test_lmod_push_okay() { - lset!(con, "mylist"); - let q = query!("lmod", "mylist", "push", "v1"); - runeq!(con, q, Element::RespCode(RespCode::Okay)); - } - /// lmod push multiple (okay) - async fn test_lmod_push_multiple_okay() { - lset!(con, "mylist"); - assert_okay!(con, query!("lmod", "mylist", "push", "v1", "v2")); - } - /// lmod push (nil) - async fn test_lmod_push_nil() { - let q = query!("lmod", "mylist", "push", "v1"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - } - /// lmod push (syntax error) - async fn test_lmod_syntax_error() { - let q = query!("lmod", "mylist", "push"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - // lmod pop - /// lmod pop (okay) - async fn test_lmod_pop_noindex_okay() { - lset!(con, "mylist", "value"); - let q = query!("lmod", "mylist", "pop"); - runeq!(con, q, Element::String("value".to_owned())); - } - /// lmod pop (good index; okay) - async fn test_lmod_pop_goodindex_okay() { - lset!(con, "mylist", "value1", "value2"); - let q = query!("lmod", "mylist", "pop", "1"); - runeq!(con, q, Element::String("value2".to_owned())); - } - /// lmod pop (bad index + existent key & non-existent key) - async fn test_lmod_pop_badindex_fail() { - lset!(con, "mylist", "v1", "v2"); - let q = query!("lmod", "mylist", "pop", "12badidx"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - - // this is post-execution; so the error must be pointed out first - let q = query!("lmod", "mymissinglist", "pop", "12badidx"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - } - /// lmod pop (nil) - async fn test_lmod_pop_nil() { - let q = query!("lmod", "mylist", "pop"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - } - /// lmod pop (syntax error) - async fn test_lmod_pop_syntax_error() { - let q = query!("lmod", "mylist", "pop", "whatthe", "whatthe2"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - // lmod clear - /// lmod clear (okay) - async fn test_lmod_clear_okay() { - lset!(con, "mylist", "v1", "v2"); - let q = query!("lmod", "mylist", "clear"); - runeq!(con, q, Element::RespCode(RespCode::Okay)); - } - /// lmod clear (nil) - async fn test_lmod_clear_nil() { - let q = query!("lmod", "mylist", "clear"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - } - /// lmod clear (syntax error) - async fn test_lmod_clear_syntax_error() { - let q = query!("lmod", "mylist", "clear", "unneeded arg"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - // lmod remove - /// lmod remove (okay) - async fn test_lmod_remove_okay() { - lset!(con, "mylist", "v1"); - let q = query!("lmod", "mylist", "remove", "0"); - runeq!(con, q, Element::RespCode(RespCode::Okay)); - } - /// lmod remove (nil) - async fn test_lmod_remove_nil() { - let q = query!("lmod", "mylist", "remove", "0"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - } - /// lmod remove (bad index; nil + existent) - async fn test_lmod_remove_bad_index() { - // non-existent key + bad idx - let q = query!("lmod", "mylist", "remove", "1a"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - // existent key + bad idx - lset!(con, "mylist"); - let q = query!("lmod", "mylist", "remove", "1a"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - } - /// lmod remove (syntax error) - async fn test_lmod_remove_syntax_error() { - let q = query!("lmod", "mylist", "remove", "a", "b"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - // lmod insert - /// lmod insert (okay) - async fn test_lmod_insert_okay() { - lset!(con, "mylist", "a", "c"); - let q = query!("lmod", "mylist", "insert", "1", "b"); - runeq!(con, q, Element::RespCode(RespCode::Okay)); - let q = query!("lget", "mylist"); - assert_skyhash_arrayeq!(str, con, q, "a", "b", "c"); - } - /// lmod insert (bad index; present + nil) - async fn test_lmod_insert_bad_index() { - // nil - let q = query!("lmod", "mylist", "insert", "1badindex", "b"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - // present - lset!(con, "mylist", "a", "c"); - let q = query!("lmod", "mylist", "insert", "1badindex", "b"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - } - /// lmod insert (syntax error) - async fn test_lmod_insert_syntax_error() { - let q = query!("lmod", "mylist", "insert", "1"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - let q = query!("lmod", "mylist", "insert"); - runeq!(con, q, Element::RespCode(RespCode::ActionError)); - } - /// lmod insert (present; non-existent index) - async fn test_lmod_insert_non_existent_index() { - lset!(con, "mylist", "a", "b"); - let q = query!( - "lmod", - "mylist", - "insert", - "125", - "my-value-that-will-never-go-in" - ); - runeq!( - con, - q, - Element::RespCode(RespCode::ErrorString("bad-list-index".to_owned())) - ) - } - /// del (existent; non-existent) - async fn test_list_del() { - // try an existent key - lset!(con, "mylist", "v1", "v2"); - let q = query!("del", "mylist"); - runeq!(con, q, Element::UnsignedInt(1)); - // try the now non-existent key - let q = query!("del", "mylist"); - runeq!(con, q, Element::UnsignedInt(0)); - } - /// exists (existent; non-existent) - async fn test_list_exists() { - lset!(con, "mylist"); - lset!(con, "myotherlist"); - let q = query!("exists", "mylist", "myotherlist", "badlist"); - runeq!(con, q, Element::UnsignedInt(2)); - } - - // tests for range - async fn test_list_range_nil() { - let q = query!("lget", "sayan", "range", "1", "10"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - let q = query!("lget", "sayan", "range", "1"); - runeq!(con, q, Element::RespCode(RespCode::NotFound)); - } - - async fn test_list_range_bounded_okay() { - lset!(con, "mylist", "1", "2", "3", "4", "5"); - let q = query!("lget", "mylist", "range", "0", "5"); - assert_skyhash_arrayeq!(str, con, q, "1", "2", "3", "4", "5"); - } - - async fn test_list_range_bounded_fail() { - lset!(con, "mylist", "1", "2", "3", "4", "5"); - let q = query!("lget", "mylist", "range", "0", "165"); - runeq!( - con, - q, - Element::RespCode(RespCode::ErrorString("bad-list-index".to_owned())) - ) - } - - async fn test_list_range_unbounded_okay() { - lset!(con, "mylist", "1", "2", "3", "4", "5"); - let q = query!("lget", "mylist", "range", "0"); - assert_skyhash_arrayeq!(str, con, q, "1", "2", "3", "4", "5"); - } - - async fn test_list_range_unbounded_fail() { - lset!(con, "mylist", "1", "2", "3", "4", "5"); - let q = query!("lget", "mylist", "range", "165"); - runeq!( - con, - q, - Element::RespCode(RespCode::ErrorString("bad-list-index".to_owned())) - ) - } - - async fn test_list_range_parse_fail() { - let q = query!("lget", "mylist", "range", "1", "2a"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - let q = query!("lget", "mylist", "range", "2a"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - // now do the same with an existing key - lset!(con, "mylist", "a", "b", "c"); - let q = query!("lget", "mylist", "range", "1", "2a"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - let q = query!("lget", "mylist", "range", "2a"); - runeq!(con, q, Element::RespCode(RespCode::Wrongtype)); - } - - // sanity tests - async fn test_get_model_error() { - query.push("GET"); - query.push("mylist"); - runeq!( - con, - query, - Element::RespCode(RespCode::ErrorString("wrong-model".to_owned())) - ); - } - async fn test_set_model_error() { - query.push("SET"); - query.push("mylist"); - query.push("myvalue"); - runeq!( - con, - query, - Element::RespCode(RespCode::ErrorString("wrong-model".to_owned())) - ); - } - async fn test_update_model_error() { - query.push("UPDATE"); - query.push("mylist"); - query.push("myvalue"); - runeq!( - con, - query, - Element::RespCode(RespCode::ErrorString("wrong-model".to_owned())) - ); - } -} diff --git a/server/src/tests/macros.rs b/server/src/tests/macros.rs deleted file mode 100644 index 14d8f8e1..00000000 --- a/server/src/tests/macros.rs +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Created on Sun Sep 05 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -macro_rules! setkeys { - ($con:ident, $($key:literal:$value:literal),*) => { - let mut q = skytable::Query::new(); - q.push("MSET"); - let mut count = 0; - $( - q.push($key); - q.push($value); - count += 1; - )* - assert_eq!( - $con.run_query_raw(&q).await.unwrap(), - Element::UnsignedInt(count) - ); - }; - ($con:ident, $($key:expr => $value:expr),*) => { - let mut q = ::skytable::Query::new(); - q.push("MSET"); - let mut count = 0; - $( - q.push($key); - q.push($value); - count += 1; - )* - assert_eq!( - $con.run_query_raw(&q).await.unwrap(), - ::skytable::Element::UnsignedInt(count) - ); - }; -} - -macro_rules! switch_entity { - ($con:expr, $entity:expr) => { - runeq!( - $con, - ::skytable::query!(format!("use {}", $entity)), - ::skytable::Element::RespCode(::skytable::RespCode::Okay) - ) - }; -} - -macro_rules! create_table_and_switch { - ($con:expr, $table:expr, $decl:expr) => {{ - runeq!( - $con, - ::skytable::query!(format!("create model {}{}", $table, $decl)), - ::skytable::Element::RespCode(::skytable::RespCode::Okay) - ); - switch_entity!($con, $table); - }}; -} - -macro_rules! push { - ($query:expr, $($val:expr),*) => {{ - $( - $query.push($val); - )* - }}; -} - -macro_rules! runeq { - ($con:expr, $query:expr, $eq:expr) => { - assert_eq!($con.run_query_raw(&$query).await.unwrap(), $eq) - }; -} - -macro_rules! runmatch { - ($con:expr, $query:expr, $match:path) => {{ - let ret = $con.run_query_raw(&$query).await.unwrap(); - assert!(matches!(ret, $match(_))) - }}; -} - -macro_rules! assert_okay { - ($con:expr, $query:expr) => { - assert_respcode!($con, $query, ::skytable::RespCode::Okay) - }; -} - -macro_rules! assert_skyhash_arrayeq { - (!str, $con:expr, $query:expr, $($val:expr),*) => { - runeq!( - $con, - $query, - skytable::Element::Array(skytable::types::Array::Str( - vec![ - $(Some($val.into()),)* - ] - )) - ) - }; - (str, $con:expr, $query:expr, $($val:expr),*) => { - runeq!( - $con, - $query, - skytable::Element::Array(skytable::types::Array::NonNullStr( - vec![ - $($val.into(),)* - ] - )) - ) - }; -} - -macro_rules! assert_respcode { - ($con:expr, $query:expr, $code:expr) => { - runeq!($con, $query, ::skytable::Element::RespCode($code)) - }; -} diff --git a/server/src/tests/mod.rs b/server/src/tests/mod.rs deleted file mode 100644 index b40608df..00000000 --- a/server/src/tests/mod.rs +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Created on Tue Aug 25 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! This module contains automated tests for queries - -#[macro_use] -mod macros; -#[cfg(not(feature = "persist-suite"))] -mod auth; -mod ddl_tests; -mod inspect_tests; -mod kvengine; -mod kvengine_encoding; -mod kvengine_list; -mod persist; -mod pipeline; -mod snapshot; -mod issue_tests; - -mod tls { - use skytable::{query, Element}; - #[sky_macros::dbtest_func(tls_cert = "cert.pem", port = 2004)] - async fn tls() { - runeq!( - con, - query!("heya", "abcd"), - Element::String("abcd".to_owned()) - ); - } -} - -mod sys { - use { - crate::protocol::{LATEST_PROTOCOL_VERSION, LATEST_PROTOCOL_VERSIONSTRING}, - libsky::VERSION, - sky_macros::dbtest_func as dbtest, - skytable::{query, Element, RespCode}, - }; - - #[dbtest] - async fn sys_info_aerr() { - runeq!( - con, - query!("sys", "info"), - Element::RespCode(RespCode::ActionError) - ); - runeq!( - con, - query!( - "sys", - "info", - "this is cool", - "but why this extra argument?" - ), - Element::RespCode(RespCode::ActionError) - ) - } - #[dbtest] - async fn sys_info_protocol() { - runeq!( - con, - query!("sys", "info", "protocol"), - Element::String(LATEST_PROTOCOL_VERSIONSTRING.to_owned()) - ) - } - #[dbtest] - async fn sys_info_protover() { - runeq!( - con, - query!("sys", "info", "protover"), - Element::Float(LATEST_PROTOCOL_VERSION) - ) - } - #[dbtest] - async fn sys_info_version() { - runeq!( - con, - query!("sys", "info", "version"), - Element::String(VERSION.to_owned()) - ) - } - #[dbtest] - async fn sys_metric_aerr() { - runeq!( - con, - query!("sys", "metric"), - Element::RespCode(RespCode::ActionError) - ); - runeq!( - con, - query!("sys", "metric", "health", "but why this extra argument?"), - Element::RespCode(RespCode::ActionError) - ) - } - #[dbtest] - async fn sys_metric_health() { - runeq!( - con, - query!("sys", "metric", "health"), - Element::String("good".to_owned()) - ) - } - #[dbtest] - async fn sys_storage_usage() { - runmatch!( - con, - query!("sys", "metric", "storage"), - Element::UnsignedInt - ) - } -} - -use skytable::{query, Element, RespCode}; - -#[sky_macros::dbtest_func] -async fn blueql_extra_args() { - runeq!( - con, - query!("use default.default", "extra useless arg"), - Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into())) - ); -} diff --git a/server/src/tests/persist/kv.rs b/server/src/tests/persist/kv.rs deleted file mode 100644 index 97fdb07e..00000000 --- a/server/src/tests/persist/kv.rs +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Created on Sat Mar 19 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{persist_load, persist_store, PERSIST_TEST_SET_SIZE}, - sky_macros::dbtest_func as dbtest, -}; - -const PERSIST_CFG_KEYMAP_BIN_BIN_TABLE: &str = "testsuite.persist_bin_bin_tbl"; -const PERSIST_DATA_KEYMAP_BIN_BIN_TABLE: [(&[u8], &[u8]); PERSIST_TEST_SET_SIZE] = [ - (bin!(b"mykey1"), bin!(b"myval1")), - (bin!(b"mykey2"), bin!(b"myval2")), - (bin!(b"mykey3"), bin!(b"myval3")), - (bin!(b"mykey4"), bin!(b"myval4")), -]; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true)] -async fn store_keymap_bin_bin() { - persist_store( - &mut con, - PERSIST_CFG_KEYMAP_BIN_BIN_TABLE, - "(binary, binary)", - PERSIST_DATA_KEYMAP_BIN_BIN_TABLE, - ) - .await; -} - -#[dbtest(run_if_cfg = "persist-suite", norun = true)] -async fn load_keymap_bin_bin() { - persist_load( - &mut con, - PERSIST_CFG_KEYMAP_BIN_BIN_TABLE, - PERSIST_DATA_KEYMAP_BIN_BIN_TABLE, - ) - .await; -} - -const PERSIST_CFG_KEYMAP_BIN_STR_TABLE: &str = "testsuite.persist_bin_str_tbl"; -const PERSIST_DATA_KEYMAP_BIN_STR_TABLE: [(&[u8], &str); PERSIST_TEST_SET_SIZE] = [ - (bin!(b"mykey1"), "myval1"), - (bin!(b"mykey2"), "myval2"), - (bin!(b"mykey3"), "myval3"), - (bin!(b"mykey4"), "myval4"), -]; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true)] -async fn store_keymap_bin_str() { - persist_store( - &mut con, - PERSIST_CFG_KEYMAP_BIN_STR_TABLE, - "(binary, string)", - PERSIST_DATA_KEYMAP_BIN_STR_TABLE, - ) - .await; -} - -#[dbtest(run_if_cfg = "persist-suite", norun = true)] -async fn load_keymap_bin_str() { - persist_load( - &mut con, - PERSIST_CFG_KEYMAP_BIN_STR_TABLE, - PERSIST_DATA_KEYMAP_BIN_STR_TABLE, - ) - .await; -} - -const PERSIST_CFG_KEYMAP_STR_STR_TABLE: &str = "testsuite.persist_str_str_tbl"; -const PERSIST_DATA_KEYMAP_STR_STR_TABLE: [(&str, &str); PERSIST_TEST_SET_SIZE] = [ - ("mykey1", "myval1"), - ("mykey2", "myval2"), - ("mykey3", "myval3"), - ("mykey4", "myval4"), -]; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true)] -async fn store_keymap_str_str() { - persist_store( - &mut con, - PERSIST_CFG_KEYMAP_STR_STR_TABLE, - "(string, string)", - PERSIST_DATA_KEYMAP_STR_STR_TABLE, - ) - .await; -} - -#[dbtest(run_if_cfg = "persist-suite", norun = true)] -async fn load_keymap_str_str() { - persist_load( - &mut con, - PERSIST_CFG_KEYMAP_STR_STR_TABLE, - PERSIST_DATA_KEYMAP_STR_STR_TABLE, - ) - .await; -} - -const PERSIST_CFG_KEYMAP_STR_BIN_TABLE: &str = "testsuite.persist_str_bin_tbl"; -const PERSIST_DATA_KEYMAP_STR_BIN_TABLE: [(&str, &[u8]); PERSIST_TEST_SET_SIZE] = [ - ("mykey1", bin!(b"myval1")), - ("mykey2", bin!(b"myval2")), - ("mykey3", bin!(b"myval3")), - ("mykey4", bin!(b"myval4")), -]; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true)] -async fn store_keymap_str_bin() { - persist_store( - &mut con, - PERSIST_CFG_KEYMAP_STR_BIN_TABLE, - "(string, binary)", - PERSIST_DATA_KEYMAP_STR_BIN_TABLE, - ) - .await; -} - -#[dbtest(run_if_cfg = "persist-suite", norun = true)] -async fn load_keymap_str_bin() { - persist_load( - &mut con, - PERSIST_CFG_KEYMAP_STR_BIN_TABLE, - PERSIST_DATA_KEYMAP_STR_BIN_TABLE, - ) - .await; -} diff --git a/server/src/tests/persist/kvlist.rs b/server/src/tests/persist/kvlist.rs deleted file mode 100644 index 0b15ec0f..00000000 --- a/server/src/tests/persist/kvlist.rs +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Created on Sat Mar 19 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{persist_load, persist_store, Bin, ListIDBin, ListIDStr, Str, PERSIST_TEST_SET_SIZE}, - sky_macros::dbtest_func as dbtest, -}; - -type ListData = [(K, [V; PERSIST_TEST_SET_SIZE]); PERSIST_TEST_SET_SIZE]; - -macro_rules! listdata { - ( - $( - $listid:expr => $element:expr - ),* - ) => { - [ - $( - ( - $listid, - $element - ), - )* - ] - }; -} - -macro_rules! binid { - ($id:expr) => { - ListIDBin(bin!($id)) - }; -} - -macro_rules! binlist { - ($($elem:expr),*) => { - [ - $( - bin!($elem), - )* - ] - }; -} - -// bin,list -const DATA_BIN_LISTBIN: ListData = listdata!( - binid!(b"list1") => binlist!(b"e1", b"e2", b"e3", b"e4"), - binid!(b"list2") => binlist!(b"e1", b"e2", b"e3", b"e4"), - binid!(b"list3") => binlist!(b"e1", b"e2", b"e3", b"e4"), - binid!(b"list4") => binlist!(b"e1", b"e2", b"e3", b"e4") -); -const TABLE_BIN_LISTBIN: &str = "testsuite.persist_bin_listbin"; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true)] -async fn store_bin_bin() { - persist_store( - &mut con, - TABLE_BIN_LISTBIN, - "(binary, list)", - DATA_BIN_LISTBIN, - ) - .await; -} - -#[dbtest(run_if_cfg = "persist-suite", norun = true)] -async fn load_bin_bin() { - persist_load(&mut con, TABLE_BIN_LISTBIN, DATA_BIN_LISTBIN).await; -} - -// bin,list -const DATA_BIN_LISTSTR: ListData = listdata!( - binid!(b"list1") => ["e1", "e2", "e3", "e4"], - binid!(b"list2") => ["e1", "e2", "e3", "e4"], - binid!(b"list3") => ["e1", "e2", "e3", "e4"], - binid!(b"list4") => ["e1", "e2", "e3", "e4"] -); - -const TABLE_BIN_LISTSTR: &str = "testsuite.persist_bin_liststr"; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true)] -async fn store_bin_str() { - persist_store( - &mut con, - TABLE_BIN_LISTSTR, - "(binary, list)", - DATA_BIN_LISTSTR, - ) - .await; -} - -#[dbtest(run_if_cfg = "persist-suite", norun = true)] -async fn load_bin_str() { - persist_load(&mut con, TABLE_BIN_LISTSTR, DATA_BIN_LISTSTR).await; -} - -// str,list -const DATA_STR_LISTBIN: ListData = listdata!( - ListIDStr("list1") => binlist!(b"e1", b"e2", b"e3", b"e4"), - ListIDStr("list2") => binlist!(b"e1", b"e2", b"e3", b"e4"), - ListIDStr("list3") => binlist!(b"e1", b"e2", b"e3", b"e4"), - ListIDStr("list4") => binlist!(b"e1", b"e2", b"e3", b"e4") -); - -const TABLE_STR_LISTBIN: &str = "testsuite.persist_str_listbin"; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true)] -async fn store_str_bin() { - persist_store( - &mut con, - TABLE_STR_LISTBIN, - "(string, list)", - DATA_STR_LISTBIN, - ) - .await; -} - -#[dbtest(run_if_cfg = "persist-suite", norun = true)] -async fn load_str_bin() { - persist_load(&mut con, TABLE_STR_LISTBIN, DATA_STR_LISTBIN).await; -} - -// str,list -const DATA_STR_LISTSTR: ListData = listdata!( - ListIDStr("list1") => ["e1", "e2", "e3", "e4"], - ListIDStr("list2") => ["e1", "e2", "e3", "e4"], - ListIDStr("list3") => ["e1", "e2", "e3", "e4"], - ListIDStr("list4") => ["e1", "e2", "e3", "e4"] -); - -const TABLE_STR_LISTSTR: &str = "testsuite.persist_str_liststr"; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true)] -async fn store_str_str() { - persist_store( - &mut con, - TABLE_STR_LISTSTR, - "(string, list)", - DATA_STR_LISTSTR, - ) - .await; -} - -#[dbtest(run_if_cfg = "persist-suite", norun = true)] -async fn load_str_str() { - persist_load(&mut con, TABLE_STR_LISTSTR, DATA_STR_LISTSTR).await; -} diff --git a/server/src/tests/persist/mod.rs b/server/src/tests/persist/mod.rs deleted file mode 100644 index 16a9eb25..00000000 --- a/server/src/tests/persist/mod.rs +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Created on Thu Mar 17 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - sky_macros::dbtest_func as dbtest, - skytable::{ - aio::Connection, - query, - types::{Array, RawString}, - Element, Query, RespCode, - }, -}; - -#[dbtest(skip_if_cfg = "persist-suite", norun = true, port = 2007)] -async fn store_keyspace() { - assert_okay!(con, query!("create space universe")); - switch_entity!(con, "universe"); - assert_okay!(con, query!("create model warp(string,string)")); - switch_entity!(con, "universe.warp"); - assert_okay!(con, query!("set", "x", "100")); -} -#[dbtest(run_if_cfg = "persist-suite", norun = true, port = 2007)] -async fn load_keyspace() { - switch_entity!(con, "universe.warp"); - runeq!(con, query!("get", "x"), Element::String("100".to_owned())); - switch_entity!(con, "default"); - assert_okay!(con, query!("drop model universe.warp force")); - assert_okay!(con, query!("drop space universe")); -} - -macro_rules! bin { - ($input:expr) => {{ - const INVALID_SEQ: [u8; 2] = *b"\x80\x81"; - const RETLEN: usize = 2 + $input.len(); - const RET0: [u8; RETLEN] = { - let mut iret: [u8; RETLEN] = [0u8; RETLEN]; - let mut idx = 0; - while idx < $input.len() { - iret[idx] = $input[idx]; - idx += 1; - } - iret[RETLEN - 2] = INVALID_SEQ[0]; - iret[RETLEN - 1] = INVALID_SEQ[1]; - iret - }; - &RET0 - }}; -} - -mod auth; -mod kv; -mod kvlist; - -const PERSIST_TEST_SET_SIZE: usize = 4; - -trait PushIntoQuery { - fn push_into(&self, query: &mut Query); -} - -impl PushIntoQuery for &str { - fn push_into(&self, q: &mut Query) { - q.push(*self); - } -} - -impl PushIntoQuery for &[u8] { - fn push_into(&self, q: &mut Query) { - q.push(RawString::from(self.to_vec())) - } -} - -impl PushIntoQuery for [T; N] { - fn push_into(&self, q: &mut Query) { - for element in self { - element.push_into(q) - } - } -} - -impl PushIntoQuery for &[T] { - fn push_into(&self, q: &mut Query) { - for element in self.iter() { - element.push_into(q) - } - } -} - -trait PersistKey: PushIntoQuery { - fn action_store() -> &'static str; - fn action_load() -> &'static str; -} - -macro_rules! impl_persist_key { - ($($ty:ty => ($store:expr, $load:expr)),*) => { - $(impl PersistKey for $ty { - fn action_store() -> &'static str { - $store - } - fn action_load() -> &'static str { - $load - } - })* - }; -} - -impl_persist_key!( - &str => ("set", "get"), - &[u8] => ("set", "get"), - ListIDBin => ("lset", "lget"), - ListIDStr => ("lset", "lget") -); - -trait PersistValue: PushIntoQuery { - fn response_store(&self) -> Element; - fn response_load(&self) -> Element; -} - -impl PersistValue for &str { - fn response_store(&self) -> Element { - Element::RespCode(RespCode::Okay) - } - fn response_load(&self) -> Element { - Element::String(self.to_string()) - } -} - -impl PersistValue for &[u8] { - fn response_store(&self) -> Element { - Element::RespCode(RespCode::Okay) - } - fn response_load(&self) -> Element { - Element::Binstr(self.to_vec()) - } -} - -impl PersistValue for [&[u8]; N] { - fn response_store(&self) -> Element { - Element::RespCode(RespCode::Okay) - } - fn response_load(&self) -> Element { - let mut flat = Vec::with_capacity(N); - for item in self { - flat.push(item.to_vec()); - } - Element::Array(Array::NonNullBin(flat)) - } -} - -impl PersistValue for [&str; N] { - fn response_store(&self) -> Element { - Element::RespCode(RespCode::Okay) - } - fn response_load(&self) -> Element { - let mut flat = Vec::with_capacity(N); - for item in self { - flat.push(item.to_string()); - } - Element::Array(Array::NonNullStr(flat)) - } -} - -type Bin = &'static [u8]; -type Str = &'static str; - -#[derive(Debug)] -struct ListIDStr(Str); -#[derive(Debug)] -struct ListIDBin(Bin); - -impl PushIntoQuery for ListIDStr { - fn push_into(&self, q: &mut Query) { - self.0.push_into(q) - } -} - -impl PushIntoQuery for ListIDBin { - fn push_into(&self, q: &mut Query) { - self.0.push_into(q) - } -} - -async fn persist_store( - con: &mut Connection, - table_id: &str, - declaration: &str, - input: [(K, V); PERSIST_TEST_SET_SIZE], -) { - create_table_and_switch!(con, table_id, declaration); - for (key, value) in input { - let mut query = Query::from(K::action_store()); - key.push_into(&mut query); - value.push_into(&mut query); - runeq!(con, query, value.response_store()) - } -} - -async fn persist_load( - con: &mut Connection, - table_id: &str, - input: [(K, V); PERSIST_TEST_SET_SIZE], -) { - switch_entity!(con, table_id); - for (key, value) in input { - let mut q = Query::from(K::action_load()); - key.push_into(&mut q); - runeq!(con, q, value.response_load()); - } - // now delete this table, freeing it up for the next suite run - switch_entity!(con, "default.default"); - runeq!( - con, - query!(format!("drop model {table_id} force")), - Element::RespCode(RespCode::Okay) - ); -} diff --git a/server/src/tests/pipeline.rs b/server/src/tests/pipeline.rs deleted file mode 100644 index f8a01aad..00000000 --- a/server/src/tests/pipeline.rs +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Created on Sat Oct 30 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#[sky_macros::dbtest_module] -mod tests { - use skytable::{query, types::Array, Element, Pipeline, RespCode}; - async fn test_pipeline_heya_echo() { - let pipe = Pipeline::new() - .append(query!("heya", "first")) - .append(query!("heya", "second")); - let ret = con.run_pipeline(pipe).await.unwrap(); - assert_eq!( - ret, - vec![ - Element::String("first".to_owned()), - Element::String("second".to_owned()) - ] - ) - } - async fn test_pipeline_basic() { - let pipe = Pipeline::new() - .append(query!("heya")) - .append(query!("get", "x")); - let ret = con.run_pipeline(pipe).await.unwrap(); - assert_eq!( - ret, - vec![ - Element::String("HEY!".to_owned()), - Element::RespCode(RespCode::NotFound) - ] - ); - } - // although an error is simply just a response, but we'll still add a test for sanity - async fn test_pipeline_with_error() { - let pipe = Pipeline::new() - .append(query!("heya")) - .append(query!("get", "x", "y")); - let ret = con.run_pipeline(pipe).await.unwrap(); - assert_eq!( - ret, - vec![ - Element::String("HEY!".to_owned()), - Element::RespCode(RespCode::ActionError) - ] - ); - } - async fn test_pipeline_with_multiple_error() { - let pipe = Pipeline::new() - .append(query!("mset", "x", "y", "z")) - .append(query!("mget", "x", "y", "z")) - .append(query!("heya", "finally")); - let ret = con.run_pipeline(pipe).await.unwrap(); - assert_eq!( - ret, - vec![ - Element::RespCode(RespCode::ActionError), - Element::Array(Array::Str(vec![None, None, None])), - Element::String("finally".to_owned()) - ] - ) - } -} diff --git a/server/src/tests/snapshot.rs b/server/src/tests/snapshot.rs deleted file mode 100644 index f6cf4152..00000000 --- a/server/src/tests/snapshot.rs +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Created on Tue Mar 29 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - sky_macros::dbtest_func as dbtest, - skytable::{query, Element, RespCode}, -}; - -const SNAPSHOT_DISABLED: &str = "err-snapshot-disabled"; - -#[dbtest] -async fn snapshot_fail_because_local_disabled() { - runeq!( - con, - query!("mksnap"), - Element::RespCode(RespCode::ErrorString(SNAPSHOT_DISABLED.to_owned())) - ) -} - -#[dbtest(skip_if_cfg = "persist-suite")] -async fn rsnap_okay() { - loop { - match con.run_query_raw(query!("mksnap", "myremo")).await.unwrap() { - Element::RespCode(RespCode::Okay) => break, - Element::RespCode(RespCode::ErrorString(estr)) if estr.eq("err-snapshot-busy") => {} - x => panic!("snapshot failed: {:?}", x), - } - } -} - -#[dbtest(port = 2007)] -async fn local_snapshot_from_remote_okay() { - assert_okay!(con, query!("mksnap")) -} - -#[dbtest(port = 2007, skip_if_cfg = "persist-suite")] -async fn remote_snapshot_okay_with_local_enabled() { - loop { - match con.run_query_raw(query!("mksnap", "myremo")).await.unwrap() { - Element::RespCode(RespCode::Okay) => break, - Element::RespCode(RespCode::ErrorString(estr)) if estr.eq("err-snapshot-busy") => {} - x => panic!("snapshot failed: {:?}", x), - } - } -} - -#[dbtest(port = 2007, skip_if_cfg = "persist-suite")] -async fn remote_snapshot_fail_because_already_exists() { - loop { - match con.run_query_raw(query!("mksnap", "dupe")).await.unwrap() { - Element::RespCode(RespCode::Okay) => break, - Element::RespCode(RespCode::ErrorString(estr)) if estr.eq("err-snapshot-busy") => {} - x => panic!("snapshot failed: {:?}", x), - } - } - loop { - match con.run_query_raw(query!("mksnap", "dupe")).await.unwrap() { - Element::RespCode(RespCode::ErrorString(estr)) => match estr.as_str() { - "err-snapshot-busy" => {} - "duplicate-snapshot" => break, - _ => panic!("Got error string: {estr} instead"), - }, - x => panic!("snapshot failed: {:?}", x), - } - } -} diff --git a/server/src/util/compiler.rs b/server/src/util/compiler.rs index 56d042ed..92567d44 100644 --- a/server/src/util/compiler.rs +++ b/server/src/util/compiler.rs @@ -27,8 +27,6 @@ //! Dark compiler arts and hackery to defy the normal. Use at your own //! risk -use core::mem; - #[cold] #[inline(never)] pub const fn cold() {} @@ -49,7 +47,13 @@ pub const fn unlikely(b: bool) -> bool { #[cold] #[inline(never)] -pub const fn cold_err(v: T) -> T { +pub fn cold_call(v: impl FnOnce() -> U) -> U { + v() +} + +#[cold] +#[inline(never)] +pub const fn cold_val(v: T) -> T { v } #[inline(always)] @@ -61,14 +65,8 @@ pub const fn hot(v: T) -> T { v } -/// # Safety -/// The caller is responsible for ensuring lifetime validity -pub const unsafe fn extend_lifetime<'a, 'b, T>(inp: &'a T) -> &'b T { - mem::transmute(inp) -} - -/// # Safety -/// The caller is responsible for ensuring lifetime validity -pub unsafe fn extend_lifetime_mut<'a, 'b, T>(inp: &'a mut T) -> &'b mut T { - mem::transmute(inp) +#[cold] +#[inline(never)] +pub fn cold_rerr(e: E) -> Result { + Err(e) } diff --git a/server/src/util/error.rs b/server/src/util/error.rs deleted file mode 100644 index 4627ef33..00000000 --- a/server/src/util/error.rs +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Created on Sat Mar 26 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::storage::v1::{error::StorageEngineError, sengine::SnapshotEngineError}, - openssl::{error::ErrorStack as SslErrorStack, ssl::Error as SslError}, - std::{fmt, io::Error as IoError}, -}; - -pub type SkyResult = Result; - -#[derive(Debug)] -pub enum Error { - Storage(StorageEngineError), - IoError(IoError), - IoErrorExtra(IoError, String), - OtherError(String), - TlsError(SslError), - SnapshotEngineError(SnapshotEngineError), -} - -impl Error { - pub fn ioerror_extra(ioe: IoError, extra: impl ToString) -> Self { - Self::IoErrorExtra(ioe, extra.to_string()) - } -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Storage(serr) => write!(f, "Storage engine error: {}", serr), - Self::IoError(nerr) => write!(f, "I/O error: {}", nerr), - Self::IoErrorExtra(ioe, extra) => write!(f, "I/O error while {extra}: {ioe}"), - Self::OtherError(oerr) => write!(f, "Error: {}", oerr), - Self::TlsError(terr) => write!(f, "TLS error: {}", terr), - Self::SnapshotEngineError(snaperr) => write!(f, "Snapshot engine error: {snaperr}"), - } - } -} - -impl From for Error { - fn from(ioe: IoError) -> Self { - Self::IoError(ioe) - } -} - -impl From for Error { - fn from(see: StorageEngineError) -> Self { - Self::Storage(see) - } -} - -impl From for Error { - fn from(sslerr: SslError) -> Self { - Self::TlsError(sslerr) - } -} - -impl From for Error { - fn from(estack: SslErrorStack) -> Self { - Self::TlsError(estack.into()) - } -} - -impl From for Error { - fn from(snaperr: SnapshotEngineError) -> Self { - Self::SnapshotEngineError(snaperr) - } -} diff --git a/server/src/util/macros.rs b/server/src/util/macros.rs index 80acde3e..8c2849e3 100644 --- a/server/src/util/macros.rs +++ b/server/src/util/macros.rs @@ -26,9 +26,17 @@ #[macro_export] macro_rules! impossible { - () => { - core::hint::unreachable_unchecked() - }; + () => {{ + if cfg!(debug_assertions) { + panic!( + "reached unreachable case at: {}:{}", + ::core::file!(), + ::core::line!() + ); + } else { + ::core::hint::unreachable_unchecked() + } + }}; } #[macro_export] @@ -70,16 +78,43 @@ macro_rules! cfg_test { #[macro_export] /// Compare two vectors irrespective of their elements' position -macro_rules! veceq { +macro_rules! veceq_transposed { ($v1:expr, $v2:expr) => { $v1.len() == $v2.len() && $v1.iter().all(|v| $v2.contains(v)) }; } #[macro_export] -macro_rules! assert_veceq { +macro_rules! assert_veceq_transposed { + ($v1:expr, $v2:expr) => {{ + if !veceq_transposed!($v1, $v2) { + panic!( + "failed to assert transposed veceq. v1: `{:#?}`, v2: `{:#?}`", + $v1, $v2 + ) + } + }}; +} + +#[cfg(test)] +macro_rules! vecstreq_exact { + ($v1:expr, $v2:expr) => { + $v1.iter() + .zip($v2.iter()) + .all(|(a, b)| a.as_bytes() == b.as_bytes()) + }; +} + +#[cfg(test)] +macro_rules! assert_vecstreq_exact { ($v1:expr, $v2:expr) => { - assert!(veceq!($v1, $v2)) + if !vecstreq_exact!($v1, $v2) { + ::core::panic!( + "failed to assert vector data equality. lhs: {:?}, rhs: {:?}", + $v1, + $v2 + ); + } }; } @@ -192,38 +227,6 @@ macro_rules! do_sleep { }}; } -#[cfg(test)] -macro_rules! tmut_bool { - ($e:expr) => {{ - *(&$e as *const _ as *const bool) - }}; - ($a:expr, $b:expr) => { - (tmut_bool!($a), tmut_bool!($b)) - }; -} - -macro_rules! ucidx { - ($base:expr, $idx:expr) => { - *($base.as_ptr().add($idx as usize)) - }; -} - -/// If you provide: [T; N] with M initialized elements, then you are given -/// [MaybeUninit; N] with M initialized elements and N-M uninit elements -macro_rules! uninit_array { - ($($vis:vis const $id:ident: [$ty:ty; $len:expr] = [$($init_element:expr),*];)*) => { - $($vis const $id: [::core::mem::MaybeUninit<$ty>; $len] = { - let mut ret = [::core::mem::MaybeUninit::uninit(); $len]; - let mut idx = 0; - $( - idx += 1; - ret[idx - 1] = ::core::mem::MaybeUninit::new($init_element); - )* - ret - };)* - }; -} - #[macro_export] macro_rules! def { ( @@ -261,3 +264,57 @@ macro_rules! bench { $vis mod $modname; }; } + +#[macro_export] +macro_rules! is_64b { + () => { + cfg!(target_pointer_width = "64") + }; +} + +#[macro_export] +macro_rules! concat_array_to_array { + ($a:expr, $b:expr) => {{ + const BUFFER_A: [u8; $a.len()] = crate::util::copy_slice_to_array($a); + const BUFFER_B: [u8; $b.len()] = crate::util::copy_slice_to_array($b); + const BUFFER: [u8; BUFFER_A.len() + BUFFER_B.len()] = unsafe { + // UNSAFE(@ohsayan): safe because align = 1 + core::mem::transmute((BUFFER_A, BUFFER_B)) + }; + BUFFER + }}; + ($a:expr, $b:expr, $c:expr) => {{ + const LA: usize = $a.len() + $b.len(); + const LB: usize = LA + $c.len(); + const S_1: [u8; LA] = concat_array_to_array!($a, $b); + const S_2: [u8; LB] = concat_array_to_array!(&S_1, $c); + S_2 + }}; +} + +#[macro_export] +macro_rules! concat_str_to_array { + ($a:expr, $b:expr) => { + concat_array_to_array!($a.as_bytes(), $b.as_bytes()) + }; + ($a:expr, $b:expr, $c:expr) => {{ + concat_array_to_array!($a.as_bytes(), $b.as_bytes(), $c.as_bytes()) + }}; +} + +#[macro_export] +macro_rules! concat_str_to_str { + ($a:expr, $b:expr) => {{ + const BUFFER: [u8; ::core::primitive::str::len($a) + ::core::primitive::str::len($b)] = + concat_str_to_array!($a, $b); + const STATIC_BUFFER: &[u8] = &BUFFER; + unsafe { + // UNSAFE(@ohsayan): all good because of restriction to str + core::str::from_utf8_unchecked(&STATIC_BUFFER) + } + }}; + ($a:expr, $b:expr, $c:expr) => {{ + const A: &str = concat_str_to_str!($a, $b); + concat_str_to_str!(A, $c) + }}; +} diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs index 2acdeab5..bbd8ce8b 100644 --- a/server/src/util/mod.rs +++ b/server/src/util/mod.rs @@ -27,19 +27,28 @@ #[macro_use] mod macros; pub mod compiler; -pub mod error; pub mod os; +#[cfg(test)] +pub mod test_utils; use { - crate::{ - actions::{ActionError, ActionResult}, - protocol::interface::ProtocolSpec, + core::{ + fmt::{self, Debug}, + marker::PhantomData, + mem::{self, MaybeUninit}, + ops::Deref, + slice, }, - core::{fmt::Debug, marker::PhantomData, ops::Deref}, std::process, }; +pub const IS_ON_CI: bool = option_env!("CI").is_some(); + const EXITCODE_ONE: i32 = 0x01; +pub fn bx_to_vec(bx: Box<[T]>) -> Vec { + Vec::from(bx) +} + /// # Unsafe unwrapping /// /// This trait provides a method `unsafe_unwrap` that is potentially unsafe and has @@ -77,20 +86,6 @@ unsafe impl Unwrappable for Option { } } -pub trait UnwrapActionError { - fn unwrap_or_custom_aerr(self, e: impl Into) -> ActionResult; - fn unwrap_or_aerr(self) -> ActionResult; -} - -impl UnwrapActionError for Option { - fn unwrap_or_custom_aerr(self, e: impl Into) -> ActionResult { - self.ok_or_else(|| e.into()) - } - fn unwrap_or_aerr(self) -> ActionResult { - self.ok_or_else(|| P::RCODE_ACTION_ERR.into()) - } -} - pub fn exit_error() -> ! { process::exit(EXITCODE_ONE) } @@ -136,7 +131,8 @@ impl Clone for Wrapper { } } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq)] +#[repr(transparent)] /// This is yet another compiler hack and has no "actual impact" in terms of memory alignment. /// /// When it's hard to have a _split mutable borrow_, all across the source we use custom @@ -156,18 +152,19 @@ impl Clone for Wrapper { /// to explicitly annotate bounds /// - this type derefs to the base type #[derive(Copy, Clone)] -pub struct Life<'a, T> { +pub struct Life<'a, T: 'a> { _lt: PhantomData<&'a T>, v: T, } -impl<'a, T> Life<'a, T> { +impl<'a, T: 'a> Life<'a, T> { /// Ensure compile-time alignment (this is just a sanity check) const _ENSURE_COMPILETIME_ALIGN: () = - assert!(std::mem::align_of::>>() == std::mem::align_of::>()); + assert!(std::mem::align_of::>() == std::mem::align_of::()); #[inline(always)] pub const fn new(v: T) -> Self { + let _ = Self::_ENSURE_COMPILETIME_ALIGN; Life { v, _lt: PhantomData, @@ -211,3 +208,219 @@ impl<'a, T: PartialEq> PartialEq for Life<'a, T> { unsafe impl<'a, T: Send> Send for Life<'a, T> {} unsafe impl<'a, T: Sync> Sync for Life<'a, T> {} + +/// [`MaybeInit`] is a structure that is like an [`Option`] in debug mode and like +/// [`MaybeUninit`] in release mode. This means that provided there are good enough test cases, most +/// incorrect `assume_init` calls should be detected in the test phase. +#[cfg_attr(not(test), repr(transparent))] +pub struct MaybeInit { + #[cfg(test)] + is_init: bool, + #[cfg(not(test))] + is_init: (), + base: MaybeUninit, +} + +impl MaybeInit { + /// Initialize a new uninitialized variant + #[inline(always)] + pub const fn uninit() -> Self { + Self { + #[cfg(test)] + is_init: false, + #[cfg(not(test))] + is_init: (), + base: MaybeUninit::uninit(), + } + } + /// Initialize with a value + #[inline(always)] + pub const fn new(val: T) -> Self { + Self { + #[cfg(test)] + is_init: true, + #[cfg(not(test))] + is_init: (), + base: MaybeUninit::new(val), + } + } + const fn ensure_init(#[cfg(test)] is_init: bool, #[cfg(not(test))] is_init: ()) { + #[cfg(test)] + { + if !is_init { + panic!("Tried to `assume_init` on uninitialized data"); + } + } + let _ = is_init; + } + /// Assume that `self` is initialized and return the inner value + /// + /// ## Safety + /// + /// Caller needs to ensure that the data is actually initialized + #[inline(always)] + pub const unsafe fn assume_init(self) -> T { + Self::ensure_init(self.is_init); + self.base.assume_init() + } + /// Assume that `self` is initialized and return a reference + /// + /// ## Safety + /// + /// Caller needs to ensure that the data is actually initialized + #[inline(always)] + pub const unsafe fn assume_init_ref(&self) -> &T { + Self::ensure_init(self.is_init); + self.base.assume_init_ref() + } + /// Assumes `self` is initialized, replaces `self` with an uninit state, returning + /// the older value + /// + /// ## Safety + pub unsafe fn take(&mut self) -> T { + Self::ensure_init(self.is_init); + let mut r = MaybeUninit::uninit(); + mem::swap(&mut r, &mut self.base); + #[cfg(test)] + { + self.is_init = false; + } + r.assume_init() + } +} + +#[cfg(test)] +impl fmt::Debug for MaybeInit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let dat_fmt = if self.is_init { + unsafe { format!("{:?}", self.base.assume_init_ref()) } + } else { + "MaybeUninit {..}".to_string() + }; + f.debug_struct("MaybeInit") + .field("is_init", &self.is_init) + .field("base", &dat_fmt) + .finish() + } +} + +#[cfg(not(test))] +impl fmt::Debug for MaybeInit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MaybeInit") + .field("base", &self.base) + .finish() + } +} + +pub unsafe trait ByteRepr { + fn repr(&self) -> &[u8]; +} + +unsafe impl ByteRepr for [u8] { + fn repr(&self) -> &[u8] { + self + } +} +unsafe impl ByteRepr for str { + fn repr(&self) -> &[u8] { + self.as_bytes() + } +} + +pub trait NumericRepr: ByteRepr { + fn be(&self) -> Self; + fn le(&self) -> Self; +} + +macro_rules! byte_repr_impls { + ($($ty:ty),*) => { + $( + unsafe impl ByteRepr for $ty { fn repr(&self) -> &[u8] { unsafe { slice::from_raw_parts(self as *const $ty as *const u8, mem::size_of::()) } } } + impl NumericRepr for $ty { fn be(&self) -> $ty { <$ty>::to_be(*self) } fn le(&self) -> $ty { <$ty>::to_le(*self) } } + )* + }; +} + +byte_repr_impls!(u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize); + +pub const fn copy_slice_to_array(bytes: &[u8]) -> [u8; N] { + assert!(bytes.len() <= N); + let mut data = [0u8; N]; + let mut i = 0; + while i < bytes.len() { + data[i] = bytes[i]; + i += 1; + } + data +} +pub const fn copy_str_to_array(str: &str) -> [u8; N] { + copy_slice_to_array(str.as_bytes()) +} +/// Copy the elements of a into b, beginning the copy at `pos` +pub const fn copy_a_into_b( + from: [u8; M], + mut to: [u8; N], + mut pos: usize, +) -> [u8; N] { + assert!(M <= N); + assert!(pos < N); + let mut i = 0; + while i < M { + to[pos] = from[i]; + i += 1; + pos += 1; + } + to +} + +pub struct Threshold { + now: usize, +} + +impl Threshold { + pub const fn new() -> Self { + Self { now: 0 } + } + pub fn bust_one(&mut self) { + self.now += 1; + } + pub fn not_busted(&self) -> bool { + self.now < N + } +} + +pub trait EndianQW { + fn u64_bytes_le(&self) -> [u8; 8]; + fn u64_bytes_be(&self) -> [u8; 8]; + fn u64_bytes_ne(&self) -> [u8; 8] { + if cfg!(target_endian = "big") { + self.u64_bytes_be() + } else { + self.u64_bytes_le() + } + } +} + +pub trait EndianDW { + fn u32_bytes_le(&self) -> [u8; 8]; + fn u32_bytes_be(&self) -> [u8; 8]; + fn u32_bytes_ne(&self) -> [u8; 8] { + if cfg!(target_endian = "big") { + self.u32_bytes_be() + } else { + self.u32_bytes_le() + } + } +} + +macro_rules! impl_endian { + ($($ty:ty),*) => { + $(impl EndianQW for $ty { + fn u64_bytes_le(&self) -> [u8; 8] { (*self as u64).to_le_bytes() } + fn u64_bytes_be(&self) -> [u8; 8] { (*self as u64).to_le_bytes() } + })* + } +} + +impl_endian!(u8, i8, u16, i16, u32, i32, u64, i64, usize, isize); diff --git a/server/src/util/os.rs b/server/src/util/os.rs index 90e4c9f5..86d09725 100644 --- a/server/src/util/os.rs +++ b/server/src/util/os.rs @@ -28,11 +28,49 @@ pub use unix::*; #[cfg(windows)] pub use windows::*; +mod flock; +mod free_memory; use { crate::IoResult, - std::{ffi::OsStr, fs, path::Path}, + std::{ + ffi::OsStr, + fmt, fs, + path::Path, + time::{SystemTime, UNIX_EPOCH}, + }, }; +pub use {flock::FileLock, free_memory::free_memory_in_bytes}; + +#[derive(Debug)] +#[repr(transparent)] +/// A wrapper around [`std`]'s I/O [Error](std::io::Error) type for simplicity with equality +pub struct SysIOError(std::io::Error); + +impl From for SysIOError { + fn from(e: std::io::Error) -> Self { + Self(e) + } +} + +impl From for SysIOError { + fn from(e: std::io::ErrorKind) -> Self { + Self(e.into()) + } +} + +impl fmt::Display for SysIOError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[cfg(test)] +impl PartialEq for SysIOError { + fn eq(&self, other: &Self) -> bool { + self.0.to_string() == other.0.to_string() + } +} #[cfg(unix)] mod unix { @@ -177,48 +215,7 @@ pub fn recursive_copy(src: impl AsRef, dst: impl AsRef) -> IoResult< Ok(()) } -#[test] -fn rcopy_okay() { - let dir_paths = [ - "testdata/backups", - "testdata/ks/default", - "testdata/ks/system", - "testdata/rsnaps", - "testdata/snaps", - ]; - let file_paths = [ - "testdata/ks/default/default", - "testdata/ks/default/PARTMAP", - "testdata/ks/PRELOAD", - "testdata/ks/system/PARTMAP", - ]; - let new_file_paths = [ - "my-backups/ks/default/default", - "my-backups/ks/default/PARTMAP", - "my-backups/ks/PRELOAD", - "my-backups/ks/system/PARTMAP", - ]; - let x = move || -> IoResult<()> { - for dir in dir_paths { - fs::create_dir_all(dir)?; - } - for file in file_paths { - fs::File::create(file)?; - } - Ok(()) - }; - x().unwrap(); - // now copy all files inside testdata/* to my-backups/* - recursive_copy("testdata", "my-backups").unwrap(); - new_file_paths - .iter() - .for_each(|path| assert!(Path::new(path).exists())); - // now remove the directories - fs::remove_dir_all("testdata").unwrap(); - fs::remove_dir_all("my-backups").unwrap(); -} - -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq)] pub enum EntryKind { Directory(String), File(String), @@ -297,3 +294,229 @@ fn dir_size_inner(dir: fs::ReadDir) -> IoResult { pub fn dirsize(path: impl AsRef) -> IoResult { dir_size_inner(fs::read_dir(path.as_ref())?) } + +/// Returns the current system uptime in milliseconds +pub fn get_uptime() -> u128 { + uptime_impl::uptime().unwrap() +} + +/// Returns the current epoch time in nanoseconds +pub fn get_epoch_time() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() +} + +/// Returns the hostname +pub fn get_hostname() -> hostname_impl::Hostname { + hostname_impl::Hostname::get() +} + +mod uptime_impl { + #[cfg(target_os = "linux")] + pub(super) fn uptime() -> std::io::Result { + let mut sysinfo: libc::sysinfo = unsafe { std::mem::zeroed() }; + let res = unsafe { libc::sysinfo(&mut sysinfo) }; + if res == 0 { + Ok(sysinfo.uptime as u128 * 1_000) + } else { + Err(std::io::Error::last_os_error()) + } + } + + #[cfg(any( + target_os = "macos", + target_os = "freebsd", + target_os = "openbsd", + target_os = "netbsd" + ))] + pub(super) fn uptime() -> std::io::Result { + use libc::{c_void, size_t, sysctl, timeval}; + use std::ptr; + + let mib = [libc::CTL_KERN, libc::KERN_BOOTTIME]; + let mut boottime = timeval { + tv_sec: 0, + tv_usec: 0, + }; + let mut size = std::mem::size_of::() as size_t; + + let result = unsafe { + sysctl( + // this cast is fine. sysctl only needs to access the ptr to array base (read) + &mib as *const _ as *mut _, + 2, + &mut boottime as *mut timeval as *mut c_void, + &mut size, + ptr::null_mut(), + 0, + ) + }; + + if result == 0 { + let current_time = unsafe { libc::time(ptr::null_mut()) }; + let uptime_secs = current_time - boottime.tv_sec; + Ok((uptime_secs as u128) * 1_000) + } else { + Err(std::io::Error::last_os_error()) + } + } + + #[cfg(target_os = "windows")] + pub(super) fn uptime() -> std::io::Result { + Ok(unsafe { winapi::um::sysinfoapi::GetTickCount64() } as u128) + } +} + +mod hostname_impl { + use std::ffi::CStr; + + pub struct Hostname { + len: u8, + raw: [u8; 255], + } + + impl Hostname { + pub fn get() -> Self { + get_hostname() + } + unsafe fn new_from_raw_buf(buf: &[u8; 256]) -> Self { + let mut raw = [0u8; 255]; + raw.copy_from_slice(&buf[..255]); + Self { + len: CStr::from_ptr(buf.as_ptr().cast()).to_bytes().len() as _, + raw, + } + } + pub fn as_str(&self) -> &str { + unsafe { + core::str::from_utf8_unchecked(core::slice::from_raw_parts( + self.raw.as_ptr(), + self.len as _, + )) + } + } + pub fn raw(&self) -> [u8; 255] { + self.raw + } + pub fn len(&self) -> u8 { + self.len + } + } + + #[cfg(target_family = "unix")] + fn get_hostname() -> Hostname { + use libc::gethostname; + + let mut buf: [u8; 256] = [0; 256]; + unsafe { + gethostname(buf.as_mut_ptr().cast(), buf.len()); + Hostname::new_from_raw_buf(&buf) + } + } + + #[cfg(target_family = "windows")] + fn get_hostname() -> Hostname { + use winapi::shared::minwindef::DWORD; + use winapi::um::sysinfoapi::{self, GetComputerNameExA}; + + let mut buf: [u8; 256] = [0; 256]; + let mut size: DWORD = buf.len() as u32; + + unsafe { + GetComputerNameExA( + sysinfoapi::ComputerNamePhysicalDnsHostname, + buf.as_mut_ptr().cast(), + &mut size, + ); + Hostname::new_from_raw_buf(&buf) + } + } + + #[cfg(test)] + mod test { + use std::process::Command; + + fn test_get_hostname() -> String { + let x = if cfg!(target_os = "windows") { + // Windows command to get hostname + Command::new("cmd") + .args(&["/C", "hostname"]) + .output() + .expect("Failed to execute command") + .stdout + } else { + // Unix command to get hostname + Command::new("uname") + .args(&["-n"]) + .output() + .expect("Failed to execute command") + .stdout + }; + String::from_utf8_lossy(&x).trim().to_string() + } + + #[test] + fn t_get_hostname() { + assert_eq!( + test_get_hostname().as_str(), + super::Hostname::get().as_str() + ); + } + } +} + +#[test] +fn rcopy_okay() { + let dir_paths = [ + "testdata/backups", + "testdata/ks/default", + "testdata/ks/system", + "testdata/rsnaps", + "testdata/snaps", + ]; + let file_paths = [ + "testdata/ks/default/default", + "testdata/ks/default/PARTMAP", + "testdata/ks/PRELOAD", + "testdata/ks/system/PARTMAP", + ]; + let new_file_paths = [ + "my-backups/ks/default/default", + "my-backups/ks/default/PARTMAP", + "my-backups/ks/PRELOAD", + "my-backups/ks/system/PARTMAP", + ]; + let x = move || -> IoResult<()> { + for dir in dir_paths { + fs::create_dir_all(dir)?; + } + for file in file_paths { + fs::File::create(file)?; + } + Ok(()) + }; + x().unwrap(); + // now copy all files inside testdata/* to my-backups/* + recursive_copy("testdata", "my-backups").unwrap(); + new_file_paths + .iter() + .for_each(|path| assert!(Path::new(path).exists())); + // now remove the directories + fs::remove_dir_all("testdata").unwrap(); + fs::remove_dir_all("my-backups").unwrap(); +} + +#[test] +fn t_uptime() { + use std::{thread, time::Duration}; + let uptime_1 = get_uptime(); + thread::sleep(Duration::from_secs(1)); + let uptime_2 = get_uptime(); + // we're putting a 10s tolerance + assert!( + Duration::from_millis(uptime_2.try_into().unwrap()) + <= (Duration::from_millis(uptime_1.try_into().unwrap()) + Duration::from_secs(10)) + ) +} diff --git a/server/src/util/os/flock.rs b/server/src/util/os/flock.rs new file mode 100644 index 00000000..8af608ca --- /dev/null +++ b/server/src/util/os/flock.rs @@ -0,0 +1,132 @@ +/* + * Created on Wed Oct 04 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +// unix imports +#[cfg(unix)] +extern crate libc; +// windows imports +#[cfg(windows)] +extern crate winapi; +#[cfg(windows)] +use std::os::windows::io::AsRawHandle; + +use std::{fs::File, io, path::Path}; + +pub struct FileLock { + _file: File, + #[cfg(windows)] + handle: winapi::um::winnt::HANDLE, +} + +impl FileLock { + pub fn new>(path: P) -> io::Result { + let file = File::create(path)?; + #[cfg(windows)] + { + use { + std::mem, + winapi::um::{ + fileapi::LockFileEx, + minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY}, + winnt::HANDLE, + }, + }; + let handle = file.as_raw_handle(); + let mut overlapped = unsafe { mem::zeroed() }; + let result = unsafe { + LockFileEx( + handle as HANDLE, + LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, + u32::MAX, + u32::MAX, + &mut overlapped, + ) + }; + if result == 0 { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "file is already locked", + )); + } + return Ok(Self { + _file: file, + handle, + }); + } + #[cfg(unix)] + { + use { + libc::{flock, LOCK_EX, LOCK_NB}, + std::os::unix::io::AsRawFd, + }; + let result = unsafe { flock(file.as_raw_fd(), LOCK_EX | LOCK_NB) }; + if result != 0 { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "file is already locked", + )); + } + return Ok(Self { _file: file }); + } + } + pub fn release(self) -> io::Result<()> { + #[cfg(windows)] + { + use { + std::mem, + winapi::um::{fileapi::UnlockFileEx, winnt::HANDLE}, + }; + + let mut overlapped = unsafe { mem::zeroed() }; + let result = unsafe { + UnlockFileEx( + self.handle as HANDLE, + 0, + u32::MAX, + u32::MAX, + &mut overlapped, + ) + }; + + if result == 0 { + return Err(io::Error::last_os_error()); + } + } + #[cfg(unix)] + { + use { + libc::{flock, LOCK_UN}, + std::os::unix::io::AsRawFd, + }; + let result = unsafe { flock(self._file.as_raw_fd(), LOCK_UN) }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) + } +} diff --git a/server/src/util/os/free_memory.rs b/server/src/util/os/free_memory.rs new file mode 100644 index 00000000..2671b307 --- /dev/null +++ b/server/src/util/os/free_memory.rs @@ -0,0 +1,90 @@ +/* + * Created on Sat Sep 02 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[cfg(target_os = "windows")] +extern crate winapi; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +extern crate libc; + +pub fn free_memory_in_bytes() -> u64 { + #[cfg(target_os = "windows")] + { + use winapi::um::sysinfoapi::{GlobalMemoryStatusEx, MEMORYSTATUSEX}; + + let mut statex: MEMORYSTATUSEX = unsafe { std::mem::zeroed() }; + statex.dwLength = std::mem::size_of::() as u32; + + unsafe { + GlobalMemoryStatusEx(&mut statex); + } + + // Return free physical memory + return statex.ullAvailPhys; + } + + #[cfg(target_os = "linux")] + { + use libc::sysinfo; + let mut info: libc::sysinfo = unsafe { core::mem::zeroed() }; + + unsafe { + if sysinfo(&mut info) == 0 { + // Return free memory + return (info.freeram as u64) * (info.mem_unit as u64); + } + } + + return 0; + } + + #[cfg(target_os = "macos")] + { + use std::mem; + unsafe { + let page_size = libc::sysconf(libc::_SC_PAGESIZE); + let mut count: u32 = libc::HOST_VM_INFO64_COUNT as _; + let mut stat: libc::vm_statistics64 = mem::zeroed(); + libc::host_statistics64( + libc::mach_host_self(), + libc::HOST_VM_INFO64, + &mut stat as *mut libc::vm_statistics64 as *mut _, + &mut count, + ); + + // see this: https://opensource.apple.com/source/xnu/xnu-4570.31.3/osfmk/mach/vm_statistics.h.auto.html + return (stat.free_count as u64) + .saturating_add(stat.inactive_count as _) + .saturating_add(stat.compressor_page_count as u64) + .saturating_mul(page_size as _); + } + } + + #[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))] + { + return 0; + } +} diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs new file mode 100644 index 00000000..a12110a1 --- /dev/null +++ b/server/src/util/test_utils.rs @@ -0,0 +1,143 @@ +/* + * Created on Sat Sep 17 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + rand::{ + distributions::{uniform::SampleUniform, Alphanumeric}, + rngs::ThreadRng, + seq::SliceRandom, + Rng, + }, + std::{ + collections::hash_map::RandomState, + hash::{BuildHasher, Hash, Hasher}, + io::Read, + }, +}; + +pub fn rng() -> ThreadRng { + rand::thread_rng() +} + +pub fn shuffle_slice(slice: &mut [T], rng: &mut impl Rng) { + slice.shuffle(rng) +} + +pub fn with_files(files: [&str; N], f: impl Fn([&str; N]) -> T) -> T { + use std::fs; + for file in files { + let _ = fs::File::create(file); + } + let r = f(files); + for file in files { + let _ = fs::remove_file(file); + } + r +} + +pub fn wait_for_key(msg: &str) { + use std::io::{self, Write}; + print!("{msg}"); + let x = || -> std::io::Result<()> { + io::stdout().flush()?; + let mut key = [0u8; 1]; + io::stdin().read_exact(&mut key)?; + Ok(()) + }; + x().unwrap(); +} + +// TODO(@ohsayan): Use my own PRNG algo here. Maybe my quadratic one? + +/// Generates a random boolean based on Bernoulli distributions +pub fn random_bool(rng: &mut impl Rng) -> bool { + rng.gen_bool(0.5) +} +/// Generate a random number within the given range +pub fn random_number(min: T, max: T, rng: &mut impl Rng) -> T { + rng.gen_range(min..max) +} + +pub fn random_string(rng: &mut impl Rng, l: usize) -> String { + rng.sample_iter(Alphanumeric) + .take(l) + .map(char::from) + .collect() +} + +pub fn random_string_checked(rng: &mut impl Rng, l: usize, ck: impl Fn(&str) -> bool) -> String { + loop { + let r = random_string(rng, l); + if ck(&r) { + break r; + } + } +} + +pub trait VecFuse { + fn fuse_append(self, arr: &mut Vec); +} + +impl VecFuse for T { + fn fuse_append(self, arr: &mut Vec) { + arr.push(self); + } +} + +impl VecFuse for [T; N] { + fn fuse_append(self, arr: &mut Vec) { + arr.extend(self) + } +} + +impl VecFuse for Vec { + fn fuse_append(self, arr: &mut Vec) { + arr.extend(self) + } +} + +#[macro_export] +macro_rules! vecfuse { + ($($expr:expr),* $(,)?) => {{ + let mut v = Vec::new(); + $(<_ as $crate::util::test_utils::VecFuse<_>>::fuse_append($expr, &mut v);)* + v + }}; +} + +pub fn randomstate() -> RandomState { + RandomState::default() +} + +pub fn hash_rs(rs: &RandomState, item: &T) -> u64 { + let mut hasher = rs.build_hasher(); + item.hash(&mut hasher); + hasher.finish() +} + +pub fn randomizer() -> ThreadRng { + rand::thread_rng() +} diff --git a/sky-bench/Cargo.toml b/sky-bench/Cargo.toml index 0aa72498..60e11f65 100644 --- a/sky-bench/Cargo.toml +++ b/sky-bench/Cargo.toml @@ -9,16 +9,11 @@ description = "The Skytable benchmark tool can be used to benchmark Skytable ins [dependencies] # internal deps -skytable = { git = "https://github.com/skytable/client-rust.git", features = [ - "sync", - "dbg", -] } -libstress = { path = "../libstress" } +skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" } +libsky = { path = "../libsky" } # external deps -clap = { version = "4.0.32", features = ["derive"] } -log = "0.4.17" -env_logger = "0.10.0" -devtimer = "4.0.1" -serde = { version = "1.0.152", features = ["derive"] } -serde_json = "1.0.91" -rand = "0.8.5" +crossbeam-channel = "0.5.8" +num_cpus = "1.16.0" +env_logger = "0.10.1" +log = "0.4.20" +tokio = { version = "1.34.0", features = ["full"] } diff --git a/sky-bench/README.md b/sky-bench/README.md index 69869217..1464d400 100644 --- a/sky-bench/README.md +++ b/sky-bench/README.md @@ -3,18 +3,31 @@ `sky-bench` is Skytable's benchmarking tool. Unlike most other benchmarking tools, Skytable's benchmark tool doesn't do anything "fancy" to make benchmarks appear better than they are. As it happens, the benchmark tool might show Skytable to be slower! +> **The benchmarking engine is currently experimental!** You can indirectly compare it to the working of `redis-benchmark` but one +> important note: **Skytable has a full fledged query language.** *Even then, you will probably enjoy the benchmarks!* +> +> Other tools like `memtier_benchmark` are far more sophisticated and use several strategies that can hugely affect benchmark +> numbers. +> +> We will upgrade the benchmark engine from time to time to improve the reporting of statistics. For example, right now the +> engine only outputs the slowest and fastest query speeds **in nanoseconds** but we plan to provide a overall distribution of +> latencies. + +## Working + Here's how the benchmark tool works (it's dead simple): -1. Depending on the configuration it launches "network pools" which are just thread pools where each worker - holds a persistent connection to the database (something like a connection pool) -2. A collection of unique, random keys are generated using a PRNG provided by the `rand` library that is - seeded using the OS' source of randomness. The values are allowed to repeat -3. The [Skytable Rust driver](https://github.com/skytable/client-rust) is used to generate _raw query packets_. To put it simply, the keys and values are turned into `Query` objects and then into the raw bytes that will be sent over the network. (This just makes it simpler to design the network pool interface) -4. For every type of benchmark (GET,SET,...) we use the network pool to send all the bytes and wait until we receive the expected response. We time how long it takes to send and receive the response for all the queries for the given test (aggregate) -5. We repeat this for all the remaining tests -6. We repeat the entire set of tests 5 times (by default, this can be changed). -7. We do the calculations and output the results. +1. We spawn up multiple client tasks (or what you can call "green threads") that can each handle tasks. These tasks are run on a threadpool which has multiple worker threads, kind of simulating multiple "application server instances" + - You can use `--connections` to set the number of client connections + - You can use `--threads` to set the number of threads to be used +2. An overall target is sent to these tasks and all tasks start executing queries until the overall target is reached + > The distribution of tasks across clients is generally unspecified, but because of Skytable's extremely low average latencies, in most common scenarios, the distribution is even. +3. Once the overall target is reached, each task relays its local execution statistics to the monitoring task +4. The monitoring task then prepares the final results, and this is returned + +### Engines -## License +There are two benchmark engines: -All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). +- `fury`: this is the new experimental engine, but also set as the default engine. It is generally more efficient and tracks statistics more effectively. At the same time, it is capable of generating larger consistent loads without crashing or blowing up CPU usage +- `rookie`: this is the old engine that's still available but is not used by default. it still uses lesser memory than prior versions (which used a very inefficient and memory hungry algorithm) but is not as resource efficient as the `fury` engine. diff --git a/sky-bench/help_text/help b/sky-bench/help_text/help new file mode 100644 index 00000000..a191d8ba --- /dev/null +++ b/sky-bench/help_text/help @@ -0,0 +1,33 @@ +sky-bench 0.8.0 +Sayan N. +Skytable benchmark tool + +USAGE: + sky-bench [OPTIONS] + +FLAGS: + --help Displays this help message + --version Displays the benchmark tool version + +REQUIRED OPTIONS: + --password Provide the password + +OPTIONS: + --endpoint Set the endpoint (defaults to tcp@127.0.0.1:2003) + --threads Set the number of threads to be used (defaults to logical + CPU count) + --connections Set the number of connections. Defaults to 8 x logical CPU + count. Only supported by the `fury` engine. + --keysize Set the default primary key size. defaults to 7 + --rowcount Set the number of rows to be manipulated for the benchmark + Defaults to 1,000,000 rows. + --engine Set the engine for benchmarking. `rookie` is the stable engine + and `fury` is the new experimental engine. Defaults to `fury` + +NOTES: + - The user for auth will be 'root' since only 'root' accounts allow the + creation and deletion of spaces and models + - A space called 'bench' will be created + - A model called 'bench' will be created in the space + created above. The created model has the structure {un: string, pw: uint8} + - The model and space will be removed once the benchmark is complete diff --git a/sky-bench/src/args.rs b/sky-bench/src/args.rs new file mode 100644 index 00000000..2aa5336d --- /dev/null +++ b/sky-bench/src/args.rs @@ -0,0 +1,231 @@ +/* + * Created on Sat Nov 18 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + crate::error::{BenchError, BenchResult}, + libsky::CliAction, + std::collections::hash_map::HashMap, +}; + +const TXT_HELP: &str = include_str!("../help_text/help"); + +#[derive(Debug)] +enum TaskInner { + HelpMsg(String), + CheckConfig(HashMap), +} + +#[derive(Debug)] +pub enum Task { + HelpMsg(String), + BenchConfig(BenchConfig), +} + +#[derive(Debug, PartialEq)] +pub enum BenchEngine { + Rookie, + Fury, +} + +#[derive(Debug)] +pub struct BenchConfig { + pub host: String, + pub port: u16, + pub root_pass: String, + pub threads: usize, + pub key_size: usize, + pub query_count: usize, + pub engine: BenchEngine, + pub connections: usize, +} + +impl BenchConfig { + pub fn new( + host: String, + port: u16, + root_pass: String, + threads: usize, + key_size: usize, + query_count: usize, + engine: BenchEngine, + connections: usize, + ) -> Self { + Self { + host, + port, + root_pass, + threads, + key_size, + query_count, + engine, + connections, + } + } +} + +fn load_env() -> BenchResult { + let action = libsky::parse_cli_args_disallow_duplicate()?; + match action { + CliAction::Help => Ok(TaskInner::HelpMsg(TXT_HELP.into())), + CliAction::Version => Ok(TaskInner::HelpMsg(libsky::version_msg("sky-bench"))), + CliAction::Action(a) => Ok(TaskInner::CheckConfig(a)), + } +} + +fn cdig(n: usize) -> usize { + if n == 0 { + 1 + } else { + (n as f64).log10().floor() as usize + 1 + } +} + +pub fn parse() -> BenchResult { + let mut args = match load_env()? { + TaskInner::HelpMsg(msg) => return Ok(Task::HelpMsg(msg)), + TaskInner::CheckConfig(args) => args, + }; + // endpoint + let (host, port) = match args.remove("--endpoint") { + None => ("127.0.0.1".to_owned(), 2003), + Some(ep) => { + // proto@host:port + let ep: Vec<&str> = ep.split("@").collect(); + if ep.len() != 2 { + return Err(BenchError::ArgsErr( + "value for --endpoint must be in the form `[protocol]@[host]:[port]`".into(), + )); + } + let protocol = ep[0]; + let host_port: Vec<&str> = ep[1].split(":").collect(); + if host_port.len() != 2 { + return Err(BenchError::ArgsErr( + "value for --endpoint must be in the form `[protocol]@[host]:[port]`".into(), + )); + } + let (host, port) = (host_port[0], host_port[1]); + let Ok(port) = port.parse::() else { + return Err(BenchError::ArgsErr( + "the value for port must be an integer in the range 0-65535".into(), + )); + }; + if protocol != "tcp" { + return Err(BenchError::ArgsErr( + "only TCP endpoints can be benchmarked at the moment".into(), + )); + } + (host.to_owned(), port) + } + }; + // password + let passsword = match args.remove("--password") { + Some(p) => p, + None => { + return Err(BenchError::ArgsErr( + "you must provide a value for `--password`".into(), + )) + } + }; + // threads + let thread_count = match args.remove("--threads") { + None => num_cpus::get(), + Some(tc) => match tc.parse() { + Ok(tc) if tc > 0 => tc, + Err(_) | Ok(_) => { + return Err(BenchError::ArgsErr( + "incorrect value for `--threads`. must be a nonzero value".into(), + )) + } + }, + }; + // query count + let query_count = match args.remove("--rowcount") { + None => 1_000_000_usize, + Some(rc) => match rc.parse() { + Ok(rc) if rc != 0 => rc, + Err(_) | Ok(_) => { + return Err(BenchError::ArgsErr(format!( + "bad value for `--rowcount` must be a nonzero value" + ))) + } + }, + }; + let need_atleast = cdig(query_count); + let key_size = match args.remove("--keysize") { + None => need_atleast, + Some(ks) => match ks.parse() { + Ok(s) if s >= need_atleast => s, + Err(_) | Ok(_) => return Err(BenchError::ArgsErr(format!("incorrect value for `--keysize`. must be set to a value that can be used to generate atleast {query_count} unique primary keys"))), + } + }; + let engine = match args.remove("--engine") { + None => { + warn!("engine unspecified. choosing 'fury'"); + BenchEngine::Fury + } + Some(engine) => match engine.as_str() { + "rookie" => BenchEngine::Rookie, + "fury" => BenchEngine::Fury, + _ => { + return Err(BenchError::ArgsErr(format!( + "bad value for `--engine`. got `{engine}` but expected warp or rookie" + ))) + } + }, + }; + let connections = match args.remove("--connections") { + None => num_cpus::get() * 8, + Some(c) => match c.parse::() { + Ok(s) if s != 0 => { + if engine == BenchEngine::Rookie { + return Err(BenchError::ArgsErr(format!( + "the 'rookie' engine does not support explicit connection count. the number of threads is the connection count" + ))); + } + s + } + _ => { + return Err(BenchError::ArgsErr(format!( + "bad value for `--connections`. must be a nonzero value" + ))) + } + }, + }; + if args.is_empty() { + Ok(Task::BenchConfig(BenchConfig::new( + host, + port, + passsword, + thread_count, + key_size, + query_count, + engine, + connections, + ))) + } else { + Err(BenchError::ArgsErr(format!("unrecognized arguments"))) + } +} diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs new file mode 100644 index 00000000..bfe18fd0 --- /dev/null +++ b/sky-bench/src/bench.rs @@ -0,0 +1,369 @@ +/* + * Created on Sat Nov 18 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use skytable::response::Value; + +use crate::args::BenchEngine; + +use { + crate::{ + args::BenchConfig, + error::{self, BenchResult}, + runtime::{fury, rookie, RuntimeStats}, + }, + skytable::{error::Error, query, response::Response, Config, Connection, Query}, + std::{fmt, time::Instant}, +}; + +pub const BENCHMARK_SPACE_ID: &'static str = "bench"; +pub const BENCHMARK_MODEL_ID: &'static str = "bench"; + +/* + task impl +*/ + +/// A bombard task used for benchmarking + +#[derive(Debug)] +pub struct BombardTask { + config: Config, +} + +impl BombardTask { + pub fn new(config: Config) -> Self { + Self { config } + } +} + +/// Errors while running a bombard +#[derive(Debug)] +pub enum BombardTaskError { + DbError(Error), + Mismatch, +} + +impl fmt::Display for BombardTaskError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DbError(e) => write!(f, "a bombard subtask failed with {e}"), + Self::Mismatch => write!(f, "got unexpected response for bombard subtask"), + } + } +} + +impl From for BombardTaskError { + fn from(dbe: Error) -> Self { + Self::DbError(dbe) + } +} + +impl rookie::ThreadedBombardTask for BombardTask { + type Worker = Connection; + type WorkerTask = (Query, (BenchmarkTask, u64)); + type WorkerTaskSpec = BenchmarkTask; + type WorkerInitError = Error; + type WorkerTaskError = BombardTaskError; + fn worker_init(&self) -> Result { + let mut db = self.config.connect()?; + db.query_parse::<()>(&skytable::query!(format!("use {BENCHMARK_SPACE_ID}"))) + .map(|_| db) + } + fn generate_task(spec: &Self::WorkerTaskSpec, current: u64) -> Self::WorkerTask { + (spec.generate_query(current), (*spec, current)) + } + fn worker_drive_timed( + worker: &mut Self::Worker, + (query, (spec, current)): Self::WorkerTask, + ) -> Result { + let start = Instant::now(); + let ret = worker.query(&query)?; + let stop = Instant::now(); + if spec.verify_response(current, ret) { + Ok(stop.duration_since(start).as_nanos()) + } else { + Err(BombardTaskError::Mismatch) + } + } +} + +/* + runner +*/ + +pub fn run(bench: BenchConfig) -> error::BenchResult<()> { + let bench_config = BombardTask::new(Config::new( + &bench.host, + bench.port, + "root", + &bench.root_pass, + )); + info!("running preliminary checks and creating model `bench.bench` with definition: `{{un: binary, pw: uint8}}`"); + let mut main_thread_db = bench_config.config.connect()?; + main_thread_db.query_parse::<()>(&query!("create space bench"))?; + main_thread_db.query_parse::<()>(&query!(format!( + "create model {BENCHMARK_SPACE_ID}.{BENCHMARK_MODEL_ID}(un: binary, pw: uint8)" + )))?; + let stats = match bench.engine { + BenchEngine::Rookie => bench_rookie(bench_config, bench), + BenchEngine::Fury => bench_fury(bench), + }; + let (total_queries, stats) = match stats { + Ok(ret) => ret, + Err(e) => { + error!("benchmarking failed. attempting to clean up"); + match cleanup(main_thread_db) { + Ok(()) => return Err(e), + Err(e_cleanup) => { + error!("failed to clean up db: {e_cleanup}. please remove model `bench.bench` manually"); + return Err(e); + } + } + } + }; + info!( + "{} queries executed. benchmark complete.", + fmt_u64(total_queries) + ); + warn!("benchmarks might appear to be slower. this tool is currently experimental"); + // print results + print_table(stats); + cleanup(main_thread_db)?; + Ok(()) +} + +/* + util +*/ + +fn cleanup(mut main_thread_db: Connection) -> Result<(), error::BenchError> { + trace!("dropping space and table"); + main_thread_db.query_parse::<()>(&query!("drop space allow not empty bench"))?; + Ok(()) +} + +fn print_table(data: Vec<(&'static str, RuntimeStats)>) { + println!( + "+---------+--------------------------+-----------------------+------------------------+" + ); + println!( + "| Query | Effective real-world QPS | Slowest Query (nanos) | Fastest Query (nanos) |" + ); + println!( + "+---------+--------------------------+-----------------------+------------------------+" + ); + for (query, RuntimeStats { qps, head, tail }) in data { + println!( + "| {:<7} | {:>24.2} | {:>21} | {:>22} |", + query, qps, tail, head + ); + } + println!( + "+---------+--------------------------+-----------------------+------------------------+" + ); +} + +/* + bench runner +*/ + +#[derive(Clone, Copy, Debug)] +pub struct BenchmarkTask { + gen_query: fn(&Self, u64) -> Query, + check_resp: fn(&Self, u64, Response) -> bool, + pk_len: usize, +} + +impl BenchmarkTask { + fn new( + pk_len: usize, + gen_query: fn(&Self, u64) -> Query, + check_resp: fn(&Self, u64, Response) -> bool, + ) -> Self { + Self { + gen_query, + check_resp, + pk_len, + } + } + fn fmt_pk(&self, current: u64) -> Vec { + format!("{:0>width$}", current, width = self.pk_len).into_bytes() + } + pub fn generate_query(&self, current: u64) -> Query { + (self.gen_query)(self, current) + } + pub fn verify_response(&self, current: u64, resp: Response) -> bool { + (self.check_resp)(self, current, resp) + } +} + +struct BenchItem { + name: &'static str, + spec: BenchmarkTask, + count: usize, +} + +impl BenchItem { + fn new(name: &'static str, spec: BenchmarkTask, count: usize) -> Self { + Self { name, spec, count } + } + fn print_log_start(&self) { + info!( + "benchmarking `{}`. average payload size = {} bytes. queries = {}", + self.name, + self.spec.generate_query(0).debug_encode_packet().len(), + self.count + ) + } + fn run(self, pool: &mut rookie::BombardPool) -> BenchResult { + pool.blocking_bombard(self.spec, self.count) + .map_err(From::from) + } + async fn run_async(self, pool: &mut fury::Fury) -> BenchResult { + pool.bombard(self.count, self.spec) + .await + .map_err(From::from) + } +} + +fn prepare_bench_spec(bench: &BenchConfig) -> Vec { + vec![ + BenchItem::new( + "INSERT", + BenchmarkTask::new( + bench.key_size, + |me, current| query!("insert into bench(?, ?)", me.fmt_pk(current), 0u64), + |_, _, actual_resp| actual_resp == Response::Empty, + ), + bench.query_count, + ), + BenchItem::new( + "SELECT", + BenchmarkTask::new( + bench.key_size, + |me, current| query!("select * from bench where un = ?", me.fmt_pk(current)), + |me, current, resp| match resp { + Response::Row(r) => { + r.into_values() == vec![Value::Binary(me.fmt_pk(current)), Value::UInt8(0)] + } + _ => false, + }, + ), + bench.query_count, + ), + BenchItem::new( + "UPDATE", + BenchmarkTask::new( + bench.key_size, + |me, current| { + query!( + "update bench set pw += ? where un = ?", + 1u64, + me.fmt_pk(current) + ) + }, + |_, _, resp| resp == Response::Empty, + ), + bench.query_count, + ), + BenchItem::new( + "DELETE", + BenchmarkTask::new( + bench.key_size, + |me, current| query!("delete from bench where un = ?", me.fmt_pk(current)), + |_, _, resp| resp == Response::Empty, + ), + bench.query_count, + ), + ] +} + +fn fmt_u64(n: u64) -> String { + let num_str = n.to_string(); + let mut result = String::new(); + let chars_rev: Vec<_> = num_str.chars().rev().collect(); + for (i, ch) in chars_rev.iter().enumerate() { + if i % 3 == 0 && i != 0 { + result.push(','); + } + result.push(*ch); + } + result.chars().rev().collect() +} + +fn bench_rookie( + task: BombardTask, + bench: BenchConfig, +) -> BenchResult<(u64, Vec<(&'static str, RuntimeStats)>)> { + // initialize pool + info!( + "initializing connections. engine=rookie, threads={}, primary key size ={} bytes", + bench.threads, bench.key_size + ); + let mut pool = rookie::BombardPool::new(bench.threads, task)?; + // prepare benches + let benches = prepare_bench_spec(&bench); + // bench + let total_queries = bench.query_count as u64 * benches.len() as u64; + let mut results = vec![]; + for task in benches { + let name = task.name; + task.print_log_start(); + let this_result = task.run(&mut pool)?; + results.push((name, this_result)); + } + Ok((total_queries, results)) +} + +fn bench_fury(bench: BenchConfig) -> BenchResult<(u64, Vec<(&'static str, RuntimeStats)>)> { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(bench.threads) + .enable_all() + .build() + .unwrap(); + rt.block_on(async move { + info!( + "initializing connections. engine=fury, threads={}, connections={}, primary key size ={} bytes", + bench.threads, bench.connections, bench.key_size + ); + let mut pool = fury::Fury::new( + bench.connections, + Config::new(&bench.host, bench.port, "root", &bench.root_pass), + ) + .await?; + // prepare benches + let benches = prepare_bench_spec(&bench); + // bench + let total_queries = bench.query_count as u64 * benches.len() as u64; + let mut results = vec![]; + for task in benches { + let name = task.name; + task.print_log_start(); + let this_result = task.run_async(&mut pool).await?; + results.push((name, this_result)); + } + Ok((total_queries,results)) + }) +} diff --git a/sky-bench/src/bench/benches.rs b/sky-bench/src/bench/benches.rs deleted file mode 100644 index 87dc93d4..00000000 --- a/sky-bench/src/bench/benches.rs +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Created on Sat Aug 13 2022 - * - * This file is a part of S{ - let ref this = loopmon; - this.current -}le (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ - report::{AggregateReport, SingleReport}, - validation, vec_with_cap, BenchmarkConfig, LoopMonitor, - }, - crate::error::BResult, - devtimer::SimpleTimer, - libstress::Workpool, - skytable::{types::RawString, Connection, Element, Query, RespCode}, - std::{ - io::{Read, Write}, - net::{Shutdown, TcpStream}, - }, -}; - -/// Run a benchmark using the given pre-loop, in-loop and post-loop closures -fn run_bench_custom( - bench_config: BenchmarkConfig, - packets: Vec>, - on_init: Lv, - on_loop: Lp, - on_loop_exit: Ex, - loopmon: LoopMonitor, - reports: &mut AggregateReport, -) -> BResult<()> -where - Ex: Clone + Fn(&mut Inp) + Send + Sync + 'static, - Inp: Sync + 'static, - Lp: Clone + Fn(&mut Inp, Box<[u8]>) + Send + Sync + 'static, - Lv: Clone + Fn() -> Inp + Send + 'static + Sync, -{ - // now do our runs - let mut loopmon = loopmon; - - while loopmon.should_continue() { - // now create our connection pool - let pool = Workpool::new( - bench_config.server.connections(), - on_init.clone(), - on_loop.clone(), - on_loop_exit.clone(), - true, - Some(bench_config.query_count()), - )?; - - // get our local copy - let this_packets = packets.clone(); - - // run and time our operations - let mut dt = SimpleTimer::new(); - dt.start(); - pool.execute_and_finish_iter(this_packets); - dt.stop(); - loopmon.incr_time(&dt); - - // cleanup - loopmon.cleanup()?; - loopmon.step(); - } - - // save time - reports.push(SingleReport::new( - loopmon.name(), - loopmon.sum() as f64 / bench_config.runs() as f64, - )); - Ok(()) -} - -#[inline(always)] -/// Init connection and buffer -fn init_connection_and_buf( - host: &str, - port: u16, - start_command: Vec, - bufsize: usize, -) -> (TcpStream, Vec) { - let mut con = TcpStream::connect((host, port)).unwrap(); - con.write_all(&start_command).unwrap(); - let mut ret = [0u8; validation::RESPCODE_OKAY.len()]; - con.read_exact(&mut ret).unwrap(); - let readbuf = vec![0; bufsize]; - (con, readbuf) -} - -/// Benchmark SET -pub fn bench_set( - keys: &[Vec], - values: &[Vec], - connection: &mut Connection, - bench_config: &BenchmarkConfig, - create_table: &[u8], - reports: &mut AggregateReport, -) -> BResult<()> { - let bench_config = bench_config.clone(); - let create_table = create_table.to_owned(); - let loopmon = LoopMonitor::new_cleanup( - bench_config.runs(), - "set", - connection, - Query::from("FLUSHDB").arg("default.tmpbench"), - Element::RespCode(RespCode::Okay), - true, - ); - let mut packets = vec_with_cap(bench_config.query_count())?; - (0..bench_config.query_count()).for_each(|i| { - packets.push( - Query::from("SET") - .arg(RawString::from(keys[i].to_owned())) - .arg(RawString::from(values[i].to_owned())) - .into_raw_query() - .into_boxed_slice(), - ) - }); - run_bench_custom( - bench_config.clone(), - packets, - move || { - init_connection_and_buf( - bench_config.server.host(), - bench_config.server.port(), - create_table.to_owned(), - validation::RESPCODE_OKAY.len(), - ) - }, - |(con, buf), packet| { - con.write_all(&packet).unwrap(); - con.read_exact(buf).unwrap(); - assert_eq!(buf, validation::RESPCODE_OKAY); - }, - |(con, _)| con.shutdown(Shutdown::Both).unwrap(), - loopmon, - reports, - ) -} - -/// Benchmark UPDATE -pub fn bench_update( - keys: &[Vec], - new_value: &[u8], - bench_config: &BenchmarkConfig, - create_table: &[u8], - reports: &mut AggregateReport, -) -> BResult<()> { - let bench_config = bench_config.clone(); - let create_table = create_table.to_owned(); - let loopmon = LoopMonitor::new(bench_config.runs(), "update"); - let mut packets = vec_with_cap(bench_config.query_count())?; - (0..bench_config.query_count()).for_each(|i| { - packets.push( - Query::from("update") - .arg(RawString::from(keys[i].clone())) - .arg(RawString::from(new_value.to_owned())) - .into_raw_query() - .into_boxed_slice(), - ) - }); - run_bench_custom( - bench_config.clone(), - packets, - move || { - init_connection_and_buf( - bench_config.server.host(), - bench_config.server.port(), - create_table.to_owned(), - validation::RESPCODE_OKAY.len(), - ) - }, - |(con, buf), packet| { - con.write_all(&packet).unwrap(); - con.read_exact(buf).unwrap(); - assert_eq!(buf, validation::RESPCODE_OKAY); - }, - |(con, _)| con.shutdown(Shutdown::Both).unwrap(), - loopmon, - reports, - ) -} - -/// Benchmark GET -pub fn bench_get( - keys: &[Vec], - bench_config: &BenchmarkConfig, - create_table: &[u8], - reports: &mut AggregateReport, -) -> BResult<()> { - let bench_config = bench_config.clone(); - let create_table = create_table.to_owned(); - let loopmon = LoopMonitor::new(bench_config.runs(), "get"); - let mut packets = vec_with_cap(bench_config.query_count())?; - (0..bench_config.query_count()).for_each(|i| { - packets.push( - Query::from("get") - .arg(RawString::from(keys[i].clone())) - .into_raw_query() - .into_boxed_slice(), - ) - }); - run_bench_custom( - bench_config.clone(), - packets, - move || { - init_connection_and_buf( - bench_config.server.host(), - bench_config.server.port(), - create_table.to_owned(), - validation::calculate_response_size(bench_config.kvsize()), - ) - }, - |(con, buf), packet| { - con.write_all(&packet).unwrap(); - con.read_exact(buf).unwrap(); - }, - |(con, _)| con.shutdown(Shutdown::Both).unwrap(), - loopmon, - reports, - ) -} diff --git a/sky-bench/src/bench/mod.rs b/sky-bench/src/bench/mod.rs deleted file mode 100644 index 79ec3642..00000000 --- a/sky-bench/src/bench/mod.rs +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Created on Tue Aug 09 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - self::report::AggregateReport, - crate::{ - config, - config::{BenchmarkConfig, ServerConfig}, - error::{BResult, Error}, - util, - }, - devtimer::SimpleTimer, - libstress::utils::{generate_random_byte_vector, ran_bytes}, - skytable::{Connection, Element, Query, RespCode}, -}; - -mod benches; -mod report; -mod validation; - -macro_rules! binfo { - ($($arg:tt)+) => { - if $crate::config::should_output_messages() { - ::log::info!($($arg)+) - } - }; -} - -/// The loop monitor can be used for maintaining a loop for a given benchmark -struct LoopMonitor<'a> { - /// cleanup instructions - inner: Option>, - /// maximum iterations - max: usize, - /// current iteration - current: usize, - /// total time - time: u128, - /// name of test - name: &'static str, -} - -impl<'a> LoopMonitor<'a> { - /// Create a benchmark loop monitor that doesn't need any cleanup - pub fn new(max: usize, name: &'static str) -> Self { - Self { - inner: None, - max, - current: 0, - time: 0, - name, - } - } - /// Create a new benchmark loop monitor that uses the given cleanup instructions: - /// - `max`: Total iterations - /// - `name`: Name of benchmark - /// - `connection`: A connection to use for cleanup instructions - /// - `query`: Query to run for cleanup - /// - `response`: Response expected when cleaned up - /// - `skip_on_last`: Skip running the cleanup instructions on the last loop - pub fn new_cleanup( - max: usize, - name: &'static str, - connection: &'a mut Connection, - query: Query, - response: Element, - skip_on_last: bool, - ) -> Self { - Self { - inner: Some(CleanupInner::new(query, response, connection, skip_on_last)), - max, - current: 0, - time: 0, - name, - } - } - /// Run cleanup - fn cleanup(&mut self) -> BResult<()> { - let last_iter = self.is_last_iter(); - if let Some(ref mut cleanup) = self.inner { - let should_run_cleanup = !(last_iter && cleanup.skip_on_last); - if should_run_cleanup { - return cleanup.cleanup(self.name); - } - } - Ok(()) - } - /// Check if this is the last iteration - fn is_last_iter(&self) -> bool { - (self.max - 1) == self.current - } - /// Step the counter ahead - fn step(&mut self) { - self.current += 1; - } - /// Determine if we should continue executing - fn should_continue(&self) -> bool { - self.current < self.max - } - /// Append a new time to the sum - fn incr_time(&mut self, dt: &SimpleTimer) { - self.time += dt.time_in_nanos().unwrap(); - } - /// Return the sum - fn sum(&self) -> u128 { - self.time - } - /// Return the name of the benchmark - fn name(&self) -> &'static str { - self.name - } -} - -/// Cleanup instructions -struct CleanupInner<'a> { - /// the connection to use for cleanup processes - connection: &'a mut Connection, - /// the query to be run - query: Query, - /// the response to expect - response: Element, - /// whether we should skip on the last loop - skip_on_last: bool, -} - -impl<'a> CleanupInner<'a> { - /// Init cleanup instructions - fn new(q: Query, r: Element, connection: &'a mut Connection, skip_on_last: bool) -> Self { - Self { - query: q, - response: r, - connection, - skip_on_last, - } - } - /// Run cleanup - fn cleanup(&mut self, name: &'static str) -> BResult<()> { - let r: Element = self.connection.run_query(&self.query)?; - if r.ne(&self.response) { - Err(Error::Runtime(format!( - "Failed to run cleanup for benchmark `{}`", - name - ))) - } else { - Ok(()) - } - } -} - -#[inline(always)] -/// Returns a vec with the given cap, ensuring that we don't overflow memory -fn vec_with_cap(cap: usize) -> BResult> { - let mut v = Vec::new(); - v.try_reserve_exact(cap)?; - Ok(v) -} - -/// Run the actual benchmarks -pub fn run_bench(servercfg: &ServerConfig, bench_config: BenchmarkConfig) -> BResult<()> { - // check if we have enough combinations for the given query count and key size - if !util::has_enough_ncr(bench_config.kvsize(), bench_config.query_count()) { - return Err(Error::Runtime( - "too low sample space for given query count. use larger kvsize".into(), - )); - } - // run sanity test; this will also set up the temporary table for benchmarking - binfo!("Running sanity test ..."); - util::run_sanity_test(&bench_config.server)?; - - // pool pre-exec setup - let servercfg = servercfg.clone(); - let switch_table = Query::from("use default.tmpbench").into_raw_query(); - - // init pool config; side_connection is for cleanups - let mut misc_connection = Connection::new(servercfg.host(), servercfg.port())?; - - // init timer and reports - let mut reports = AggregateReport::new(bench_config.query_count()); - - // init test data - binfo!("Initializing test data ..."); - let mut rng = rand::thread_rng(); - let keys = generate_random_byte_vector( - bench_config.query_count(), - bench_config.kvsize(), - &mut rng, - true, - )?; - let values = generate_random_byte_vector( - bench_config.query_count(), - bench_config.kvsize(), - &mut rng, - false, - )?; - let new_updated_key = ran_bytes(bench_config.kvsize(), &mut rng); - - // run tests; the idea here is to run all tests one-by-one instead of generating all packets at once - // such an approach helps us keep memory usage low - // bench set - binfo!("Benchmarking SET ..."); - benches::bench_set( - &keys, - &values, - &mut misc_connection, - &bench_config, - &switch_table, - &mut reports, - )?; - - // bench update - binfo!("Benchmarking UPDATE ..."); - benches::bench_update( - &keys, - &new_updated_key, - &bench_config, - &switch_table, - &mut reports, - )?; - - // bench get - binfo!("Benchmarking GET ..."); - benches::bench_get(&keys, &bench_config, &switch_table, &mut reports)?; - - // remove all test data - binfo!("Finished benchmarks. Cleaning up ..."); - let r: Element = misc_connection.run_query(Query::from("drop model default.tmpbench force"))?; - if r != Element::RespCode(RespCode::Okay) { - return Err(Error::Runtime("failed to clean up after benchmarks".into())); - } - - if config::should_output_messages() { - // normal output - println!("===========RESULTS==========="); - let (maxpad, reports) = reports.finish(); - for report in reports { - let padding = " ".repeat(maxpad - report.name().len()); - println!( - "{}{} {:.6}/sec", - report.name().to_uppercase(), - padding, - report.stat(), - ); - } - println!("============================="); - } else { - // JSON - println!("{}", reports.into_json()) - } - Ok(()) -} diff --git a/sky-bench/src/bench/report.rs b/sky-bench/src/bench/report.rs deleted file mode 100644 index 4d0edbab..00000000 --- a/sky-bench/src/bench/report.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Created on Wed Aug 10 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use serde::Serialize; - -#[derive(Serialize)] -pub struct SingleReport { - name: &'static str, - stat: f64, -} - -impl SingleReport { - pub fn new(name: &'static str, stat: f64) -> Self { - Self { name, stat } - } - - pub fn stat(&self) -> f64 { - self.stat - } - - pub fn name(&self) -> &str { - self.name - } -} - -pub struct AggregateReport { - names: Vec, - query_count: usize, -} - -impl AggregateReport { - pub fn new(query_count: usize) -> Self { - Self { - names: Vec::new(), - query_count, - } - } - pub fn push(&mut self, report: SingleReport) { - self.names.push(report) - } - pub(crate) fn into_json(self) -> String { - let (_, report) = self.finish(); - serde_json::to_string(&report).unwrap() - } - - pub(crate) fn finish(self) -> (usize, Vec) { - let mut maxpad = self.names[0].name.len(); - let mut reps = self.names; - reps.iter_mut().for_each(|rep| { - let total_time = rep.stat; - let qps = (self.query_count as f64 / total_time) * 1_000_000_000_f64; - rep.stat = qps; - if rep.name.len() > maxpad { - maxpad = rep.name.len(); - } - }); - (maxpad, reps) - } -} diff --git a/sky-bench/src/config.rs b/sky-bench/src/config.rs deleted file mode 100644 index 1bf64e15..00000000 --- a/sky-bench/src/config.rs +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Created on Mon Aug 08 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::{util, Cli}; - -static mut OUTPUT_JSON: bool = false; - -#[derive(Clone)] -pub struct ServerConfig { - /// host - host: String, - /// port - port: u16, - /// connection count for network pool - connections: usize, -} - -impl ServerConfig { - pub fn host(&self) -> &str { - self.host.as_ref() - } - pub fn port(&self) -> u16 { - self.port - } - pub fn connections(&self) -> usize { - self.connections - } -} - -/// Benchmark configuration -#[derive(Clone)] -pub struct BenchmarkConfig { - pub server: ServerConfig, - kvsize: usize, - queries: usize, - runs: usize, -} - -impl BenchmarkConfig { - pub fn kvsize(&self) -> usize { - self.kvsize - } - pub fn query_count(&self) -> usize { - self.queries - } - pub fn runs(&self) -> usize { - self.runs - } -} - -pub fn should_output_messages() -> bool { - util::ensure_main_thread(); - unsafe { !OUTPUT_JSON } -} - -impl From<(&ServerConfig, &Cli)> for BenchmarkConfig { - fn from(tuple: (&ServerConfig, &Cli)) -> Self { - let (server_config, cli) = tuple; - unsafe { - OUTPUT_JSON = cli.json; - } - BenchmarkConfig { - server: server_config.clone(), - queries: cli.query_count, - kvsize: cli.kvsize, - runs: cli.runs, - } - } -} - -impl From<&Cli> for ServerConfig { - fn from(cli: &Cli) -> Self { - ServerConfig { - connections: cli.connections, - host: cli.host.clone(), - port: cli.port, - } - } -} diff --git a/sky-bench/src/error.rs b/sky-bench/src/error.rs index b077e222..1f012149 100644 --- a/sky-bench/src/error.rs +++ b/sky-bench/src/error.rs @@ -1,5 +1,5 @@ /* - * Created on Mon Aug 08 2022 + * Created on Sat Nov 18 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2022, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -25,44 +25,83 @@ */ use { - libstress::WorkpoolError, - skytable::error::Error as SkyError, - std::{collections::TryReserveError, fmt::Display}, + crate::{ + bench::BombardTask, + runtime::{fury, rookie::BombardError}, + }, + core::fmt, + skytable::error::Error, }; -pub type BResult = Result; +pub type BenchResult = Result; -/// Benchmark tool errors -pub enum Error { - /// An error originating from the Skytable client - Client(SkyError), - /// A runtime error - Runtime(String), +#[derive(Debug)] +pub enum BenchError { + ArgsErr(String), + RookieEngineError(BombardError), + FuryEngineError(fury::FuryError), + DirectDbError(Error), } -impl From for Error { - fn from(e: SkyError) -> Self { - Self::Client(e) +impl From for BenchError { + fn from(e: fury::FuryError) -> Self { + Self::FuryEngineError(e) } } -impl Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl From for BenchError { + fn from(e: libsky::ArgParseError) -> Self { + match e { + libsky::ArgParseError::Duplicate(d) => { + Self::ArgsErr(format!("duplicate value for `{d}`")) + } + libsky::ArgParseError::MissingValue(m) => { + Self::ArgsErr(format!("missing value for `{m}`")) + } + } + } +} + +impl From for BenchError { + fn from(e: Error) -> Self { + Self::DirectDbError(e) + } +} + +impl From> for BenchError { + fn from(e: BombardError) -> Self { + Self::RookieEngineError(e) + } +} + +impl fmt::Display for BenchError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Error::Client(e) => write!(f, "client error: {}", e), - Error::Runtime(e) => write!(f, "runtime error: {}", e), + Self::ArgsErr(e) => write!(f, "args error: {e}"), + Self::DirectDbError(e) => write!(f, "direct operation on db failed. {e}"), + Self::RookieEngineError(e) => write!(f, "benchmark failed (rookie engine): {e}"), + Self::FuryEngineError(e) => write!(f, "benchmark failed (fury engine): {e}"), } } } -impl From for Error { - fn from(e: TryReserveError) -> Self { - Error::Runtime(format!("memory reserve error: {}", e)) +impl std::error::Error for BenchError {} + +#[derive(Debug)] +pub enum BenchmarkTaskWorkerError { + DbError(Error), +} + +impl From for BenchmarkTaskWorkerError { + fn from(e: Error) -> Self { + Self::DbError(e) } } -impl From for Error { - fn from(e: WorkpoolError) -> Self { - Error::Runtime(format!("threadpool error: {}", e)) +impl fmt::Display for BenchmarkTaskWorkerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DbError(e) => write!(f, "worker failed due to DB error. {e}"), + } } } diff --git a/sky-bench/src/main.rs b/sky-bench/src/main.rs index b938d992..07e5f522 100644 --- a/sky-bench/src/main.rs +++ b/sky-bench/src/main.rs @@ -1,5 +1,5 @@ /* - * Created on Mon Aug 08 2022 + * Created on Wed Nov 15 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2022, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -23,41 +23,32 @@ * along with this program. If not, see . * */ -use { - crate::cli::Cli, - clap::Parser, - env_logger::Builder, - std::{env, process}, -}; #[macro_use] extern crate log; - +mod args; mod bench; -mod cli; -mod config; mod error; -mod util; +mod runtime; fn main() { - Builder::new() - .parse_filters(&env::var("SKYBENCH_LOG").unwrap_or_else(|_| "info".to_owned())) + env_logger::Builder::new() + .parse_filters(&std::env::var("SKYBENCH_LOG").unwrap_or_else(|_| "info".to_owned())) .init(); - if let Err(e) = run() { - error!("sky-bench exited with error: {}", e); - process::exit(0x01); + match run() { + Ok(()) => {} + Err(e) => { + error!("bench error: {e}"); + std::process::exit(0x01); + } } } -fn run() -> error::BResult<()> { - // Init CLI arg parser - let cli = &Cli::parse(); - - // Parse args and initialize configs - let server_config = &cli.into(); - let bench_config = (server_config, cli).into(); - - // Run our task - bench::run_bench(server_config, bench_config)?; - util::cleanup(server_config) +fn run() -> error::BenchResult<()> { + let task = args::parse()?; + match task { + args::Task::HelpMsg(msg) => println!("{msg}"), + args::Task::BenchConfig(bench) => bench::run(bench)?, + } + Ok(()) } diff --git a/server/src/services/mod.rs b/sky-bench/src/runtime.rs similarity index 52% rename from server/src/services/mod.rs rename to sky-bench/src/runtime.rs index b294b186..2077d6a7 100644 --- a/server/src/services/mod.rs +++ b/sky-bench/src/runtime.rs @@ -1,5 +1,5 @@ /* - * Created on Sun May 16 2021 + * Created on Sun Nov 19 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2021, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,32 +24,46 @@ * */ -pub mod bgsave; -pub mod snapshot; -use crate::{ - corestore::memstore::Memstore, diskstore::flock::FileLock, storage, util::os, IoResult, -}; +pub mod fury; +pub mod rookie; -pub fn restore_data(src: Option) -> IoResult<()> { - if let Some(src) = src { - // hmm, so restore it - os::recursive_copy(src, "data")?; - log::info!("Successfully restored data from snapshot"); - } - Ok(()) +use std::time::Instant; + +fn qps(query_count: usize, time_taken_in_nanos: u128) -> f64 { + const NANOS_PER_SECOND: u128 = 1_000_000_000; + let time_taken_in_nanos_f64 = time_taken_in_nanos as f64; + let query_count_f64 = query_count as f64; + (query_count_f64 / time_taken_in_nanos_f64) * NANOS_PER_SECOND as f64 } -pub fn pre_shutdown_cleanup(mut pid_file: FileLock, mr: Option<&Memstore>) -> bool { - if let Err(e) = pid_file.unlock() { - log::error!("Shutdown failure: Failed to unlock pid file: {}", e); - return false; - } - if let Some(mr) = mr { - log::info!("Compacting tree"); - if let Err(e) = storage::v1::interface::cleanup_tree(mr) { - log::error!("Failed to compact tree: {}", e); - return false; +#[derive(Debug, Clone)] +pub(self) enum WorkerTask { + Task(T), + Exit, +} + +#[derive(Debug)] +pub struct RuntimeStats { + pub qps: f64, + pub head: u128, + pub tail: u128, +} + +#[derive(Debug)] +struct WorkerLocalStats { + start: Instant, + elapsed: u128, + head: u128, + tail: u128, +} + +impl WorkerLocalStats { + fn new(start: Instant, elapsed: u128, head: u128, tail: u128) -> Self { + Self { + start, + elapsed, + head, + tail, } } - true } diff --git a/sky-bench/src/runtime/fury.rs b/sky-bench/src/runtime/fury.rs new file mode 100644 index 00000000..02fa7a36 --- /dev/null +++ b/sky-bench/src/runtime/fury.rs @@ -0,0 +1,351 @@ +/* + * Created on Wed Nov 22 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{RuntimeStats, WorkerLocalStats, WorkerTask}, + crate::bench::{BenchmarkTask, BENCHMARK_SPACE_ID}, + skytable::Config, + std::{ + fmt, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, + time::{Duration, Instant}, + }, + tokio::sync::{broadcast, mpsc, RwLock}, +}; + +/* + state +*/ + +static GLOBAL_START: RwLock<()> = RwLock::const_new(()); +static GLOBAL_TARGET: AtomicUsize = AtomicUsize::new(0); +static GLOBAL_EXIT: AtomicBool = AtomicBool::new(false); + +fn gset_target(target: usize) { + GLOBAL_TARGET.store(target, Ordering::Release) +} +fn gset_exit() { + GLOBAL_EXIT.store(true, Ordering::Release) +} +fn grefresh_target() -> usize { + let mut current = GLOBAL_TARGET.load(Ordering::Acquire); + loop { + if current == 0 { + return 0; + } + match GLOBAL_TARGET.compare_exchange( + current, + current - 1, + Ordering::Release, + Ordering::Acquire, + ) { + Ok(prev) => return prev, + Err(new) => current = new, + } + } +} +fn grefresh_early_exit() -> bool { + GLOBAL_EXIT.load(Ordering::Acquire) +} + +/* + errors +*/ + +pub type FuryResult = Result; + +#[derive(Debug)] +pub enum FuryError { + Init(skytable::error::Error), + Worker(FuryWorkerError), + Dead, +} + +impl fmt::Display for FuryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Init(e) => write!(f, "fury init failed. {e}"), + Self::Worker(e) => write!(f, "worker failed. {e}"), + Self::Dead => write!(f, "all workers offline"), + } + } +} + +impl From for FuryError { + fn from(e: FuryWorkerError) -> Self { + Self::Worker(e) + } +} + +#[derive(Debug)] +pub enum FuryWorkerError { + DbError(skytable::error::Error), + Mismatch, +} + +impl fmt::Display for FuryWorkerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DbError(e) => write!(f, "client errored. {e}"), + Self::Mismatch => write!(f, "server response did not match expected response"), + } + } +} + +/* + impl +*/ + +#[derive(Debug)] +pub struct Fury { + tx_task: broadcast::Sender>, + rx_task_result: mpsc::Receiver>, + client_count: usize, +} + +impl Fury { + pub async fn new(client_count: usize, config: Config) -> FuryResult { + let (tx_task, rx_task) = broadcast::channel(1); + let (tx_task_result, rx_task_result) = mpsc::channel(client_count); + let (tx_ack, mut rx_ack) = mpsc::channel(1); + for id in 0..client_count { + let rx_task = tx_task.subscribe(); + let tx_task_result = tx_task_result.clone(); + let tx_ack = tx_ack.clone(); + let config = config.clone(); + tokio::spawn( + async move { worker_svc(id, rx_task, tx_task_result, tx_ack, config).await }, + ); + } + drop((tx_ack, rx_task)); + match rx_ack.recv().await { + None => {} + Some(e) => return Err(FuryError::Init(e)), + } + info!("all workers online. ready for event loop"); + Ok(Self { + tx_task, + rx_task_result, + client_count, + }) + } + pub async fn bombard(&mut self, count: usize, task: BenchmarkTask) -> FuryResult { + // pause workers and set target + let start_guard = GLOBAL_START.write().await; + gset_target(count); + // send tasks + if self.tx_task.send(WorkerTask::Task(task)).is_err() { + return Err(FuryError::Dead); + } + // begin work + drop(start_guard); + // init stats + let mut global_start = None; + let mut global_stop = None; + let mut global_head = u128::MAX; + let mut global_tail = 0u128; + let mut remaining = self.client_count; + while remaining != 0 { + let WorkerLocalStats { + start: this_start, + elapsed: this_elapsed, + head: this_head, + tail: this_tail, + } = match self.rx_task_result.recv().await { + None => { + return Err(FuryError::Dead); + } + Some(res) => res, + }?; + match global_start.as_mut() { + None => global_start = Some(this_start), + Some(current_start) => { + if this_start < *current_start { + *current_start = this_start; + } + } + } + let this_stop = this_start + Duration::from_nanos(this_elapsed.try_into().unwrap()); + match global_stop.as_mut() { + None => global_stop = Some(this_stop), + Some(current_gstop) => { + if this_stop > *current_gstop { + *current_gstop = this_stop; + } + } + } + if this_head < global_head { + global_head = this_head; + } + if this_tail > global_tail { + global_tail = this_tail; + } + remaining -= 1; + } + Ok(RuntimeStats { + qps: super::qps( + count, + global_stop + .unwrap() + .duration_since(global_start.unwrap()) + .as_nanos(), + ), + head: global_head, + tail: global_tail, + }) + } +} + +async fn worker_svc( + id: usize, + mut rx_task: broadcast::Receiver>, + tx_task_result: mpsc::Sender>, + tx_ack: mpsc::Sender, + connection_cfg: Config, +) { + let mut db = match connection_cfg.connect_async().await { + Ok(c) => c, + Err(e) => { + if tx_ack.send(e).await.is_err() { + error!("worker-{id} failed to ack because main thread exited"); + } + return; + } + }; + // set DB in connections + match db + .query_parse::<()>(&skytable::query!(format!("use {BENCHMARK_SPACE_ID}"))) + .await + { + Ok(()) => {} + Err(e) => { + if tx_ack.send(e).await.is_err() { + error!("worker-{id} failed to report error because main thread exited"); + } + return; + } + } + // we're connected and ready to server + drop(tx_ack); + 'wait: loop { + let task = match rx_task.recv().await { + Err(_) => { + error!("worked-{id} is exiting because main thread exited"); + return; + } + Ok(WorkerTask::Exit) => return, + Ok(WorkerTask::Task(t)) => t, + }; + // received a task; ready to roll; wait for begin signal + let permit = GLOBAL_START.read().await; + // off to the races + let mut current = grefresh_target(); + let mut exit_now = grefresh_early_exit(); + // init local stats + let mut local_start = None; + let mut local_elapsed = 0u128; + let mut local_head = u128::MAX; + let mut local_tail = 0u128; + while (current != 0) && !exit_now { + // prepare query + let query = task.generate_query(current as _); + // execute timed + let start = Instant::now(); + let ret = db.query(&query).await; + let stop = Instant::now(); + // check response + let resp = match ret { + Ok(resp) => resp, + Err(e) => { + gset_exit(); + if tx_task_result + .send(Err(FuryError::Worker(FuryWorkerError::DbError(e)))) + .await + .is_err() + { + error!( + "worker-{id} failed to report worker error because main thread exited" + ); + return; + } + continue 'wait; + } + }; + if !task.verify_response(current as _, resp.clone()) { + gset_exit(); + if tx_task_result + .send(Err(FuryError::Worker(FuryWorkerError::Mismatch))) + .await + .is_err() + { + error!( + "worker-{id} failed to report mismatch error because main thread exited" + ); + return; + } + continue 'wait; + } + // update stats + if local_start.is_none() { + local_start = Some(start); + } + let elapsed = stop.duration_since(start).as_nanos(); + local_elapsed += elapsed; + if elapsed > local_tail { + local_tail = elapsed; + } + if elapsed < local_head { + local_head = elapsed; + } + current = grefresh_target(); + exit_now = grefresh_early_exit(); + } + if exit_now { + continue 'wait; + } + // good! send these results + if tx_task_result + .send(Ok(WorkerLocalStats::new( + local_start.unwrap(), + local_elapsed, + local_head, + local_tail, + ))) + .await + .is_err() + { + error!("worker-{id} failed to send results because main thread exited"); + return; + } + drop(permit); + } +} + +impl Drop for Fury { + fn drop(&mut self) { + let _ = self.tx_task.send(WorkerTask::Exit); + } +} diff --git a/sky-bench/src/runtime/rookie.rs b/sky-bench/src/runtime/rookie.rs new file mode 100644 index 00000000..000891f3 --- /dev/null +++ b/sky-bench/src/runtime/rookie.rs @@ -0,0 +1,378 @@ +/* + * Created on Tue Nov 21 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use { + super::{RuntimeStats, WorkerLocalStats, WorkerTask}, + crossbeam_channel::{unbounded, Receiver, Sender}, + std::{ + fmt::{self, Display}, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + RwLock, RwLockReadGuard, RwLockWriteGuard, + }, + thread::{self, JoinHandle}, + time::{Duration, Instant}, + }, +}; + +pub type BombardResult = Result>; + +/* + state mgmt +*/ + +#[derive(Debug)] +/// The pool state. Be warned **ONLY ONE POOL AT A TIME!** +struct GPState { + current: AtomicU64, + state: AtomicBool, + occupied: AtomicBool, + start_sig: RwLock<()>, +} + +impl GPState { + #[inline(always)] + fn get() -> &'static Self { + static STATE: GPState = GPState::zero(); + &STATE + } + const fn zero() -> Self { + Self { + current: AtomicU64::new(0), + state: AtomicBool::new(true), + occupied: AtomicBool::new(false), + start_sig: RwLock::new(()), + } + } + fn wait_for_global_begin(&self) -> RwLockReadGuard<'_, ()> { + self.start_sig.read().unwrap() + } + fn occupy(&self) { + assert!(!self.occupied.swap(true, Ordering::Release)); + } + fn vacate(&self) { + assert!(self.occupied.swap(false, Ordering::Release)); + } + fn guard(f: impl FnOnce(RwLockWriteGuard<'static, ()>) -> T) -> T { + let slf = Self::get(); + slf.occupy(); + let ret = f(slf.start_sig.write().unwrap()); + slf.vacate(); + ret + } + fn post_failure(&self) { + self.state.store(false, Ordering::Release) + } + fn post_target(&self, target: u64) { + self.current.store(target, Ordering::Release) + } + /// WARNING: this is not atomic! only sensible to run a quiescent state + fn post_reset(&self) { + self.current.store(0, Ordering::Release); + self.state.store(true, Ordering::Release); + } + fn update_target(&self) -> u64 { + let mut current = self.current.load(Ordering::Acquire); + loop { + if current == 0 { + return 0; + } + match self.current.compare_exchange( + current, + current - 1, + Ordering::Release, + Ordering::Acquire, + ) { + Ok(last) => { + return last; + } + Err(new) => { + current = new; + } + } + } + } + fn load_okay(&self) -> bool { + self.state.load(Ordering::Acquire) + } +} + +/* + task spec +*/ + +/// A threaded bombard task specification which drives a global pool of threads towards a common goal +pub trait ThreadedBombardTask: Send + Sync + 'static { + /// The per-task worker that is initialized once in every thread (not to be confused with the actual thread worker!) + type Worker: Send + Sync; + /// The task that the [`ThreadedBombardTask::TaskWorker`] performs + type WorkerTask: Send + Sync; + type WorkerTaskSpec: Clone + Send + Sync + 'static; + /// Errors while running a task + type WorkerTaskError: Send + Sync; + /// Errors while initializing a task worker + type WorkerInitError: Send + Sync; + /// Initialize a task worker + fn worker_init(&self) -> Result; + fn generate_task(spec: &Self::WorkerTaskSpec, current: u64) -> Self::WorkerTask; + /// Drive a single subtask + fn worker_drive_timed( + worker: &mut Self::Worker, + task: Self::WorkerTask, + ) -> Result; +} + +/* + worker +*/ + +#[derive(Debug)] +enum WorkerResult { + Completed(WorkerLocalStats), + Errored(Bt::WorkerTaskError), +} + +#[derive(Debug)] +struct Worker { + handle: JoinHandle<()>, +} + +impl Worker { + fn start( + id: usize, + driver: Bt::Worker, + rx_work: Receiver>, + tx_res: Sender>, + ) -> Self { + Self { + handle: thread::Builder::new() + .name(format!("worker-{id}")) + .spawn(move || { + let mut worker_driver = driver; + 'blocking_wait: loop { + let task = match rx_work.recv().unwrap() { + WorkerTask::Exit => return, + WorkerTask::Task(spec) => spec, + }; + let guard = GPState::get().wait_for_global_begin(); + // check global state + let mut global_okay = GPState::get().load_okay(); + let mut global_position = GPState::get().update_target(); + // init local state + let mut local_start = None; + let mut local_elapsed = 0u128; + let mut local_head = u128::MAX; + let mut local_tail = 0; + // bombard + while (global_position != 0) & global_okay { + let task = Bt::generate_task(&task, global_position); + if local_start.is_none() { + local_start = Some(Instant::now()); + } + let this_elapsed = + match Bt::worker_drive_timed(&mut worker_driver, task) { + Ok(elapsed) => elapsed, + Err(e) => { + GPState::get().post_failure(); + tx_res.send(WorkerResult::Errored(e)).unwrap(); + continue 'blocking_wait; + } + }; + local_elapsed += this_elapsed; + if this_elapsed < local_head { + local_head = this_elapsed; + } + if this_elapsed > local_tail { + local_tail = this_elapsed; + } + global_position = GPState::get().update_target(); + global_okay = GPState::get().load_okay(); + } + if global_okay { + // we're done + tx_res + .send(WorkerResult::Completed(WorkerLocalStats::new( + local_start.unwrap(), + local_elapsed, + local_head, + local_tail, + ))) + .unwrap(); + } + drop(guard); + } + }) + .expect("failed to start thread"), + } + } +} + +/* + pool +*/ + +#[derive(Debug)] +pub enum BombardError { + InitError(Bt::WorkerInitError), + WorkerTaskError(Bt::WorkerTaskError), + AllWorkersOffline, +} + +impl fmt::Display for BombardError +where + Bt::WorkerInitError: fmt::Display, + Bt::WorkerTaskError: Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AllWorkersOffline => write!( + f, + "bombard failed because all workers went offline indicating catastrophic failure" + ), + Self::WorkerTaskError(e) => write!(f, "worker task failed. {e}"), + Self::InitError(e) => write!(f, "worker init failed. {e}"), + } + } +} + +#[derive(Debug)] +pub struct BombardPool { + workers: Vec<(Worker, Sender>)>, + rx_res: Receiver>, + _config: Bt, +} + +impl BombardPool { + pub fn new(size: usize, config: Bt) -> BombardResult { + assert_ne!(size, 0, "pool can't be empty"); + let mut workers = Vec::with_capacity(size); + let (tx_res, rx_res) = unbounded(); + for id in 0..size { + let (tx_work, rx_work) = unbounded(); + let driver = config.worker_init().map_err(BombardError::InitError)?; + workers.push((Worker::start(id, driver, rx_work, tx_res.clone()), tx_work)); + } + Ok(Self { + workers, + rx_res, + _config: config, + }) + } + /// Bombard queries to the workers + pub fn blocking_bombard( + &mut self, + task_description: Bt::WorkerTaskSpec, + count: usize, + ) -> BombardResult { + GPState::guard(|paused| { + GPState::get().post_target(count as _); + let mut global_start = None; + let mut global_stop = None; + let mut global_head = u128::MAX; + let mut global_tail = 0u128; + for (_, sender) in self.workers.iter() { + sender + .send(WorkerTask::Task(task_description.clone())) + .unwrap(); + } + // now let them begin! + drop(paused); + // wait for all workers to complete + let mut received = 0; + while received != self.workers.len() { + let results = match self.rx_res.recv() { + Err(_) => return Err(BombardError::AllWorkersOffline), + Ok(r) => r, + }; + let WorkerLocalStats { + start: this_start, + elapsed, + head, + tail, + } = match results { + WorkerResult::Completed(r) => r, + WorkerResult::Errored(e) => return Err(BombardError::WorkerTaskError(e)), + }; + // update start if required + match global_start.as_mut() { + None => { + global_start = Some(this_start); + } + Some(start) => { + if this_start < *start { + *start = this_start; + } + } + } + let this_task_stopped_at = + this_start + Duration::from_nanos(elapsed.try_into().unwrap()); + match global_stop.as_mut() { + None => { + global_stop = Some(this_task_stopped_at); + } + Some(stop) => { + if this_task_stopped_at > *stop { + // this task stopped later than the previous one + *stop = this_task_stopped_at; + } + } + } + if head < global_head { + global_head = head; + } + if tail > global_tail { + global_tail = tail; + } + received += 1; + } + // reset global pool state + GPState::get().post_reset(); + // compute results + let global_elapsed = global_stop + .unwrap() + .duration_since(global_start.unwrap()) + .as_nanos(); + Ok(RuntimeStats { + qps: super::qps(count, global_elapsed), + head: global_head, + tail: global_tail, + }) + }) + } +} + +impl Drop for BombardPool { + fn drop(&mut self) { + info!("taking all workers offline"); + for (_, sender) in self.workers.iter() { + sender.send(WorkerTask::Exit).unwrap(); + } + for (worker, _) in self.workers.drain(..) { + worker.handle.join().unwrap(); + } + info!("all workers now offline"); + } +} diff --git a/sky-bench/src/util.rs b/sky-bench/src/util.rs deleted file mode 100644 index fc2ab9bb..00000000 --- a/sky-bench/src/util.rs +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Created on Tue Aug 09 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - config::ServerConfig, - error::{BResult, Error}, - }, - skytable::{Connection, Element, Query, RespCode}, - std::thread, -}; - -/// Check if the provided keysize has enough combinations to support the given `queries` count -/// -/// This function is heavily optimized and should take Θ(1) time. The `ALWAYS_TRUE_FACTOR` is -/// dependent on pointer width (more specifically the virtual address space size). -/// - For 64-bit address spaces: `(256!)/r!(256-r!)`; for a value of r >= 12, we'll hit the maximum -/// of the address space and hence this will always return true (because of the size of `usize`) -/// > The value for r = 12 is `1.27309515e+20` which largely exceeds `1.8446744e+19` -/// - For 32-bit address spaces: `(256!)/r!(256-r!)`; for a value of r >= 5, we'll hit the maximum -/// of the address space and hence this will always return true (because of the size of `usize`) -/// > The value for r = 5 is `8.81e+9` which largely exceeds `4.3e+9` -pub const fn has_enough_ncr(keysize: usize, queries: usize) -> bool { - const LUT: [u64; 11] = [ - // 1B - 256, - // 2B - 32640, - // 3B - 2763520, - // 4B - 174792640, - // 5B - 8809549056, - // 6B - 368532802176, - // 7B - 13161885792000, - // 8B - 409663695276000, - // 9B - 11288510714272000, - // 10B - 278826214642518400, - // 11B - 6235568072914502400, - ]; - #[cfg(target_pointer_width = "64")] - const ALWAYS_TRUE_FACTOR: usize = 12; - #[cfg(target_pointer_width = "32")] - const ALWAYS_TRUE_FACTOR: usize = 5; - keysize >= ALWAYS_TRUE_FACTOR || (LUT[keysize - 1] >= queries as _) -} - -/// Run a sanity test, making sure that the server is ready for benchmarking. This function will do the -/// following tests: -/// - Connect to the instance -/// - Run a `heya` as a preliminary test -/// - Create a new table `tmpbench`. This is where we're supposed to run all the benchmarks. -/// - Switch to the new table -/// - Set a key, and get it checking the equality of the returned value -pub fn run_sanity_test(server_config: &ServerConfig) -> BResult<()> { - let mut con = Connection::new(server_config.host(), server_config.port())?; - let tests: [(Query, Element, &str); 5] = [ - ( - Query::from("HEYA"), - Element::String("HEY!".to_owned()), - "heya", - ), - ( - Query::from("CREATE MODEL default.tmpbench(binary, binary)"), - Element::RespCode(RespCode::Okay), - "create model", - ), - ( - Query::from("use default.tmpbench"), - Element::RespCode(RespCode::Okay), - "use", - ), - ( - Query::from("set").arg("x").arg("100"), - Element::RespCode(RespCode::Okay), - "set", - ), - ( - Query::from("get").arg("x"), - Element::Binstr("100".as_bytes().to_owned()), - "get", - ), - ]; - for (query, expected, test_kind) in tests { - let r: Element = con.run_query(query)?; - if r != expected { - return Err(Error::Runtime(format!( - "sanity test for `{test_kind}` failed" - ))); - } - } - Ok(()) -} - -/// Ensures that the current thread is the main thread. If not, this function will panic -pub fn ensure_main_thread() { - assert_eq!( - thread::current().name().unwrap(), - "main", - "unsafe function called from non-main thread" - ) -} - -/// Run a cleanup. This function attempts to remove the `default.tmpbench` entity -pub fn cleanup(server_config: &ServerConfig) -> BResult<()> { - let mut c = Connection::new(server_config.host(), server_config.port())?; - let r: Element = c.run_query(Query::from("drop model default.tmpbench force"))?; - if r == Element::RespCode(RespCode::Okay) { - Err(Error::Runtime("failed to run cleanup".into())) - } else { - Ok(()) - } -} diff --git a/sky-macros/Cargo.toml b/sky-macros/Cargo.toml index c099b339..1f24cf3f 100644 --- a/sky-macros/Cargo.toml +++ b/sky-macros/Cargo.toml @@ -11,7 +11,7 @@ proc-macro = true [dependencies] # external deps -proc-macro2 = "1.0.49" -quote = "1.0.23" -rand = "0.8.5" -syn = { version = "1.0.107", features = ["full"] } +proc-macro2 = "1.0.70" +quote = "1.0.33" +syn = { version = "1.0.109", features = ["full"] } +libsky = { path = "../libsky" } diff --git a/sky-macros/src/dbtest.rs b/sky-macros/src/dbtest.rs new file mode 100644 index 00000000..52c882a7 --- /dev/null +++ b/sky-macros/src/dbtest.rs @@ -0,0 +1,289 @@ +/* + * Created on Wed Nov 29 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use quote::quote; + +use { + crate::util::{self, AttributeKind}, + proc_macro::TokenStream, + std::collections::HashMap, + syn::{parse_macro_input, AttributeArgs, ItemFn}, +}; + +/* + host setup +*/ + +#[derive(Debug)] +enum DbTestClient { + Skyhash, + Tcp, +} + +struct DbConfig { + client: DbTestClient, + port: u16, + host: String, +} + +impl Default for DbConfig { + fn default() -> Self { + Self { + client: DbTestClient::Skyhash, + port: libsky::test_utils::DEFAULT_PORT, + host: libsky::test_utils::DEFAULT_HOST.into(), + } + } +} + +/* + client setup +*/ + +#[derive(Debug)] +struct ClientConfig { + username: String, + password: String, +} + +impl Default for ClientConfig { + fn default() -> Self { + Self { + username: libsky::test_utils::DEFAULT_USER_NAME.into(), + password: libsky::test_utils::DEFAULT_USER_PASS.into(), + } + } +} + +/* + test setup +*/ + +#[derive(Debug)] +enum TestStrategy { + Standard, + Relogin { username: String, password: String }, +} + +impl TestStrategy { + fn is_relogin(&self) -> bool { + matches!(self, TestStrategy::Relogin { .. }) + } +} + +struct TestSetup { + client: ClientConfig, + db: DbConfig, + strategy: TestStrategy, +} + +fn parse_attrs(attrs: AttributeArgs) -> TestSetup { + let mut db_config = DbConfig::default(); + let mut client_config = ClientConfig::default(); + let mut collected_attrs = HashMap::new(); + let mut strategy = TestStrategy::Standard; + for attr in attrs { + match util::extract_attribute(&attr) { + AttributeKind::Pair(k, v) => { + assert!( + collected_attrs.insert(k.to_string(), v).is_none(), + "duplicate key: {}", + k.to_string() + ); + continue; + } + AttributeKind::NestedAttrs { name, attrs } => match name.to_string().as_str() { + "switch_user" => { + if strategy.is_relogin() { + panic!("already set `switch_user` strategy"); + } + let mut username = None; + let mut password = None; + for (key, data) in attrs.into_iter().map(AttributeKind::into_pair) { + match key.to_string().as_str() { + "username" => { + assert!(username.is_none(), "username already set"); + username = Some(util::extract_str_from_lit(&data).unwrap()); + } + "password" => { + assert!(password.is_none(), "password already set"); + password = Some(util::extract_str_from_lit(&data).unwrap()); + } + unknown_subattr => panic!( + "unknown sub-attribute for `switch_user`: `{unknown_subattr}`" + ), + } + } + assert!(username.is_some(), "username must be set"); + strategy = TestStrategy::Relogin { + username: username.unwrap(), + password: password.unwrap_or(libsky::test_utils::DEFAULT_USER_PASS.into()), + }; + } + unknown => panic!("unknown nested attribute `{unknown}`"), + }, + AttributeKind::Path(_) | AttributeKind::Lit(_) => { + panic!("unexpected tokens") + } + } + } + for (attr_name, attr_val) in collected_attrs { + match attr_name.as_str() { + "client" => match util::extract_str_from_lit(&attr_val).unwrap().as_str() { + "skyhash" => db_config.client = DbTestClient::Skyhash, + "tcp" => db_config.client = DbTestClient::Tcp, + unknown_client => panic!("unknown client mode {unknown_client}"), + }, + "port" => db_config.port = util::extract_int_from_lit(&attr_val).unwrap(), + "host" => db_config.host = util::extract_str_from_lit(&attr_val).unwrap(), + "username" => { + assert!( + !strategy.is_relogin(), + "`username` makes no sense when used with strategy `switch_user`. instead, set dbtest(switch_user(username = ...))" + ); + client_config.username = util::extract_str_from_lit(&attr_val).unwrap() + } + "password" => { + assert!( + !strategy.is_relogin(), + "`password` makes no sense when used with strategy `switch_user`. instead, set dbtest(switch_user(password = ...))" + ); + client_config.password = util::extract_str_from_lit(&attr_val).unwrap(); + } + unknown_attr => panic!("unknown dbtest attribute `{unknown_attr}`"), + } + } + TestSetup { + client: client_config, + db: db_config, + strategy, + } +} + +pub fn dbtest(attrs: TokenStream, item: TokenStream) -> TokenStream { + let attr_args = parse_macro_input!(attrs as AttributeArgs); + let input_fn = parse_macro_input!(item as ItemFn); + let TestSetup { + client: + ClientConfig { + username: login_username, + password: login_password, + }, + db: DbConfig { client, port, host }, + strategy, + } = parse_attrs(attr_args); + + let function_attrs = &input_fn.attrs; + let function_vis = &input_fn.vis; + let function_sig = &input_fn.sig; + let function_block = &input_fn.block; + + let retfn = quote!( + #(#function_attrs)* #function_vis #function_sig + ); + let mut block = quote! { + const __DBTEST_HOST: &str = #host; + const __DBTEST_PORT: u16 = #port; + }; + match strategy { + TestStrategy::Standard => { + block = quote! { + #block + /// username set by [`sky_macros::dbtest`] + const __DBTEST_USER: &str = #login_username; + /// password set by [`sky_macros::dbtest`] + const __DBTEST_PASS: &str = #login_password; + }; + } + TestStrategy::Relogin { + username: ref new_username, + password: ref new_password, + } => { + // we need to create an user, log in and log out + block = quote! { + #block + /// username set by [`sky_macros::dbtest`] (relogin) + const __DBTEST_USER: &str = #new_username; + /// password set by [`sky_macros::dbtest`] (relogin) + const __DBTEST_PASS: &str = #new_password; + { + let query_assembled = format!("sysctl create user {} with {{ password: ? }}", #new_username); + let mut db = skytable::Config::new(#host, #port, #login_username, #login_password).connect().unwrap(); + db.query_parse::<()>(&skytable::query!(query_assembled, #new_password)).unwrap(); + } + }; + } + } + match client { + DbTestClient::Skyhash => { + block = quote! { + #block + /// Get a Skyhash connection the database (defined by [`sky_macros::dbtest`]) + macro_rules! db { + () => {{ + skytable::Config::new(__DBTEST_HOST, __DBTEST_PORT, __DBTEST_USER, __DBTEST_PASS).connect().unwrap() + }} + } + }; + } + DbTestClient::Tcp => { + block = quote! { + #block + /// Get a TCP connection the database (defined by [`sky_macros::dbtest`]) + macro_rules! tcp { + () => {{ + std::net::TcpStream::connect((__DBTEST_HOST, __DBTEST_PORT)).unwrap() + }} + } + }; + } + } + let mut ret_block = quote! { + #block + #function_block + }; + match strategy { + TestStrategy::Relogin { ref username, .. } => { + let new_username = username; + ret_block = quote! { + #ret_block + { + let query_assembled = format!("sysctl drop user {}", #new_username); + let mut db = skytable::Config::new(#host, #port, #login_username, #login_password).connect().unwrap(); + db.query_parse::<()>(&skytable::query!(query_assembled)).unwrap(); + } + }; + } + TestStrategy::Standard => {} + } + let ret = quote! { + #[core::prelude::v1::test] + #retfn { + #ret_block + } + }; + ret.into() +} diff --git a/sky-macros/src/dbtest_fn.rs b/sky-macros/src/dbtest_fn.rs deleted file mode 100644 index e1f0e8e1..00000000 --- a/sky-macros/src/dbtest_fn.rs +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Created on Wed Mar 09 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use {crate::util, proc_macro::TokenStream, quote::quote, syn::AttributeArgs}; - -type OptString = Option; - -pub struct DBTestFunctionConfig { - table_decl: String, - port: u16, - host: String, - tls_cert: OptString, - login: (OptString, OptString), - testuser: bool, - rootuser: bool, - norun: bool, - skip_cfg: quote::__private::TokenStream, -} - -impl DBTestFunctionConfig { - pub fn default() -> Self { - Self { - table_decl: "(string, string)".to_owned(), - port: 2003, - host: "127.0.0.1".to_owned(), - tls_cert: None, - login: (None, None), - testuser: false, - rootuser: false, - norun: false, - skip_cfg: quote! {}, - } - } - pub fn get_connection_tokens(&self) -> impl quote::ToTokens { - let DBTestFunctionConfig { - port, - host, - tls_cert, - .. - } = &self; - match tls_cert { - Some(cert) => { - quote! { - let certpath = ::std::format!("{}/{}", crate::ROOT_DIR, #cert); - let mut con = skytable::aio::TlsConnection::new( - #host, #port, &certpath - ).await.unwrap(); - } - } - None => quote! { - let mut con = skytable::AsyncConnection::new(#host, #port).await.unwrap(); - }, - } - } - pub fn get_create_table_tokens(&self, table_name: &str) -> impl quote::ToTokens { - let Self { table_decl, .. } = self; - quote! { - con.run_query_raw( - &skytable::query!(format!("create model {}{} volatile", #table_name, #table_decl)) - ).await.unwrap() - } - } - pub fn get_skip_cfg_tokens(&self) -> &impl quote::ToTokens { - &self.skip_cfg - } - pub fn get_login_tokens(&self) -> Option { - let Self { - login, - testuser, - rootuser, - .. - } = self; - let conflict = (*rootuser && *testuser) - || ((*rootuser || *testuser) && (login.0.is_some() || login.1.is_some())); - if conflict { - panic!("Expected either of `username` and `password`, or `auth_rootuser`, or `auth_testuser`"); - } - let ret; - if *testuser { - ret = quote! { - let __username__ = crate::auth::provider::testsuite_data::TESTSUITE_TEST_USER; - let __password__ = crate::auth::provider::testsuite_data::TESTSUITE_TEST_TOKEN; - }; - } else if *rootuser { - ret = quote! { - let __username__ = crate::auth::provider::testsuite_data::TESTSUITE_ROOT_USER; - let __password__ = crate::auth::provider::testsuite_data::TESTSUITE_ROOT_TOKEN; - }; - } else { - let (username, password) = login; - match (username, password) { - (Some(username), Some(password)) => { - ret = quote! { - let __username__ = #username; - let __password__ = #password; - } - } - (None, None) => return None, - _ => panic!("Expected both `username` and `password`"), - } - } - Some(quote! { - #ret - let __loginquery__ = ::skytable::query!("auth", "login", __username__, __password__); - assert_eq!( - con.run_query_raw(&__loginquery__).await.unwrap(), - ::skytable::Element::RespCode(::skytable::RespCode::Okay), - "Failed to login" - ); - }) - } -} - -pub fn parse_dbtest_func_args( - arg: &str, - lit: &syn::Lit, - span: proc_macro2::Span, - fcfg: &mut DBTestFunctionConfig, -) { - match arg { - "table" => { - // check if the user wants some special table declaration - fcfg.table_decl = - util::parse_string(lit, span, "table").expect("Expected a value for `table`"); - } - "port" => { - // check if we need a custom port - fcfg.port = util::parse_number(lit, span, "port").expect("Expected a u16"); - } - "host" => { - fcfg.host = util::parse_string(lit, span, "host").expect("Expected a string"); - } - "tls_cert" => { - fcfg.tls_cert = Some(util::parse_string(lit, span, "host").expect("Expected a string")); - } - "username" => { - fcfg.login.0 = - Some(util::parse_string(lit, span, "username").expect("Expected a string")) - } - "password" => { - fcfg.login.1 = - Some(util::parse_string(lit, span, "password").expect("Expected a string")) - } - "auth_testuser" => { - fcfg.testuser = util::parse_bool(lit, span, "auth_testuser").expect("Expected a bool") - } - "auth_rootuser" => { - fcfg.rootuser = util::parse_bool(lit, span, "auth_testuser").expect("Expected a bool") - } - "norun" => fcfg.norun = util::parse_bool(lit, span, "norun").expect("Expected a bool"), - "run_if_cfg" => { - let cfg_name = util::parse_string(lit, span, "run_if_cfg").expect("Expected a string"); - fcfg.skip_cfg = quote! { - #[cfg_attr(not(feature = #cfg_name), ignore)] - }; - } - "skip_if_cfg" => { - let cfg_name = util::parse_string(lit, span, "run_if_cfg").expect("Expected a string"); - fcfg.skip_cfg = quote! { - #[cfg_attr(feature = #cfg_name, ignore)] - }; - } - x => panic!("unknown attribute {x} specified"), - } -} - -/// This parses a function within a `dbtest` module -/// -/// This accepts an `async` function and returns a non-`async` version of it - by -/// making the body of the function use the `tokio` runtime -fn generate_dbtest( - mut input: syn::ItemFn, - rng: &mut impl rand::Rng, - fcfg: &DBTestFunctionConfig, -) -> Result { - let sig = &mut input.sig; - let fname = sig.ident.to_string(); - let testbody = &input.block; - let attrs = &input.attrs; - let vis = &input.vis; - let header = quote! { - #[::core::prelude::v1::test] - }; - if sig.asyncness.is_none() { - let msg = "`dbtest` functions need to be async"; - return Err(syn::Error::new_spanned(sig.fn_token, msg)); - } - sig.asyncness = None; - let rand_string = util::get_rand_string(rng); - let mut body = quote! {}; - - // first add connection tokens - let connection_tokens = fcfg.get_connection_tokens(); - body = quote! { - #body - #connection_tokens - }; - - // check if we need to log in - if let Some(login_tokens) = fcfg.get_login_tokens() { - body = quote! { - #body - #login_tokens - }; - } - - if !fcfg.norun { - // now create keyspace - body = quote! { - #body - let __create_ks = - con.run_query_raw( - &skytable::query!("create space testsuite") - ).await.unwrap(); - if !( - __create_ks == skytable::Element::RespCode(skytable::RespCode::Okay) || - __create_ks == skytable::Element::RespCode( - skytable::RespCode::ErrorString( - skytable::error::errorstring::ERR_ALREADY_EXISTS.to_owned() - ) - ) - ) { - panic!("Failed to create keyspace: {:?}", __create_ks); - } - }; - // now switch keyspace - body = quote! { - #body - let __switch_ks = - con.run_query_raw( - &skytable::query!("use testsuite") - ).await.unwrap(); - if (__switch_ks != skytable::Element::RespCode(skytable::RespCode::Okay)) { - panic!("Failed to switch keyspace: {:?}", __switch_ks); - } - }; - // now create table - let create_table_tokens = fcfg.get_create_table_tokens(&rand_string); - body = quote! { - #body - assert_eq!( - #create_table_tokens, - skytable::Element::RespCode(skytable::RespCode::Okay), - "Failed to create table" - ); - }; - // now generate the __MYENTITY__ string - body = quote! { - #body - let mut __concat_entity = std::string::String::new(); - __concat_entity.push_str("testsuite."); - __concat_entity.push_str(&#rand_string); - let __MYTABLE__ : String = #rand_string.to_string(); - let __MYKS__: String = "testsuite".to_owned(); - let __MYENTITY__: String = __concat_entity.clone(); - }; - // now switch to the temporary table we created - body = quote! { - #body - let __switch_entity = - con.run_query_raw( - &skytable::query!(format!("use {}", __concat_entity)) - ).await.unwrap(); - assert_eq!( - __switch_entity, skytable::Element::RespCode(skytable::RespCode::Okay), "Failed to switch" - ); - }; - } - // now give the query ghost variable - body = quote! { - #body - let mut query = skytable::Query::new(); - }; - // IMPORTANT: now append the actual test body - body = quote! { - #body - #testbody - }; - if !fcfg.norun { - // now we're done with the test so flush the table - body = quote! { - #body - { - let mut __flush__ = skytable::Query::from("flushdb"); - std::assert_eq!( - con.run_query_raw(&__flush__).await.unwrap(), - skytable::Element::RespCode(skytable::RespCode::Okay) - ); - } - }; - } - let skip_cfg = fcfg.get_skip_cfg_tokens(); - let result = quote! { - #skip_cfg - #header - #(#attrs)* - #vis #sig { - tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .thread_name(#fname) - .thread_stack_size(3 * 1024 * 1024) - .enable_all() - .build() - .unwrap() - .block_on(async { #body }); - } - }; - Ok(result.into()) -} - -/// This function checks if the current function is eligible to be a test and if so, returns -/// the generated test -pub fn generate_test( - input: syn::ItemFn, - rng: &mut impl rand::Rng, - fcfg: &DBTestFunctionConfig, -) -> TokenStream { - for attr in &input.attrs { - if attr.path.is_ident("test") { - let msg = "second test attribute is supplied"; - return syn::Error::new_spanned(attr, msg).to_compile_error().into(); - } - } - - if !input.sig.inputs.is_empty() { - let msg = "the test function cannot accept arguments"; - return syn::Error::new_spanned(&input.sig.inputs, msg) - .to_compile_error() - .into(); - } - generate_dbtest(input, rng, fcfg).unwrap_or_else(|e| e.to_compile_error().into()) -} - -fn parse_dbtest_func_attrs(attrs: AttributeArgs) -> DBTestFunctionConfig { - let mut fcfg = DBTestFunctionConfig::default(); - attrs.iter().for_each(|arg| { - if let syn::NestedMeta::Meta(syn::Meta::NameValue(namevalue)) = arg { - let (ident, lit, span) = util::get_metanamevalue_data(namevalue); - parse_dbtest_func_args(&ident, lit, span, &mut fcfg) - } - }); - fcfg -} - -pub fn dbtest_func(args: TokenStream, item: TokenStream) -> TokenStream { - let input = syn::parse_macro_input!(item as syn::ItemFn); - let attrs = syn::parse_macro_input!(args as AttributeArgs); - let mut rng = rand::thread_rng(); - let fcfg = parse_dbtest_func_attrs(attrs); - generate_dbtest(input, &mut rng, &fcfg).unwrap() -} diff --git a/sky-macros/src/dbtest_mod.rs b/sky-macros/src/dbtest_mod.rs deleted file mode 100644 index d57a1754..00000000 --- a/sky-macros/src/dbtest_mod.rs +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Created on Wed Mar 09 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - dbtest_fn::{self, DBTestFunctionConfig}, - util, - }, - proc_macro::TokenStream, - quote::quote, - std::collections::HashSet, - syn::{self, AttributeArgs}, -}; - -struct DBTestModuleConfig { - fcfg: DBTestFunctionConfig, - skips: HashSet, -} - -impl DBTestModuleConfig { - fn default() -> Self { - Self { - skips: HashSet::new(), - fcfg: DBTestFunctionConfig::default(), - } - } -} - -fn parse_dbtest_module_args(args: AttributeArgs) -> DBTestModuleConfig { - let mut modcfg = DBTestModuleConfig::default(); - for arg in args { - if let syn::NestedMeta::Meta(syn::Meta::NameValue(namevalue)) = arg { - let (ident, lit, span) = util::get_metanamevalue_data(&namevalue); - match ident.as_str() { - "skip" => { - modcfg.skips = util::parse_string(lit, span, "skip") - .expect("Expected a value for argument `skip`") - .split_whitespace() - .map(|val| val.to_string()) - .collect(); - } - possibly_func_arg => dbtest_fn::parse_dbtest_func_args( - possibly_func_arg, - lit, - span, - &mut modcfg.fcfg, - ), - } - } - } - modcfg -} - -/// This function accepts an entire **inline** module which comprises of `dbtest` functions. -/// It takes each function in turn, and generates `#[test]`able functions for them -pub fn parse_test_module(args: TokenStream, item: TokenStream) -> TokenStream { - let input = syn::parse_macro_input!(item as syn::ItemMod); - let content = match input.content { - Some((_, c)) => c, - None => { - return syn::Error::new_spanned(&input, "Couldn't get the module content") - .to_compile_error() - .into() - } - }; - let modcfg = parse_dbtest_module_args(syn::parse_macro_input!(args as AttributeArgs)); - let mut result = quote! {}; - let mut rng = rand::thread_rng(); - for item in content { - match item { - // We just care about functions, so parse functions and ignore everything - // else - syn::Item::Fn(function) if !modcfg.skips.contains(&function.sig.ident.to_string()) => { - let generated_fn = dbtest_fn::generate_test(function, &mut rng, &modcfg.fcfg); - let __tok: syn::ItemFn = syn::parse_macro_input!(generated_fn as syn::ItemFn); - let tok = quote! { - #__tok - }; - result = quote! { - #result - #tok - }; - } - token => { - result = quote! { - #result - #token - }; - } - } - } - result.into() -} diff --git a/sky-macros/src/lib.rs b/sky-macros/src/lib.rs index 13b0dedd..3d517caf 100644 --- a/sky-macros/src/lib.rs +++ b/sky-macros/src/lib.rs @@ -43,137 +43,132 @@ //! - `__MYENTITY__` - `String` with entity //! -use {proc_macro::TokenStream, quote::quote, syn::Lit}; +use { + proc_macro::TokenStream, + proc_macro2::TokenStream as TokenStream2, + quote::quote, + syn::{parse_macro_input, Data, DataStruct, DeriveInput, Fields, Meta, NestedMeta}, +}; -mod dbtest_fn; -mod dbtest_mod; +mod dbtest; mod util; #[proc_macro_attribute] -/// The `dbtest_module` function accepts an inline module of `dbtest_func` compatible functions, -/// unpacking each function into a dbtest -pub fn dbtest_module(args: TokenStream, item: TokenStream) -> TokenStream { - dbtest_mod::parse_test_module(args, item) +pub fn dbtest(attrs: TokenStream, item: TokenStream) -> TokenStream { + dbtest::dbtest(attrs, item) } -/// The `dbtest_func` macro starts an async server in the background and is meant for -/// use within the `skyd` or `WORKSPACEROOT/server/` crate. If you use this compiler -/// macro in any other crate, you'll simply get compilation errors -/// -/// All tests will clean up all values once a single test is over -/// -/// ## Arguments -/// - `table -> str`: Custom table declaration -/// - `port -> u16`: Custom port -/// - `host -> str`: Custom host -/// - `tls_cert -> str`: TLS cert (makes the connection a TLS one) -/// - `username -> str`: Username for authn -/// - `password -> str`: Password for authn -/// - `auth_testuser -> bool`: Login as the test user -/// - `auth_rootuser -> bool`: Login as the root user -/// - `norun -> bool`: Don't execute anything on the connection -/// -/// ## _Ghost_ values -/// This macro gives: -/// - `con`: a `skytable::AsyncConnection` -/// - `query`: a mutable `skytable::Query` -/// - `__MYENTITY__`: the entity set on launch -/// - `__MYTABLE__`: the table set on launch -/// - `__MYKS__`: the keyspace set on launch -/// -/// ## Requirements -/// -/// The `#[dbtest]` macro expects several things. The calling crate: -/// - should have the `tokio` crate as a dependency and should have the -/// `features` set to full -/// - should have the `skytable` crate as a dependency and should have the `features` set to `async` and version -/// upstreamed to `next` on skytable/client-rust -/// -/// ## Collisions -/// -/// The sample space for table name generation is so large (in the order of 4.3 to the 50) that collisions -/// are practially impossible. Hence we do not bother with a global random string table and instead proceed -/// to generate tables randomly at the point of invocation -/// -#[proc_macro_attribute] -pub fn dbtest_func(args: TokenStream, item: TokenStream) -> TokenStream { - dbtest_fn::dbtest_func(args, item) -} - -#[proc_macro] -/// Get a compile time respcode/respstring array. For example, if you pass: "Unknown action", -/// it will return: `!14\nUnknown Action\n` -pub fn compiled_eresp_array(tokens: TokenStream) -> TokenStream { - _get_eresp_array(tokens, false) +#[proc_macro_derive(Wrapper)] +/// Implements necessary traits for some type `T` to make it identify as a different type but mimic the functionality +/// as the inner type it wraps around +pub fn derive_wrapper(t: TokenStream) -> TokenStream { + let item = syn::parse_macro_input!(t as DeriveInput); + let r = wrapper(item); + r.into() } -#[proc_macro] -/// Get a compile time respcode/respstring array. For example, if you pass: "Unknown action", -/// it will return: `!14\n14\nUnknown Action\n` -pub fn compiled_eresp_array_v1(tokens: TokenStream) -> TokenStream { - _get_eresp_array(tokens, true) +fn wrapper(item: DeriveInput) -> TokenStream2 { + let st_name = &item.ident; + let fields = match item.data { + Data::Struct(DataStruct { + fields: Fields::Unnamed(ref f), + .. + }) if f.unnamed.len() == 1 => f, + _ => panic!("only works on tuple structs with one field"), + }; + let field = &fields.unnamed[0]; + let ty = &field.ty; + let (impl_generics, ty_generics, where_clause) = item.generics.split_for_impl(); + quote! { + #[automatically_derived] + impl #impl_generics #st_name #ty_generics #where_clause { pub fn into_inner(self) -> #ty { self.0 } } + #[automatically_derived] + impl #impl_generics ::core::ops::Deref for #st_name #ty_generics #where_clause { + type Target = #ty; + fn deref(&self) -> &Self::Target { &self.0 } + } + #[automatically_derived] + impl #impl_generics ::core::ops::DerefMut for #st_name #ty_generics #where_clause { + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } + } + #[automatically_derived] + impl #impl_generics ::core::cmp::PartialEq<#ty> for #st_name #ty_generics #where_clause { + fn eq(&self, other: &#ty) -> bool { ::core::cmp::PartialEq::eq(&self.0, other) } + } + #[automatically_derived] + impl #impl_generics ::core::cmp::PartialEq<#st_name #ty_generics> for #ty #where_clause { + fn eq(&self, other: &#st_name #ty_generics) -> bool { ::core::cmp::PartialEq::eq(self, &other.0) } + } + } } -fn _get_eresp_array(tokens: TokenStream, sizeline: bool) -> TokenStream { - let payload_str = match syn::parse_macro_input!(tokens as Lit) { - Lit::Str(st) => st.value(), - _ => panic!("Expected a string literal"), - }; - let mut processed = quote! { - b'!', - }; - if sizeline { - let payload_len = payload_str.as_bytes().len(); - let payload_len_str = payload_len.to_string(); - let payload_len_bytes = payload_len_str.as_bytes(); - for byte in payload_len_bytes { - processed = quote! { - #processed - #byte, - }; +#[proc_macro_derive(EnumMethods)] +pub fn derive_value_methods(input: TokenStream) -> TokenStream { + let ast = parse_macro_input!(input as DeriveInput); + let enum_name = &ast.ident; + let mut repr_type = None; + // Get repr attribute + for attr in &ast.attrs { + if attr.path.is_ident("repr") { + if let Meta::List(list) = attr.parse_meta().unwrap() { + if let Some(NestedMeta::Meta(Meta::Path(path))) = list.nested.first() { + repr_type = Some(path.get_ident().unwrap().to_string()); + } + } } - processed = quote! { - #processed - b'\n', - }; } - let payload_bytes = payload_str.as_bytes(); - for byte in payload_bytes { - processed = quote! { - #processed - #byte, + let repr_type = repr_type.expect("Must have repr(u8) or repr(u16) etc."); + let mut dscr_expressions = vec![]; + // Ensure all variants have explicit discriminants + if let Data::Enum(data) = &ast.data { + for variant in &data.variants { + match &variant.fields { + Fields::Unit => { + let (_, dscr_expr) = variant + .discriminant + .as_ref() + .expect("All enum variants must have explicit discriminants"); + dscr_expressions.push(dscr_expr.clone()); + } + _ => panic!("All enum variants must be unit variants"), + } } + } else { + panic!("This derive macro only works on enums"); } - processed = quote! { - #processed - b'\n', - }; - processed = quote! { - [#processed] + + let value_expressions = quote! { + [#(#dscr_expressions),*] }; - processed.into() -} -#[proc_macro] -/// Get a compile time respcode/respstring slice. For example, if you pass: "Unknown action", -/// it will return: `!14\nUnknown Action\n` -pub fn compiled_eresp_bytes(tokens: TokenStream) -> TokenStream { - let ret = compiled_eresp_array(tokens); - let ret = syn::parse_macro_input!(ret as syn::Expr); - quote! { - &#ret - } - .into() -} + let variant_len = dscr_expressions.len(); -#[proc_macro] -/// Get a compile time respcode/respstring slice. For example, if you pass: "Unknown action", -/// it will return: `!14\nUnknown Action\n` -pub fn compiled_eresp_bytes_v1(tokens: TokenStream) -> TokenStream { - let ret = compiled_eresp_array_v1(tokens); - let ret = syn::parse_macro_input!(ret as syn::Expr); - quote! { - &#ret - } - .into() + let repr_type_ident = syn::Ident::new(&repr_type, proc_macro2::Span::call_site()); + let repr_type_ident_func = syn::Ident::new( + &format!("value_{repr_type}"), + proc_macro2::Span::call_site(), + ); + + let gen = quote! { + impl #enum_name { + pub const MAX: #repr_type_ident = Self::max_value(); + pub const VARIANTS: usize = #variant_len; + pub const fn #repr_type_ident_func(&self) -> #repr_type_ident { unsafe { core::mem::transmute(*self) } } + pub const fn value_word(&self) -> usize { self.#repr_type_ident_func() as usize } + pub const fn value_qword(&self) -> u64 { self.#repr_type_ident_func() as u64 } + pub const fn max_value() -> #repr_type_ident { + let values = #value_expressions; + let mut i = 1; + let mut max = values[0]; + while i < values.len() { + if values[i] > max { + max = values[i]; + } + i = i + 1; + } + max + } + } + }; + gen.into() } diff --git a/sky-macros/src/util.rs b/sky-macros/src/util.rs index b55c029b..540dcd7a 100644 --- a/sky-macros/src/util.rs +++ b/sky-macros/src/util.rs @@ -1,5 +1,5 @@ /* - * Created on Wed Mar 09 2022 + * Created on Wed Nov 29 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2022, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,66 +24,55 @@ * */ -use { - core::{fmt::Display, str::FromStr}, - proc_macro2::Span, - rand::Rng, - syn::{Lit, MetaNameValue}, -}; +use proc_macro2::Ident; +use syn::{Lit, Meta, MetaNameValue, NestedMeta, Path}; -const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - -pub fn get_rand_string(rng: &mut impl Rng) -> String { - (0..64) - .map(|_| { - let idx = rng.gen_range(0..CHARSET.len()); - CHARSET[idx] as char - }) - .collect() +pub enum AttributeKind { + Lit(Lit), + NestedAttrs { name: Ident, attrs: Vec }, + Pair(Ident, Lit), + Path(Path), } -pub fn parse_string(int: &Lit, span: Span, field: &str) -> Result { - match int { - syn::Lit::Str(s) => Ok(s.value()), - syn::Lit::Verbatim(s) => Ok(s.to_string()), - _ => Err(syn::Error::new( - span, - format!("Failed to parse {} into a string.", field), - )), +impl AttributeKind { + pub fn into_pair(self) -> (Ident, Lit) { + match self { + Self::Pair(i, l) => (i, l), + _ => panic!("expected attribute name pair"), + } } } -pub fn parse_number, E: Display>( - int: &Lit, - span: Span, - field: &str, -) -> Result { - match int { - syn::Lit::Int(int) => int.base10_parse::(), - _ => Err(syn::Error::new( - span, - format!("Failed to parse {} into an int.", field), - )), +pub fn extract_attribute(attr: &NestedMeta) -> AttributeKind { + match attr { + NestedMeta::Lit(l) => AttributeKind::Lit(l.clone()), + NestedMeta::Meta(m) => match m { + Meta::List(l) => AttributeKind::NestedAttrs { + name: l.path.get_ident().unwrap().clone(), + attrs: l.nested.iter().map(extract_attribute).collect(), + }, + Meta::NameValue(MetaNameValue { path, lit, .. }) => { + AttributeKind::Pair(path.get_ident().unwrap().clone(), lit.clone()) + } + Meta::Path(p) => AttributeKind::Path(p.clone()), + }, } } -pub fn parse_bool(boolean: &Lit, span: Span, field: &str) -> Result { - match boolean { - Lit::Bool(boolean) => Ok(boolean.value), - _ => Err(syn::Error::new( - span, - format!("Failed to parse {} into a boolean.", field), - )), +pub fn extract_str_from_lit(l: &Lit) -> Option { + match l { + Lit::Str(s) => Some(s.value()), + _ => None, } } -pub fn get_metanamevalue_data(namevalue: &MetaNameValue) -> (String, &Lit, Span) { - match namevalue - .path - .get_ident() - .map(|ident| ident.to_string().to_lowercase()) - { - None => panic!("Must have specified ident!"), - Some(ident) => (ident, &namevalue.lit, namevalue.lit.span()), +pub fn extract_int_from_lit(l: &Lit) -> Option +where + I: std::str::FromStr, + I::Err: std::fmt::Display, +{ + match l { + Lit::Int(i) => i.base10_parse::().ok(), + _ => None, } } diff --git a/sky-migrate/Cargo.toml b/sky-migrate/Cargo.toml deleted file mode 100644 index e6ed9690..00000000 --- a/sky-migrate/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "sky-migrate" -version = "0.8.0" -authors = ["Sayan Nandan "] -edition = "2021" -description = "The Skytable migration tool allows users coming from older versions (>=0.8.0) to upgrade their datasets to the latest Skytable version. This tool currently supports versions >= 0.8.0 and upgrading it to 0.7.0." - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -skytable = { git = "https://github.com/skytable/client-rust.git" } -env_logger = "0.10.0" -bincode = "1.3.3" -log = "0.4.17" -clap = { version = "4.0.32", features = ["derive"] } diff --git a/sky-migrate/README.md b/sky-migrate/README.md deleted file mode 100644 index f5c7429a..00000000 --- a/sky-migrate/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Skytable migration tool - -## Introduction - -The Skytable migration tool can be used to perform migrations between database versions. The basic -idea is: "read all data from the older machine and send it over to the newer one". - -For migrating versions from 0.6 to 0.7, the database administrator has to launch the tool in the -old data directory of the old instance and then pass the new instance host/port information. The -tool will then read the data from the directory (this was possible because 0.6 used a very simple -disk format than newer versions). This approach however has the advantage of not having to start -the database server for the migration to happen. - -## How To Use - -To upgrade, one needs to simply run: -```shell -# Here `` is the path to the last installation's data directory and -# `` and `` is the hostname and port for the new server instance -sky-migrate --prevdir --new : -``` - -## License - -All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/sky-migrate/src/cli.rs b/sky-migrate/src/cli.rs deleted file mode 100644 index 2b92081a..00000000 --- a/sky-migrate/src/cli.rs +++ /dev/null @@ -1,104 +0,0 @@ -use clap::Parser; - -const HELP_TEMPLATE: &str = r#" -{before-help}{name} {version} -{author-with-newline}{about-with-newline} -{usage-heading} {usage} - -{all-args}{after-help} -"#; - -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None, help_template = HELP_TEMPLATE, arg_required_else_help = true)] -pub struct Cli { - #[arg( - short = 'n', - long = "new", - help = "The : combo for the new instance", - value_name = "HOST:PORT" - )] - pub new: String, - - #[arg( - short = 'p', - long = "prevdir", - help = "Path to the previous installation location", - value_name = "PREVDIR" - )] - pub prevdir: String, - - #[arg( - short = 's', - long, - help = "Transfer entries one-by-one instead of all at once to save memory" - )] - pub serial: bool, -} - -#[cfg(test)] -mod tests { - use crate::Cli; - use clap::error::ErrorKind; - use clap::Parser; - - #[test] - fn test_mandatory_args_success() { - let args = vec!["sky-migrate", "-n", "localhost:1234", "-p", "/tmp/skyd1"]; - let cli = Cli::parse_from(args.into_iter()); - assert_eq!(cli.new, "localhost:1234"); - assert_eq!(cli.prevdir, "/tmp/skyd1"); - assert!(!cli.serial); - } - - #[test] - fn test_serial_enabled_success() { - let args = vec![ - "sky-migrate", - "-n", - "localhost:1234", - "-p", - "/tmp/skyd1", - "-s", - ]; - let cli = Cli::parse_from(args.into_iter()); - assert_eq!(cli.new, "localhost:1234"); - assert_eq!(cli.prevdir, "/tmp/skyd1"); - assert!(cli.serial); - } - - #[test] - fn test_host_port_missing_failure() { - let args = vec!["sky-migrate", "-p", "/tmp/skyd1"]; - let cli_result: Result = Cli::try_parse_from(args.into_iter()); - - assert!(cli_result.is_err()); - assert_eq!( - cli_result.unwrap_err().kind(), - ErrorKind::MissingRequiredArgument - ); - } - - #[test] - fn test_prevdir_missing_failure() { - let args = vec!["sky-migrate", "-n", "localhost:8083"]; - let cli_result: Result = Cli::try_parse_from(args.into_iter()); - - assert!(cli_result.is_err()); - assert_eq!( - cli_result.unwrap_err().kind(), - ErrorKind::MissingRequiredArgument - ); - } - - #[test] - fn test_display_help_when_all_args_missing() { - let args = vec!["sky-migrate"]; - let cli_result: Result = Cli::try_parse_from(args.into_iter()); - - assert!(cli_result.is_err()); - assert_eq!( - cli_result.unwrap_err().kind(), - ErrorKind::DisplayHelpOnMissingArgumentOrSubcommand - ); - } -} diff --git a/sky-migrate/src/main.rs b/sky-migrate/src/main.rs deleted file mode 100644 index c0f78ef4..00000000 --- a/sky-migrate/src/main.rs +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Created on Tue Aug 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![allow(clippy::unit_arg)] - -mod cli; - -use { - crate::cli::Cli, - clap::Parser, - env_logger::Builder, - log::{error as err, info}, - skytable::{query, sync::Connection, Element, Query, RespCode}, - std::{collections::HashMap, env, fs, process}, -}; - -type Bytes = Vec; - -fn main() { - // first evaluate config - let cli = Cli::parse(); - Builder::new() - .parse_filters(&env::var("SKY_LOG").unwrap_or_else(|_| "info".to_owned())) - .init(); - let serial = cli.serial; - let hostsplit: Vec<&str> = cli.new.split(':').collect(); - if hostsplit.len() != 2 { - err(err!("Bad value for --new")); - } - let (host, port) = unsafe { (hostsplit.get_unchecked(0), hostsplit.get_unchecked(1)) }; - let port = match port.parse() { - Ok(p) => p, - Err(e) => err(err!("Bad value for port in --new: {}", e)), - }; - let mut old_dir = cli.prevdir; - old_dir.push_str("data.bin"); - // now connect - let mut con = match Connection::new(host, port) { - Ok(con) => con, - Err(e) => err(err!("Failed to connect to new instance with error: {}", e)), - }; - // run sanity test - let q = query!("HEYA"); - match con.run_query_raw(&q) { - Ok(Element::String(s)) if s.eq("HEY!") => {} - Ok(_) => err(err!("Unknown response from server")), - Err(e) => err(err!( - "An I/O error occurred while running sanity test: {}", - e - )), - } - info!("Sanity test complete"); - - // now de old file - let read = match fs::read(old_dir) { - Ok(r) => r, - Err(e) => err(err!( - "Failed to read data.bin file from old directory: {}", - e - )), - }; - let de: HashMap = match bincode::deserialize(&read) { - Ok(r) => r, - Err(e) => err(err!("Failed to unpack old file with: {}", e)), - }; - unsafe { - if serial { - // transfer serially - for (key, value) in de.into_iter() { - let q = query!( - "USET", - String::from_utf8_unchecked(key), - String::from_utf8_unchecked(value) - ); - okay(&mut con, q) - } - } else { - // transfer all at once - let mut query = Query::from("USET"); - for (key, value) in de.into_iter() { - query.push(String::from_utf8_unchecked(key)); - query.push(String::from_utf8_unchecked(value)); - } - okay(&mut con, query) - } - } - info!("Finished migration"); -} - -fn err(_i: ()) -> ! { - process::exit(0x01) -} - -fn okay(con: &mut Connection, q: Query) { - match con.run_query_raw(&q) { - Ok(Element::RespCode(RespCode::Okay)) => {} - Err(e) => err(err!("An I/O error occurred while running query: {}", e)), - Ok(_) => err(err!("Unknown response from server")), - } -} diff --git a/stress-test/Cargo.toml b/stress-test/Cargo.toml deleted file mode 100644 index cf8e678c..00000000 --- a/stress-test/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "stress-test" -version = "0.1.0" -authors = ["Sayan Nandan "] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -# internal deps -libstress = { path = "../libstress" } -skytable = { git = "https://github.com/skytable/client-rust.git", branch = "next", features = [ - "dbg", -] } -devtimer = "4.0.1" -# external deps -sysinfo = "0.27.4" -env_logger = "0.10.0" -log = "0.4.17" -rand = "0.8.5" -crossbeam-channel = "0.5.6" diff --git a/stress-test/README.md b/stress-test/README.md deleted file mode 100644 index 5008306b..00000000 --- a/stress-test/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Skytable stress-test tool - -This is a tool in its infancy, but its ultimate goal is to provide a stress testing framework for Skytable. For now, it tests linearity (core scalability) with increasing clients. - -## License - -All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/stress-test/src/linearity_client.rs b/stress-test/src/linearity_client.rs deleted file mode 100644 index d97e37b7..00000000 --- a/stress-test/src/linearity_client.rs +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Created on Fri Jun 18 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # Client linearity tests -//! -//! This module contains functions to test the linearity of the database with increasing number -//! of clients, i.e how the number of queries scale with increasing clients. These functions -//! however, DO NOT focus on benchmarking and instead focus on correctness under load from -//! concurrent clients. -//! - -use { - crate::{logstress, DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV}, - crossbeam_channel::bounded, - devtimer::SimpleTimer, - libstress::{rayon::prelude::*, utils::generate_random_string_vector, Workpool}, - skytable::{actions::Actions, query, Connection, Element, Query, RespCode}, -}; - -macro_rules! log_client_linearity { - ($stressid:expr, $counter:expr, $what:expr) => { - log::info!( - "Stress ({}{}) [{}]: Clients: {}; K/V size: {}; Queries: {}", - $stressid, - $counter, - $what, - $counter, - DEFAULT_SIZE_KV, - DEFAULT_QUERY_COUNT - ); - }; -} - -/// This object provides methods to measure the percentage change in the slope -/// of a function that is expected to have linearity. -/// -/// For example, we can think of it to work in the following way: -/// Let h(x) be a function with linearity (not proportionality because that isn't -/// applicable in our case). h(x) gives us the time taken to run a given number of -/// queries (invariant; not plotted on axes), where x is the number of concurrent -/// clients. As we would want our database to scale with increasing clients (and cores), -/// we'd expect linearity, hence the gradient should continue to fall with increasing -/// values in the +ve x-axis effectively producing a constantly decreasing slope, reflected -/// by increasing values of abs(get_delta(h(x))). -/// -/// TODO(@ohsayan): Of course, some unexpected kernel errors/scheduler hiccups et al can -/// cause there to be a certain epsilon that must be tolerated with a tolerance factor -/// -pub struct LinearityMeter { - init: Option, - measure: Vec, -} - -impl LinearityMeter { - pub const fn new() -> Self { - Self { - init: None, - measure: Vec::new(), - } - } - pub fn get_delta(&mut self, current: u128) -> f32 { - if let Some(u) = self.init { - let cur = ((current as f32 - u as f32) / u as f32) * 100.00_f32; - self.measure.push(cur); - cur - } else { - // if init is not initialized, initialize it - self.init = Some(current); - // no change when at base - 0.00 - } - } -} - -pub fn stress_linearity_concurrent_clients_set( - mut rng: &mut impl rand::Rng, - max_workers: usize, - temp_con: &mut Connection, -) { - logstress!( - "A [SET]", - "Linearity test with monotonically increasing clients" - ); - let mut current_thread_count = 1usize; - - // generate the random k/v pairs - let keys = generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, true); - let values = - generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, false); - let (keys, values) = match (keys, values) { - (Ok(k), Ok(v)) => (k, v), - _ => { - eprintln!("Allocation error"); - std::process::exit(0x01); - } - }; - // make sure the database is empty - temp_con.flushdb().unwrap(); - - // initialize the linearity counter - let mut linearity = LinearityMeter::new(); - while current_thread_count <= max_workers { - log_client_linearity!("A", current_thread_count, "SET"); - // generate the set packets - let set_packs: Vec = keys - .par_iter() - .zip(values.par_iter()) - .map(|(k, v)| query!("SET", k, v)) - .collect(); - let workpool = Workpool::new( - current_thread_count, - || Connection::new("127.0.0.1", 2003).unwrap(), - move |sock, query| { - assert_eq!( - sock.run_query_raw(&query).unwrap(), - Element::RespCode(RespCode::Okay) - ); - }, - |_| {}, - true, - Some(DEFAULT_QUERY_COUNT), - ) - .unwrap(); - let mut timer = SimpleTimer::new(); - timer.start(); - workpool.execute_and_finish_iter(set_packs); - timer.stop(); - log::info!( - "Delta: {}%", - linearity.get_delta(timer.time_in_nanos().unwrap()) - ); - // clean up the database - temp_con.flushdb().unwrap(); - current_thread_count += 1; - } -} - -pub fn stress_linearity_concurrent_clients_get( - mut rng: &mut impl rand::Rng, - max_workers: usize, - temp_con: &mut Connection, -) { - logstress!( - "A [GET]", - "Linearity test with monotonically increasing clients" - ); - let mut current_thread_count = 1usize; - - // generate the random k/v pairs - let keys = generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, true); - let values = - generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, false); - let (keys, values) = match (keys, values) { - (Ok(k), Ok(v)) => (k, v), - _ => { - eprintln!("Allocation error"); - std::process::exit(0x01); - } - }; - - // Make sure that the database is empty - temp_con.flushdb().unwrap(); - - // First set the keys - let set_packs: Vec = keys - .par_iter() - .zip(values.par_iter()) - .map(|(k, v)| query!("SET", k, v)) - .collect(); - let workpool = Workpool::new_default_threads( - || Connection::new("127.0.0.1", 2003).unwrap(), - move |sock, query| { - assert_eq!( - sock.run_query_raw(&query).unwrap(), - Element::RespCode(RespCode::Okay) - ); - }, - |_| {}, - true, - Some(DEFAULT_QUERY_COUNT), - ) - .unwrap(); - workpool.execute_and_finish_iter(set_packs); - - // initialize the linearity counter - let mut linearity = LinearityMeter::new(); - while current_thread_count <= max_workers { - log_client_linearity!("A", current_thread_count, "GET"); - /* - We create a mpmc to receive the results returned. This avoids us using - any kind of locking on the surface which can slow down things - */ - let (tx, rx) = bounded::(DEFAULT_QUERY_COUNT); - - // generate the get packets - let get_packs: Vec = keys.iter().map(|k| query!("GET", k)).collect(); - let wp = Workpool::new( - current_thread_count, - || Connection::new("127.0.0.1", 2003).unwrap(), - move |sock, query| { - let tx = tx.clone(); - tx.send(sock.run_query_raw(&query).unwrap()).unwrap(); - }, - |_| {}, - true, - Some(DEFAULT_QUERY_COUNT), - ) - .unwrap(); - let mut timer = SimpleTimer::new(); - timer.start(); - wp.execute_and_finish_iter(get_packs); - timer.stop(); - log::info!( - "Delta: {}%", - linearity.get_delta(timer.time_in_nanos().unwrap()) - ); - let rets: Vec = rx - .into_iter() - .map(|v| { - if let Element::String(val) = v { - val - } else { - panic!("Unexpected response from server"); - } - }) - .collect(); - assert_eq!( - rets.len(), - values.len(), - "Incorrect number of values returned by server" - ); - - // now evaluate them - assert!( - rets.into_par_iter().all(|v| values.contains(&v)), - "Values returned by the server don't match what was sent" - ); - current_thread_count += 1; - } - temp_con.flushdb().unwrap(); -} diff --git a/stress-test/src/main.rs b/stress-test/src/main.rs deleted file mode 100644 index e59045ef..00000000 --- a/stress-test/src/main.rs +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Created on Wed Jun 16 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![deny(unused_crate_dependencies)] -#![deny(unused_imports)] - -use std::thread::available_parallelism; - -use { - libstress::traits::ExitError, - log::{info, trace, warn}, - rand::thread_rng, - skytable::Connection, - std::env, - sysinfo::{RefreshKind, System, SystemExt}, -}; -mod linearity_client; -mod utils; - -pub const DEFAULT_SIZE_KV: usize = 4; -pub const DEFAULT_QUERY_COUNT: usize = 100_000_usize; - -#[macro_export] -macro_rules! logstress { - ($stressid:expr, $extra:expr) => { - log::info!("Stress ({}): {}", $stressid, $extra); - }; -} - -fn main() { - // Build the logger - env_logger::Builder::new() - .parse_filters(&env::var("SKY_STRESS_LOG").unwrap_or_else(|_| "trace".to_owned())) - .init(); - warn!("The stress test checks correctness under load and DOES NOT show the true throughput"); - - // get the rng and refresh sysinfo - let mut rng = thread_rng(); - // we only need to refresh memory and CPU info; don't waste time syncing other things - let to_refresh = RefreshKind::new().with_memory(); - let mut sys = System::new_with_specifics(to_refresh); - sys.refresh_specifics(to_refresh); - let core_count = available_parallelism().map_or(1, usize::from); - let max_workers = core_count * 2; - trace!( - "This host has {} logical cores. Will spawn a maximum of {} threads", - core_count, - max_workers * 2 - ); - - // establish a connection to ensure sanity - let mut temp_con = Connection::new("127.0.0.1", 2003).exit_error("Failed to connect to server"); - - // calculate the maximum keylen - let max_keylen = utils::calculate_max_keylen(DEFAULT_QUERY_COUNT, &mut sys); - info!( - "This host can support a maximum theoretical keylen of: {}", - max_keylen - ); - - // run the actual stress tests - linearity_client::stress_linearity_concurrent_clients_set(&mut rng, max_workers, &mut temp_con); - linearity_client::stress_linearity_concurrent_clients_get(&mut rng, max_workers, &mut temp_con); - - // done, exit - info!("SUCCESS. Stress test complete!"); -} diff --git a/stress-test/src/utils.rs b/stress-test/src/utils.rs deleted file mode 100644 index 76cbbc8f..00000000 --- a/stress-test/src/utils.rs +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Created on Fri Jun 18 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ -use { - log::trace, - skytable::Query, - sysinfo::{System, SystemExt}, -}; - -pub fn calculate_max_keylen(expected_queries: usize, sys: &mut System) -> usize { - let total_mem_in_bytes = (sys.total_memory() * 1024) as usize; - trace!( - "This host has a total memory of: {} Bytes", - total_mem_in_bytes - ); - // av_mem gives us 90% of the memory size - let ninety_percent_of_memory = (0.90_f32 * total_mem_in_bytes as f32) as usize; - let mut highest_len = 1usize; - loop { - let set_pack_len = Query::array_packet_size_hint(vec![3, highest_len, highest_len]); - let get_pack_len = Query::array_packet_size_hint(vec![3, highest_len]); - let resulting_size = expected_queries - * ( - // for the set packets - set_pack_len + - // for the get packets - get_pack_len + - // for the keys themselves - highest_len - ); - if resulting_size >= ninety_percent_of_memory as usize { - break; - } - // increase the length by 5% every time to get the maximum possible length - // now this 5% increment is a tradeoff, but it's worth it to not wait for - // so long - highest_len = (highest_len as f32 * 1.05_f32).ceil() as usize; - } - highest_len -}