From fa00a9923b0820aadf5e74da400471f8c2dbedd8 Mon Sep 17 00:00:00 2001 From: jkomyno Date: Wed, 18 Dec 2024 20:59:47 +0100 Subject: [PATCH 1/5] feat(schema-engine): let "schema-connector" be wasm-compatible, fix ORM-457 --- .../connectors/schema-connector/Cargo.toml | 34 ++++++++++++++++--- .../src/introspection_context.rs | 5 +++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/schema-engine/connectors/schema-connector/Cargo.toml b/schema-engine/connectors/schema-connector/Cargo.toml index 5c40185f2a3b..e3cebd1270d9 100644 --- a/schema-engine/connectors/schema-connector/Cargo.toml +++ b/schema-engine/connectors/schema-connector/Cargo.toml @@ -3,14 +3,40 @@ name = "schema-connector" version = "0.1.0" edition = "2021" +[features] +postgresql = ["relation_joins", "quaint/postgresql", "psl/postgresql"] +postgresql-native = ["postgresql", "quaint/postgresql-native", "quaint/pooled"] +mysql = ["relation_joins", "quaint/mysql", "psl/mysql"] +mysql-native = ["mysql", "quaint/mysql-native", "quaint/pooled"] +sqlite = ["quaint/sqlite", "psl/sqlite"] +sqlite-native = ["sqlite", "quaint/sqlite-native", "quaint/pooled"] +mssql = ["quaint/mssql"] +mssql-native = ["mssql", "quaint/mssql-native", "quaint/pooled"] +cockroachdb = ["relation_joins", "quaint/postgresql", "psl/cockroachdb"] +cockroachdb-native = [ + "cockroachdb", + "quaint/postgresql-native", + "quaint/pooled", +] +vendored-openssl = ["quaint/vendored-openssl"] +all-native = [ + "sqlite-native", + "mysql-native", + "postgresql-native", + "mssql-native", + "cockroachdb-native", +] +# TODO: At the moment of writing (rustc 1.77.0), can_have_capability from psl does not eliminate joins +# code from bundle for some reason, so we are doing it explicitly. Check with a newer version of compiler - if elimination +# happens successfully, we don't need this feature anymore +relation_joins = [] + [dependencies] psl.workspace = true -quaint = { workspace = true, features = ["all-native", "pooled"] } +quaint.workspace = true serde.workspace = true serde_json.workspace = true -user-facing-errors = { path = "../../../libs/user-facing-errors", features = [ - "all-native", -] } +user-facing-errors = { path = "../../../libs/user-facing-errors" } chrono.workspace = true enumflags2.workspace = true diff --git a/schema-engine/connectors/schema-connector/src/introspection_context.rs b/schema-engine/connectors/schema-connector/src/introspection_context.rs index d19e7b7acf83..bfd7078ea983 100644 --- a/schema-engine/connectors/schema-connector/src/introspection_context.rs +++ b/schema-engine/connectors/schema-connector/src/introspection_context.rs @@ -109,10 +109,15 @@ impl IntrospectionContext { /// The SQL family we're using currently. pub fn sql_family(&self) -> SqlFamily { match self.datasource().active_provider { + #[cfg(feature = "postgresql")] "postgresql" => SqlFamily::Postgres, + #[cfg(feature = "cockroachdb")] "cockroachdb" => SqlFamily::Postgres, + #[cfg(feature = "sqlite")] "sqlite" => SqlFamily::Sqlite, + #[cfg(feature = "mssql")] "sqlserver" => SqlFamily::Mssql, + #[cfg(feature = "mysql")] "mysql" => SqlFamily::Mysql, name => unreachable!("The name `{}` for the datamodel connector is not known", name), } From 6d879e6162d060bed70f005206ce798a0c06687f Mon Sep 17 00:00:00 2001 From: jkomyno Date: Wed, 18 Dec 2024 21:02:36 +0100 Subject: [PATCH 2/5] feat(schema-engine): create "schema-engine-wasm" skeleton, close ORM-474 --- schema-engine/schema-engine-wasm/.gitignore | 7 + schema-engine/schema-engine-wasm/.nvmrc | 1 + schema-engine/schema-engine-wasm/Cargo.toml | 35 +++++ schema-engine/schema-engine-wasm/build.rs | 3 + schema-engine/schema-engine-wasm/build.sh | 128 ++++++++++++++++++ schema-engine/schema-engine-wasm/package.json | 5 + .../schema-engine-wasm/src/engine.rs | 89 ++++++++++++ schema-engine/schema-engine-wasm/src/lib.rs | 1 + 8 files changed, 269 insertions(+) create mode 100644 schema-engine/schema-engine-wasm/.gitignore create mode 100644 schema-engine/schema-engine-wasm/.nvmrc create mode 100644 schema-engine/schema-engine-wasm/Cargo.toml create mode 100644 schema-engine/schema-engine-wasm/build.rs create mode 100755 schema-engine/schema-engine-wasm/build.sh create mode 100644 schema-engine/schema-engine-wasm/package.json create mode 100644 schema-engine/schema-engine-wasm/src/engine.rs create mode 100644 schema-engine/schema-engine-wasm/src/lib.rs diff --git a/schema-engine/schema-engine-wasm/.gitignore b/schema-engine/schema-engine-wasm/.gitignore new file mode 100644 index 000000000000..a6f0e4dca125 --- /dev/null +++ b/schema-engine/schema-engine-wasm/.gitignore @@ -0,0 +1,7 @@ +/target +**/*.rs.bk +Cargo.lock +bin/ +pkg/ +wasm-pack.log +node_modules/ \ No newline at end of file diff --git a/schema-engine/schema-engine-wasm/.nvmrc b/schema-engine/schema-engine-wasm/.nvmrc new file mode 100644 index 000000000000..6569dfa4f323 --- /dev/null +++ b/schema-engine/schema-engine-wasm/.nvmrc @@ -0,0 +1 @@ +20.8.1 diff --git a/schema-engine/schema-engine-wasm/Cargo.toml b/schema-engine/schema-engine-wasm/Cargo.toml new file mode 100644 index 000000000000..b1c77eeb1dcc --- /dev/null +++ b/schema-engine/schema-engine-wasm/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "schema-engine-wasm" +version = "0.1.0" +edition = "2021" + +[lib] +doc = false +crate-type = ["cdylib"] +name = "schema_engine_wasm" + +[features] +sqlite = ["driver-adapters/sqlite", "psl/sqlite"] +postgresql = ["driver-adapters/postgresql", "psl/postgresql"] +mysql = ["driver-adapters/mysql", "psl/mysql"] + +[dependencies] +psl.workspace = true +quaint.workspace = true +tracing.workspace = true + +js-sys.workspace = true +tsify.workspace = true +wasm-bindgen.workspace = true +wasm-bindgen-futures.workspace = true +wasm-rs-dbg.workspace = true +driver-adapters = { path = "../../query-engine/driver-adapters" } + +[build-dependencies] +build-utils.path = "../../libs/build-utils" + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false # use wasm-opt explicitly in `./build.sh` + +[package.metadata.wasm-pack.profile.profiling] +wasm-opt = false # use wasm-opt explicitly in `./build.sh` diff --git a/schema-engine/schema-engine-wasm/build.rs b/schema-engine/schema-engine-wasm/build.rs new file mode 100644 index 000000000000..33aded23a4a5 --- /dev/null +++ b/schema-engine/schema-engine-wasm/build.rs @@ -0,0 +1,3 @@ +fn main() { + build_utils::store_git_commit_hash_in_env(); +} diff --git a/schema-engine/schema-engine-wasm/build.sh b/schema-engine/schema-engine-wasm/build.sh new file mode 100755 index 000000000000..ab751e4f8f1d --- /dev/null +++ b/schema-engine/schema-engine-wasm/build.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Call this script as `./build.sh ` +set -euo pipefail + +CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +REPO_ROOT="$( cd "$( dirname "$CURRENT_DIR/../../../" )" >/dev/null 2>&1 && pwd )" +OUT_VERSION="${1:-"0.0.0"}" +OUT_FOLDER="${2:-"schema-engine/schema-engine-wasm/pkg"}" +OUT_TARGET="bundler" +# wasm-opt pass +WASM_OPT_ARGS=( + "-Os" # execute size-focused optimization passes (-Oz actually increases size by 1KB) + "--vacuum" # removes obviously unneeded code + "--duplicate-function-elimination" # removes duplicate functions + "--duplicate-import-elimination" # removes duplicate imports + "--remove-unused-module-elements" # removes unused module elements + "--dae-optimizing" # removes arguments to calls in an lto-like manner + "--remove-unused-names" # removes names from location that are never branched to + "--rse" # removes redundant local.sets + "--gsi" # global struct inference, to optimize constant values + "--gufa-optimizing" # optimize the entire program using type monomorphization + "--strip-dwarf" # removes DWARF debug information + "--strip-producers" # removes the "producers" section + "--strip-target-features" # removes the "target_features" section +) + +# if it's a relative path, let it be relative to the repo root +if [[ "$OUT_FOLDER" != /* ]]; then + OUT_FOLDER="$REPO_ROOT/$OUT_FOLDER" +fi +OUT_JSON="${OUT_FOLDER}/package.json" + +echo "ℹ️ target version: $OUT_VERSION" +echo "ℹ️ out folder: $OUT_FOLDER" + +if [[ -z "${WASM_BUILD_PROFILE:-}" ]]; then + if [[ -z "${BUILDKITE:-}" ]] && [[ -z "${GITHUB_ACTIONS:-}" ]]; then + WASM_BUILD_PROFILE="dev" + else + WASM_BUILD_PROFILE="release" + fi +fi + +if [ "$WASM_BUILD_PROFILE" = "dev" ]; then + WASM_TARGET_SUBDIR="debug" +else + WASM_TARGET_SUBDIR="$WASM_BUILD_PROFILE" +fi + + + +build() { + echo "ℹ️ Note that schema-engine compiled to WASM uses a different Rust toolchain" + cargo --version + + local CONNECTOR="$1" + local CARGO_TARGET_DIR + CARGO_TARGET_DIR=$(cargo metadata --format-version 1 | jq -r .target_directory) + echo "🔨 Building $CONNECTOR" + CARGO_PROFILE_RELEASE_OPT_LEVEL="z" cargo build \ + -p schema-engine-wasm \ + --profile "$WASM_BUILD_PROFILE" \ + --features "$CONNECTOR" \ + --target wasm32-unknown-unknown + + local IN_FILE="$CARGO_TARGET_DIR/wasm32-unknown-unknown/$WASM_TARGET_SUBDIR/schema_engine_wasm.wasm" + local OUT_FILE="$OUT_FOLDER/$CONNECTOR/schema_engine_bg.wasm" + + wasm-bindgen --target "$OUT_TARGET" --out-name schema_engine --out-dir "$OUT_FOLDER/$CONNECTOR" "$IN_FILE" + optimize "$OUT_FILE" + + if ! command -v wasm2wat &> /dev/null; then + echo "Skipping wasm2wat, as it is not installed." + else + wasm2wat "$OUT_FILE" -o "./schema_engine.$CONNECTOR.wat" + fi +} + +optimize() { + local OUT_FILE="$1" + case "$WASM_BUILD_PROFILE" in + release) + # In release mode, we want to strip the debug symbols. + wasm-opt "${WASM_OPT_ARGS[@]}" \ + "--strip-debug" \ + "$OUT_FILE" \ + -o "$OUT_FILE" + ;; + profiling) + # In profiling mode, we want to keep the debug symbols. + wasm-opt "${WASM_OPT_ARGS[@]}" \ + "--debuginfo" \ + "${OUT_FILE}" \ + -o "${OUT_FILE}" + ;; + *) + # In other modes (e.g., "dev"), skip wasm-opt. + echo "Skipping wasm-opt." + ;; + esac +} + +report_size() { + local CONNECTOR + local GZ_SIZE + local FORMATTED_GZ_SIZE + + CONNECTOR="$1" + GZ_SIZE=$(gzip -c "${OUT_FOLDER}/$CONNECTOR/schema_engine_bg.wasm" | wc -c) + FORMATTED_GZ_SIZE=$(echo "$GZ_SIZE"|numfmt --format '%.3f' --to=iec-i --suffix=B) + + echo "$CONNECTOR:" + echo "ℹ️ raw: $(du -h "${OUT_FOLDER}/$CONNECTOR/schema_engine_bg.wasm")" + echo "ℹ️ zip: $GZ_SIZE bytes ($FORMATTED_GZ_SIZE)" + echo "" +} + +echo "Building schema-engine-wasm using $WASM_BUILD_PROFILE profile" + +build "postgresql" +build "sqlite" +build "mysql" + +jq '.version=$version' --arg version "$OUT_VERSION" package.json > "$OUT_JSON" + +report_size "postgresql" +report_size "sqlite" +report_size "mysql" diff --git a/schema-engine/schema-engine-wasm/package.json b/schema-engine/schema-engine-wasm/package.json new file mode 100644 index 000000000000..782528ba0925 --- /dev/null +++ b/schema-engine/schema-engine-wasm/package.json @@ -0,0 +1,5 @@ +{ + "name": "@prisma/schema-engine-wasm", + "version": "0.0.0", + "type": "module" +} \ No newline at end of file diff --git a/schema-engine/schema-engine-wasm/src/engine.rs b/schema-engine/schema-engine-wasm/src/engine.rs new file mode 100644 index 000000000000..495ddb72d17f --- /dev/null +++ b/schema-engine/schema-engine-wasm/src/engine.rs @@ -0,0 +1,89 @@ +#![allow(dead_code)] +#![allow(unused_variables)] + +use driver_adapters::JsObject; +use psl::{ConnectorRegistry, ValidatedSchema}; +use quaint::connector::ExternalConnector; +use std::sync::Arc; +// use quaint::connector::ExternalConnector; +use wasm_bindgen::prelude::wasm_bindgen; + +const CONNECTOR_REGISTRY: ConnectorRegistry<'_> = &[ + #[cfg(feature = "postgresql")] + psl::builtin_connectors::POSTGRES, + #[cfg(feature = "mysql")] + psl::builtin_connectors::MYSQL, + #[cfg(feature = "sqlite")] + psl::builtin_connectors::SQLITE, +]; + +#[wasm_bindgen] +extern "C" { + /// This function registers the reason for a Wasm panic via the + /// JS function `globalThis.PRISMA_WASM_PANIC_REGISTRY.set_message()` + #[wasm_bindgen(js_namespace = ["global", "PRISMA_WASM_PANIC_REGISTRY"], js_name = "set_message")] + fn prisma_set_wasm_panic_message(s: &str); +} + +/// Registers a singleton panic hook that will register the reason for the Wasm panic in JS. +/// Without this, the panic message would be lost: you'd see `RuntimeError: unreachable` message in JS, +/// with no reference to the Rust function and line that panicked. +/// This function should be manually called before any other public function in this module. +/// Note: no method is safe to call after a panic has occurred. +fn register_panic_hook() { + use std::sync::Once; + static SET_HOOK: Once = Once::new(); + + SET_HOOK.call_once(|| { + std::panic::set_hook(Box::new(|info| { + let message = &info.to_string(); + prisma_set_wasm_panic_message(message); + })); + }); +} + +/// The main query engine used by JS +#[wasm_bindgen] +pub struct SchemaEngine { + schema: ValidatedSchema, + adapter: Arc, +} + +#[wasm_bindgen] +pub struct SchemaEngineParams { + // TODO: support multiple datamodels + datamodel: String, +} + +#[wasm_bindgen] +impl SchemaEngine { + #[wasm_bindgen(constructor)] + pub fn new(params: SchemaEngineParams, adapter: JsObject) -> Result { + let SchemaEngineParams { datamodel, .. } = params; + + // Note: if we used `psl::validate`, we'd add ~1MB to the Wasm artifact (before gzip). + let schema = psl::parse_without_validation(datamodel.into(), CONNECTOR_REGISTRY); + let js_queryable = Arc::new(driver_adapters::from_js(adapter)); + + tracing::info!(git_hash = env!("GIT_HASH"), "Starting schema-engine-wasm"); + + Ok(Self { + schema, + adapter: js_queryable, + }) + } + + #[wasm_bindgen] + pub async fn debug_panic(&self) { + register_panic_hook(); + panic!("This is the debugPanic artificial panic") + } + + #[wasm_bindgen] + pub async fn can_connect_to_database(&self, datasource_url: String) -> Result<(), wasm_bindgen::JsError> { + register_panic_hook(); + Err(wasm_bindgen::JsError::new( + "You need to enable driverAdapters first in Wasm.", + )) + } +} diff --git a/schema-engine/schema-engine-wasm/src/lib.rs b/schema-engine/schema-engine-wasm/src/lib.rs new file mode 100644 index 000000000000..863383918949 --- /dev/null +++ b/schema-engine/schema-engine-wasm/src/lib.rs @@ -0,0 +1 @@ +mod engine; From 27befa7d302f661addb53cbbf84a63f38c10688a Mon Sep 17 00:00:00 2001 From: jkomyno Date: Wed, 18 Dec 2024 21:03:49 +0100 Subject: [PATCH 3/5] feat(schema-engine): create test playground for schema-engine-wasm, close ORM-460 --- query-engine/connector-test-kit-rs/README.md | 2 +- .../query-tests-setup/src/config.rs | 2 +- .../driver-adapters/executor/package.json | 10 +- .../executor/script/testd-qe.sh | 2 + .../executor/script/testd-se.sh | 2 + .../driver-adapters/executor/script/testd.sh | 2 - .../driver-adapters/executor/src/bench.ts | 2 +- .../src/{wasm.ts => query-engine-wasm.ts} | 4 +- .../executor/src/{qe.ts => query-engine.ts} | 2 +- .../executor/src/schema-engine-wasm.ts | 30 ++++ .../executor/src/schema-engine.ts | 23 +++ .../driver-adapters/executor/src/setup.ts | 33 ++++ .../executor/src/{testd.ts => testd-qe.ts} | 56 +------ .../driver-adapters/executor/src/testd-se.ts | 143 ++++++++++++++++++ .../driver-adapters/executor/src/utils.ts | 16 +- 15 files changed, 265 insertions(+), 64 deletions(-) create mode 100755 query-engine/driver-adapters/executor/script/testd-qe.sh create mode 100755 query-engine/driver-adapters/executor/script/testd-se.sh delete mode 100755 query-engine/driver-adapters/executor/script/testd.sh rename query-engine/driver-adapters/executor/src/{wasm.ts => query-engine-wasm.ts} (90%) rename query-engine/driver-adapters/executor/src/{qe.ts => query-engine.ts} (95%) create mode 100644 query-engine/driver-adapters/executor/src/schema-engine-wasm.ts create mode 100644 query-engine/driver-adapters/executor/src/schema-engine.ts create mode 100644 query-engine/driver-adapters/executor/src/setup.ts rename query-engine/driver-adapters/executor/src/{testd.ts => testd-qe.ts} (79%) create mode 100644 query-engine/driver-adapters/executor/src/testd-se.ts diff --git a/query-engine/connector-test-kit-rs/README.md b/query-engine/connector-test-kit-rs/README.md index ef8396f48045..440445cf5398 100644 --- a/query-engine/connector-test-kit-rs/README.md +++ b/query-engine/connector-test-kit-rs/README.md @@ -89,7 +89,7 @@ To run tests through a driver adapters, you should also configure the following Example: ```shell -export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/query-engine/driver-adapters/executor/script/testd.sh" +export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/query-engine/driver-adapters/executor/script/testd-qe.sh" export DRIVER_ADAPTER=neon export ENGINE=wasm export DRIVER_ADAPTER_CONFIG ='{ "proxyUrl": "127.0.0.1:5488/v1" }' diff --git a/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs b/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs index f287f3e4782b..20e530697aa9 100644 --- a/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs +++ b/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs @@ -348,7 +348,7 @@ impl TestConfig { } pub fn external_test_executor_path(&self) -> Option { - const DEFAULT_TEST_EXECUTOR: &str = "query-engine/driver-adapters/executor/script/testd.sh"; + const DEFAULT_TEST_EXECUTOR: &str = "query-engine/driver-adapters/executor/script/testd-qe.sh"; self.with_driver_adapter() .and_then(|_| { Self::workspace_root().or_else(|| { diff --git a/query-engine/driver-adapters/executor/package.json b/query-engine/driver-adapters/executor/package.json index 5d770fb484f0..35a4405d7c9b 100644 --- a/query-engine/driver-adapters/executor/package.json +++ b/query-engine/driver-adapters/executor/package.json @@ -8,15 +8,19 @@ "description": "", "private": true, "scripts": { - "build": "tsup ./src/testd.ts ./src/bench.ts --format esm --dts", - "test": "node --import tsx ./src/testd.ts", + "build": "tsup ./src/testd-qe.ts ./src/testd-se.ts ./src/bench.ts --format esm --dts", + "test:qe": "node --import tsx ./src/testd-qe.ts", + "test:se": "node --import tsx ./src/testd-se.ts", "clean:d1": "rm -rf ../../connector-test-kit-rs/query-engine-tests/.wrangler" }, "tsup": { "external": [ "../../../query-engine-wasm/pkg/postgresql/query_engine_bg.js", "../../../query-engine-wasm/pkg/mysql/query_engine_bg.js", - "../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js" + "../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js", + "../../../schema-engine-wasm/pkg/postgresql/schema_engine_bg.js", + "../../../schema-engine-wasm/pkg/mysql/schema_engine_bg.js", + "../../../schema-engine-wasm/pkg/sqlite/schema_engine_bg.js" ] }, "keywords": [], diff --git a/query-engine/driver-adapters/executor/script/testd-qe.sh b/query-engine/driver-adapters/executor/script/testd-qe.sh new file mode 100755 index 000000000000..a5b1a27faa96 --- /dev/null +++ b/query-engine/driver-adapters/executor/script/testd-qe.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +node "$(dirname "${BASH_SOURCE[0]}")/../dist/testd-qe.mjs" diff --git a/query-engine/driver-adapters/executor/script/testd-se.sh b/query-engine/driver-adapters/executor/script/testd-se.sh new file mode 100755 index 000000000000..a5b1a27faa96 --- /dev/null +++ b/query-engine/driver-adapters/executor/script/testd-se.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +node "$(dirname "${BASH_SOURCE[0]}")/../dist/testd-qe.mjs" diff --git a/query-engine/driver-adapters/executor/script/testd.sh b/query-engine/driver-adapters/executor/script/testd.sh deleted file mode 100755 index b61fb5deb981..000000000000 --- a/query-engine/driver-adapters/executor/script/testd.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -node "$(dirname "${BASH_SOURCE[0]}")/../dist/testd.mjs" diff --git a/query-engine/driver-adapters/executor/src/bench.ts b/query-engine/driver-adapters/executor/src/bench.ts index 7aaccc20d488..bd0a4827cdeb 100644 --- a/query-engine/driver-adapters/executor/src/bench.ts +++ b/query-engine/driver-adapters/executor/src/bench.ts @@ -7,7 +7,7 @@ import * as fs from "node:fs/promises"; import path from "node:path"; import { __dirname } from './utils' -import * as qe from "./qe"; +import * as qe from "./query-engine"; import { pg } from "@prisma/bundled-js-drivers"; import * as prismaPg from "@prisma/adapter-pg"; diff --git a/query-engine/driver-adapters/executor/src/wasm.ts b/query-engine/driver-adapters/executor/src/query-engine-wasm.ts similarity index 90% rename from query-engine/driver-adapters/executor/src/wasm.ts rename to query-engine/driver-adapters/executor/src/query-engine-wasm.ts index c60d54f398c3..675329c6154c 100644 --- a/query-engine/driver-adapters/executor/src/wasm.ts +++ b/query-engine/driver-adapters/executor/src/query-engine-wasm.ts @@ -3,7 +3,7 @@ import * as wasmMysql from '../../../query-engine-wasm/pkg/mysql/query_engine_bg import * as wasmSqlite from '../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js' import fs from 'node:fs/promises' import path from 'node:path' -import { __dirname } from './utils' +import { __dirname } from './utils.js' const wasm = { postgres: wasmPostgres, @@ -15,7 +15,7 @@ type EngineName = keyof typeof wasm const initializedModules = new Set() -export async function getEngineForProvider(provider: EngineName) { +export async function getQueryEngineForProvider(provider: EngineName) { const engine = wasm[provider] if (!initializedModules.has(provider)) { const subDir = provider === 'postgres' ? 'postgresql' : provider diff --git a/query-engine/driver-adapters/executor/src/qe.ts b/query-engine/driver-adapters/executor/src/query-engine.ts similarity index 95% rename from query-engine/driver-adapters/executor/src/qe.ts rename to query-engine/driver-adapters/executor/src/query-engine.ts index a3f385413e33..c3838aa274a4 100644 --- a/query-engine/driver-adapters/executor/src/qe.ts +++ b/query-engine/driver-adapters/executor/src/query-engine.ts @@ -33,7 +33,7 @@ export async function initQueryEngine( const options = queryEngineOptions(datamodel); if (engineType === "Wasm") { - const { getEngineForProvider } = await import("./wasm"); + const { getQueryEngineForProvider: getEngineForProvider } = await import("./query-engine-wasm"); const WasmQueryEngine = await getEngineForProvider(adapter.provider) return new WasmQueryEngine(options, logCallback, adapter); } else { diff --git a/query-engine/driver-adapters/executor/src/schema-engine-wasm.ts b/query-engine/driver-adapters/executor/src/schema-engine-wasm.ts new file mode 100644 index 000000000000..bb43db85ebb0 --- /dev/null +++ b/query-engine/driver-adapters/executor/src/schema-engine-wasm.ts @@ -0,0 +1,30 @@ +import * as wasmPostgres from '../../../../schema-engine/schema-engine-wasm/pkg/postgresql/schema_engine_bg.js' +import * as wasmMysql from "../../../../schema-engine/schema-engine-wasm/pkg/mysql/schema_engine_bg.js"; +import * as wasmSqlite from "../../../../schema-engine/schema-engine-wasm/pkg/sqlite/schema_engine_bg.js"; +import fs from 'node:fs/promises' +import path from 'node:path' +import { __dirname } from './utils.js' + +const wasm = { + postgres: wasmPostgres, + mysql: wasmMysql, + sqlite: wasmSqlite +} + +type EngineName = keyof typeof wasm + +const initializedModules = new Set() + +export async function getSchemaEngineForProvider(provider: EngineName) { + const engine = wasm[provider] + if (!initializedModules.has(provider)) { + const subDir = provider === 'postgres' ? 'postgresql' : provider + const bytes = await fs.readFile(path.resolve(__dirname, '..', '..', '..', 'schema-engine-wasm', 'pkg', subDir, 'schema_engine_bg.wasm')) + const module = new WebAssembly.Module(bytes) + const instance = new WebAssembly.Instance(module, { './schema_engine_bg.js': engine }) + engine.__wbg_set_wasm(instance.exports); + initializedModules.add(provider) + } + + return engine.SchemaEngine +} diff --git a/query-engine/driver-adapters/executor/src/schema-engine.ts b/query-engine/driver-adapters/executor/src/schema-engine.ts new file mode 100644 index 000000000000..10c1282b3c97 --- /dev/null +++ b/query-engine/driver-adapters/executor/src/schema-engine.ts @@ -0,0 +1,23 @@ +import type { DriverAdapter } from "@prisma/driver-adapter-utils"; +import { __dirname } from './utils' + +export interface SchemaEngine { + connect(trace: string, requestId: string): Promise; + disconnect(trace: string, requestId: string): Promise; + query(body: string, trace: string, tx_id: string | undefined, requestId: string): Promise; + startTransaction(input: string, trace: string, requestId: string): Promise; + commitTransaction(tx_id: string, trace: string, requestId: string): Promise; + rollbackTransaction(tx_id: string, trace: string, requestId: string): Promise; +} + +export type QueryLogCallback = (log: string) => void; + +export async function initSchemaEngine( + adapter: DriverAdapter, + datamodel: string, + debug: (...args: any[]) => void +): Promise { + const { getSchemaEngineForProvider: getEngineForProvider } = await import("./schema-engine-wasm"); + const WasmQueryEngine = await getEngineForProvider(adapter.provider) + return new WasmQueryEngine(adapter); +} diff --git a/query-engine/driver-adapters/executor/src/setup.ts b/query-engine/driver-adapters/executor/src/setup.ts new file mode 100644 index 000000000000..c57a7922a522 --- /dev/null +++ b/query-engine/driver-adapters/executor/src/setup.ts @@ -0,0 +1,33 @@ +import { match } from 'ts-pattern'; +import { type DriverAdaptersManager } from './driver-adapters-manager'; +import type { Env } from './types'; +import { PgManager } from "./driver-adapters-manager/pg"; +import { NeonWsManager } from "./driver-adapters-manager/neon.ws"; +import { LibSQLManager } from "./driver-adapters-manager/libsql"; +import { PlanetScaleManager } from "./driver-adapters-manager/planetscale"; +import { D1Manager } from "./driver-adapters-manager/d1"; + +export async function setupDriverAdaptersManager( + env: Env, + migrationScript?: string +): Promise { + return match(env) + .with({ DRIVER_ADAPTER: "pg" }, async (env) => await PgManager.setup(env)) + .with( + { DRIVER_ADAPTER: "neon:ws" }, + async (env) => await NeonWsManager.setup(env) + ) + .with( + { DRIVER_ADAPTER: "libsql" }, + async (env) => await LibSQLManager.setup(env) + ) + .with( + { DRIVER_ADAPTER: "planetscale" }, + async (env) => await PlanetScaleManager.setup(env) + ) + .with( + { DRIVER_ADAPTER: "d1" }, + async (env) => await D1Manager.setup(env, migrationScript) + ) + .exhaustive(); +} diff --git a/query-engine/driver-adapters/executor/src/testd.ts b/query-engine/driver-adapters/executor/src/testd-qe.ts similarity index 79% rename from query-engine/driver-adapters/executor/src/testd.ts rename to query-engine/driver-adapters/executor/src/testd-qe.ts index 6a5a3665da25..52a8fdc5dd47 100644 --- a/query-engine/driver-adapters/executor/src/testd.ts +++ b/query-engine/driver-adapters/executor/src/testd-qe.ts @@ -1,65 +1,17 @@ import * as readline from "node:readline"; -import { match } from "ts-pattern"; import * as S from "@effect/schema/Schema"; import { bindAdapter, ErrorCapturingDriverAdapter, } from "@prisma/driver-adapter-utils"; -import { webcrypto } from "node:crypto"; import type { DriverAdaptersManager } from "./driver-adapters-manager"; import { jsonRpc, Env } from "./types"; -import * as qe from "./qe"; -import { PgManager } from "./driver-adapters-manager/pg"; -import { NeonWsManager } from "./driver-adapters-manager/neon.ws"; -import { LibSQLManager } from "./driver-adapters-manager/libsql"; -import { PlanetScaleManager } from "./driver-adapters-manager/planetscale"; -import { D1Manager } from "./driver-adapters-manager/d1"; +import * as qe from "./query-engine"; import { nextRequestId } from "./requestId"; import { createRNEngineConnector } from "./rn"; - -if (!global.crypto) { - global.crypto = webcrypto as Crypto; -} - -async function initialiseDriverAdapterManager( - env: Env, - migrationScript?: string -): Promise { - return match(env) - .with({ DRIVER_ADAPTER: "pg" }, async (env) => await PgManager.setup(env)) - .with( - { DRIVER_ADAPTER: "neon:ws" }, - async (env) => await NeonWsManager.setup(env) - ) - .with( - { DRIVER_ADAPTER: "libsql" }, - async (env) => await LibSQLManager.setup(env) - ) - .with( - { DRIVER_ADAPTER: "planetscale" }, - async (env) => await PlanetScaleManager.setup(env) - ) - .with( - { DRIVER_ADAPTER: "d1" }, - async (env) => await D1Manager.setup(env, migrationScript) - ) - .exhaustive(); -} - -// conditional debug logging based on LOG_LEVEL env var -const debug = (() => { - if ((process.env.LOG_LEVEL ?? "").toLowerCase() != "debug") { - return (...args: any[]) => {}; - } - - return (...args: any[]) => { - console.error("[nodejs] DEBUG:", ...args); - }; -})(); - -// error logger -const err = (...args: any[]) => console.error("[nodejs] ERROR:", ...args); +import { debug, err } from "./utils"; +import { setupDriverAdaptersManager } from "./setup"; async function main(): Promise { const env = S.decodeUnknownSync(Env)(process.env); @@ -116,7 +68,7 @@ async function handleRequest( logs.push(log); }; - const driverAdapterManager = await initialiseDriverAdapterManager( + const driverAdapterManager = await setupDriverAdaptersManager( env, migrationScript ); diff --git a/query-engine/driver-adapters/executor/src/testd-se.ts b/query-engine/driver-adapters/executor/src/testd-se.ts new file mode 100644 index 000000000000..c36f341a3ff3 --- /dev/null +++ b/query-engine/driver-adapters/executor/src/testd-se.ts @@ -0,0 +1,143 @@ +import * as readline from "node:readline"; +import * as S from "@effect/schema/Schema"; +import { + bindAdapter, + ErrorCapturingDriverAdapter, +} from "@prisma/driver-adapter-utils"; + +import type { DriverAdaptersManager } from "./driver-adapters-manager"; +import { jsonRpc, Env } from "./types"; +import * as se from "./schema-engine"; +import { debug, err } from "./utils"; +import { setupDriverAdaptersManager } from "./setup"; + +async function main(): Promise { + const env = S.decodeUnknownSync(Env)(process.env); + console.log("[env]", env); + + const iface = readline.createInterface({ + input: process.stdin, + output: process.stdout, + terminal: false, + }); + + iface.on("line", async (line) => { + try { + const request = S.decodeSync(jsonRpc.RequestFromString)(line); + debug(`Got a request: ${line}`); + + try { + const response = await handleRequest(request, env); + respondOk(request.id, response); + } catch (err) { + debug("[nodejs] Error from request handler: ", err); + respondErr(request.id, { + code: 1, + message: err.stack ?? err.toString(), + }); + } + } catch (err) { + debug("Received non-json line: ", line); + console.error(err); + } + }); +} + +const state: Record< + number, + { + engine: se.SchemaEngine; + driverAdapterManager: DriverAdaptersManager; + adapter: ErrorCapturingDriverAdapter | null; + logs: string[]; + } +> = {}; + +async function handleRequest( + { method, params }: jsonRpc.Request, + env: Env +): Promise { + switch (method) { + case "initializeSchema": { + const { url, schema, schemaId, migrationScript } = params; + const logs = [] as string[]; + + const logCallback = (log) => { + logs.push(log); + }; + + const driverAdapterManager = await setupDriverAdaptersManager( + env, + migrationScript + ); + + const { engine, adapter } = await initSe({ + env, + url, + driverAdapterManager, + schema, + }); + + state[schemaId] = { + engine, + driverAdapterManager, + adapter, + logs, + }; + + if (adapter && adapter.getConnectionInfo) { + const maxBindValuesResult = adapter.getConnectionInfo().map(info => info.maxBindValues) + if (maxBindValuesResult.ok) { + return { maxBindValues: maxBindValuesResult.value } + } + } + + return { maxBindValues: null } + } + default: { + throw new Error(`Unknown method: \`${method}\``); + } + } +} + +function respondErr(requestId: number, error: jsonRpc.RpcError) { + const msg: jsonRpc.ErrResponse = { + jsonrpc: "2.0", + id: requestId, + error, + }; + console.log(JSON.stringify(msg)); +} + +function respondOk(requestId: number, payload: unknown) { + const msg: jsonRpc.OkResponse = { + jsonrpc: "2.0", + id: requestId, + result: payload, + }; + console.log(JSON.stringify(msg)); +} + +type InitSchemaEngineParams = { + env: Env; + driverAdapterManager: DriverAdaptersManager; + url: string; + schema: string; +}; + +async function initSe({ env, driverAdapterManager, url, schema }: InitSchemaEngineParams) { + const adapter = await driverAdapterManager.connect({ url }) + const errorCapturingAdapter = bindAdapter(adapter) + const engineInstance = await se.initSchemaEngine( + errorCapturingAdapter, + schema, + debug, + ) + + return { + engine: engineInstance, + adapter: errorCapturingAdapter, + } +} + +main().catch(err); diff --git a/query-engine/driver-adapters/executor/src/utils.ts b/query-engine/driver-adapters/executor/src/utils.ts index f46e44dd2d95..ef9db446a6c3 100644 --- a/query-engine/driver-adapters/executor/src/utils.ts +++ b/query-engine/driver-adapters/executor/src/utils.ts @@ -25,7 +25,7 @@ export function postgres_options(url: string): PostgresOptions { const schemaName = postgresSchemaName(url) if (schemaName != null) { - args.options = `--search_path="${schemaName}"` + args.options = `--search_path='${schemaName}'` } return args @@ -40,3 +40,17 @@ export async function runBatch(D1_DATABASE: D1Database, statements: return D1_DATABASE.batch(statements) } + +// conditional debug logging based on LOG_LEVEL env var +export const debug = (() => { + if ((process.env.LOG_LEVEL ?? '').toLowerCase() != 'debug') { + return (...args: any[]) => {} + } + + return (...args: any[]) => { + console.error('[nodejs] DEBUG:', ...args) + } +})() + +// error logger +export const err = (...args: any[]) => console.error("[nodejs] ERROR:", ...args); From e3a93c9a759a6cf165acbfd0ad68537c80d8687c Mon Sep 17 00:00:00 2001 From: jkomyno Date: Wed, 18 Dec 2024 21:04:07 +0100 Subject: [PATCH 4/5] chore: update Cargo.toml --- Cargo.lock | 16 ++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 8167930b59e1..bc153aa63cd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4576,6 +4576,22 @@ dependencies = [ "user-facing-errors", ] +[[package]] +name = "schema-engine-wasm" +version = "0.1.0" +dependencies = [ + "build-utils", + "driver-adapters", + "js-sys", + "psl", + "quaint", + "tracing", + "tsify", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-rs-dbg", +] + [[package]] name = "scopeguard" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index 397b1c70e471..91c1421b6bdd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ members = [ "prisma-fmt", "prisma-schema-wasm", "psl/*", - "quaint", + "quaint", "schema-engine/schema-engine-wasm", ] [workspace.dependencies] From 359d66035edabec005351c4976cefefa9557f3c8 Mon Sep 17 00:00:00 2001 From: jkomyno Date: Wed, 18 Dec 2024 21:35:38 +0100 Subject: [PATCH 5/5] feat(schema-engine): let "sql-schema-describer" be wasm-compatible; partial revert of prisma/prisma-engines/pull/3093; close ORM-456 --- schema-engine/sql-schema-describer/Cargo.toml | 37 ++++++++++--- schema-engine/sql-schema-describer/src/lib.rs | 8 +++ .../sql-schema-describer/src/sqlite.rs | 54 ++++--------------- 3 files changed, 48 insertions(+), 51 deletions(-) diff --git a/schema-engine/sql-schema-describer/Cargo.toml b/schema-engine/sql-schema-describer/Cargo.toml index 17b8eae63684..31e74cf5f835 100644 --- a/schema-engine/sql-schema-describer/Cargo.toml +++ b/schema-engine/sql-schema-describer/Cargo.toml @@ -3,9 +3,38 @@ edition = "2021" name = "sql-schema-describer" version = "0.1.0" +[features] +postgresql = ["relation_joins", "quaint/postgresql", "psl/postgresql"] +postgresql-native = ["postgresql", "quaint/postgresql-native", "quaint/pooled"] +mysql = ["relation_joins", "quaint/mysql", "psl/mysql"] +mysql-native = ["mysql", "quaint/mysql-native", "quaint/pooled"] +sqlite = ["quaint/sqlite", "psl/sqlite"] +sqlite-native = ["sqlite", "quaint/sqlite-native", "quaint/pooled"] +mssql = ["quaint/mssql"] +mssql-native = ["mssql", "quaint/mssql-native", "quaint/pooled"] +cockroachdb = ["relation_joins", "quaint/postgresql", "psl/cockroachdb"] +cockroachdb-native = [ + "cockroachdb", + "quaint/postgresql-native", + "quaint/pooled", +] +vendored-openssl = ["quaint/vendored-openssl"] +all-native = [ + "sqlite-native", + "mysql-native", + "postgresql-native", + "mssql-native", + "cockroachdb-native", +] +# TODO: At the moment of writing (rustc 1.77.0), can_have_capability from psl does not eliminate joins +# code from bundle for some reason, so we are doing it explicitly. Check with a newer version of compiler - if elimination +# happens successfully, we don't need this feature anymore +relation_joins = [] + [dependencies] prisma-value = { path = "../../libs/prisma-value" } -psl = { workspace = true, features = ["all"] } +psl.workspace = true +quaint.workspace = true either = "1.8.0" async-trait.workspace = true @@ -19,12 +48,6 @@ serde.workspace = true tracing.workspace = true tracing-error = "0.2" tracing-futures.workspace = true -quaint = { workspace = true, features = [ - "all-native", - "pooled", - "expose-drivers", - "fmt-sql", -] } [dev-dependencies] expect-test = "1.2.2" diff --git a/schema-engine/sql-schema-describer/src/lib.rs b/schema-engine/sql-schema-describer/src/lib.rs index 8f65c175b2a3..b7c86bf72e6c 100644 --- a/schema-engine/sql-schema-describer/src/lib.rs +++ b/schema-engine/sql-schema-describer/src/lib.rs @@ -3,10 +3,18 @@ #![deny(rust_2018_idioms, unsafe_code)] #![allow(clippy::derive_partial_eq_without_eq)] +#[cfg(feature = "mssql")] pub mod mssql; + +#[cfg(feature = "mysql")] pub mod mysql; + +#[cfg(any(feature = "postgresql", feature = "cockroachdb"))] pub mod postgres; + +#[cfg(feature = "sqlite")] pub mod sqlite; + pub mod walkers; mod connector_data; diff --git a/schema-engine/sql-schema-describer/src/sqlite.rs b/schema-engine/sql-schema-describer/src/sqlite.rs index bd82c52fce0e..9130d1622c67 100644 --- a/schema-engine/sql-schema-describer/src/sqlite.rs +++ b/schema-engine/sql-schema-describer/src/sqlite.rs @@ -8,9 +8,9 @@ use crate::{ use either::Either; use indexmap::IndexMap; use quaint::{ - ast::{Value, ValueType}, - connector::{ColumnType as QuaintColumnType, GetRow, ToColumnNames}, - prelude::ResultRow, + ast::Value, + prelude::{Queryable, ResultRow}, + ValueType, }; use std::{any::type_name, borrow::Cow, collections::BTreeMap, convert::TryInto, fmt::Debug, path::Path}; use tracing::trace; @@ -24,44 +24,8 @@ pub trait Connection { ) -> quaint::Result; } -#[async_trait::async_trait] -impl Connection for std::sync::Mutex { - async fn query_raw<'a>( - &'a self, - sql: &'a str, - params: &'a [quaint::prelude::Value<'a>], - ) -> quaint::Result { - let conn = self.lock().unwrap(); - let mut stmt = conn.prepare_cached(sql)?; - let column_types = stmt.columns().iter().map(QuaintColumnType::from).collect::>(); - let mut rows = stmt.query(quaint::connector::rusqlite::params_from_iter(params.iter()))?; - let column_names = rows.to_column_names(); - let mut converted_rows = Vec::new(); - while let Some(row) = rows.next()? { - converted_rows.push(row.get_result_row().unwrap()); - } - - Ok(quaint::prelude::ResultSet::new( - column_names, - column_types, - converted_rows, - )) - } -} - -#[async_trait::async_trait] -impl Connection for quaint::single::Quaint { - async fn query_raw<'a>( - &'a self, - sql: &'a str, - params: &'a [quaint::prelude::Value<'a>], - ) -> quaint::Result { - quaint::prelude::Queryable::query_raw(self, sql, params).await - } -} - pub struct SqlSchemaDescriber<'a> { - conn: &'a (dyn Connection + Send + Sync), + conn: &'a dyn Queryable, } impl Debug for SqlSchemaDescriber<'_> { @@ -92,7 +56,9 @@ impl SqlSchemaDescriberBackend for SqlSchemaDescriber<'_> { } async fn version(&self) -> DescriberResult> { - Ok(Some(quaint::connector::sqlite_version().to_owned())) + // TODO: implement `SELECT version` via Driver Adapters + // Ok(Some(quaint::connector::sqlite_version().to_owned())) + Ok(None) } } @@ -100,7 +66,7 @@ impl Parser for SqlSchemaDescriber<'_> {} impl<'a> SqlSchemaDescriber<'a> { /// Constructor. - pub fn new(conn: &'a (dyn Connection + Send + Sync)) -> SqlSchemaDescriber<'a> { + pub fn new(conn: &'a dyn Queryable) -> SqlSchemaDescriber<'a> { SqlSchemaDescriber { conn } } @@ -330,7 +296,7 @@ async fn push_columns( table_name: &str, container_id: Either, schema: &mut SqlSchema, - conn: &(dyn Connection + Send + Sync), + conn: &dyn Queryable, ) -> DescriberResult<()> { let sql = format!(r#"PRAGMA table_info ("{table_name}")"#); let result_set = conn.query_raw(&sql, &[]).await?; @@ -469,7 +435,7 @@ async fn push_indexes( table: &str, table_id: TableId, schema: &mut SqlSchema, - conn: &(dyn Connection + Send + Sync), + conn: &dyn Queryable, ) -> DescriberResult<()> { let sql = format!(r#"PRAGMA index_list("{table}");"#); let result_set = conn.query_raw(&sql, &[]).await?;