Main function for ParcelV3 and FileSystem implementation (#9758)

* checkpoint

* getting somewhere

* async for napi

* tidy up

* Tidy up

* Tidy

* integraiton tests

* reverting send/sync

* Update

* Unused dependency

* linting

* lock file

* Refactor to use thread-safe functions directly

* unused imports, utf8

* integration tests

* Fix warnings and add logs

* Remove some log lines

* Revert +nightly bound from rustfmt

---------

Co-authored-by: pyamada (Remote Dev Environment) <[email protected]>

crates/json-comments-rs/Cargo.toml

@@ -2,7 +2,7 @@
 name = "json_comments"
 version = "0.2.1"
 authors = ["Thayne McCombs <[email protected]>"]
-edition = "2018"
+edition = "2021"
 license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/tmccombs/json-comments-rs"

crates/node-bindings/Cargo.toml

@@ -23,6 +23,8 @@ parcel_napi_helpers = { path = "../parcel_napi_helpers" }
 dashmap = "5.4.0"
 xxhash-rust = { version = "0.8.2", features = ["xxh3"] }
 log = "0.4.21"
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
 
 glob = "0.3.1"
 serde = "1.0.198"
@@ -33,11 +35,23 @@ mockall = "0.12.1"
 parking_lot = "0.12"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-sentry = { version = "0.32.2", optional = true, default-features = false, features = ["backtrace", "contexts", "panic", "reqwest", "debug-images", "anyhow"]}
+sentry = { version = "0.32.2", optional = true, default-features = false, features = [
+    "backtrace",
+    "contexts",
+    "panic",
+    "reqwest",
+    "debug-images",
+    "anyhow",
+] }
 whoami = { version = "1.5.1", optional = true }
 once_cell = { version = "1.19.0", optional = true }
 
-napi = { version = "2.16.4", features = ["serde-json", "napi4", "napi5"] }
+napi = { version = "2.16.4", features = [
+    "serde-json",
+    "napi4",
+    "napi5",
+    "async",
+] }
 parcel-dev-dep-resolver = { path = "../../packages/utils/dev-dep-resolver" }
 parcel-macros = { path = "../macros", features = ["napi"] }
 oxipng = "8.0.0"

crates/node-bindings/src/file_system/file_system_napi.rs

@@ -0,0 +1,93 @@
+use std::path::Path;
+use std::path::PathBuf;
+
+use napi::bindgen_prelude::FromNapiValue;
+use napi::threadsafe_function::ThreadsafeFunctionCallMode;
+use napi::Env;
+use napi::JsFunction;
+use napi::JsObject;
+use parcel_filesystem::FileSystem;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+// TODO error handling
+
+pub struct FileSystemNapi {
+  read_file_fn: Box<dyn Fn((PathBuf, String)) -> String + Send + Sync>,
+  is_file_fn: Box<dyn Fn(PathBuf) -> bool + Send + Sync>,
+  is_dir_fn: Box<dyn Fn(PathBuf) -> bool + Send + Sync>,
+}
+
+impl FileSystemNapi {
+  pub fn new(env: &Env, js_file_system: JsObject) -> napi::Result<Self> {
+    Ok(Self {
+      read_file_fn: Box::new(create_js_thread_safe_method(
+        &env,
+        &js_file_system,
+        "readFileSync",
+      )?),
+      is_file_fn: Box::new(create_js_thread_safe_method(
+        &env,
+        &js_file_system,
+        "isFile",
+      )?),
+      is_dir_fn: Box::new(create_js_thread_safe_method(
+        &env,
+        &js_file_system,
+        "isDir",
+      )?),
+    })
+  }
+}
+
+// These methods must be run off the nodejs main/worker
+// thread or they will cause JavaScript to deadlock
+impl FileSystem for FileSystemNapi {
+  fn read_to_string(&self, path: &Path) -> std::io::Result<String> {
+    Ok((*self.read_file_fn)((
+      path.to_path_buf(),
+      "utf8".to_string(),
+    )))
+  }
+
+  fn is_file(&self, path: &Path) -> bool {
+    (*self.is_file_fn)(path.to_path_buf())
+  }
+
+  fn is_dir(&self, path: &Path) -> bool {
+    (*self.is_dir_fn)(path.to_path_buf())
+  }
+}
+
+fn create_js_thread_safe_method<
+  Params: Send + Serialize + 'static,
+  Response: Send + DeserializeOwned + 'static + FromNapiValue,
+>(
+  env: &Env,
+  js_file_system: &JsObject,
+  method_name: &str,
+) -> Result<impl Fn(Params) -> Response, napi::Error> {
+  let jsfn: JsFunction = js_file_system.get_property(env.create_string(method_name)?)?;
+
+  let threadsafe_function = env.create_threadsafe_function(
+    &jsfn,
+    0,
+    |ctx: napi::threadsafe_function::ThreadSafeCallContext<Params>| {
+      Ok(vec![ctx.env.to_js_value(&ctx.value)?])
+    },
+  )?;
+  let result = move |params| {
+    let (tx, rx) = std::sync::mpsc::sync_channel(1);
+    threadsafe_function.call_with_return_value(
+      Ok(params),
+      ThreadsafeFunctionCallMode::Blocking,
+      move |result: Response| {
+        let _ = tx.send(result);
+        Ok(())
+      },
+    );
+    rx.recv().unwrap()
+  };
+
+  Ok(result)
+}

crates/node-bindings/src/file_system/mod.rs

@@ -0,0 +1,2 @@
+pub mod file_system_napi;
+pub use self::file_system_napi::*;

crates/node-bindings/src/lib.rs

@@ -15,12 +15,15 @@ static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
+mod file_system;
+
 /// napi versions of `crate::core::requests`
 #[cfg(not(target_arch = "wasm32"))]
 mod fs_search;
 mod hash;
 #[cfg(not(target_arch = "wasm32"))]
 mod image;
+pub mod parcel;
 mod resolver;
 mod transformer;
 

crates/node-bindings/src/parcel/mod.rs

@@ -0,0 +1 @@
+pub mod parcel;

crates/node-bindings/src/parcel/parcel.rs

@@ -0,0 +1,56 @@
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use napi::Env;
+use napi::JsObject;
+use napi_derive::napi;
+use parcel_core::FileSystemRef;
+use parcel_core::Parcel;
+use parcel_core::ParcelOptions;
+
+use crate::file_system::FileSystemNapi;
+
+#[napi]
+pub struct ParcelNapi {
+  internal: Arc<Parcel>,
+}
+
+#[napi]
+impl ParcelNapi {
+  #[napi(constructor)]
+  pub fn new(env: Env, options: JsObject) -> napi::Result<Self> {
+    let _ = tracing_subscriber::fmt::try_init();
+
+    let thread_id = std::thread::current().id();
+    tracing::trace!(?thread_id, "parcel-napi initialize");
+
+    let mut fs = None::<FileSystemRef>;
+
+    if options.has_named_property("fs")? {
+      let fs_raw: JsObject = options.get_named_property("fs")?;
+      fs.replace(Arc::new(FileSystemNapi::new(&env, fs_raw)?));
+    }
+
+    let parcel = Parcel::new(ParcelOptions { fs });
+
+    Ok(Self {
+      internal: Arc::new(parcel),
+    })
+  }
+
+  // Temporary, for testing
+  #[napi]
+  pub async fn _testing_temp_fs_read_to_string(&self, path: String) -> napi::Result<String> {
+    Ok(self.internal.fs.read_to_string(&PathBuf::from(path))?)
+  }
+
+  #[napi]
+  pub async fn _testing_temp_fs_is_file(&self, path: String) -> napi::Result<bool> {
+    Ok(self.internal.fs.is_file(&PathBuf::from(path)))
+  }
+
+  #[napi]
+  pub async fn _testing_temp_fs_is_dir(&self, path: String) -> napi::Result<bool> {
+    Ok(self.internal.fs.is_dir(&PathBuf::from(path)))
+  }
+}

crates/parcel_core/src/lib.rs

@@ -3,8 +3,9 @@
 
 pub mod bundle_graph;
 pub mod hash;
+pub mod parcel;
 pub mod plugin;
 pub mod request_tracker;
 pub mod types;
 
-// pub use parcel::*;
+pub use parcel::*;

crates/parcel_core/src/parcel/parcel.rs

@@ -1,12 +1,24 @@
-// TODO eventual public API for Parcel
-pub struct Parcel {}
+use std::sync::Arc;
+
+use parcel_filesystem::os_file_system::OsFileSystem;
+use parcel_filesystem::FileSystem;
+
+pub type FileSystemRef = Arc<dyn FileSystem + Send + Sync>;
+
+pub struct Parcel {
+  pub fs: FileSystemRef,
+}
+
+pub struct ParcelOptions {
+  pub fs: Option<FileSystemRef>,
+}
 
 impl Parcel {
-  pub fn run() {
-    todo!();
-  }
+  pub fn new(options: ParcelOptions) -> Self {
+    let fs = options
+      .fs
+      .unwrap_or_else(|| Arc::new(OsFileSystem::default()));
 
-  pub fn watch() {
-    todo!();
+    Self { fs }
   }
 }

crates/parcel_core/src/types.rs

@@ -1,3 +1,6 @@
+// Re-export this from core, probably want to move this type here
+pub use parcel_filesystem::FileSystem;
+
 mod asset;
 pub use self::asset::*;
 

packages/core/integration-tests/test/parcel-v3.js

@@ -0,0 +1,40 @@
+// @flow
+
+import assert from 'assert';
+import path from 'path';
+import {bundle, run} from '@parcel/test-utils';
+import * as napi from '@parcel/rust';
+import {inputFS} from '@parcel/test-utils';
+
+describe('parcel-v3', function () {
+  // Duplicated temporarily for convenience, will remove once the Rust stuff works
+  it.skip('should produce a basic JS bundle with CommonJS requires', async function () {
+    let b = await bundle(
+      path.join(__dirname, '/integration/commonjs/index.js'),
+      {
+        featureFlags: {parcelV3: true},
+      },
+    );
+
+    // assert.equal(b.assets.size, 8);
+    // assert.equal(b.childBundles.size, 1);
+
+    let output = await run(b);
+    assert.equal(typeof output, 'function');
+    assert.equal(output(), 3);
+  });
+
+  it('should run the main-thread bootstrap function', async function () {
+    let p = new napi.ParcelNapi({
+      fs: {
+        readFileSync: (_, [...args]) => inputFS.readFileSync(...args),
+        isFile: (_, path) => inputFS.statSync(path).isFile(),
+        isDir: (_, path) => inputFS.statSync(path).isDirectory(),
+      },
+    });
+
+    assert(typeof (await p.testingTempFsReadToString(__filename)) === 'string');
+    assert(!(await p.testingTempFsIsDir(__filename)));
+    assert(await p.testingTempFsIsFile(__filename));
+  });
+});

packages/core/rust/index.js.flow

@@ -5,27 +5,48 @@ declare export var init: void | (() => void);
 
 export type ProjectPath = any;
 export interface ConfigRequest {
-  id: string,
-  invalidateOnFileChange: Array<ProjectPath>,
-  invalidateOnConfigKeyChange: Array<any>,
-  invalidateOnFileCreate: Array<any>,
-  invalidateOnEnvChange: Array<string>,
-  invalidateOnOptionChange: Array<string>,
-  invalidateOnStartup: boolean,
-  invalidateOnBuild: boolean,
+  id: string;
+  invalidateOnFileChange: Array<ProjectPath>;
+  invalidateOnConfigKeyChange: Array<any>;
+  invalidateOnFileCreate: Array<any>;
+  invalidateOnEnvChange: Array<string>;
+  invalidateOnOptionChange: Array<string>;
+  invalidateOnStartup: boolean;
+  invalidateOnBuild: boolean;
+}
+export interface RequestOptions {}
+
+export interface ParcelNapiOptions {
+  fs?: any;
 }
-export interface RequestOptions {
+
+declare export class ParcelNapi {
+  constructor(options: ParcelNapiOptions): ParcelNapi;
+  testingTempFsReadToString(path: string): string;
+  testingTempFsIsDir(path: string): boolean;
+  testingTempFsIsFile(path: string): boolean;
 }
 
 declare export function initSentry(): void;
 declare export function closeSentry(): void;
-declare export function napiRunConfigRequest(configRequest: ConfigRequest, api: any, options: any): void
-declare export function findAncestorFile(filenames: Array<string>, from: string, root: string): string | null
-declare export function findFirstFile(names: Array<string>): string | null
-declare export function findNodeModule(module: string, from: string): string | null
-declare export function hashString(s: string): string
-declare export function hashBuffer(buf: Buffer): string
-declare export function optimizeImage(kind: string, buf: Buffer): Buffer
+declare export function napiRunConfigRequest(
+  configRequest: ConfigRequest,
+  api: any,
+  options: any,
+): void;
+declare export function findAncestorFile(
+  filenames: Array<string>,
+  from: string,
+  root: string,
+): string | null;
+declare export function findFirstFile(names: Array<string>): string | null;
+declare export function findNodeModule(
+  module: string,
+  from: string,
+): string | null;
+declare export function hashString(s: string): string;
+declare export function hashBuffer(buf: Buffer): string;
+declare export function optimizeImage(kind: string, buf: Buffer): Buffer;
 export interface JsFileSystemOptions {
   canonicalize: string => string;
   read: string => Buffer;

rustfmt.toml

@@ -1 +1,2 @@
+edition = "2021"
 tab_spaces = 2