From c391ad315ef75fb32298e87f9b80781fb11043ac Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 30 Dec 2024 12:38:20 -0500 Subject: [PATCH] refactor: use sys_traits (#27480) --- Cargo.lock | 79 +++++-- Cargo.toml | 9 +- cli/Cargo.toml | 3 +- cli/args/deno_json.rs | 56 ----- cli/args/lockfile.rs | 6 +- cli/args/mod.rs | 8 +- cli/cache/deno_dir.rs | 153 ++---------- cli/cache/disk_cache.rs | 10 +- cli/cache/mod.rs | 137 +---------- cli/factory.rs | 40 ++-- cli/file_fetcher.rs | 55 +++-- cli/graph_util.rs | 162 ++++--------- cli/lsp/cache.rs | 3 +- cli/lsp/config.rs | 93 +------- cli/lsp/diagnostics.rs | 4 +- cli/lsp/language_server.rs | 5 +- cli/lsp/registries.rs | 3 +- cli/lsp/resolver.rs | 37 ++- cli/module_loader.rs | 79 ++++--- cli/node.rs | 25 +- cli/npm/byonm.rs | 8 +- cli/npm/managed/mod.rs | 100 ++++---- cli/npm/managed/resolvers/common.rs | 13 +- cli/npm/managed/resolvers/global.rs | 6 +- cli/npm/managed/resolvers/local.rs | 38 +-- cli/npm/managed/resolvers/mod.rs | 8 +- cli/npm/mod.rs | 53 +---- cli/resolver.rs | 96 +++----- cli/standalone/binary.rs | 1 - cli/standalone/code_cache.rs | 6 +- cli/standalone/file_system.rs | 1 + cli/standalone/mod.rs | 47 ++-- cli/tools/coverage/mod.rs | 3 +- cli/tools/doc.rs | 5 +- cli/tools/fmt.rs | 3 +- cli/tools/lint/linter.rs | 6 +- cli/tools/lint/mod.rs | 3 +- cli/tools/registry/paths.rs | 7 +- cli/tools/registry/unfurl.rs | 6 +- cli/tsc/mod.rs | 12 +- cli/util/fs.rs | 318 +------------------------ cli/util/path.rs | 14 -- ext/fs/Cargo.toml | 2 + ext/fs/interface.rs | 342 ++++++++++++++++++++++++++- ext/fs/lib.rs | 2 + ext/fs/sync.rs | 6 + ext/node/lib.rs | 96 ++------ resolvers/deno/Cargo.toml | 1 + resolvers/deno/cjs.rs | 21 +- resolvers/deno/fs.rs | 22 -- resolvers/deno/lib.rs | 41 ++-- resolvers/deno/npm/byonm.rs | 117 +++++---- resolvers/deno/npm/mod.rs | 48 ++-- resolvers/node/Cargo.toml | 1 + resolvers/node/analyze.rs | 41 ++-- resolvers/node/env.rs | 39 --- resolvers/node/lib.rs | 2 +- resolvers/node/package_json.rs | 25 +- resolvers/node/resolution.rs | 101 ++++---- resolvers/npm_cache/Cargo.toml | 2 + resolvers/npm_cache/fs_util.rs | 99 ++++++++ resolvers/npm_cache/lib.rs | 76 ++++-- resolvers/npm_cache/registry_info.rs | 95 ++++++-- resolvers/npm_cache/tarball.rs | 61 ++++- tests/Cargo.toml | 1 + tests/integration/jsr_tests.rs | 2 +- 66 files changed, 1372 insertions(+), 1592 deletions(-) delete mode 100644 resolvers/deno/fs.rs delete mode 100644 resolvers/node/env.rs create mode 100644 resolvers/npm_cache/fs_util.rs diff --git a/Cargo.lock b/Cargo.lock index 5e86a7647c04ed..0fb94936291999 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -867,6 +867,7 @@ dependencies = [ "regex", "reqwest", "serde", + "sys_traits", "test_server", "tokio", "url", @@ -1274,7 +1275,7 @@ dependencies = [ "deno_npm", "deno_npm_cache", "deno_package_json", - "deno_path_util", + "deno_path_util 0.3.0", "deno_resolver", "deno_runtime", "deno_semver", @@ -1340,6 +1341,7 @@ dependencies = [ "spki", "sqlformat", "strsim", + "sys_traits", "tar", "tempfile", "test_server", @@ -1452,9 +1454,9 @@ dependencies = [ [[package]] name = "deno_cache_dir" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54df1c5177ace01d92b872584ab9af8290681bb150fd9b423c37a494ad5ddbdc" +checksum = "e73ed17f285731a23df9779ca1e0e721de866db6776ed919ebd9235e0a107c4c" dependencies = [ "async-trait", "base32", @@ -1465,7 +1467,7 @@ dependencies = [ "data-url", "deno_error", "deno_media_type", - "deno_path_util", + "deno_path_util 0.3.0", "http 1.1.0", "indexmap 2.3.0", "log", @@ -1474,6 +1476,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "sys_traits", "thiserror 1.0.64", "url", ] @@ -1491,13 +1494,13 @@ dependencies = [ [[package]] name = "deno_config" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afa3beb6b9e0604cfe0380d30f88c5b758d44e228d5a5fc42ae637ccfb7d089" +checksum = "b45aaf31e58ca915d5c0746bf8e2d07b94635154ad9e5afe5ff265cae6187b19" dependencies = [ "anyhow", "deno_package_json", - "deno_path_util", + "deno_path_util 0.3.0", "deno_semver", "glob", "ignore", @@ -1509,6 +1512,7 @@ dependencies = [ "phf", "serde", "serde_json", + "sys_traits", "thiserror 1.0.64", "url", ] @@ -1623,7 +1627,7 @@ dependencies = [ "comrak", "deno_ast", "deno_graph", - "deno_path_util", + "deno_path_util 0.2.2", "handlebars", "html-escape", "import_map", @@ -1673,7 +1677,7 @@ dependencies = [ "bytes", "data-url", "deno_core", - "deno_path_util", + "deno_path_util 0.3.0", "deno_permissions", "deno_tls", "dyn-clone", @@ -1730,15 +1734,17 @@ dependencies = [ "boxed_error", "deno_core", "deno_io", - "deno_path_util", + "deno_path_util 0.3.0", "deno_permissions", "filetime", + "getrandom", "junction", "libc", "nix", "rand", "rayon", "serde", + "sys_traits", "thiserror 2.0.3", "winapi", "windows-sys 0.59.0", @@ -1746,15 +1752,16 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.86.5" +version = "0.86.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f669d96d63841d9ba10f86b161d898678ce05bc1e3c9ee1c1f7449a68eed2b64" +checksum = "83af194ca492ea7b624d21055f933676d3f3d27586de93be31c8f1babcc73510" dependencies = [ "anyhow", "async-trait", "capacity_builder 0.5.0", "data-url", "deno_ast", + "deno_path_util 0.3.0", "deno_semver", "deno_unsync", "encoding_rs", @@ -1769,6 +1776,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "sys_traits", "thiserror 2.0.3", "twox-hash", "url", @@ -1847,7 +1855,7 @@ dependencies = [ "chrono", "deno_core", "deno_fetch", - "deno_path_util", + "deno_path_util 0.3.0", "deno_permissions", "deno_tls", "denokv_proto", @@ -1976,7 +1984,7 @@ dependencies = [ "deno_media_type", "deno_net", "deno_package_json", - "deno_path_util", + "deno_path_util 0.3.0", "deno_permissions", "deno_whoami", "der", @@ -2078,6 +2086,7 @@ dependencies = [ "deno_core", "deno_error", "deno_npm", + "deno_path_util 0.3.0", "deno_semver", "deno_unsync", "faster-hex", @@ -2090,6 +2099,7 @@ dependencies = [ "rand", "ring", "serde_json", + "sys_traits", "tar", "tempfile", "thiserror 2.0.3", @@ -2114,17 +2124,18 @@ dependencies = [ [[package]] name = "deno_package_json" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d72db99fdebfc371d7be16972c18a47daa7a29cb5fbb3900ab2114b1f42d96" +checksum = "e1d3c0f699ba2040669204ce24ab73720499fc290af843e4ce0fc8a9b3d67735" dependencies = [ "boxed_error", "deno_error", - "deno_path_util", + "deno_path_util 0.3.0", "deno_semver", "indexmap 2.3.0", "serde", "serde_json", + "sys_traits", "thiserror 2.0.3", "url", ] @@ -2141,13 +2152,26 @@ dependencies = [ "url", ] +[[package]] +name = "deno_path_util" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420e8211aaba7fde83ccaa9a5dad855c3b940ed988d70c95159acd600a70dc87" +dependencies = [ + "deno_error", + "percent-encoding", + "sys_traits", + "thiserror 2.0.3", + "url", +] + [[package]] name = "deno_permissions" version = "0.43.0" dependencies = [ "capacity_builder 0.5.0", "deno_core", - "deno_path_util", + "deno_path_util 0.3.0", "deno_terminal 0.2.0", "fqdn", "libc", @@ -2171,9 +2195,10 @@ dependencies = [ "deno_config", "deno_media_type", "deno_package_json", - "deno_path_util", + "deno_path_util 0.3.0", "deno_semver", "node_resolver", + "sys_traits", "test_server", "thiserror 2.0.3", "url", @@ -2201,7 +2226,7 @@ dependencies = [ "deno_napi", "deno_net", "deno_node", - "deno_path_util", + "deno_path_util 0.3.0", "deno_permissions", "deno_telemetry", "deno_terminal 0.2.0", @@ -5049,13 +5074,14 @@ dependencies = [ "boxed_error", "deno_media_type", "deno_package_json", - "deno_path_util", + "deno_path_util 0.3.0", "futures", "lazy-regex", "once_cell", "path-clean", "regex", "serde_json", + "sys_traits", "thiserror 2.0.3", "tokio", "url", @@ -7652,6 +7678,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "sys_traits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5a12729b699487bb50163466e87be7197871d83d04cc6815d430cf7c893bbd7" +dependencies = [ + "getrandom", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "tagptr" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 81c750f0af12ea..442680c332b579 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,11 +51,11 @@ deno_ast = { version = "=0.44.0", features = ["transpiling"] } deno_core = { version = "0.327.0" } deno_bench_util = { version = "0.178.0", path = "./bench_util" } -deno_config = { version = "=0.41.0", features = ["workspace", "sync"] } +deno_config = { version = "=0.42.0", features = ["workspace", "sync"] } deno_lockfile = "=0.24.0" deno_media_type = { version = "0.2.0", features = ["module_specifier"] } deno_npm = "=0.27.0" -deno_path_util = "=0.2.2" +deno_path_util = "=0.3.0" deno_permissions = { version = "0.43.0", path = "./runtime/permissions" } deno_runtime = { version = "0.192.0", path = "./runtime" } deno_semver = "=0.7.1" @@ -118,9 +118,9 @@ console_static_text = "=0.8.1" dashmap = "5.5.3" data-encoding = "2.3.3" data-url = "=0.3.1" -deno_cache_dir = "=0.15.0" +deno_cache_dir = "=0.16.0" deno_error = "=0.5.2" -deno_package_json = { version = "0.3.0", default-features = false } +deno_package_json = { version = "0.4.0", default-features = false } deno_unsync = "0.4.2" dlopen2 = "0.6.1" ecb = "=0.1.2" @@ -193,6 +193,7 @@ slab = "0.4" smallvec = "1.8" socket2 = { version = "0.5.3", features = ["all"] } spki = "0.7.2" +sys_traits = "=0.1.1" tar = "=0.4.40" tempfile = "3.4.0" termcolor = "1.1.3" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ad0e4fa95d7fee..2cf12f14d41a10 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -74,7 +74,7 @@ deno_config.workspace = true deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = { version = "=0.161.3", features = ["rust", "comrak"] } deno_error.workspace = true -deno_graph = { version = "=0.86.5" } +deno_graph = { version = "=0.86.6" } deno_lint = { version = "=0.68.2", features = ["docs"] } deno_lockfile.workspace = true deno_npm.workspace = true @@ -158,6 +158,7 @@ shell-escape = "=0.1.5" spki = { version = "0.7", features = ["pem"] } sqlformat = "=0.3.2" strsim = "0.11.1" +sys_traits = { workspace = true, features = ["libc", "real", "winapi"] } tar.workspace = true tempfile.workspace = true text-size = "=1.1.0" diff --git a/cli/args/deno_json.rs b/cli/args/deno_json.rs index 8853107eef08c7..47dcbb91ea7b8b 100644 --- a/cli/args/deno_json.rs +++ b/cli/args/deno_json.rs @@ -8,62 +8,6 @@ use deno_semver::jsr::JsrDepPackageReq; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; -#[cfg(test)] // happens to only be used by the tests at the moment -pub struct DenoConfigFsAdapter<'a>( - pub &'a dyn deno_runtime::deno_fs::FileSystem, -); - -#[cfg(test)] -impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> { - fn read_to_string_lossy( - &self, - path: &std::path::Path, - ) -> Result, std::io::Error> { - self - .0 - .read_text_file_lossy_sync(path, None) - .map_err(|err| err.into_io_error()) - } - - fn stat_sync( - &self, - path: &std::path::Path, - ) -> Result { - self - .0 - .stat_sync(path) - .map(|stat| deno_config::fs::FsMetadata { - is_file: stat.is_file, - is_directory: stat.is_directory, - is_symlink: stat.is_symlink, - }) - .map_err(|err| err.into_io_error()) - } - - fn read_dir( - &self, - path: &std::path::Path, - ) -> Result, std::io::Error> { - self - .0 - .read_dir_sync(path) - .map_err(|err| err.into_io_error()) - .map(|entries| { - entries - .into_iter() - .map(|e| deno_config::fs::FsDirEntry { - path: path.join(e.name), - metadata: deno_config::fs::FsMetadata { - is_file: e.is_file, - is_directory: e.is_directory, - is_symlink: e.is_symlink, - }, - }) - .collect() - }) - } -} - pub fn import_map_deps( import_map: &serde_json::Value, ) -> HashSet { diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index 1075f93a6fdd76..0648b6e5ed6887 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -12,12 +12,13 @@ use deno_core::parking_lot::MutexGuard; use deno_core::serde_json; use deno_lockfile::WorkspaceMemberConfig; use deno_package_json::PackageJsonDepValue; +use deno_path_util::fs::atomic_write_file_with_retries; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::PackageJson; use deno_semver::jsr::JsrDepPackageReq; use crate::args::deno_json::import_map_deps; use crate::cache; -use crate::util::fs::atomic_write_file_with_retries; use crate::Flags; use crate::args::DenoSubcommand; @@ -91,8 +92,9 @@ impl CliLockfile { // do an atomic write to reduce the chance of multiple deno // processes corrupting the file atomic_write_file_with_retries( + &FsSysTraitsAdapter::new_real(), &lockfile.filename, - bytes, + &bytes, cache::CACHE_PERM, ) .context("Failed writing lockfile.")?; diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 3f42a6b9d4442e..aa3a251105092c 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -30,6 +30,7 @@ use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; use deno_path_util::normalize_path; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_semver::npm::NpmPackageReqReference; use deno_semver::StackString; use deno_telemetry::OtelConfig; @@ -83,9 +84,9 @@ use std::num::NonZeroUsize; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use sys_traits::EnvHomeDir; use thiserror::Error; -use crate::cache; use crate::cache::DenoDirProvider; use crate::file_fetcher::CliFileFetcher; use crate::util::fs::canonicalize_path_maybe_not_exists; @@ -572,7 +573,7 @@ fn discover_npmrc( // TODO(bartlomieju): update to read both files - one in the project root and one and // home dir and then merge them. // 3. Try `.npmrc` in the user's home directory - if let Some(home_dir) = cache::home_dir() { + if let Some(home_dir) = sys_traits::impls::RealSys.env_home_dir() { match try_to_read_npmrc(&home_dir) { Ok(Some((source, path))) => { return try_to_parse_npmrc(source, &path).map(|r| (r, Some(path))); @@ -845,7 +846,6 @@ impl CliOptions { log::debug!("package.json auto-discovery is disabled"); } WorkspaceDiscoverOptions { - fs: Default::default(), // use real fs deno_json_cache: None, pkg_json_cache: Some(&node_resolver::PackageJsonThreadLocalCache), workspace_cache: None, @@ -867,6 +867,7 @@ impl CliOptions { ConfigFlag::Discover => { if let Some(start_paths) = flags.config_path_args(&initial_cwd) { WorkspaceDirectory::discover( + &FsSysTraitsAdapter::new_real(), WorkspaceDiscoverStart::Paths(&start_paths), &resolve_workspace_discover_options(), )? @@ -877,6 +878,7 @@ impl CliOptions { ConfigFlag::Path(path) => { let config_path = normalize_path(initial_cwd.join(path)); WorkspaceDirectory::discover( + &FsSysTraitsAdapter::new_real(), WorkspaceDiscoverStart::ConfigFile(&config_path), &resolve_workspace_discover_options(), )? diff --git a/cli/cache/deno_dir.rs b/cli/cache/deno_dir.rs index 7b7059c224df3c..d83ea8ebd5dea7 100644 --- a/cli/cache/deno_dir.rs +++ b/cli/cache/deno_dir.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use deno_cache_dir::DenoDirResolutionError; use once_cell::sync::OnceCell; use super::DiskCache; @@ -11,7 +12,7 @@ use std::path::PathBuf; /// where functionality wants to continue if the DENO_DIR can't be created. pub struct DenoDirProvider { maybe_custom_root: Option, - deno_dir: OnceCell>, + deno_dir: OnceCell>, } impl DenoDirProvider { @@ -22,12 +23,21 @@ impl DenoDirProvider { } } - pub fn get_or_create(&self) -> Result<&DenoDir, std::io::Error> { + pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> { self .deno_dir .get_or_init(|| DenoDir::new(self.maybe_custom_root.clone())) .as_ref() - .map_err(|err| std::io::Error::new(err.kind(), err.to_string())) + .map_err(|err| match err { + DenoDirResolutionError::NoCacheOrHomeDir => { + DenoDirResolutionError::NoCacheOrHomeDir + } + DenoDirResolutionError::FailedCwd { source } => { + DenoDirResolutionError::FailedCwd { + source: std::io::Error::new(source.kind(), source.to_string()), + } + } + }) } } @@ -42,27 +52,13 @@ pub struct DenoDir { } impl DenoDir { - pub fn new(maybe_custom_root: Option) -> std::io::Result { - let maybe_custom_root = - maybe_custom_root.or_else(|| env::var("DENO_DIR").map(String::into).ok()); - let root: PathBuf = if let Some(root) = maybe_custom_root { - root - } else if let Some(cache_dir) = dirs::cache_dir() { - // We use the OS cache dir because all files deno writes are cache files - // Once that changes we need to start using different roots if DENO_DIR - // is not set, and keep a single one if it is. - cache_dir.join("deno") - } else if let Some(home_dir) = dirs::home_dir() { - // fallback path - home_dir.join(".deno") - } else { - panic!("Could not set the Deno root directory") - }; - let root = if root.is_absolute() { - root - } else { - std::env::current_dir()?.join(root) - }; + pub fn new( + maybe_custom_root: Option, + ) -> Result { + let root = deno_cache_dir::resolve_deno_dir( + &sys_traits::impls::RealSys, + maybe_custom_root, + )?; assert!(root.is_absolute()); let gen_path = root.join("gen"); @@ -166,112 +162,3 @@ impl DenoDir { self.root.join("dl") } } - -/// To avoid the poorly managed dirs crate -#[cfg(not(windows))] -pub mod dirs { - use std::path::PathBuf; - - pub fn cache_dir() -> Option { - if cfg!(target_os = "macos") { - home_dir().map(|h| h.join("Library/Caches")) - } else { - std::env::var_os("XDG_CACHE_HOME") - .map(PathBuf::from) - .or_else(|| home_dir().map(|h| h.join(".cache"))) - } - } - - pub fn home_dir() -> Option { - std::env::var_os("HOME") - .and_then(|h| if h.is_empty() { None } else { Some(h) }) - .or_else(|| { - // TODO(bartlomieju): - #[allow(clippy::undocumented_unsafe_blocks)] - unsafe { - fallback() - } - }) - .map(PathBuf::from) - } - - // This piece of code is taken from the deprecated home_dir() function in Rust's standard library: https://github.com/rust-lang/rust/blob/master/src/libstd/sys/unix/os.rs#L579 - // The same code is used by the dirs crate - unsafe fn fallback() -> Option { - let amt = match libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) { - n if n < 0 => 512_usize, - n => n as usize, - }; - let mut buf = Vec::with_capacity(amt); - let mut passwd: libc::passwd = std::mem::zeroed(); - let mut result = std::ptr::null_mut(); - match libc::getpwuid_r( - libc::getuid(), - &mut passwd, - buf.as_mut_ptr(), - buf.capacity(), - &mut result, - ) { - 0 if !result.is_null() => { - let ptr = passwd.pw_dir as *const _; - let bytes = std::ffi::CStr::from_ptr(ptr).to_bytes().to_vec(); - Some(std::os::unix::ffi::OsStringExt::from_vec(bytes)) - } - _ => None, - } - } -} - -/// To avoid the poorly managed dirs crate -// Copied from -// https://github.com/dirs-dev/dirs-sys-rs/blob/ec7cee0b3e8685573d847f0a0f60aae3d9e07fa2/src/lib.rs#L140-L164 -// MIT license. Copyright (c) 2018-2019 dirs-rs contributors -#[cfg(windows)] -pub mod dirs { - use std::ffi::OsString; - use std::os::windows::ffi::OsStringExt; - use std::path::PathBuf; - use winapi::shared::winerror; - use winapi::um::combaseapi; - use winapi::um::knownfolders; - use winapi::um::shlobj; - use winapi::um::shtypes; - use winapi::um::winbase; - use winapi::um::winnt; - - fn known_folder(folder_id: shtypes::REFKNOWNFOLDERID) -> Option { - // SAFETY: winapi calls - unsafe { - let mut path_ptr: winnt::PWSTR = std::ptr::null_mut(); - let result = shlobj::SHGetKnownFolderPath( - folder_id, - 0, - std::ptr::null_mut(), - &mut path_ptr, - ); - if result == winerror::S_OK { - let len = winbase::lstrlenW(path_ptr) as usize; - let path = std::slice::from_raw_parts(path_ptr, len); - let ostr: OsString = OsStringExt::from_wide(path); - combaseapi::CoTaskMemFree(path_ptr as *mut winapi::ctypes::c_void); - Some(PathBuf::from(ostr)) - } else { - None - } - } - } - - pub fn cache_dir() -> Option { - known_folder(&knownfolders::FOLDERID_LocalAppData) - } - - pub fn home_dir() -> Option { - if let Some(userprofile) = std::env::var_os("USERPROFILE") { - if !userprofile.is_empty() { - return Some(PathBuf::from(userprofile)); - } - } - - known_folder(&knownfolders::FOLDERID_Profile) - } -} diff --git a/cli/cache/disk_cache.rs b/cli/cache/disk_cache.rs index 2fee1efe09bef9..b22b2e3cc770f4 100644 --- a/cli/cache/disk_cache.rs +++ b/cli/cache/disk_cache.rs @@ -1,11 +1,12 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use super::CACHE_PERM; -use crate::util::fs::atomic_write_file_with_retries; use deno_cache_dir::url_to_filename; use deno_core::url::Host; use deno_core::url::Url; +use deno_path_util::fs::atomic_write_file_with_retries; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use std::ffi::OsStr; use std::fs; use std::path::Component; @@ -120,7 +121,12 @@ impl DiskCache { pub fn set(&self, filename: &Path, data: &[u8]) -> std::io::Result<()> { let path = self.location.join(filename); - atomic_write_file_with_retries(&path, data, CACHE_PERM) + atomic_write_file_with_retries( + &FsSysTraitsAdapter::new_real(), + &path, + data, + CACHE_PERM, + ) } } diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 31968be0c2908f..c8b0eacaa49547 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -5,9 +5,6 @@ use crate::file_fetcher::CliFetchNoFollowErrorKind; use crate::file_fetcher::CliFileFetcher; use crate::file_fetcher::FetchNoFollowOptions; use crate::file_fetcher::FetchPermissionsOptionRef; -use crate::util::fs::atomic_write_file_with_retries; -use crate::util::fs::atomic_write_file_with_retries_and_fs; -use crate::util::fs::AtomicWriteFileFsAdapter; use deno_ast::MediaType; use deno_cache_dir::file_fetcher::CacheSetting; @@ -21,15 +18,12 @@ use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; use deno_graph::source::LoadResponse; use deno_graph::source::Loader; -use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_permissions::PermissionsContainer; use node_resolver::InNpmPackageChecker; -use std::borrow::Cow; use std::collections::HashMap; -use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use std::time::SystemTime; mod cache_db; mod caches; @@ -50,7 +44,6 @@ pub use caches::Caches; pub use check::TypeCheckCache; pub use code_cache::CodeCache; pub use common::FastInsecureHasher; -pub use deno_dir::dirs::home_dir; pub use deno_dir::DenoDir; pub use deno_dir::DenoDirProvider; pub use disk_cache::DiskCache; @@ -63,121 +56,12 @@ pub use parsed_source::LazyGraphSourceParser; pub use parsed_source::ParsedSourceCache; /// Permissions used to save a file in the disk caches. -pub const CACHE_PERM: u32 = 0o644; +pub use deno_cache_dir::CACHE_PERM; -#[derive(Debug, Clone)] -pub struct RealDenoCacheEnv; - -impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv { - fn read_file_bytes( - &self, - path: &Path, - ) -> std::io::Result> { - std::fs::read(path).map(Cow::Owned) - } - - fn atomic_write_file( - &self, - path: &Path, - bytes: &[u8], - ) -> std::io::Result<()> { - atomic_write_file_with_retries(path, bytes, CACHE_PERM) - } - - fn canonicalize_path(&self, path: &Path) -> std::io::Result { - crate::util::fs::canonicalize_path(path) - } - - fn create_dir_all(&self, path: &Path) -> std::io::Result<()> { - std::fs::create_dir_all(path) - } - - fn modified(&self, path: &Path) -> std::io::Result> { - match std::fs::metadata(path) { - Ok(metadata) => Ok(Some( - metadata.modified().unwrap_or_else(|_| SystemTime::now()), - )), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - } - } - - fn is_file(&self, path: &Path) -> bool { - path.is_file() - } - - fn time_now(&self) -> SystemTime { - SystemTime::now() - } -} - -#[derive(Debug, Clone)] -pub struct DenoCacheEnvFsAdapter<'a>( - pub &'a dyn deno_runtime::deno_fs::FileSystem, -); - -impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> { - fn read_file_bytes( - &self, - path: &Path, - ) -> std::io::Result> { - self - .0 - .read_file_sync(path, None) - .map_err(|err| err.into_io_error()) - } - - fn atomic_write_file( - &self, - path: &Path, - bytes: &[u8], - ) -> std::io::Result<()> { - atomic_write_file_with_retries_and_fs( - &AtomicWriteFileFsAdapter { - fs: self.0, - write_mode: CACHE_PERM, - }, - path, - bytes, - ) - } - - fn canonicalize_path(&self, path: &Path) -> std::io::Result { - self.0.realpath_sync(path).map_err(|e| e.into_io_error()) - } - - fn create_dir_all(&self, path: &Path) -> std::io::Result<()> { - self - .0 - .mkdir_sync(path, true, None) - .map_err(|e| e.into_io_error()) - } - - fn modified(&self, path: &Path) -> std::io::Result> { - self - .0 - .stat_sync(path) - .map(|stat| { - stat - .mtime - .map(|ts| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(ts)) - }) - .map_err(|e| e.into_io_error()) - } - - fn is_file(&self, path: &Path) -> bool { - self.0.is_file_sync(path) - } - - fn time_now(&self) -> SystemTime { - SystemTime::now() - } -} - -pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache; -pub type LocalHttpCache = deno_cache_dir::LocalHttpCache; +pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache; +pub type LocalHttpCache = deno_cache_dir::LocalHttpCache; pub type LocalLspHttpCache = - deno_cache_dir::LocalLspHttpCache; + deno_cache_dir::LocalLspHttpCache; pub use deno_cache_dir::HttpCache; pub struct FetchCacherOptions { @@ -192,11 +76,11 @@ pub struct FetchCacherOptions { pub struct FetchCacher { pub file_header_overrides: HashMap>, file_fetcher: Arc, - fs: Arc, global_http_cache: Arc, in_npm_pkg_checker: Arc, module_info_cache: Arc, permissions: PermissionsContainer, + sys: FsSysTraitsAdapter, is_deno_publish: bool, cache_info_enabled: bool, } @@ -204,18 +88,18 @@ pub struct FetchCacher { impl FetchCacher { pub fn new( file_fetcher: Arc, - fs: Arc, global_http_cache: Arc, in_npm_pkg_checker: Arc, module_info_cache: Arc, + sys: FsSysTraitsAdapter, options: FetchCacherOptions, ) -> Self { Self { file_fetcher, - fs, global_http_cache, in_npm_pkg_checker, module_info_cache, + sys, file_header_overrides: options.file_header_overrides, permissions: options.permissions, is_deno_publish: options.is_deno_publish, @@ -277,9 +161,8 @@ impl Loader for FetchCacher { // symlinked to `/my-project-2/node_modules`), so first we checked if the path // is in a node_modules dir to avoid needlessly canonicalizing, then now compare // against the canonicalized specifier. - let specifier = crate::node::resolve_specifier_into_node_modules( - specifier, - self.fs.as_ref(), + let specifier = node_resolver::resolve_specifier_into_node_modules( + &self.sys, specifier, ); if self.in_npm_pkg_checker.in_npm_package(&specifier) { return Box::pin(futures::future::ready(Ok(Some( diff --git a/cli/factory.rs b/cli/factory.rs index d6940d6df11d15..5c73537743cf77 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -11,7 +11,6 @@ use crate::args::StorageKeyResolver; use crate::args::TsConfigType; use crate::cache::Caches; use crate::cache::CodeCache; -use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::DenoDir; use crate::cache::DenoDirProvider; use crate::cache::EmitCache; @@ -43,7 +42,6 @@ use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::CreateInNpmPkgCheckerOptions; use crate::resolver::CjsTracker; use crate::resolver::CliDenoResolver; -use crate::resolver::CliDenoResolverFs; use crate::resolver::CliNpmReqResolver; use crate::resolver::CliResolver; use crate::resolver::CliResolverOptions; @@ -76,9 +74,10 @@ use deno_resolver::npm::NpmReqResolverOptions; use deno_resolver::DenoResolverOptions; use deno_resolver::NodeAndNpmReqResolver; use deno_runtime::deno_fs; -use deno_runtime::deno_node::DenoFsNodeResolverEnv; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_node::PackageJsonResolver; +use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use deno_runtime::deno_permissions::Permissions; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_tls::rustls::RootCertStore; @@ -318,8 +317,8 @@ impl CliFactory { pub fn global_http_cache(&self) -> Result<&Arc, AnyError> { self.services.global_http_cache.get_or_try_init(|| { Ok(Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter(self.fs().clone()), self.deno_dir()?.remote_folder_path(), - crate::cache::RealDenoCacheEnv, ))) }) } @@ -396,7 +395,7 @@ impl CliFactory { let global_path = self.deno_dir()?.npm_folder_path(); let cli_options = self.cli_options()?; Ok(Arc::new(NpmCacheDir::new( - &DenoCacheEnvFsAdapter(fs.as_ref()), + &FsSysTraitsAdapter(fs.clone()), global_path, cli_options.npmrc().get_all_known_registries_urls(), ))) @@ -416,7 +415,7 @@ impl CliFactory { create_cli_npm_resolver(if cli_options.use_byonm() { CliNpmResolverCreateOptions::Byonm( CliByonmNpmResolverCreateOptions { - fs: CliDenoResolverFs(fs.clone()), + sys: FsSysTraitsAdapter(fs.clone()), pkg_json_resolver: self.pkg_json_resolver().clone(), root_node_modules_dir: Some( match cli_options.node_modules_dir_path() { @@ -434,6 +433,13 @@ impl CliFactory { } else { CliNpmResolverCreateOptions::Managed( CliManagedNpmResolverCreateOptions { + http_client_provider: self.http_client_provider().clone(), + npm_install_deps_provider: Arc::new( + NpmInstallDepsProvider::from_workspace( + cli_options.workspace(), + ), + ), + sys: FsSysTraitsAdapter(self.fs().clone()), snapshot: match cli_options.resolve_npm_resolution_snapshot()? { Some(snapshot) => { CliNpmResolverManagedSnapshotOption::Specified(Some( @@ -452,19 +458,12 @@ impl CliFactory { }, }, maybe_lockfile: cli_options.maybe_lockfile().cloned(), - fs: fs.clone(), - http_client_provider: self.http_client_provider().clone(), npm_cache_dir: self.npm_cache_dir()?.clone(), cache_setting: cli_options.cache_setting(), text_only_progress_bar: self.text_only_progress_bar().clone(), maybe_node_modules_path: cli_options .node_modules_dir_path() .cloned(), - npm_install_deps_provider: Arc::new( - NpmInstallDepsProvider::from_workspace( - cli_options.workspace(), - ), - ), npm_system_info: cli_options.npm_system_info(), npmrc: cli_options.npmrc().clone(), lifecycle_scripts: cli_options.lifecycle_scripts_config(), @@ -487,7 +486,7 @@ impl CliFactory { .get_or_try_init(|| { Ok(self.cli_options()?.unstable_sloppy_imports().then(|| { Arc::new(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new( - self.fs().clone(), + FsSysTraitsAdapter(self.fs().clone()), ))) })) }) @@ -655,14 +654,15 @@ impl CliFactory { .get_or_try_init_async( async { Ok(Arc::new(NodeResolver::new( - DenoFsNodeResolverEnv::new(self.fs().clone()), self.in_npm_pkg_checker()?.clone(), + RealIsBuiltInNodeModuleChecker, self .npm_resolver() .await? .clone() .into_npm_pkg_folder_resolver(), self.pkg_json_resolver().clone(), + FsSysTraitsAdapter(self.fs().clone()), ))) } .boxed_local(), @@ -690,7 +690,6 @@ impl CliFactory { Ok(Arc::new(NodeCodeTranslator::new( cjs_esm_analyzer, - DenoFsNodeResolverEnv::new(self.fs().clone()), self.in_npm_pkg_checker()?.clone(), node_resolver, self @@ -699,6 +698,7 @@ impl CliFactory { .clone() .into_npm_pkg_folder_resolver(), self.pkg_json_resolver().clone(), + FsSysTraitsAdapter(self.fs().clone()), ))) }) .await @@ -714,7 +714,7 @@ impl CliFactory { let npm_resolver = self.npm_resolver().await?; Ok(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions { byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(), - fs: CliDenoResolverFs(self.fs().clone()), + sys: FsSysTraitsAdapter(self.fs().clone()), in_npm_pkg_checker: self.in_npm_pkg_checker()?.clone(), node_resolver: self.node_resolver().await?.clone(), npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(), @@ -725,7 +725,7 @@ impl CliFactory { pub fn pkg_json_resolver(&self) -> &Arc { self.services.pkg_json_resolver.get_or_init(|| { - Arc::new(PackageJsonResolver::new(DenoFsNodeResolverEnv::new( + Arc::new(PackageJsonResolver::new(FsSysTraitsAdapter( self.fs().clone(), ))) }) @@ -765,7 +765,6 @@ impl CliFactory { self.cjs_tracker()?.clone(), cli_options.clone(), self.file_fetcher()?.clone(), - self.fs().clone(), self.global_http_cache()?.clone(), self.in_npm_pkg_checker()?.clone(), cli_options.maybe_lockfile().cloned(), @@ -775,6 +774,7 @@ impl CliFactory { self.parsed_source_cache().clone(), self.resolver().await?.clone(), self.root_permissions_container()?.clone(), + FsSysTraitsAdapter(self.fs().clone()), ))) }) .await @@ -960,7 +960,6 @@ impl CliFactory { None }, self.emitter()?.clone(), - fs.clone(), in_npm_pkg_checker.clone(), self.main_module_graph_container().await?.clone(), self.module_load_preparer().await?.clone(), @@ -975,6 +974,7 @@ impl CliFactory { ), self.parsed_source_cache().clone(), self.resolver().await?.clone(), + FsSysTraitsAdapter(self.fs().clone()), )), node_resolver.clone(), npm_resolver.clone(), diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 1b286c76b71e5f..38f3dd1847a8be 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -1,11 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::cache::HttpCache; -use crate::cache::RealDenoCacheEnv; -use crate::colors; -use crate::http_util::get_response_body_with_progress; -use crate::http_util::HttpClientProvider; -use crate::util::progress_bar::ProgressBar; +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::Arc; use boxed_error::Boxed; use deno_ast::MediaType; @@ -27,7 +24,7 @@ use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_error::JsError; use deno_graph::source::LoaderChecksum; - +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_permissions::CheckSpecifierKind; use deno_runtime::deno_permissions::PermissionCheckError; use deno_runtime::deno_permissions::PermissionsContainer; @@ -35,12 +32,14 @@ use deno_runtime::deno_web::BlobStore; use http::header; use http::HeaderMap; use http::StatusCode; -use std::borrow::Cow; -use std::collections::HashMap; -use std::env; -use std::sync::Arc; use thiserror::Error; +use crate::cache::HttpCache; +use crate::colors; +use crate::http_util::get_response_body_with_progress; +use crate::http_util::HttpClientProvider; +use crate::util::progress_bar::ProgressBar; + #[derive(Debug, Clone, Eq, PartialEq)] pub struct TextDecodedFile { pub media_type: MediaType, @@ -268,7 +267,7 @@ pub struct FetchNoFollowOptions<'a> { type DenoCacheDirFileFetcher = deno_cache_dir::file_fetcher::FileFetcher< BlobStoreAdapter, - RealDenoCacheEnv, + FsSysTraitsAdapter, HttpClientAdapter, >; @@ -290,9 +289,11 @@ impl CliFileFetcher { download_log_level: log::Level, ) -> Self { let memory_files = Arc::new(MemoryFiles::default()); + let sys = FsSysTraitsAdapter::new_real(); + let auth_tokens = AuthTokens::new_from_sys(&sys); let file_fetcher = DenoCacheDirFileFetcher::new( BlobStoreAdapter(blob_store), - RealDenoCacheEnv, + sys, http_cache, HttpClientAdapter { http_client_provider: http_client_provider.clone(), @@ -303,7 +304,7 @@ impl CliFileFetcher { FileFetcherOptions { allow_remote, cache_setting, - auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()), + auth_tokens, }, ); Self { @@ -497,7 +498,6 @@ fn validate_scheme(specifier: &Url) -> Result<(), UnsupportedSchemeError> { #[cfg(test)] mod tests { use crate::cache::GlobalHttpCache; - use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClientProvider; use super::*; @@ -538,7 +538,10 @@ mod tests { let temp_dir = maybe_temp_dir.unwrap_or_default(); let location = temp_dir.path().join("remote").to_path_buf(); let blob_store: Arc = Default::default(); - let cache = Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)); + let cache = Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), + location, + )); let file_fetcher = CliFileFetcher::new( cache.clone(), Arc::new(HttpClientProvider::new(None, None)), @@ -752,8 +755,8 @@ mod tests { let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher = CliFileFetcher::new( Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), location, - crate::cache::RealDenoCacheEnv, )), Arc::new(HttpClientProvider::new(None, None)), Default::default(), @@ -781,8 +784,8 @@ mod tests { resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap(); let http_cache = Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), location.clone(), - crate::cache::RealDenoCacheEnv, )); let file_modified_01 = { let file_fetcher = CliFileFetcher::new( @@ -808,8 +811,8 @@ mod tests { let file_modified_02 = { let file_fetcher = CliFileFetcher::new( Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), location, - crate::cache::RealDenoCacheEnv, )), Arc::new(HttpClientProvider::new(None, None)), Default::default(), @@ -938,8 +941,8 @@ mod tests { let redirected_specifier = resolve_url("http://localhost:4546/subdir/mismatch_ext.ts").unwrap(); let http_cache = Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), location.clone(), - crate::cache::RealDenoCacheEnv, )); let metadata_file_modified_01 = { @@ -1073,8 +1076,8 @@ mod tests { let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher = CliFileFetcher::new( Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), location, - crate::cache::RealDenoCacheEnv, )), Arc::new(HttpClientProvider::new(None, None)), Default::default(), @@ -1110,7 +1113,10 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher_01 = CliFileFetcher::new( - Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)), + Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), + location.clone(), + )), Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, @@ -1119,7 +1125,10 @@ mod tests { log::Level::Info, ); let file_fetcher_02 = CliFileFetcher::new( - Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), + Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), + location, + )), Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 88c412b65a1ce8..a21d055adcc24b 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -1,53 +1,35 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::args::config_to_deno_graph_workspace_member; -use crate::args::jsr_url; -use crate::args::CliLockfile; -use crate::args::CliOptions; -pub use crate::args::NpmCachingStrategy; -use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; -use crate::cache; -use crate::cache::FetchCacher; -use crate::cache::GlobalHttpCache; -use crate::cache::ModuleInfoCache; -use crate::cache::ParsedSourceCache; -use crate::colors; -use crate::errors::get_error_class_name; -use crate::file_fetcher::CliFileFetcher; -use crate::npm::CliNpmResolver; -use crate::resolver::CjsTracker; -use crate::resolver::CliResolver; -use crate::resolver::CliSloppyImportsResolver; -use crate::resolver::SloppyImportsCachedFs; -use crate::tools::check; -use crate::tools::check::TypeChecker; -use crate::util::file_watcher::WatcherCommunicator; -use crate::util::fs::canonicalize_path; +use std::collections::HashSet; +use std::error::Error; +use std::ops::Deref; +use std::path::PathBuf; +use std::sync::Arc; + use deno_config::deno_json::JsxImportSourceConfig; use deno_config::workspace::JsrPackageConfig; use deno_core::anyhow::bail; -use deno_graph::source::LoaderChecksum; -use deno_graph::source::ResolutionKind; -use deno_graph::FillFromLockfileOptions; -use deno_graph::JsrLoadError; -use deno_graph::ModuleLoadError; -use deno_graph::WorkspaceFastCheckOption; - use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; use deno_graph::source::Loader; +use deno_graph::source::LoaderChecksum; +use deno_graph::source::ResolutionKind; use deno_graph::source::ResolveError; +use deno_graph::FillFromLockfileOptions; use deno_graph::GraphKind; +use deno_graph::JsrLoadError; use deno_graph::ModuleError; use deno_graph::ModuleGraph; use deno_graph::ModuleGraphError; +use deno_graph::ModuleLoadError; use deno_graph::ResolutionError; use deno_graph::SpecifierError; +use deno_graph::WorkspaceFastCheckOption; use deno_path_util::url_to_file_path; use deno_resolver::sloppy_imports::SloppyImportsResolutionKind; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node; use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::jsr::JsrDepPackageReq; @@ -55,11 +37,30 @@ use deno_semver::package::PackageNv; use deno_semver::SmallStackString; use import_map::ImportMapError; use node_resolver::InNpmPackageChecker; -use std::collections::HashSet; -use std::error::Error; -use std::ops::Deref; -use std::path::PathBuf; -use std::sync::Arc; + +use crate::args::config_to_deno_graph_workspace_member; +use crate::args::jsr_url; +use crate::args::CliLockfile; +use crate::args::CliOptions; +pub use crate::args::NpmCachingStrategy; +use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; +use crate::cache; +use crate::cache::FetchCacher; +use crate::cache::GlobalHttpCache; +use crate::cache::ModuleInfoCache; +use crate::cache::ParsedSourceCache; +use crate::colors; +use crate::errors::get_error_class_name; +use crate::file_fetcher::CliFileFetcher; +use crate::npm::CliNpmResolver; +use crate::resolver::CjsTracker; +use crate::resolver::CliResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; +use crate::tools::check; +use crate::tools::check::TypeChecker; +use crate::util::file_watcher::WatcherCommunicator; +use crate::util::fs::canonicalize_path; #[derive(Clone)] pub struct GraphValidOptions { @@ -80,7 +81,7 @@ pub struct GraphValidOptions { /// for the CLI. pub fn graph_valid( graph: &ModuleGraph, - fs: &Arc, + sys: &FsSysTraitsAdapter, roots: &[ModuleSpecifier], options: GraphValidOptions, ) -> Result<(), AnyError> { @@ -90,7 +91,7 @@ pub fn graph_valid( let mut errors = graph_walk_errors( graph, - fs, + sys, roots, GraphWalkErrorsOptions { check_js: options.check_js, @@ -140,7 +141,7 @@ pub struct GraphWalkErrorsOptions { /// and enhances them with CLI information. pub fn graph_walk_errors<'a>( graph: &'a ModuleGraph, - fs: &'a Arc, + sys: &'a FsSysTraitsAdapter, roots: &'a [ModuleSpecifier], options: GraphWalkErrorsOptions, ) -> impl Iterator + 'a { @@ -175,7 +176,7 @@ pub fn graph_walk_errors<'a>( } ModuleGraphError::ModuleError(error) => { enhanced_integrity_error_message(error) - .or_else(|| enhanced_sloppy_imports_error_message(fs, error)) + .or_else(|| enhanced_sloppy_imports_error_message(sys, error)) .unwrap_or_else(|| format_deno_graph_error(error)) } }; @@ -433,7 +434,6 @@ pub struct ModuleGraphBuilder { cjs_tracker: Arc, cli_options: Arc, file_fetcher: Arc, - fs: Arc, global_http_cache: Arc, in_npm_pkg_checker: Arc, lockfile: Option>, @@ -443,6 +443,7 @@ pub struct ModuleGraphBuilder { parsed_source_cache: Arc, resolver: Arc, root_permissions_container: PermissionsContainer, + sys: FsSysTraitsAdapter, } impl ModuleGraphBuilder { @@ -452,7 +453,6 @@ impl ModuleGraphBuilder { cjs_tracker: Arc, cli_options: Arc, file_fetcher: Arc, - fs: Arc, global_http_cache: Arc, in_npm_pkg_checker: Arc, lockfile: Option>, @@ -462,13 +462,13 @@ impl ModuleGraphBuilder { parsed_source_cache: Arc, resolver: Arc, root_permissions_container: PermissionsContainer, + sys: FsSysTraitsAdapter, ) -> Self { Self { caches, cjs_tracker, cli_options, file_fetcher, - fs, global_http_cache, in_npm_pkg_checker, lockfile, @@ -478,6 +478,7 @@ impl ModuleGraphBuilder { parsed_source_cache, resolver, root_permissions_container, + sys, } } @@ -593,7 +594,7 @@ impl ModuleGraphBuilder { is_dynamic: options.is_dynamic, passthrough_jsr_specifiers: false, executor: Default::default(), - file_system: &DenoGraphFsAdapter(self.fs.as_ref()), + file_system: &self.sys, jsr_url_provider: &CliJsrUrlProvider, npm_resolver: Some(&graph_npm_resolver), module_analyzer: &analyzer, @@ -747,10 +748,10 @@ impl ModuleGraphBuilder { ) -> cache::FetchCacher { cache::FetchCacher::new( self.file_fetcher.clone(), - self.fs.clone(), self.global_http_cache.clone(), self.in_npm_pkg_checker.clone(), self.module_info_cache.clone(), + self.sys.clone(), cache::FetchCacherOptions { file_header_overrides: self.cli_options.resolve_file_header_overrides(), permissions, @@ -779,7 +780,7 @@ impl ModuleGraphBuilder { ) -> Result<(), AnyError> { graph_valid( graph, - &self.fs, + &self.sys, roots, GraphValidOptions { kind: if self.cli_options.type_check_mode().is_true() { @@ -835,13 +836,13 @@ pub fn enhanced_resolution_error_message(error: &ResolutionError) -> String { } fn enhanced_sloppy_imports_error_message( - fs: &Arc, + sys: &FsSysTraitsAdapter, error: &ModuleError, ) -> Option { match error { ModuleError::LoadingErr(specifier, _, ModuleLoadError::Loader(_)) // ex. "Is a directory" error | ModuleError::Missing(specifier, _) => { - let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(fs.clone())) + let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(sys.clone())) .resolve(specifier, SloppyImportsResolutionKind::Execution)? .as_suggestion_message(); Some(format!( @@ -1082,71 +1083,6 @@ impl deno_graph::source::Reporter for FileWatcherReporter { } } -pub struct DenoGraphFsAdapter<'a>( - pub &'a dyn deno_runtime::deno_fs::FileSystem, -); - -impl<'a> deno_graph::source::FileSystem for DenoGraphFsAdapter<'a> { - fn read_dir( - &self, - dir_url: &deno_graph::ModuleSpecifier, - ) -> Vec { - use deno_core::anyhow; - use deno_graph::source::DirEntry; - use deno_graph::source::DirEntryKind; - - let dir_path = match dir_url.to_file_path() { - Ok(path) => path, - // ignore, treat as non-analyzable - Err(()) => return vec![], - }; - let entries = match self.0.read_dir_sync(&dir_path) { - Ok(dir) => dir, - Err(err) - if matches!( - err.kind(), - std::io::ErrorKind::PermissionDenied | std::io::ErrorKind::NotFound - ) => - { - return vec![]; - } - Err(err) => { - return vec![DirEntry { - kind: DirEntryKind::Error( - anyhow::Error::from(err) - .context("Failed to read directory.".to_string()), - ), - url: dir_url.clone(), - }]; - } - }; - let mut dir_entries = Vec::with_capacity(entries.len()); - for entry in entries { - let entry_path = dir_path.join(&entry.name); - dir_entries.push(if entry.is_directory { - DirEntry { - kind: DirEntryKind::Dir, - url: ModuleSpecifier::from_directory_path(&entry_path).unwrap(), - } - } else if entry.is_file { - DirEntry { - kind: DirEntryKind::File, - url: ModuleSpecifier::from_file_path(&entry_path).unwrap(), - } - } else if entry.is_symlink { - DirEntry { - kind: DirEntryKind::Symlink, - url: ModuleSpecifier::from_file_path(&entry_path).unwrap(), - } - } else { - continue; - }); - } - - dir_entries - } -} - pub fn format_range_with_colors(referrer: &deno_graph::Range) -> String { format!( "{}:{}:{}", diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index fbf9ea6f1b12f9..c6d1b39ef43b25 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -11,6 +11,7 @@ use crate::lsp::logging::lsp_warn; use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_path_util::url_to_file_path; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use std::collections::BTreeMap; use std::fs; use std::path::Path; @@ -94,8 +95,8 @@ impl LspCache { let deno_dir = DenoDir::new(global_cache_path) .expect("should be infallible with absolute custom root"); let global = Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), deno_dir.remote_folder_path(), - crate::cache::RealDenoCacheEnv, )); Self { deno_dir, diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index 3efebe63b1f356..a43e774934dee4 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -9,8 +9,6 @@ use deno_config::deno_json::LintConfig; use deno_config::deno_json::NodeModulesDirMode; use deno_config::deno_json::TestConfig; use deno_config::deno_json::TsConfig; -use deno_config::fs::DenoConfigFs; -use deno_config::fs::RealDenoConfigFs; use deno_config::glob::FilePatterns; use deno_config::glob::PathOrPatternSet; use deno_config::workspace::CreateResolverOptions; @@ -38,10 +36,10 @@ use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::ResolvedNpmRc; use deno_package_json::PackageJsonCache; use deno_path_util::url_to_file_path; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::PackageJson; use indexmap::IndexSet; use lsp_types::ClientCapabilities; -use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; @@ -1220,7 +1218,6 @@ impl ConfigData { settings: &Settings, file_fetcher: &Arc, // sync requirement is because the lsp requires sync - cached_deno_config_fs: &(dyn DenoConfigFs + Sync), deno_json_cache: &(dyn DenoJsonCache + Sync), pkg_json_cache: &(dyn PackageJsonCache + Sync), workspace_cache: &(dyn WorkspaceCache + Sync), @@ -1230,6 +1227,7 @@ impl ConfigData { Ok(scope_dir_path) => { let paths = [scope_dir_path]; WorkspaceDirectory::discover( + &FsSysTraitsAdapter::new_real(), match specified_config { Some(config_path) => { deno_config::workspace::WorkspaceDiscoverStart::ConfigFile( @@ -1241,7 +1239,6 @@ impl ConfigData { } }, &WorkspaceDiscoverOptions { - fs: cached_deno_config_fs, additional_config_file_names: &[], deno_json_cache: Some(deno_json_cache), pkg_json_cache: Some(pkg_json_cache), @@ -1618,9 +1615,9 @@ impl ConfigData { || unstable.contains("sloppy-imports"); let sloppy_imports_resolver = unstable_sloppy_imports.then(|| { Arc::new(CliSloppyImportsResolver::new( - SloppyImportsCachedFs::new_without_stat_cache(Arc::new( - deno_runtime::deno_fs::RealFs, - )), + SloppyImportsCachedFs::new_without_stat_cache( + FsSysTraitsAdapter::new_real(), + ), )) }); let resolver = Arc::new(resolver); @@ -1840,7 +1837,6 @@ impl ConfigTree { // since we're resolving a workspace multiple times in different // folders, we want to cache all the lookups and config files across // ConfigData::load calls - let cached_fs = CachedDenoConfigFs::default(); let deno_json_cache = DenoJsonMemCache::default(); let pkg_json_cache = PackageJsonMemCache::default(); let workspace_cache = WorkspaceMemCache::default(); @@ -1865,7 +1861,6 @@ impl ConfigTree { folder_uri, settings, file_fetcher, - &cached_fs, &deno_json_cache, &pkg_json_cache, &workspace_cache, @@ -1896,7 +1891,6 @@ impl ConfigTree { &scope, settings, file_fetcher, - &cached_fs, &deno_json_cache, &pkg_json_cache, &workspace_cache, @@ -1913,7 +1907,6 @@ impl ConfigTree { member_scope, settings, file_fetcher, - &cached_fs, &deno_json_cache, &pkg_json_cache, &workspace_cache, @@ -1930,7 +1923,7 @@ impl ConfigTree { pub async fn inject_config_file(&mut self, config_file: ConfigFile) { let scope = config_file.specifier.join(".").unwrap(); let json_text = serde_json::to_string(&config_file.json).unwrap(); - let test_fs = deno_runtime::deno_fs::InMemoryFs::default(); + let test_fs = Arc::new(deno_runtime::deno_fs::InMemoryFs::default()); let config_path = url_to_file_path(&config_file.specifier).unwrap(); test_fs.setup_text_files(vec![( config_path.to_string_lossy().to_string(), @@ -1938,11 +1931,11 @@ impl ConfigTree { )]); let workspace_dir = Arc::new( WorkspaceDirectory::discover( + &FsSysTraitsAdapter(test_fs.clone()), deno_config::workspace::WorkspaceDiscoverStart::ConfigFile( &config_path, ), &deno_config::workspace::WorkspaceDiscoverOptions { - fs: &crate::args::deno_json::DenoConfigFsAdapter(&test_fs), ..Default::default() }, ) @@ -2076,78 +2069,6 @@ impl deno_config::workspace::WorkspaceCache for WorkspaceMemCache { } } -#[derive(Default)] -struct CachedFsItems { - items: HashMap>, -} - -impl CachedFsItems { - pub fn get( - &mut self, - path: &Path, - action: impl FnOnce(&Path) -> Result, - ) -> Result { - let value = if let Some(value) = self.items.get(path) { - value - } else { - let value = action(path); - // just in case this gets really large for some reason - if self.items.len() == 16_384 { - return value; - } - self.items.insert(path.to_owned(), value); - self.items.get(path).unwrap() - }; - value - .as_ref() - .map(|v| (*v).clone()) - .map_err(|e| std::io::Error::new(e.kind(), e.to_string())) - } -} - -#[derive(Default)] -struct InnerData { - stat_calls: CachedFsItems, - read_to_string_calls: CachedFsItems>, -} - -#[derive(Default)] -struct CachedDenoConfigFs(Mutex); - -impl DenoConfigFs for CachedDenoConfigFs { - fn stat_sync( - &self, - path: &Path, - ) -> Result { - self - .0 - .lock() - .stat_calls - .get(path, |path| RealDenoConfigFs.stat_sync(path)) - } - - fn read_to_string_lossy( - &self, - path: &Path, - ) -> Result, std::io::Error> { - self - .0 - .lock() - .read_to_string_calls - .get(path, |path| RealDenoConfigFs.read_to_string_lossy(path)) - } - - fn read_dir( - &self, - path: &Path, - ) -> Result, std::io::Error> { - // no need to cache these because the workspace cache will ensure - // we only do read_dir calls once (read_dirs are only used for - // npm workspace resolution) - RealDenoConfigFs.read_dir(path) - } -} - #[cfg(test)] mod tests { use deno_config::deno_json::ConfigParseOptions; diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index 804cebfb9b98f0..33fd4897c46ca8 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -48,7 +48,7 @@ use deno_graph::SpecifierError; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_resolver::sloppy_imports::SloppyImportsResolution; use deno_resolver::sloppy_imports::SloppyImportsResolutionKind; -use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node; use deno_runtime::tokio_util::create_basic_runtime; use deno_semver::jsr::JsrPackageReqReference; @@ -1281,7 +1281,7 @@ impl DenoDiagnostic { Self::NotInstalledNpm(pkg_req, specifier) => (lsp::DiagnosticSeverity::ERROR, format!("npm package \"{pkg_req}\" is not installed or doesn't exist."), Some(json!({ "specifier": specifier }))), Self::NoLocal(specifier) => { let maybe_sloppy_resolution = CliSloppyImportsResolver::new( - SloppyImportsCachedFs::new(Arc::new(deno_fs::RealFs)) + SloppyImportsCachedFs::new(FsSysTraitsAdapter::new_real()) ).resolve(specifier, SloppyImportsResolutionKind::Execution); let data = maybe_sloppy_resolution.as_ref().map(|res| { json!({ diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index a7a0a59743bee7..1cc26aff5824df 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -17,6 +17,7 @@ use deno_core::ModuleSpecifier; use deno_graph::GraphKind; use deno_graph::Resolution; use deno_path_util::url_to_file_path; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::RootCertStoreProvider; use deno_semver::jsr::JsrPackageReqReference; @@ -279,7 +280,7 @@ impl LanguageServer { .await?; graph_util::graph_valid( &graph, - factory.fs(), + &FsSysTraitsAdapter(factory.fs().clone()), &roots, graph_util::GraphValidOptions { kind: GraphKind::All, @@ -3612,11 +3613,11 @@ impl Inner { let workspace = match config_data { Some(d) => d.member_dir.clone(), None => Arc::new(WorkspaceDirectory::discover( + &FsSysTraitsAdapter::new_real(), deno_config::workspace::WorkspaceDiscoverStart::Paths(&[ initial_cwd.clone() ]), &WorkspaceDiscoverOptions { - fs: Default::default(), // use real fs, deno_json_cache: None, pkg_json_cache: None, workspace_cache: None, diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index 067f2018294d46..488e333e9d330f 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -32,6 +32,7 @@ use deno_core::url::Position; use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::Dependency; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use log::error; use once_cell::sync::Lazy; use std::borrow::Cow; @@ -430,8 +431,8 @@ impl ModuleRegistry { ) -> Self { // the http cache should always be the global one for registry completions let http_cache = Arc::new(GlobalHttpCache::new( + FsSysTraitsAdapter::new_real(), location.clone(), - crate::cache::RealDenoCacheEnv, )); let file_fetcher = CliFileFetcher::new( http_cache.clone(), diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index 482f2ddb4006d3..addecc9a613776 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -19,9 +19,10 @@ use deno_resolver::cjs::IsCjsResolutionMode; use deno_resolver::npm::NpmReqResolverOptions; use deno_resolver::DenoResolverOptions; use deno_resolver::NodeAndNpmReqResolver; -use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_node::PackageJsonResolver; +use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; @@ -42,7 +43,6 @@ use super::jsr::JsrCacheResolver; use crate::args::create_default_npmrc; use crate::args::CliLockfile; use crate::args::NpmInstallDepsProvider; -use crate::cache::DenoCacheEnvFsAdapter; use crate::factory::Deferred; use crate::graph_util::to_node_resolution_kind; use crate::graph_util::to_node_resolution_mode; @@ -61,7 +61,6 @@ use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::CreateInNpmPkgCheckerOptions; use crate::npm::ManagedCliNpmResolver; use crate::resolver::CliDenoResolver; -use crate::resolver::CliDenoResolverFs; use crate::resolver::CliNpmReqResolver; use crate::resolver::CliResolver; use crate::resolver::CliResolverOptions; @@ -599,21 +598,19 @@ struct ResolverFactoryServices { struct ResolverFactory<'a> { config_data: Option<&'a Arc>, - fs: Arc, pkg_json_resolver: Arc, + sys: FsSysTraitsAdapter, services: ResolverFactoryServices, } impl<'a> ResolverFactory<'a> { pub fn new(config_data: Option<&'a Arc>) -> Self { - let fs = Arc::new(deno_fs::RealFs); - let pkg_json_resolver = Arc::new(PackageJsonResolver::new( - deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), - )); + let sys = FsSysTraitsAdapter::new_real(); + let pkg_json_resolver = Arc::new(PackageJsonResolver::new(sys.clone())); Self { config_data, - fs, pkg_json_resolver, + sys, services: Default::default(), } } @@ -624,9 +621,10 @@ impl<'a> ResolverFactory<'a> { cache: &LspCache, ) { let enable_byonm = self.config_data.map(|d| d.byonm).unwrap_or(false); + let sys = FsSysTraitsAdapter::new_real(); let options = if enable_byonm { CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { - fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)), + sys, pkg_json_resolver: self.pkg_json_resolver.clone(), root_node_modules_dir: self.config_data.and_then(|config_data| { config_data.node_modules_dir.clone().or_else(|| { @@ -642,12 +640,14 @@ impl<'a> ResolverFactory<'a> { .and_then(|d| d.npmrc.clone()) .unwrap_or_else(create_default_npmrc); let npm_cache_dir = Arc::new(NpmCacheDir::new( - &DenoCacheEnvFsAdapter(self.fs.as_ref()), + &sys, cache.deno_dir().npm_folder_path(), npmrc.get_all_known_registries_urls(), )); CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions { http_client_provider: http_client_provider.clone(), + // only used for top level install, so we can ignore this + npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()), snapshot: match self.config_data.and_then(|d| d.lockfile.as_ref()) { Some(lockfile) => { CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( @@ -656,10 +656,7 @@ impl<'a> ResolverFactory<'a> { } None => CliNpmResolverManagedSnapshotOption::Specified(None), }, - // Don't provide the lockfile. We don't want these resolvers - // updating it. Only the cache request should update the lockfile. - maybe_lockfile: None, - fs: Arc::new(deno_fs::RealFs), + sys: FsSysTraitsAdapter::new_real(), npm_cache_dir, // Use an "only" cache setting in order to make the // user do an explicit "cache" command and prevent @@ -667,11 +664,12 @@ impl<'a> ResolverFactory<'a> { // the user is typing. cache_setting: CacheSetting::Only, text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly), + // Don't provide the lockfile. We don't want these resolvers + // updating it. Only the cache request should update the lockfile. + maybe_lockfile: None, maybe_node_modules_path: self .config_data .and_then(|d| d.node_modules_dir.clone()), - // only used for top level install, so we can ignore this - npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()), npmrc, npm_system_info: NpmSystemInfo::default(), lifecycle_scripts: Default::default(), @@ -779,10 +777,11 @@ impl<'a> ResolverFactory<'a> { .get_or_init(|| { let npm_resolver = self.services.npm_resolver.as_ref()?; Some(Arc::new(NodeResolver::new( - deno_runtime::deno_node::DenoFsNodeResolverEnv::new(self.fs.clone()), self.in_npm_pkg_checker().clone(), + RealIsBuiltInNodeModuleChecker, npm_resolver.clone().into_npm_pkg_folder_resolver(), self.pkg_json_resolver.clone(), + self.sys.clone(), ))) }) .as_ref() @@ -797,10 +796,10 @@ impl<'a> ResolverFactory<'a> { let npm_resolver = self.npm_resolver()?; Some(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions { byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(), - fs: CliDenoResolverFs(self.fs.clone()), in_npm_pkg_checker: self.in_npm_pkg_checker().clone(), node_resolver: node_resolver.clone(), npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(), + sys: self.sys.clone(), }))) }) .as_ref() diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 3ba8753335a02e..3e81dbc881b0f3 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -11,36 +11,6 @@ use std::sync::atomic::AtomicU16; use std::sync::atomic::Ordering; use std::sync::Arc; -use crate::args::jsr_url; -use crate::args::CliLockfile; -use crate::args::CliOptions; -use crate::args::DenoSubcommand; -use crate::args::TsTypeLib; -use crate::cache::CodeCache; -use crate::cache::FastInsecureHasher; -use crate::cache::ParsedSourceCache; -use crate::emit::Emitter; -use crate::graph_container::MainModuleGraphContainer; -use crate::graph_container::ModuleGraphContainer; -use crate::graph_container::ModuleGraphUpdatePermit; -use crate::graph_util::CreateGraphOptions; -use crate::graph_util::ModuleGraphBuilder; -use crate::node; -use crate::node::CliNodeCodeTranslator; -use crate::npm::CliNpmResolver; -use crate::resolver::CjsTracker; -use crate::resolver::CliNpmReqResolver; -use crate::resolver::CliResolver; -use crate::resolver::ModuleCodeStringSource; -use crate::resolver::NotSupportedKindInNpmError; -use crate::resolver::NpmModuleLoader; -use crate::tools::check; -use crate::tools::check::TypeChecker; -use crate::util::progress_bar::ProgressBar; -use crate::util::text_encoding::code_without_source_map; -use crate::util::text_encoding::source_map_from_code; -use crate::worker::CreateModuleLoaderResult; -use crate::worker::ModuleLoaderFactory; use deno_ast::MediaType; use deno_ast::ModuleKind; use deno_core::anyhow::anyhow; @@ -69,7 +39,7 @@ use deno_graph::ModuleGraph; use deno_graph::Resolution; use deno_graph::WasmModule; use deno_runtime::code_cache; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::create_host_defined_options; use deno_runtime::deno_node::NodeRequireLoader; use deno_runtime::deno_node::NodeResolver; @@ -79,6 +49,37 @@ use node_resolver::errors::ClosestPkgJsonError; use node_resolver::InNpmPackageChecker; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; +use sys_traits::FsRead; + +use crate::args::jsr_url; +use crate::args::CliLockfile; +use crate::args::CliOptions; +use crate::args::DenoSubcommand; +use crate::args::TsTypeLib; +use crate::cache::CodeCache; +use crate::cache::FastInsecureHasher; +use crate::cache::ParsedSourceCache; +use crate::emit::Emitter; +use crate::graph_container::MainModuleGraphContainer; +use crate::graph_container::ModuleGraphContainer; +use crate::graph_container::ModuleGraphUpdatePermit; +use crate::graph_util::CreateGraphOptions; +use crate::graph_util::ModuleGraphBuilder; +use crate::node::CliNodeCodeTranslator; +use crate::npm::CliNpmResolver; +use crate::resolver::CjsTracker; +use crate::resolver::CliNpmReqResolver; +use crate::resolver::CliResolver; +use crate::resolver::ModuleCodeStringSource; +use crate::resolver::NotSupportedKindInNpmError; +use crate::resolver::NpmModuleLoader; +use crate::tools::check; +use crate::tools::check::TypeChecker; +use crate::util::progress_bar::ProgressBar; +use crate::util::text_encoding::code_without_source_map; +use crate::util::text_encoding::source_map_from_code; +use crate::worker::CreateModuleLoaderResult; +use crate::worker::ModuleLoaderFactory; pub struct ModuleLoadPreparer { options: Arc, @@ -215,7 +216,6 @@ struct SharedCliModuleLoaderState { cjs_tracker: Arc, code_cache: Option>, emitter: Arc, - fs: Arc, in_npm_pkg_checker: Arc, main_module_graph_container: Arc, module_load_preparer: Arc, @@ -226,6 +226,7 @@ struct SharedCliModuleLoaderState { npm_module_loader: NpmModuleLoader, parsed_source_cache: Arc, resolver: Arc, + sys: FsSysTraitsAdapter, in_flight_loads_tracker: InFlightModuleLoadsTracker, } @@ -275,7 +276,6 @@ impl CliModuleLoaderFactory { cjs_tracker: Arc, code_cache: Option>, emitter: Arc, - fs: Arc, in_npm_pkg_checker: Arc, main_module_graph_container: Arc, module_load_preparer: Arc, @@ -286,6 +286,7 @@ impl CliModuleLoaderFactory { npm_module_loader: NpmModuleLoader, parsed_source_cache: Arc, resolver: Arc, + sys: FsSysTraitsAdapter, ) -> Self { Self { shared: Arc::new(SharedCliModuleLoaderState { @@ -301,7 +302,6 @@ impl CliModuleLoaderFactory { cjs_tracker, code_cache, emitter, - fs, in_npm_pkg_checker, main_module_graph_container, module_load_preparer, @@ -312,6 +312,7 @@ impl CliModuleLoaderFactory { npm_module_loader, parsed_source_cache, resolver, + sys, in_flight_loads_tracker: InFlightModuleLoadsTracker { loads_number: Arc::new(AtomicU16::new(0)), cleanup_task_timeout: 10_000, @@ -344,7 +345,7 @@ impl CliModuleLoaderFactory { let node_require_loader = Rc::new(CliNodeRequireLoader { cjs_tracker: self.shared.cjs_tracker.clone(), emitter: self.shared.emitter.clone(), - fs: self.shared.fs.clone(), + sys: self.shared.sys.clone(), graph_container, in_npm_pkg_checker: self.shared.in_npm_pkg_checker.clone(), npm_resolver: self.shared.npm_resolver.clone(), @@ -593,9 +594,9 @@ impl Some(Module::Json(module)) => module.specifier.clone(), Some(Module::Wasm(module)) => module.specifier.clone(), Some(Module::External(module)) => { - node::resolve_specifier_into_node_modules( + node_resolver::resolve_specifier_into_node_modules( + &self.shared.sys, &module.specifier, - self.shared.fs.as_ref(), ) } None => specifier.into_owned(), @@ -1091,7 +1092,7 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { struct CliNodeRequireLoader { cjs_tracker: Arc, emitter: Arc, - fs: Arc, + sys: FsSysTraitsAdapter, graph_container: TGraphContainer, in_npm_pkg_checker: Arc, npm_resolver: Arc, @@ -1120,7 +1121,7 @@ impl NodeRequireLoader ) -> Result, AnyError> { // todo(dsherret): use the preloaded module from the graph if available? let media_type = MediaType::from_path(path); - let text = self.fs.read_text_file_lossy_sync(path, None)?; + let text = self.sys.fs_read_to_string_lossy(path)?; if media_type.is_emittable() { let specifier = deno_path_util::url_from_file_path(path)?; if self.in_npm_pkg_checker.in_npm_package(&specifier) { diff --git a/cli/node.rs b/cli/node.rs index 11959df6b9ba3c..480da506c89c6b 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -8,7 +8,8 @@ use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_graph::ParsedSourceStore; use deno_runtime::deno_fs; -use deno_runtime::deno_node::DenoFsNodeResolverEnv; +use deno_runtime::deno_fs::FsSysTraitsAdapter; +use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; use node_resolver::analyze::CjsAnalysisExports; use node_resolver::analyze::CjsCodeAnalyzer; @@ -21,23 +22,11 @@ use crate::cache::NodeAnalysisCache; use crate::cache::ParsedSourceCache; use crate::resolver::CjsTracker; -pub type CliNodeCodeTranslator = - NodeCodeTranslator; - -/// Resolves a specifier that is pointing into a node_modules folder. -/// -/// Note: This should be called whenever getting the specifier from -/// a Module::External(module) reference because that module might -/// not be fully resolved at the time deno_graph is analyzing it -/// because the node_modules folder might not exist at that time. -pub fn resolve_specifier_into_node_modules( - specifier: &ModuleSpecifier, - fs: &dyn deno_fs::FileSystem, -) -> ModuleSpecifier { - node_resolver::resolve_specifier_into_node_modules(specifier, &|path| { - fs.realpath_sync(path).map_err(|err| err.into_io_error()) - }) -} +pub type CliNodeCodeTranslator = NodeCodeTranslator< + CliCjsCodeAnalyzer, + RealIsBuiltInNodeModuleChecker, + FsSysTraitsAdapter, +>; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum CliCjsAnalysis { diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs index eca399251b0e31..218f33989dcc1b 100644 --- a/cli/npm/byonm.rs +++ b/cli/npm/byonm.rs @@ -9,22 +9,20 @@ use deno_core::serde_json; use deno_resolver::npm::ByonmNpmResolver; use deno_resolver::npm::ByonmNpmResolverCreateOptions; use deno_resolver::npm::CliNpmReqResolver; -use deno_runtime::deno_node::DenoFsNodeResolverEnv; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodePermissions; use deno_runtime::ops::process::NpmProcessStateProvider; use node_resolver::NpmPackageFolderResolver; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; -use crate::resolver::CliDenoResolverFs; use super::CliNpmResolver; use super::InnerCliNpmResolverRef; pub type CliByonmNpmResolverCreateOptions = - ByonmNpmResolverCreateOptions; -pub type CliByonmNpmResolver = - ByonmNpmResolver; + ByonmNpmResolverCreateOptions; +pub type CliByonmNpmResolver = ByonmNpmResolver; // todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple. #[derive(Debug)] diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 5006902aa1de20..5b0a304de8232b 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -21,9 +21,10 @@ use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; use deno_npm_cache::NpmCacheSetting; +use deno_path_util::fs::canonicalize_path_maybe_not_exists; use deno_resolver::npm::CliNpmReqResolver; use deno_runtime::colors; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodePermissions; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; @@ -41,7 +42,6 @@ use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; use crate::args::PackageJsonDepValueParseWithLocationError; use crate::cache::FastInsecureHasher; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; use crate::util::progress_bar::ProgressBar; use crate::util::sync::AtomicFlag; @@ -50,7 +50,7 @@ use self::resolvers::create_npm_fs_resolver; use self::resolvers::NpmPackageFsResolver; use super::CliNpmCache; -use super::CliNpmCacheEnv; +use super::CliNpmCacheHttpClient; use super::CliNpmRegistryInfoProvider; use super::CliNpmResolver; use super::CliNpmTarballCache; @@ -68,9 +68,9 @@ pub enum CliNpmResolverManagedSnapshotOption { pub struct CliManagedNpmResolverCreateOptions { pub snapshot: CliNpmResolverManagedSnapshotOption, pub maybe_lockfile: Option>, - pub fs: Arc, pub http_client_provider: Arc, pub npm_cache_dir: Arc, + pub sys: FsSysTraitsAdapter, pub cache_setting: deno_cache_dir::file_fetcher::CacheSetting, pub text_only_progress_bar: crate::util::progress_bar::ProgressBar, pub maybe_node_modules_path: Option, @@ -83,9 +83,12 @@ pub struct CliManagedNpmResolverCreateOptions { pub async fn create_managed_npm_resolver_for_lsp( options: CliManagedNpmResolverCreateOptions, ) -> Arc { - let cache_env = create_cache_env(&options); - let npm_cache = create_cache(cache_env.clone(), &options); - let npm_api = create_api(npm_cache.clone(), cache_env.clone(), &options); + let npm_cache = create_cache(&options); + let http_client = Arc::new(CliNpmCacheHttpClient::new( + options.http_client_provider.clone(), + options.text_only_progress_bar.clone(), + )); + let npm_api = create_api(npm_cache.clone(), http_client.clone(), &options); // spawn due to the lsp's `Send` requirement deno_core::unsync::spawn(async move { let snapshot = match resolve_snapshot(&npm_api, options.snapshot).await { @@ -96,14 +99,14 @@ pub async fn create_managed_npm_resolver_for_lsp( } }; create_inner( - cache_env, - options.fs, - options.maybe_lockfile, - npm_api, + http_client, npm_cache, - options.npmrc, options.npm_install_deps_provider, + npm_api, + options.sys, options.text_only_progress_bar, + options.maybe_lockfile, + options.npmrc, options.maybe_node_modules_path, options.npm_system_info, snapshot, @@ -117,19 +120,22 @@ pub async fn create_managed_npm_resolver_for_lsp( pub async fn create_managed_npm_resolver( options: CliManagedNpmResolverCreateOptions, ) -> Result, AnyError> { - let npm_cache_env = create_cache_env(&options); - let npm_cache = create_cache(npm_cache_env.clone(), &options); - let api = create_api(npm_cache.clone(), npm_cache_env.clone(), &options); + let npm_cache = create_cache(&options); + let http_client = Arc::new(CliNpmCacheHttpClient::new( + options.http_client_provider.clone(), + options.text_only_progress_bar.clone(), + )); + let api = create_api(npm_cache.clone(), http_client.clone(), &options); let snapshot = resolve_snapshot(&api, options.snapshot).await?; Ok(create_inner( - npm_cache_env, - options.fs, - options.maybe_lockfile, - api, + http_client, npm_cache, - options.npmrc, options.npm_install_deps_provider, + api, + options.sys, options.text_only_progress_bar, + options.maybe_lockfile, + options.npmrc, options.maybe_node_modules_path, options.npm_system_info, snapshot, @@ -139,14 +145,14 @@ pub async fn create_managed_npm_resolver( #[allow(clippy::too_many_arguments)] fn create_inner( - env: Arc, - fs: Arc, - maybe_lockfile: Option>, - registry_info_provider: Arc, + http_client: Arc, npm_cache: Arc, - npm_rc: Arc, npm_install_deps_provider: Arc, + registry_info_provider: Arc, + sys: FsSysTraitsAdapter, text_only_progress_bar: crate::util::progress_bar::ProgressBar, + maybe_lockfile: Option>, + npm_rc: Arc, node_modules_dir_path: Option, npm_system_info: NpmSystemInfo, snapshot: Option, @@ -159,28 +165,29 @@ fn create_inner( )); let tarball_cache = Arc::new(CliNpmTarballCache::new( npm_cache.clone(), - env, + http_client, + sys.clone(), npm_rc.clone(), )); let fs_resolver = create_npm_fs_resolver( - fs.clone(), npm_cache.clone(), &npm_install_deps_provider, &text_only_progress_bar, resolution.clone(), + sys.clone(), tarball_cache.clone(), node_modules_dir_path, npm_system_info.clone(), lifecycle_scripts.clone(), ); Arc::new(ManagedCliNpmResolver::new( - fs, fs_resolver, maybe_lockfile, registry_info_provider, npm_cache, npm_install_deps_provider, resolution, + sys, tarball_cache, text_only_progress_bar, npm_system_info, @@ -188,36 +195,25 @@ fn create_inner( )) } -fn create_cache_env( - options: &CliManagedNpmResolverCreateOptions, -) -> Arc { - Arc::new(CliNpmCacheEnv::new( - options.fs.clone(), - options.http_client_provider.clone(), - options.text_only_progress_bar.clone(), - )) -} - fn create_cache( - env: Arc, options: &CliManagedNpmResolverCreateOptions, ) -> Arc { Arc::new(CliNpmCache::new( options.npm_cache_dir.clone(), + options.sys.clone(), NpmCacheSetting::from_cache_setting(&options.cache_setting), - env, options.npmrc.clone(), )) } fn create_api( cache: Arc, - env: Arc, + http_client: Arc, options: &CliManagedNpmResolverCreateOptions, ) -> Arc { Arc::new(CliNpmRegistryInfoProvider::new( cache, - env, + http_client, options.npmrc.clone(), )) } @@ -306,12 +302,12 @@ pub enum PackageCaching<'a> { /// An npm resolver where the resolution is managed by Deno rather than /// the user bringing their own node_modules (BYONM) on the file system. pub struct ManagedCliNpmResolver { - fs: Arc, fs_resolver: Arc, maybe_lockfile: Option>, registry_info_provider: Arc, npm_cache: Arc, npm_install_deps_provider: Arc, + sys: FsSysTraitsAdapter, resolution: Arc, tarball_cache: Arc, text_only_progress_bar: ProgressBar, @@ -331,20 +327,19 @@ impl std::fmt::Debug for ManagedCliNpmResolver { impl ManagedCliNpmResolver { #[allow(clippy::too_many_arguments)] pub fn new( - fs: Arc, fs_resolver: Arc, maybe_lockfile: Option>, registry_info_provider: Arc, npm_cache: Arc, npm_install_deps_provider: Arc, resolution: Arc, + sys: FsSysTraitsAdapter, tarball_cache: Arc, text_only_progress_bar: ProgressBar, npm_system_info: NpmSystemInfo, lifecycle_scripts: LifecycleScriptsConfig, ) -> Self { Self { - fs, fs_resolver, maybe_lockfile, registry_info_provider, @@ -352,6 +347,7 @@ impl ManagedCliNpmResolver { npm_install_deps_provider, text_only_progress_bar, resolution, + sys, tarball_cache, npm_system_info, top_level_install_flag: Default::default(), @@ -364,8 +360,7 @@ impl ManagedCliNpmResolver { pkg_id: &NpmPackageId, ) -> Result { let path = self.fs_resolver.package_folder(pkg_id)?; - let path = - canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())?; + let path = canonicalize_path_maybe_not_exists(&self.sys, &path)?; log::debug!( "Resolved package folder of {} to {}", pkg_id.as_serialized(), @@ -667,12 +662,13 @@ impl NpmPackageFolderResolver for ManagedCliNpmResolver { .fs_resolver .resolve_package_folder_from_package(name, referrer)?; let path = - canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref()) - .map_err(|err| PackageFolderResolveIoError { + canonicalize_path_maybe_not_exists(&self.sys, &path).map_err(|err| { + PackageFolderResolveIoError { package_name: name.to_string(), referrer: referrer.clone(), source: err, - })?; + } + })?; log::debug!("Resolved {} from {} to {}", name, referrer, path.display()); Ok(path) } @@ -728,13 +724,12 @@ impl CliNpmResolver for ManagedCliNpmResolver { )); Arc::new(ManagedCliNpmResolver::new( - self.fs.clone(), create_npm_fs_resolver( - self.fs.clone(), self.npm_cache.clone(), &self.npm_install_deps_provider, &self.text_only_progress_bar, npm_resolution.clone(), + self.sys.clone(), self.tarball_cache.clone(), self.root_node_modules_path().map(ToOwned::to_owned), self.npm_system_info.clone(), @@ -745,6 +740,7 @@ impl CliNpmResolver for ManagedCliNpmResolver { self.npm_cache.clone(), self.npm_install_deps_provider.clone(), npm_resolution, + self.sys.clone(), self.tarball_cache.clone(), self.text_only_progress_bar.clone(), self.npm_system_info.clone(), diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 68e95fb39ab3a0..83081d3b8eada5 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -11,7 +11,6 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; -use super::super::PackageCaching; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_core::anyhow::Context; @@ -21,10 +20,12 @@ use deno_core::futures::StreamExt; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodePermissions; use node_resolver::errors::PackageFolderResolveError; +use sys_traits::FsCanonicalize; +use super::super::PackageCaching; use crate::npm::CliNpmTarballCache; /// Part of the resolution that interacts with the file system. @@ -73,15 +74,15 @@ pub trait NpmPackageFsResolver: Send + Sync { #[derive(Debug)] pub struct RegistryReadPermissionChecker { - fs: Arc, + sys: FsSysTraitsAdapter, cache: Mutex>, registry_path: PathBuf, } impl RegistryReadPermissionChecker { - pub fn new(fs: Arc, registry_path: PathBuf) -> Self { + pub fn new(fs: FsSysTraitsAdapter, registry_path: PathBuf) -> Self { Self { - fs, + sys: fs, registry_path, cache: Default::default(), } @@ -108,7 +109,7 @@ impl RegistryReadPermissionChecker { |path: &Path| -> Result, AnyError> { match cache.get(path) { Some(canon) => Ok(Some(canon.clone())), - None => match self.fs.realpath_sync(path) { + None => match self.sys.fs_canonicalize(path) { Ok(canon) => { cache.insert(path.to_path_buf(), canon.clone()); Ok(Some(canon)) diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 4e79941af6515f..f56f0124078f79 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -18,7 +18,7 @@ use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodePermissions; use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageNotFoundError; @@ -47,15 +47,15 @@ pub struct GlobalNpmPackageResolver { impl GlobalNpmPackageResolver { pub fn new( cache: Arc, - fs: Arc, tarball_cache: Arc, resolution: Arc, + sys: FsSysTraitsAdapter, system_info: NpmSystemInfo, lifecycle_scripts: LifecycleScriptsConfig, ) -> Self { Self { registry_read_permission_checker: RegistryReadPermissionChecker::new( - fs, + sys, cache.root_dir_path().to_path_buf(), ), cache, diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index 788d6569ae60ef..8bbaf6c51cd803 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -15,11 +15,6 @@ use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; -use crate::args::LifecycleScriptsConfig; -use crate::colors; -use crate::npm::managed::PackageCaching; -use crate::npm::CliNpmCache; -use crate::npm::CliNpmTarballCache; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_cache_dir::npm::mixed_case_package_name_decode; @@ -34,8 +29,10 @@ use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; +use deno_path_util::fs::atomic_write_file_with_retries; +use deno_path_util::fs::canonicalize_path_maybe_not_exists; use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder; -use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodePermissions; use deno_semver::package::PackageNv; use deno_semver::StackString; @@ -45,11 +42,15 @@ use node_resolver::errors::PackageNotFoundError; use node_resolver::errors::ReferrerNotFoundError; use serde::Deserialize; use serde::Serialize; +use sys_traits::FsMetadata; +use crate::args::LifecycleScriptsConfig; use crate::args::NpmInstallDepsProvider; use crate::cache::CACHE_PERM; -use crate::util::fs::atomic_write_file_with_retries; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; +use crate::colors; +use crate::npm::managed::PackageCaching; +use crate::npm::CliNpmCache; +use crate::npm::CliNpmTarballCache; use crate::util::fs::clone_dir_recursive; use crate::util::fs::symlink_dir; use crate::util::fs::LaxSingleProcessFsFlag; @@ -66,10 +67,10 @@ use super::common::RegistryReadPermissionChecker; #[derive(Debug)] pub struct LocalNpmPackageResolver { cache: Arc, - fs: Arc, npm_install_deps_provider: Arc, progress_bar: ProgressBar, resolution: Arc, + sys: FsSysTraitsAdapter, tarball_cache: Arc, root_node_modules_path: PathBuf, root_node_modules_url: Url, @@ -82,10 +83,10 @@ impl LocalNpmPackageResolver { #[allow(clippy::too_many_arguments)] pub fn new( cache: Arc, - fs: Arc, npm_install_deps_provider: Arc, progress_bar: ProgressBar, resolution: Arc, + sys: FsSysTraitsAdapter, tarball_cache: Arc, node_modules_folder: PathBuf, system_info: NpmSystemInfo, @@ -93,15 +94,15 @@ impl LocalNpmPackageResolver { ) -> Self { Self { cache, - fs: fs.clone(), npm_install_deps_provider, progress_bar, resolution, tarball_cache, registry_read_permission_checker: RegistryReadPermissionChecker::new( - fs, + sys.clone(), node_modules_folder.clone(), ), + sys, root_node_modules_url: Url::from_directory_path(&node_modules_folder) .unwrap(), root_node_modules_path: node_modules_folder, @@ -140,8 +141,7 @@ impl LocalNpmPackageResolver { }; // Canonicalize the path so it's not pointing to the symlinked directory // in `node_modules` directory of the referrer. - canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref()) - .map(Some) + canonicalize_path_maybe_not_exists(&self.sys, &path).map(Some) } fn resolve_package_folder_from_specifier( @@ -210,7 +210,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver { }; let sub_dir = join_package_name(&node_modules_folder, name); - if self.fs.is_dir_sync(&sub_dir) { + if self.sys.fs_is_dir_no_err(&sub_dir) { return Ok(sub_dir); } @@ -925,7 +925,13 @@ impl SetupCache { } bincode::serialize(&self.current).ok().and_then(|data| { - atomic_write_file_with_retries(&self.file_path, data, CACHE_PERM).ok() + atomic_write_file_with_retries( + &FsSysTraitsAdapter::new_real(), + &self.file_path, + &data, + CACHE_PERM, + ) + .ok() }); true } diff --git a/cli/npm/managed/resolvers/mod.rs b/cli/npm/managed/resolvers/mod.rs index 736270749f8b38..2d6d37798f04f1 100644 --- a/cli/npm/managed/resolvers/mod.rs +++ b/cli/npm/managed/resolvers/mod.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use std::sync::Arc; use deno_npm::NpmSystemInfo; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use crate::args::LifecycleScriptsConfig; use crate::args::NpmInstallDepsProvider; @@ -25,11 +25,11 @@ use super::resolution::NpmResolution; #[allow(clippy::too_many_arguments)] pub fn create_npm_fs_resolver( - fs: Arc, npm_cache: Arc, npm_install_deps_provider: &Arc, progress_bar: &ProgressBar, resolution: Arc, + sys: FsSysTraitsAdapter, tarball_cache: Arc, maybe_node_modules_path: Option, system_info: NpmSystemInfo, @@ -38,10 +38,10 @@ pub fn create_npm_fs_resolver( match maybe_node_modules_path { Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new( npm_cache, - fs, npm_install_deps_provider.clone(), progress_bar.clone(), resolution, + sys, tarball_cache, node_modules_folder, system_info, @@ -49,9 +49,9 @@ pub fn create_npm_fs_resolver( )), None => Arc::new(GlobalNpmPackageResolver::new( npm_cache, - fs, tarball_cache, resolution, + sys, system_info, lifecycle_scripts, )), diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 312ea2055b5d0a..6f686c3553bcb2 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -17,7 +17,7 @@ use deno_resolver::npm::ByonmInNpmPackageChecker; use deno_resolver::npm::ByonmNpmResolver; use deno_resolver::npm::CliNpmReqResolver; use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodePermissions; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; @@ -30,9 +30,6 @@ use node_resolver::NpmPackageFolderResolver; use crate::file_fetcher::CliFileFetcher; use crate::http_util::HttpClientProvider; -use crate::util::fs::atomic_write_file_with_retries_and_fs; -use crate::util::fs::hard_link_dir_recursive; -use crate::util::fs::AtomicWriteFileFsAdapter; use crate::util::progress_bar::ProgressBar; pub use self::byonm::CliByonmNpmResolver; @@ -43,26 +40,26 @@ pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::ManagedCliNpmResolver; pub use self::managed::PackageCaching; -pub type CliNpmTarballCache = deno_npm_cache::TarballCache; -pub type CliNpmCache = deno_npm_cache::NpmCache; -pub type CliNpmRegistryInfoProvider = - deno_npm_cache::RegistryInfoProvider; +pub type CliNpmTarballCache = + deno_npm_cache::TarballCache; +pub type CliNpmCache = deno_npm_cache::NpmCache; +pub type CliNpmRegistryInfoProvider = deno_npm_cache::RegistryInfoProvider< + CliNpmCacheHttpClient, + FsSysTraitsAdapter, +>; #[derive(Debug)] -pub struct CliNpmCacheEnv { - fs: Arc, +pub struct CliNpmCacheHttpClient { http_client_provider: Arc, progress_bar: ProgressBar, } -impl CliNpmCacheEnv { +impl CliNpmCacheHttpClient { pub fn new( - fs: Arc, http_client_provider: Arc, progress_bar: ProgressBar, ) -> Self { Self { - fs, http_client_provider, progress_bar, } @@ -70,35 +67,7 @@ impl CliNpmCacheEnv { } #[async_trait::async_trait(?Send)] -impl deno_npm_cache::NpmCacheEnv for CliNpmCacheEnv { - fn exists(&self, path: &Path) -> bool { - self.fs.exists_sync(path) - } - - fn hard_link_dir_recursive( - &self, - from: &Path, - to: &Path, - ) -> Result<(), AnyError> { - // todo(dsherret): use self.fs here instead - hard_link_dir_recursive(from, to) - } - - fn atomic_write_file_with_retries( - &self, - file_path: &Path, - data: &[u8], - ) -> std::io::Result<()> { - atomic_write_file_with_retries_and_fs( - &AtomicWriteFileFsAdapter { - fs: self.fs.as_ref(), - write_mode: crate::cache::CACHE_PERM, - }, - file_path, - data, - ) - } - +impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient { async fn download_with_retries_on_any_tokio_runtime( &self, url: Url, diff --git a/cli/resolver.rs b/cli/resolver.rs index f5c3f68f36ad86..0a4fd78686c7c6 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -1,5 +1,10 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + use async_trait::async_trait; use dashmap::DashMap; use dashmap::DashSet; @@ -20,16 +25,14 @@ use deno_npm::resolution::NpmResolutionError; use deno_resolver::sloppy_imports::SloppyImportsResolver; use deno_runtime::colors; use deno_runtime::deno_fs; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::is_builtin_node_module; -use deno_runtime::deno_node::DenoFsNodeResolverEnv; +use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use deno_semver::package::PackageReq; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; -use std::borrow::Cow; -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; +use sys_traits::FsMetadata; +use sys_traits::FsMetadataValue; use thiserror::Error; use crate::args::NpmCachingStrategy; @@ -40,18 +43,19 @@ use crate::npm::InnerCliNpmResolverRef; use crate::util::sync::AtomicFlag; use crate::util::text_encoding::from_utf8_lossy_cow; -pub type CjsTracker = deno_resolver::cjs::CjsTracker; -pub type IsCjsResolver = - deno_resolver::cjs::IsCjsResolver; +pub type CjsTracker = deno_resolver::cjs::CjsTracker; +pub type IsCjsResolver = deno_resolver::cjs::IsCjsResolver; pub type CliSloppyImportsResolver = SloppyImportsResolver; pub type CliDenoResolver = deno_resolver::DenoResolver< - CliDenoResolverFs, - DenoFsNodeResolverEnv, + RealIsBuiltInNodeModuleChecker, SloppyImportsCachedFs, + FsSysTraitsAdapter, +>; +pub type CliNpmReqResolver = deno_resolver::npm::NpmReqResolver< + RealIsBuiltInNodeModuleChecker, + FsSysTraitsAdapter, >; -pub type CliNpmReqResolver = - deno_resolver::npm::NpmReqResolver; pub struct ModuleCodeStringSource { pub code: ModuleSourceCode, @@ -59,53 +63,6 @@ pub struct ModuleCodeStringSource { pub media_type: MediaType, } -#[derive(Debug, Clone)] -pub struct CliDenoResolverFs(pub Arc); - -impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs { - fn read_to_string_lossy( - &self, - path: &Path, - ) -> std::io::Result> { - self - .0 - .read_text_file_lossy_sync(path, None) - .map_err(|e| e.into_io_error()) - } - - fn realpath_sync(&self, path: &Path) -> std::io::Result { - self.0.realpath_sync(path).map_err(|e| e.into_io_error()) - } - - fn exists_sync(&self, path: &Path) -> bool { - self.0.exists_sync(path) - } - - fn is_dir_sync(&self, path: &Path) -> bool { - self.0.is_dir_sync(path) - } - - fn read_dir_sync( - &self, - dir_path: &Path, - ) -> std::io::Result> { - self - .0 - .read_dir_sync(dir_path) - .map(|entries| { - entries - .into_iter() - .map(|e| deno_resolver::fs::DirEntry { - name: e.name, - is_file: e.is_file, - is_directory: e.is_directory, - }) - .collect::>() - }) - .map_err(|err| err.into_io_error()) - } -} - #[derive(Debug, Error)] #[error("{media_type} files are not supported in npm packages: {specifier}")] pub struct NotSupportedKindInNpmError { @@ -440,7 +397,7 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> { #[derive(Debug)] pub struct SloppyImportsCachedFs { - fs: Arc, + sys: FsSysTraitsAdapter, cache: Option< DashMap< PathBuf, @@ -450,15 +407,18 @@ pub struct SloppyImportsCachedFs { } impl SloppyImportsCachedFs { - pub fn new(fs: Arc) -> Self { + pub fn new(sys: FsSysTraitsAdapter) -> Self { Self { - fs, + sys, cache: Some(Default::default()), } } - pub fn new_without_stat_cache(fs: Arc) -> Self { - Self { fs, cache: None } + pub fn new_without_stat_cache(fs: FsSysTraitsAdapter) -> Self { + Self { + sys: fs, + cache: None, + } } } @@ -475,10 +435,10 @@ impl deno_resolver::sloppy_imports::SloppyImportResolverFs } } - let entry = self.fs.stat_sync(path).ok().and_then(|stat| { - if stat.is_file { + let entry = self.sys.fs_metadata(path).ok().and_then(|stat| { + if stat.file_type().is_file() { Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::File) - } else if stat.is_directory { + } else if stat.file_type().is_dir() { Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::Dir) } else { None diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 3707543eb055bf..91187c48d19ece 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -37,7 +37,6 @@ use deno_core::futures::AsyncReadExt; use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; -use deno_graph::source::RealFileSystem; use deno_graph::ModuleGraph; use deno_npm::resolution::SerializedNpmResolutionSnapshot; use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; diff --git a/cli/standalone/code_cache.rs b/cli/standalone/code_cache.rs index 9580b9b44e16c1..a44c920328e983 100644 --- a/cli/standalone/code_cache.rs +++ b/cli/standalone/code_cache.rs @@ -15,11 +15,12 @@ use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::unsync::sync::AtomicFlag; +use deno_path_util::get_atomic_path; use deno_runtime::code_cache::CodeCache; use deno_runtime::code_cache::CodeCacheType; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use crate::cache::FastInsecureHasher; -use crate::util::path::get_atomic_file_path; use crate::worker::CliCodeCache; enum CodeCacheStrategy { @@ -189,7 +190,8 @@ impl FirstRunCodeCacheStrategy { cache_data: &HashMap, ) { let count = cache_data.len(); - let temp_file = get_atomic_file_path(&self.file_path); + let temp_file = + get_atomic_path(&FsSysTraitsAdapter::new_real(), &self.file_path); match serialize(&temp_file, self.cache_key, cache_data) { Ok(()) => { if let Err(err) = std::fs::rename(&temp_file, &self.file_path) { diff --git a/cli/standalone/file_system.rs b/cli/standalone/file_system.rs index 48dc907570babd..4b1024db6aafab 100644 --- a/cli/standalone/file_system.rs +++ b/cli/standalone/file_system.rs @@ -9,6 +9,7 @@ use deno_runtime::deno_fs::AccessCheckCb; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::FsDirEntry; use deno_runtime::deno_fs::FsFileType; +use deno_runtime::deno_fs::FsStatSlim; use deno_runtime::deno_fs::OpenOptions; use deno_runtime::deno_fs::RealFs; use deno_runtime::deno_io::fs::File; diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 75247a98c07f97..0ce25ff62f08ae 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -36,10 +36,12 @@ use deno_package_json::PackageJsonDepValue; use deno_resolver::cjs::IsCjsResolutionMode; use deno_resolver::npm::NpmReqResolverOptions; use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::create_host_defined_options; use deno_runtime::deno_node::NodeRequireLoader; use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_node::PackageJsonResolver; +use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker; use deno_runtime::deno_permissions::Permissions; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_tls::rustls::RootCertStore; @@ -69,11 +71,9 @@ use crate::args::CaData; use crate::args::NpmInstallDepsProvider; use crate::args::StorageKeyResolver; use crate::cache::Caches; -use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::DenoDirProvider; use crate::cache::FastInsecureHasher; use crate::cache::NodeAnalysisCache; -use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClientProvider; use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; @@ -87,7 +87,6 @@ use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::CreateInNpmPkgCheckerOptions; use crate::resolver::CjsTracker; -use crate::resolver::CliDenoResolverFs; use crate::resolver::CliNpmReqResolver; use crate::resolver::NpmModuleLoader; use crate::util::progress_bar::ProgressBar; @@ -656,9 +655,8 @@ pub async fn run(data: StandaloneData) -> Result { let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap(); let npm_global_cache_dir = root_path.join(".deno_compile_node_modules"); let cache_setting = CacheSetting::Only; - let pkg_json_resolver = Arc::new(PackageJsonResolver::new( - deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), - )); + let sys = FsSysTraitsAdapter(fs.clone()); + let pkg_json_resolver = Arc::new(PackageJsonResolver::new(sys.clone())); let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules { Some(binary::NodeModules::Managed { node_modules_dir }) => { // create an npmrc that uses the fake npm_registry_url to resolve packages @@ -671,7 +669,7 @@ pub async fn run(data: StandaloneData) -> Result { registry_configs: Default::default(), }); let npm_cache_dir = Arc::new(NpmCacheDir::new( - &DenoCacheEnvFsAdapter(fs.as_ref()), + &sys, npm_global_cache_dir, npmrc.get_all_known_registries_urls(), )); @@ -692,17 +690,17 @@ pub async fn run(data: StandaloneData) -> Result { snapshot, )), maybe_lockfile: None, - fs: fs.clone(), http_client_provider: http_client_provider.clone(), npm_cache_dir, - cache_setting, - text_only_progress_bar: progress_bar, - maybe_node_modules_path, - npm_system_info: Default::default(), npm_install_deps_provider: Arc::new( // this is only used for installing packages, which isn't necessary with deno compile NpmInstallDepsProvider::empty(), ), + sys: sys.clone(), + text_only_progress_bar: progress_bar, + cache_setting, + maybe_node_modules_path, + npm_system_info: Default::default(), npmrc, lifecycle_scripts: Default::default(), }, @@ -719,7 +717,7 @@ pub async fn run(data: StandaloneData) -> Result { create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm); let npm_resolver = create_cli_npm_resolver( CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { - fs: CliDenoResolverFs(fs.clone()), + sys: sys.clone(), pkg_json_resolver: pkg_json_resolver.clone(), root_node_modules_dir, }), @@ -732,7 +730,7 @@ pub async fn run(data: StandaloneData) -> Result { // so no need to create actual `.npmrc` configuration. let npmrc = create_default_npmrc(); let npm_cache_dir = Arc::new(NpmCacheDir::new( - &DenoCacheEnvFsAdapter(fs.as_ref()), + &sys, npm_global_cache_dir, npmrc.get_all_known_registries_urls(), )); @@ -747,18 +745,18 @@ pub async fn run(data: StandaloneData) -> Result { create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( CliManagedNpmResolverCreateOptions { snapshot: CliNpmResolverManagedSnapshotOption::Specified(None), - maybe_lockfile: None, - fs: fs.clone(), http_client_provider: http_client_provider.clone(), - npm_cache_dir, - cache_setting, - text_only_progress_bar: progress_bar, - maybe_node_modules_path: None, - npm_system_info: Default::default(), npm_install_deps_provider: Arc::new( // this is only used for installing packages, which isn't necessary with deno compile NpmInstallDepsProvider::empty(), ), + sys: sys.clone(), + cache_setting, + text_only_progress_bar: progress_bar, + npm_cache_dir, + maybe_lockfile: None, + maybe_node_modules_path: None, + npm_system_info: Default::default(), npmrc: create_default_npmrc(), lifecycle_scripts: Default::default(), }, @@ -770,10 +768,11 @@ pub async fn run(data: StandaloneData) -> Result { let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some(); let node_resolver = Arc::new(NodeResolver::new( - deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), in_npm_pkg_checker.clone(), + RealIsBuiltInNodeModuleChecker, npm_resolver.clone().into_npm_pkg_folder_resolver(), pkg_json_resolver.clone(), + sys.clone(), )); let cjs_tracker = Arc::new(CjsTracker::new( in_npm_pkg_checker.clone(), @@ -791,7 +790,7 @@ pub async fn run(data: StandaloneData) -> Result { let npm_req_resolver = Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions { byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(), - fs: CliDenoResolverFs(fs.clone()), + sys: sys.clone(), in_npm_pkg_checker: in_npm_pkg_checker.clone(), node_resolver: node_resolver.clone(), npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(), @@ -804,11 +803,11 @@ pub async fn run(data: StandaloneData) -> Result { ); let node_code_translator = Arc::new(NodeCodeTranslator::new( cjs_esm_code_analyzer, - deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), in_npm_pkg_checker, node_resolver.clone(), npm_resolver.clone().into_npm_pkg_folder_resolver(), pkg_json_resolver.clone(), + sys, )); let workspace_resolver = { let import_map = match metadata.workspace_resolver.import_map { diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 2b36a8ddd8cc99..4736bdab416751 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -26,6 +26,7 @@ use deno_core::serde_json; use deno_core::sourcemap::SourceMap; use deno_core::url::Url; use deno_core::LocalInspectorSession; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use node_resolver::InNpmPackageChecker; use regex::Regex; use std::fs; @@ -428,7 +429,7 @@ fn collect_coverages( .ignore_git_folder() .ignore_node_modules() .set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned)) - .collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)?; + .collect_file_patterns(&FsSysTraitsAdapter::new_real(), file_patterns)?; let coverage_patterns = FilePatterns { base: initial_cwd.to_path_buf(), diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index 90d54f6cd96e28..c33b988de0ed48 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -28,6 +28,7 @@ use deno_graph::EsParser; use deno_graph::GraphKind; use deno_graph::ModuleAnalyzer; use deno_graph::ModuleSpecifier; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use doc::html::ShortPath; use doc::DocDiagnostic; use indexmap::IndexMap; @@ -114,7 +115,7 @@ pub async fn doc( } DocSourceFileFlag::Paths(ref source_files) => { let module_graph_creator = factory.module_graph_creator().await?; - let fs = factory.fs(); + let fs = FsSysTraitsAdapter(factory.fs().clone()); let module_specifiers = collect_specifiers( FilePatterns { @@ -141,7 +142,7 @@ pub async fn doc( graph_exit_integrity_errors(&graph); let errors = graph_walk_errors( &graph, - fs, + &fs, &module_specifiers, GraphWalkErrorsOptions { check_js: false, diff --git a/cli/tools/fmt.rs b/cli/tools/fmt.rs index e29627345c8a73..55046155c0856b 100644 --- a/cli/tools/fmt.rs +++ b/cli/tools/fmt.rs @@ -34,6 +34,7 @@ use deno_core::futures; use deno_core::parking_lot::Mutex; use deno_core::unsync::spawn_blocking; use deno_core::url::Url; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use log::debug; use log::info; use log::warn; @@ -230,7 +231,7 @@ fn collect_fmt_files( .ignore_node_modules() .use_gitignore() .set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned)) - .collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files) + .collect_file_patterns(&FsSysTraitsAdapter::new_real(), files) } /// Formats markdown (using ) and its code blocks diff --git a/cli/tools/lint/linter.rs b/cli/tools/lint/linter.rs index 2c2bc43acb2caa..a10ad6479e5308 100644 --- a/cli/tools/lint/linter.rs +++ b/cli/tools/lint/linter.rs @@ -15,8 +15,9 @@ use deno_lint::linter::LintConfig as DenoLintConfig; use deno_lint::linter::LintFileOptions; use deno_lint::linter::Linter as DenoLintLinter; use deno_lint::linter::LinterOptions; +use deno_path_util::fs::atomic_write_file_with_retries; +use deno_runtime::deno_fs::FsSysTraitsAdapter; -use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::specifier_from_file_path; use super::rules::FileOrPackageLintRule; @@ -176,8 +177,9 @@ impl CliLinter { if fix_iterations > 0 { // everything looks good and the file still parses, so write it out atomic_write_file_with_retries( + &FsSysTraitsAdapter::new_real(), file_path, - source.text().as_ref(), + source.text().as_bytes(), crate::cache::CACHE_PERM, ) .context("Failed writing fix to file.")?; diff --git a/cli/tools/lint/mod.rs b/cli/tools/lint/mod.rs index 50fc16799aa4cd..4071f30e7ece7b 100644 --- a/cli/tools/lint/mod.rs +++ b/cli/tools/lint/mod.rs @@ -21,6 +21,7 @@ use deno_core::unsync::future::SharedLocal; use deno_graph::ModuleGraph; use deno_lint::diagnostic::LintDiagnostic; use deno_lint::linter::LintConfig as DenoLintConfig; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use log::debug; use reporters::create_reporter; use reporters::LintReporter; @@ -452,7 +453,7 @@ fn collect_lint_files( .ignore_node_modules() .use_gitignore() .set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned)) - .collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files) + .collect_file_patterns(&FsSysTraitsAdapter::new_real(), files) } #[allow(clippy::print_stdout)] diff --git a/cli/tools/registry/paths.rs b/cli/tools/registry/paths.rs index 8b6c05fc01e561..b607c0924cbcf3 100644 --- a/cli/tools/registry/paths.rs +++ b/cli/tools/registry/paths.rs @@ -11,6 +11,7 @@ use deno_ast::ModuleSpecifier; use deno_config::glob::FileCollector; use deno_config::glob::FilePatterns; use deno_core::error::AnyError; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use thiserror::Error; use crate::args::CliOptions; @@ -323,11 +324,11 @@ fn collect_paths( file_patterns: FilePatterns, ) -> Result, AnyError> { FileCollector::new(|e| { - if !e.metadata.is_file { + if !e.metadata.file_type().is_file() { if let Ok(specifier) = ModuleSpecifier::from_file_path(e.path) { diagnostics_collector.push(PublishDiagnostic::UnsupportedFileType { specifier, - kind: if e.metadata.is_symlink { + kind: if e.metadata.file_type().is_symlink() { "symlink".to_string() } else { "Unknown".to_string() @@ -345,5 +346,5 @@ fn collect_paths( .ignore_node_modules() .set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned)) .use_gitignore() - .collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns) + .collect_file_patterns(&FsSysTraitsAdapter::new_real(), file_patterns) } diff --git a/cli/tools/registry/unfurl.rs b/cli/tools/registry/unfurl.rs index bf6aaaf50d7958..ca507757179989 100644 --- a/cli/tools/registry/unfurl.rs +++ b/cli/tools/registry/unfurl.rs @@ -663,6 +663,7 @@ mod tests { use deno_config::workspace::ResolverWorkspaceJsrPackage; use deno_core::serde_json::json; use deno_core::url::Url; + use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_fs::RealFs; use deno_runtime::deno_node::PackageJson; use deno_semver::Version; @@ -722,10 +723,9 @@ mod tests { vec![Arc::new(package_json)], deno_config::workspace::PackageJsonDepResolution::Enabled, ); - let fs = Arc::new(RealFs); let unfurler = SpecifierUnfurler::new( Some(Arc::new(CliSloppyImportsResolver::new( - SloppyImportsCachedFs::new(fs), + SloppyImportsCachedFs::new(FsSysTraitsAdapter::new_real()), ))), Arc::new(workspace_resolver), true, @@ -863,7 +863,7 @@ const warn2 = await import(`${expr}`); ], deno_config::workspace::PackageJsonDepResolution::Enabled, ); - let fs = Arc::new(RealFs); + let fs = FsSysTraitsAdapter(Arc::new(RealFs)); let unfurler = SpecifierUnfurler::new( Some(Arc::new(CliSloppyImportsResolver::new( SloppyImportsCachedFs::new(fs), diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 4c18d1a2b0b23f..26cf3857343aa1 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -4,7 +4,6 @@ use crate::args::TsConfig; use crate::args::TypeCheckMode; use crate::cache::FastInsecureHasher; use crate::cache::ModuleInfoCache; -use crate::node; use crate::npm::CliNpmResolver; use crate::resolver::CjsTracker; use crate::util::checksum; @@ -35,12 +34,13 @@ use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::ResolutionResolved; use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; -use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FsSysTraitsAdapter; use deno_runtime::deno_node::NodeResolver; use deno_semver::npm::NpmPackageReqReference; use node_resolver::errors::NodeJsErrorCode; use node_resolver::errors::NodeJsErrorCoded; use node_resolver::errors::PackageSubpathResolveError; +use node_resolver::resolve_specifier_into_node_modules; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; use once_cell::sync::Lazy; @@ -660,9 +660,9 @@ fn op_load_inner( None } else { // means it's Deno code importing an npm module - let specifier = node::resolve_specifier_into_node_modules( + let specifier = resolve_specifier_into_node_modules( + &FsSysTraitsAdapter::new_real(), &module.specifier, - &deno_fs::RealFs, ); Some(Cow::Owned(load_from_node_modules( &specifier, @@ -924,9 +924,9 @@ fn resolve_graph_specifier_types( Some(Module::External(module)) => { // we currently only use "External" for when the module is in an npm package Ok(state.maybe_npm.as_ref().map(|_| { - let specifier = node::resolve_specifier_into_node_modules( + let specifier = resolve_specifier_into_node_modules( + &FsSysTraitsAdapter::new_real(), &module.specifier, - &deno_fs::RealFs, ); into_specifier_and_media_type(Some(specifier)) })) diff --git a/cli/util/fs.rs b/cli/util/fs.rs index ba84a0e8f33d06..58b5fc72c61527 100644 --- a/cli/util/fs.rs +++ b/cli/util/fs.rs @@ -1,9 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use std::fs::OpenOptions; use std::io::Error; use std::io::ErrorKind; -use std::io::Write; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -19,185 +17,12 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::unsync::spawn_blocking; use deno_core::ModuleSpecifier; -use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::FsSysTraitsAdapter; -use crate::util::path::get_atomic_file_path; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; use crate::util::progress_bar::ProgressMessagePrompt; -/// Writes the file to the file system at a temporary path, then -/// renames it to the destination in a single sys call in order -/// to never leave the file system in a corrupted state. -/// -/// This also handles creating the directory if a NotFound error -/// occurs. -pub fn atomic_write_file_with_retries>( - file_path: &Path, - data: T, - mode: u32, -) -> std::io::Result<()> { - struct RealAtomicWriteFileFs { - mode: u32, - } - - impl AtomicWriteFileFs for RealAtomicWriteFileFs { - fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> { - write_file(path, bytes, self.mode) - } - fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> { - std::fs::rename(from, to) - } - fn remove_file(&self, path: &Path) -> std::io::Result<()> { - std::fs::remove_file(path) - } - fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> { - std::fs::create_dir_all(dir_path) - } - fn path_exists(&self, path: &Path) -> bool { - path.exists() - } - } - - atomic_write_file_with_retries_and_fs( - &RealAtomicWriteFileFs { mode }, - file_path, - data.as_ref(), - ) -} - -pub trait AtomicWriteFileFs { - fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()>; - fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()>; - fn remove_file(&self, path: &Path) -> std::io::Result<()>; - fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()>; - fn path_exists(&self, path: &Path) -> bool; -} - -pub struct AtomicWriteFileFsAdapter<'a> { - pub fs: &'a dyn FileSystem, - pub write_mode: u32, -} - -impl<'a> AtomicWriteFileFs for AtomicWriteFileFsAdapter<'a> { - fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> { - self - .fs - .write_file_sync( - path, - deno_runtime::deno_fs::OpenOptions::write( - true, - false, - false, - Some(self.write_mode), - ), - None, - bytes, - ) - .map_err(|e| e.into_io_error()) - } - - fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> { - self.fs.rename_sync(from, to).map_err(|e| e.into_io_error()) - } - - fn remove_file(&self, path: &Path) -> std::io::Result<()> { - self - .fs - .remove_sync(path, false) - .map_err(|e| e.into_io_error()) - } - - fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> { - self - .fs - .mkdir_sync(dir_path, /* recursive */ true, None) - .map_err(|e| e.into_io_error()) - } - - fn path_exists(&self, path: &Path) -> bool { - self.fs.exists_sync(path) - } -} - -pub fn atomic_write_file_with_retries_and_fs>( - fs: &impl AtomicWriteFileFs, - file_path: &Path, - data: T, -) -> std::io::Result<()> { - let mut count = 0; - loop { - match atomic_write_file(fs, file_path, data.as_ref()) { - Ok(()) => return Ok(()), - Err(err) => { - if count >= 5 { - // too many retries, return the error - return Err(err); - } - count += 1; - let sleep_ms = std::cmp::min(50, 10 * count); - std::thread::sleep(std::time::Duration::from_millis(sleep_ms)); - } - } - } -} - -/// Writes the file to the file system at a temporary path, then -/// renames it to the destination in a single sys call in order -/// to never leave the file system in a corrupted state. -/// -/// This also handles creating the directory if a NotFound error -/// occurs. -fn atomic_write_file( - fs: &impl AtomicWriteFileFs, - file_path: &Path, - data: &[u8], -) -> std::io::Result<()> { - fn atomic_write_file_raw( - fs: &impl AtomicWriteFileFs, - temp_file_path: &Path, - file_path: &Path, - data: &[u8], - ) -> std::io::Result<()> { - fs.write_file(temp_file_path, data)?; - fs.rename_file(temp_file_path, file_path) - .inspect_err(|_err| { - // clean up the created temp file on error - let _ = fs.remove_file(temp_file_path); - }) - } - - let temp_file_path = get_atomic_file_path(file_path); - - if let Err(write_err) = - atomic_write_file_raw(fs, &temp_file_path, file_path, data) - { - if write_err.kind() == ErrorKind::NotFound { - let parent_dir_path = file_path.parent().unwrap(); - match fs.create_dir_all(parent_dir_path) { - Ok(()) => { - return atomic_write_file_raw(fs, &temp_file_path, file_path, data) - .map_err(|err| add_file_context_to_err(file_path, err)); - } - Err(create_err) => { - if !fs.path_exists(parent_dir_path) { - return Err(Error::new( - create_err.kind(), - format!( - "{:#} (for '{}')\nCheck the permission of the directory.", - create_err, - parent_dir_path.display() - ), - )); - } - } - } - } - return Err(add_file_context_to_err(file_path, write_err)); - } - Ok(()) -} - /// Creates a std::fs::File handling if the parent does not exist. pub fn create_file(file_path: &Path) -> std::io::Result { match std::fs::File::create(file_path) { @@ -236,45 +61,6 @@ fn add_file_context_to_err(file_path: &Path, err: Error) -> Error { ) } -pub fn write_file>( - filename: &Path, - data: T, - mode: u32, -) -> std::io::Result<()> { - write_file_2(filename, data, true, mode, true, false) -} - -pub fn write_file_2>( - filename: &Path, - data: T, - update_mode: bool, - mode: u32, - is_create: bool, - is_append: bool, -) -> std::io::Result<()> { - let mut file = OpenOptions::new() - .read(false) - .write(true) - .append(is_append) - .truncate(!is_append) - .create(is_create) - .open(filename)?; - - if update_mode { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode = mode & 0o777; - let permissions = PermissionsExt::from_mode(mode); - file.set_permissions(permissions)?; - } - #[cfg(not(unix))] - let _ = mode; - } - - file.write_all(data.as_ref()) -} - /// Similar to `std::fs::canonicalize()` but strips UNC prefixes on Windows. pub fn canonicalize_path(path: &Path) -> Result { Ok(deno_path_util::strip_unc_prefix(path.canonicalize()?)) @@ -289,16 +75,10 @@ pub fn canonicalize_path(path: &Path) -> Result { pub fn canonicalize_path_maybe_not_exists( path: &Path, ) -> Result { - deno_path_util::canonicalize_path_maybe_not_exists(path, &canonicalize_path) -} - -pub fn canonicalize_path_maybe_not_exists_with_fs( - path: &Path, - fs: &dyn FileSystem, -) -> Result { - deno_path_util::canonicalize_path_maybe_not_exists(path, &|path| { - fs.realpath_sync(path).map_err(|err| err.into_io_error()) - }) + deno_path_util::fs::canonicalize_path_maybe_not_exists( + &FsSysTraitsAdapter::new_real(), + path, + ) } /// Collects module specifiers that satisfy the given predicate as a file path, by recursively walking `include`. @@ -346,7 +126,7 @@ pub fn collect_specifiers( .ignore_git_folder() .ignore_node_modules() .set_vendor_folder(vendor_folder) - .collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)?; + .collect_file_patterns(&FsSysTraitsAdapter::new_real(), files)?; let mut collected_files_as_urls = collected_files .iter() .map(|f| specifier_from_file_path(f).unwrap()) @@ -418,7 +198,13 @@ mod clone_dir_imp { from: &std::path::Path, to: &std::path::Path, ) -> Result<(), deno_core::error::AnyError> { - if let Err(e) = super::hard_link_dir_recursive(from, to) { + use deno_runtime::deno_fs::FsSysTraitsAdapter; + + if let Err(e) = deno_npm_cache::hard_link_dir_recursive( + &FsSysTraitsAdapter::new_real(), + from, + to, + ) { log::debug!("Failed to hard link dir {:?} to {:?}: {}", from, to, e); super::copy_dir_recursive(from, to)?; } @@ -465,84 +251,6 @@ pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> { Ok(()) } -/// Hardlinks the files in one directory to another directory. -/// -/// Note: Does not handle symlinks. -pub fn hard_link_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> { - std::fs::create_dir_all(to) - .with_context(|| format!("Creating {}", to.display()))?; - let read_dir = std::fs::read_dir(from) - .with_context(|| format!("Reading {}", from.display()))?; - - for entry in read_dir { - let entry = entry?; - let file_type = entry.file_type()?; - let new_from = from.join(entry.file_name()); - let new_to = to.join(entry.file_name()); - - if file_type.is_dir() { - hard_link_dir_recursive(&new_from, &new_to).with_context(|| { - format!("Dir {} to {}", new_from.display(), new_to.display()) - })?; - } else if file_type.is_file() { - // note: chance for race conditions here between attempting to create, - // then removing, then attempting to create. There doesn't seem to be - // a way to hard link with overwriting in Rust, but maybe there is some - // way with platform specific code. The workaround here is to handle - // scenarios where something else might create or remove files. - if let Err(err) = std::fs::hard_link(&new_from, &new_to) { - if err.kind() == ErrorKind::AlreadyExists { - if let Err(err) = std::fs::remove_file(&new_to) { - if err.kind() == ErrorKind::NotFound { - // Assume another process/thread created this hard link to the file we are wanting - // to remove then sleep a little bit to let the other process/thread move ahead - // faster to reduce contention. - std::thread::sleep(Duration::from_millis(10)); - } else { - return Err(err).with_context(|| { - format!( - "Removing file to hard link {} to {}", - new_from.display(), - new_to.display() - ) - }); - } - } - - // Always attempt to recreate the hardlink. In contention scenarios, the other process - // might have been killed or exited after removing the file, but before creating the hardlink - if let Err(err) = std::fs::hard_link(&new_from, &new_to) { - // Assume another process/thread created this hard link to the file we are wanting - // to now create then sleep a little bit to let the other process/thread move ahead - // faster to reduce contention. - if err.kind() == ErrorKind::AlreadyExists { - std::thread::sleep(Duration::from_millis(10)); - } else { - return Err(err).with_context(|| { - format!( - "Hard linking {} to {}", - new_from.display(), - new_to.display() - ) - }); - } - } - } else { - return Err(err).with_context(|| { - format!( - "Hard linking {} to {}", - new_from.display(), - new_to.display() - ) - }); - } - } - } - } - - Ok(()) -} - pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), Error> { let err_mapper = |err: Error, kind: Option| { Error::new( diff --git a/cli/util/path.rs b/cli/util/path.rs index de72843406efbd..539e1235a80c31 100644 --- a/cli/util/path.rs +++ b/cli/util/path.rs @@ -1,7 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::borrow::Cow; -use std::fmt::Write; use std::path::Path; use std::path::PathBuf; @@ -52,19 +51,6 @@ pub fn get_extension(file_path: &Path) -> Option { .map(|e| e.to_lowercase()); } -pub fn get_atomic_file_path(file_path: &Path) -> PathBuf { - let rand = gen_rand_path_component(); - let extension = format!("{rand}.tmp"); - file_path.with_extension(extension) -} - -fn gen_rand_path_component() -> String { - (0..4).fold(String::with_capacity(8), |mut output, _| { - write!(&mut output, "{:02x}", rand::random::()).unwrap(); - output - }) -} - /// TypeScript figures out the type of file based on the extension, but we take /// other factors into account like the file headers. The hack here is to map the /// specifier passed to TypeScript to a new specifier with the file extension. diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index 1d0b6237183dbd..8692f04a737007 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -25,10 +25,12 @@ deno_io.workspace = true deno_path_util.workspace = true deno_permissions.workspace = true filetime.workspace = true +getrandom = "0.2" libc.workspace = true rand.workspace = true rayon = "1.8.0" serde.workspace = true +sys_traits.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/ext/fs/interface.rs b/ext/fs/interface.rs index 28a49c5d9b887a..304c2636147c41 100644 --- a/ext/fs/interface.rs +++ b/ext/fs/interface.rs @@ -5,6 +5,8 @@ use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use std::rc::Rc; +use std::time::Duration; +use std::time::SystemTime; use serde::Deserialize; use serde::Serialize; @@ -12,6 +14,8 @@ use serde::Serialize; use deno_io::fs::File; use deno_io::fs::FsResult; use deno_io::fs::FsStat; +use sys_traits::FsFile; +use sys_traits::FsFileSetPermissions; use crate::sync::MaybeSend; use crate::sync::MaybeSync; @@ -71,7 +75,7 @@ pub enum FsFileType { } /// WARNING: This is part of the public JS Deno API. -#[derive(Serialize)] +#[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct FsDirEntry { pub name: String, @@ -100,6 +104,56 @@ impl AccessCheckFn for T where { } +#[derive(Debug)] +pub struct FsStatSlim { + file_type: sys_traits::FileType, + modified: Result, +} + +impl FsStatSlim { + pub fn from_std(metadata: &std::fs::Metadata) -> Self { + Self { + file_type: metadata.file_type().into(), + modified: metadata.modified(), + } + } + + pub fn from_deno_fs_stat(data: &FsStat) -> Self { + FsStatSlim { + file_type: if data.is_file { + sys_traits::FileType::File + } else if data.is_directory { + sys_traits::FileType::Dir + } else if data.is_symlink { + sys_traits::FileType::Symlink + } else { + sys_traits::FileType::Unknown + }, + modified: data + .mtime + .map(|ms| SystemTime::UNIX_EPOCH + Duration::from_millis(ms)) + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "No mtime") + }), + } + } +} + +impl sys_traits::FsMetadataValue for FsStatSlim { + #[inline] + fn file_type(&self) -> sys_traits::FileType { + self.file_type + } + + fn modified(&self) -> Result { + self + .modified + .as_ref() + .copied() + .map_err(|err| std::io::Error::new(err.kind(), err.to_string())) + } +} + pub type AccessCheckCb<'a> = &'a mut (dyn AccessCheckFn + 'a); #[async_trait::async_trait(?Send)] @@ -361,3 +415,289 @@ fn string_from_utf8_lossy(buf: Vec) -> String { Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(buf) }, } } + +// todo(dsherret): this is temporary. Instead of using the `FileSystem` trait implementation +// in the CLI, the CLI should instead create it's own file system using `sys_traits` traits +// then that can implement the `FileSystem` trait. Then this `FileSystem` trait can stay here +// for use only for `ext/fs` and not the entire CLI. +#[derive(Debug, Clone)] +pub struct FsSysTraitsAdapter(pub FileSystemRc); + +impl FsSysTraitsAdapter { + pub fn new_real() -> Self { + Self(crate::sync::new_rc(crate::RealFs)) + } +} + +impl sys_traits::BaseFsHardLink for FsSysTraitsAdapter { + #[inline] + fn base_fs_hard_link(&self, src: &Path, dst: &Path) -> std::io::Result<()> { + self + .0 + .link_sync(src, dst) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsRead for FsSysTraitsAdapter { + #[inline] + fn base_fs_read(&self, path: &Path) -> std::io::Result> { + self + .0 + .read_file_sync(path, None) + .map_err(|err| err.into_io_error()) + } +} + +#[derive(Debug)] +pub struct FsSysTraitsAdapterReadDirEntry { + path: PathBuf, + entry: FsDirEntry, +} + +impl sys_traits::FsDirEntry for FsSysTraitsAdapterReadDirEntry { + type Metadata = FsStatSlim; + + fn file_name(&self) -> Cow { + Cow::Borrowed(self.entry.name.as_ref()) + } + + fn file_type(&self) -> std::io::Result { + if self.entry.is_file { + Ok(sys_traits::FileType::File) + } else if self.entry.is_directory { + Ok(sys_traits::FileType::Dir) + } else if self.entry.is_symlink { + Ok(sys_traits::FileType::Symlink) + } else { + Ok(sys_traits::FileType::Unknown) + } + } + + fn metadata(&self) -> std::io::Result { + Ok(FsStatSlim { + file_type: self.file_type().unwrap(), + modified: Err(std::io::Error::new( + std::io::ErrorKind::Other, + "not supported", + )), + }) + } + + fn path(&self) -> Cow { + Cow::Borrowed(&self.path) + } +} + +impl sys_traits::BaseFsReadDir for FsSysTraitsAdapter { + type ReadDirEntry = FsSysTraitsAdapterReadDirEntry; + + fn base_fs_read_dir( + &self, + path: &Path, + ) -> std::io::Result< + Box>>, + > { + // todo(dsherret): needs to actually be iterable and not allocate a vector + let entries = self + .0 + .read_dir_sync(path) + .map_err(|err| err.into_io_error())?; + let parent_dir = path.to_path_buf(); + Ok(Box::new(entries.into_iter().map(move |entry| { + Ok(FsSysTraitsAdapterReadDirEntry { + path: parent_dir.join(&entry.name), + entry, + }) + }))) + } +} + +impl sys_traits::BaseFsCanonicalize for FsSysTraitsAdapter { + #[inline] + fn base_fs_canonicalize(&self, path: &Path) -> std::io::Result { + self + .0 + .realpath_sync(path) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsMetadata for FsSysTraitsAdapter { + type Metadata = FsStatSlim; + + #[inline] + fn base_fs_metadata(&self, path: &Path) -> std::io::Result { + self + .0 + .stat_sync(path) + .map(|data| FsStatSlim::from_deno_fs_stat(&data)) + .map_err(|err| err.into_io_error()) + } + + #[inline] + fn base_fs_symlink_metadata( + &self, + path: &Path, + ) -> std::io::Result { + self + .0 + .lstat_sync(path) + .map(|data| FsStatSlim::from_deno_fs_stat(&data)) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsCreateDir for FsSysTraitsAdapter { + #[inline] + fn base_fs_create_dir( + &self, + path: &Path, + options: &sys_traits::CreateDirOptions, + ) -> std::io::Result<()> { + self + .0 + .mkdir_sync(path, options.recursive, options.mode) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsRemoveFile for FsSysTraitsAdapter { + #[inline] + fn base_fs_remove_file(&self, path: &Path) -> std::io::Result<()> { + self + .0 + .remove_sync(path, false) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsRename for FsSysTraitsAdapter { + #[inline] + fn base_fs_rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self + .0 + .rename_sync(from, to) + .map_err(|err| err.into_io_error()) + } +} + +pub struct FsFileAdapter(pub Rc); + +impl FsFile for FsFileAdapter {} + +impl FsFileSetPermissions for FsFileAdapter { + #[inline] + fn fs_file_set_permissions(&mut self, mode: u32) -> std::io::Result<()> { + if cfg!(windows) { + Ok(()) // ignore + } else { + self + .0 + .clone() + .chmod_sync(mode) + .map_err(|err| err.into_io_error()) + } + } +} + +impl std::io::Read for FsFileAdapter { + #[inline] + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self + .0 + .clone() + .read_sync(buf) + .map_err(|err| err.into_io_error()) + } +} + +impl std::io::Seek for FsFileAdapter { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self + .0 + .clone() + .seek_sync(pos) + .map_err(|err| err.into_io_error()) + } +} + +impl std::io::Write for FsFileAdapter { + #[inline] + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self + .0 + .clone() + .write_sync(buf) + .map_err(|err| err.into_io_error()) + } + + #[inline] + fn flush(&mut self) -> std::io::Result<()> { + self + .0 + .clone() + .sync_sync() + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::BaseFsOpen for FsSysTraitsAdapter { + type File = FsFileAdapter; + + fn base_fs_open( + &self, + path: &Path, + options: &sys_traits::OpenOptions, + ) -> std::io::Result { + self + .0 + .open_sync( + path, + OpenOptions { + read: options.read, + write: options.write, + create: options.create, + truncate: options.truncate, + append: options.append, + create_new: options.create_new, + mode: options.mode, + }, + None, + ) + .map(FsFileAdapter) + .map_err(|err| err.into_io_error()) + } +} + +impl sys_traits::SystemRandom for FsSysTraitsAdapter { + #[inline] + fn sys_random(&self, buf: &mut [u8]) -> std::io::Result<()> { + getrandom::getrandom(buf).map_err(|err| { + std::io::Error::new(std::io::ErrorKind::Other, err.to_string()) + }) + } +} + +impl sys_traits::SystemTimeNow for FsSysTraitsAdapter { + #[inline] + fn sys_time_now(&self) -> SystemTime { + SystemTime::now() + } +} + +impl sys_traits::ThreadSleep for FsSysTraitsAdapter { + #[inline] + fn thread_sleep(&self, dur: Duration) { + std::thread::sleep(dur); + } +} + +impl sys_traits::BaseEnvVar for FsSysTraitsAdapter { + fn base_env_var_os( + &self, + key: &std::ffi::OsStr, + ) -> Option { + std::env::var_os(key) + } +} diff --git a/ext/fs/lib.rs b/ext/fs/lib.rs index 26fac1e79f5b02..cfcf249783de43 100644 --- a/ext/fs/lib.rs +++ b/ext/fs/lib.rs @@ -13,6 +13,8 @@ pub use crate::interface::FileSystem; pub use crate::interface::FileSystemRc; pub use crate::interface::FsDirEntry; pub use crate::interface::FsFileType; +pub use crate::interface::FsStatSlim; +pub use crate::interface::FsSysTraitsAdapter; pub use crate::interface::OpenOptions; pub use crate::ops::FsOpsError; pub use crate::ops::FsOpsErrorKind; diff --git a/ext/fs/sync.rs b/ext/fs/sync.rs index 6a913f658a7923..06694f1dc4296b 100644 --- a/ext/fs/sync.rs +++ b/ext/fs/sync.rs @@ -21,3 +21,9 @@ mod inner { pub trait MaybeSend {} impl MaybeSend for T where T: ?Sized {} } + +#[allow(clippy::disallowed_types)] +#[inline] +pub fn new_rc(value: T) -> MaybeArc { + MaybeArc::new(value) +} diff --git a/ext/node/lib.rs b/ext/node/lib.rs index 1e6c920c9e6d54..b9b459efc135c4 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -14,7 +14,9 @@ use deno_core::url::Url; #[allow(unused_imports)] use deno_core::v8; use deno_core::v8::ExternalReference; +use deno_fs::FsSysTraitsAdapter; use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::IsBuiltInNodeModuleChecker; use node_resolver::NpmPackageFolderResolverRc; use once_cell::sync::Lazy; @@ -807,92 +809,28 @@ deno_core::extension!(deno_node, }, ); -pub type NodeResolver = node_resolver::NodeResolver; -#[allow(clippy::disallowed_types)] -pub type NodeResolverRc = - deno_fs::sync::MaybeArc>; -pub type PackageJsonResolver = - node_resolver::PackageJsonResolver; -#[allow(clippy::disallowed_types)] -pub type PackageJsonResolverRc = deno_fs::sync::MaybeArc< - node_resolver::PackageJsonResolver, ->; - #[derive(Debug)] -pub struct DenoFsNodeResolverEnv { - fs: deno_fs::FileSystemRc, -} - -impl DenoFsNodeResolverEnv { - pub fn new(fs: deno_fs::FileSystemRc) -> Self { - Self { fs } - } -} +pub struct RealIsBuiltInNodeModuleChecker; -impl node_resolver::env::NodeResolverEnv for DenoFsNodeResolverEnv { +impl IsBuiltInNodeModuleChecker for RealIsBuiltInNodeModuleChecker { + #[inline] fn is_builtin_node_module(&self, specifier: &str) -> bool { is_builtin_node_module(specifier) } - - fn realpath_sync( - &self, - path: &std::path::Path, - ) -> std::io::Result { - self - .fs - .realpath_sync(path) - .map_err(|err| err.into_io_error()) - } - - fn stat_sync( - &self, - path: &std::path::Path, - ) -> std::io::Result { - self - .fs - .stat_sync(path) - .map(|stat| node_resolver::env::NodeResolverFsStat { - is_file: stat.is_file, - is_dir: stat.is_directory, - is_symlink: stat.is_symlink, - }) - .map_err(|err| err.into_io_error()) - } - - fn exists_sync(&self, path: &std::path::Path) -> bool { - self.fs.exists_sync(path) - } - - fn pkg_json_fs(&self) -> &dyn deno_package_json::fs::DenoPkgJsonFs { - self - } } -impl deno_package_json::fs::DenoPkgJsonFs for DenoFsNodeResolverEnv { - fn read_to_string_lossy( - &self, - path: &std::path::Path, - ) -> Result, std::io::Error> { - self - .fs - .read_text_file_lossy_sync(path, None) - .map_err(|err| err.into_io_error()) - } -} - -pub struct DenoPkgJsonFsAdapter<'a>(pub &'a dyn deno_fs::FileSystem); - -impl<'a> deno_package_json::fs::DenoPkgJsonFs for DenoPkgJsonFsAdapter<'a> { - fn read_to_string_lossy( - &self, - path: &Path, - ) -> Result, std::io::Error> { - self - .0 - .read_text_file_lossy_sync(path, None) - .map_err(|err| err.into_io_error()) - } -} +pub type NodeResolver = node_resolver::NodeResolver< + RealIsBuiltInNodeModuleChecker, + FsSysTraitsAdapter, +>; +#[allow(clippy::disallowed_types)] +pub type NodeResolverRc = deno_fs::sync::MaybeArc; +pub type PackageJsonResolver = + node_resolver::PackageJsonResolver; +#[allow(clippy::disallowed_types)] +pub type PackageJsonResolverRc = deno_fs::sync::MaybeArc< + node_resolver::PackageJsonResolver, +>; pub fn create_host_defined_options<'s>( scope: &mut v8::HandleScope<'s>, diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml index a7273c7e73837f..12c18d4452d62a 100644 --- a/resolvers/deno/Cargo.toml +++ b/resolvers/deno/Cargo.toml @@ -29,6 +29,7 @@ deno_path_util.workspace = true deno_semver.workspace = true node_resolver.workspace = true node_resolver.features = ["sync"] +sys_traits.workspace = true thiserror.workspace = true url.workspace = true diff --git a/resolvers/deno/cjs.rs b/resolvers/deno/cjs.rs index 6ae648deab32c6..2ec253d41a7fef 100644 --- a/resolvers/deno/cjs.rs +++ b/resolvers/deno/cjs.rs @@ -1,29 +1,30 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::sync::MaybeDashMap; use deno_media_type::MediaType; -use node_resolver::env::NodeResolverEnv; use node_resolver::errors::ClosestPkgJsonError; use node_resolver::InNpmPackageCheckerRc; use node_resolver::PackageJsonResolverRc; use node_resolver::ResolutionMode; +use sys_traits::FsRead; use url::Url; +use crate::sync::MaybeDashMap; + /// Keeps track of what module specifiers were resolved as CJS. /// /// Modules that are `.js`, `.ts`, `.jsx`, and `tsx` are only known to /// be CJS or ESM after they're loaded based on their contents. So these /// files will be "maybe CJS" until they're loaded. #[derive(Debug)] -pub struct CjsTracker { - is_cjs_resolver: IsCjsResolver, +pub struct CjsTracker { + is_cjs_resolver: IsCjsResolver, known: MaybeDashMap, } -impl CjsTracker { +impl CjsTracker { pub fn new( in_npm_pkg_checker: InNpmPackageCheckerRc, - pkg_json_resolver: PackageJsonResolverRc, + pkg_json_resolver: PackageJsonResolverRc, mode: IsCjsResolutionMode, ) -> Self { Self { @@ -124,16 +125,16 @@ pub enum IsCjsResolutionMode { /// Resolves whether a module is CJS or ESM. #[derive(Debug)] -pub struct IsCjsResolver { +pub struct IsCjsResolver { in_npm_pkg_checker: InNpmPackageCheckerRc, - pkg_json_resolver: PackageJsonResolverRc, + pkg_json_resolver: PackageJsonResolverRc, mode: IsCjsResolutionMode, } -impl IsCjsResolver { +impl IsCjsResolver { pub fn new( in_npm_pkg_checker: InNpmPackageCheckerRc, - pkg_json_resolver: PackageJsonResolverRc, + pkg_json_resolver: PackageJsonResolverRc, mode: IsCjsResolutionMode, ) -> Self { Self { diff --git a/resolvers/deno/fs.rs b/resolvers/deno/fs.rs deleted file mode 100644 index f2021a73a9ca87..00000000000000 --- a/resolvers/deno/fs.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::borrow::Cow; -use std::path::Path; -use std::path::PathBuf; - -pub struct DirEntry { - pub name: String, - pub is_file: bool, - pub is_directory: bool, -} - -pub trait DenoResolverFs { - fn read_to_string_lossy( - &self, - path: &Path, - ) -> std::io::Result>; - fn realpath_sync(&self, path: &Path) -> std::io::Result; - fn exists_sync(&self, path: &Path) -> bool; - fn is_dir_sync(&self, path: &Path) -> bool; - fn read_dir_sync(&self, dir_path: &Path) -> std::io::Result>; -} diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs index 05fa416da1b7e4..c943aacdaea14e 100644 --- a/resolvers/deno/lib.rs +++ b/resolvers/deno/lib.rs @@ -14,11 +14,10 @@ use deno_config::workspace::WorkspaceResolver; use deno_package_json::PackageJsonDepValue; use deno_package_json::PackageJsonDepValueParseError; use deno_semver::npm::NpmPackageReqReference; -use fs::DenoResolverFs; -use node_resolver::env::NodeResolverEnv; use node_resolver::errors::NodeResolveError; use node_resolver::errors::PackageSubpathResolveError; use node_resolver::InNpmPackageCheckerRc; +use node_resolver::IsBuiltInNodeModuleChecker; use node_resolver::NodeResolution; use node_resolver::NodeResolutionKind; use node_resolver::NodeResolverRc; @@ -32,11 +31,14 @@ use npm::ResolveReqWithSubPathErrorKind; use sloppy_imports::SloppyImportResolverFs; use sloppy_imports::SloppyImportsResolutionKind; use sloppy_imports::SloppyImportsResolverRc; +use sys_traits::FsCanonicalize; +use sys_traits::FsMetadata; +use sys_traits::FsRead; +use sys_traits::FsReadDir; use thiserror::Error; use url::Url; pub mod cjs; -pub mod fs; pub mod npm; pub mod sloppy_imports; mod sync; @@ -80,22 +82,22 @@ pub enum DenoResolveErrorKind { #[derive(Debug)] pub struct NodeAndNpmReqResolver< - Fs: DenoResolverFs, - TNodeResolverEnv: NodeResolverEnv, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { - pub node_resolver: NodeResolverRc, - pub npm_req_resolver: NpmReqResolverRc, + pub node_resolver: NodeResolverRc, + pub npm_req_resolver: NpmReqResolverRc, } pub struct DenoResolverOptions< 'a, - Fs: DenoResolverFs, - TNodeResolverEnv: NodeResolverEnv, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, TSloppyImportResolverFs: SloppyImportResolverFs, + TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { pub in_npm_pkg_checker: InNpmPackageCheckerRc, pub node_and_req_resolver: - Option>, + Option>, pub sloppy_imports_resolver: Option>, pub workspace_resolver: WorkspaceResolverRc, @@ -110,12 +112,13 @@ pub struct DenoResolverOptions< /// import map, JSX settings. #[derive(Debug)] pub struct DenoResolver< - Fs: DenoResolverFs, - TNodeResolverEnv: NodeResolverEnv, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, TSloppyImportResolverFs: SloppyImportResolverFs, + TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { in_npm_pkg_checker: InNpmPackageCheckerRc, - node_and_npm_resolver: Option>, + node_and_npm_resolver: + Option>, sloppy_imports_resolver: Option>, workspace_resolver: WorkspaceResolverRc, @@ -124,13 +127,17 @@ pub struct DenoResolver< } impl< - Fs: DenoResolverFs, - TNodeResolverEnv: NodeResolverEnv, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, TSloppyImportResolverFs: SloppyImportResolverFs, - > DenoResolver + TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, + > DenoResolver { pub fn new( - options: DenoResolverOptions, + options: DenoResolverOptions< + TIsBuiltInNodeModuleChecker, + TSloppyImportResolverFs, + TSys, + >, ) -> Self { Self { in_npm_pkg_checker: options.in_npm_pkg_checker, diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs index c6f5d1cd041a0d..3056a70f612d41 100644 --- a/resolvers/deno/npm/byonm.rs +++ b/resolvers/deno/npm/byonm.rs @@ -11,7 +11,6 @@ use deno_path_util::url_to_file_path; use deno_semver::package::PackageReq; use deno_semver::StackString; use deno_semver::Version; -use node_resolver::env::NodeResolverEnv; use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageFolderResolveIoError; use node_resolver::errors::PackageJsonLoadError; @@ -19,11 +18,14 @@ use node_resolver::errors::PackageNotFoundError; use node_resolver::InNpmPackageChecker; use node_resolver::NpmPackageFolderResolver; use node_resolver::PackageJsonResolverRc; +use sys_traits::FsCanonicalize; +use sys_traits::FsDirEntry; +use sys_traits::FsMetadata; +use sys_traits::FsRead; +use sys_traits::FsReadDir; use thiserror::Error; use url::Url; -use crate::fs::DenoResolverFs; - use super::local::normalize_pkg_name_for_node_modules_deno_folder; use super::CliNpmReqResolver; use super::ResolvePkgFolderFromDenoReqError; @@ -40,44 +42,45 @@ pub enum ByonmResolvePkgFolderFromDenoReqError { Io(#[from] std::io::Error), } -pub struct ByonmNpmResolverCreateOptions< - Fs: DenoResolverFs, - TEnv: NodeResolverEnv, -> { +pub struct ByonmNpmResolverCreateOptions { // todo(dsherret): investigate removing this pub root_node_modules_dir: Option, - pub fs: Fs, - pub pkg_json_resolver: PackageJsonResolverRc, + pub sys: TSys, + pub pkg_json_resolver: PackageJsonResolverRc, } #[allow(clippy::disallowed_types)] -pub type ByonmNpmResolverRc = - crate::sync::MaybeArc>; +pub type ByonmNpmResolverRc = + crate::sync::MaybeArc>; #[derive(Debug)] -pub struct ByonmNpmResolver { - fs: Fs, - pkg_json_resolver: PackageJsonResolverRc, +pub struct ByonmNpmResolver< + TSys: FsCanonicalize + FsRead + FsMetadata + FsReadDir, +> { + sys: TSys, + pkg_json_resolver: PackageJsonResolverRc, root_node_modules_dir: Option, } -impl Clone - for ByonmNpmResolver +impl Clone + for ByonmNpmResolver { fn clone(&self) -> Self { Self { - fs: self.fs.clone(), + sys: self.sys.clone(), pkg_json_resolver: self.pkg_json_resolver.clone(), root_node_modules_dir: self.root_node_modules_dir.clone(), } } } -impl ByonmNpmResolver { - pub fn new(options: ByonmNpmResolverCreateOptions) -> Self { +impl + ByonmNpmResolver +{ + pub fn new(options: ByonmNpmResolverCreateOptions) -> Self { Self { root_node_modules_dir: options.root_node_modules_dir, - fs: options.fs, + sys: options.sys, pkg_json_resolver: options.pkg_json_resolver, } } @@ -129,19 +132,20 @@ impl ByonmNpmResolver { req: &PackageReq, referrer: &Url, ) -> Result { - fn node_resolve_dir( - fs: &Fs, + fn node_resolve_dir( + sys: &TSys, alias: &str, start_dir: &Path, ) -> std::io::Result> { for ancestor in start_dir.ancestors() { let node_modules_folder = ancestor.join("node_modules"); let sub_dir = join_package_name(&node_modules_folder, alias); - if fs.is_dir_sync(&sub_dir) { - return Ok(Some(deno_path_util::canonicalize_path_maybe_not_exists( - &sub_dir, - &|path| fs.realpath_sync(path), - )?)); + if sys.fs_is_dir_no_err(&sub_dir) { + return Ok(Some( + deno_path_util::fs::canonicalize_path_maybe_not_exists( + sys, &sub_dir, + )?, + )); } } Ok(None) @@ -154,7 +158,7 @@ impl ByonmNpmResolver { Some((pkg_json, alias)) => { // now try node resolution if let Some(resolved) = - node_resolve_dir(&self.fs, &alias, pkg_json.dir_path())? + node_resolve_dir(&self.sys, &alias, pkg_json.dir_path())? { return Ok(resolved); } @@ -298,7 +302,7 @@ impl ByonmNpmResolver { // now check if node_modules/.deno/ matches this constraint let root_node_modules_dir = self.root_node_modules_dir.as_ref()?; let node_modules_deno_dir = root_node_modules_dir.join(".deno"); - let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else { + let Ok(entries) = self.sys.fs_read_dir(&node_modules_deno_dir) else { return None; }; let search_prefix = format!( @@ -311,10 +315,17 @@ impl ByonmNpmResolver { // - @denotest+add@1.0.0 // - @denotest+add@1.0.0_1 for entry in entries { - if !entry.is_directory { + let Ok(entry) = entry else { + continue; + }; + let Ok(file_type) = entry.file_type() else { + continue; + }; + if !file_type.is_dir() { continue; } - let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix) + let entry_name = entry.file_name().to_string_lossy().into_owned(); + let Some(version_and_copy_idx) = entry_name.strip_prefix(&search_prefix) else { continue; }; @@ -327,8 +338,8 @@ impl ByonmNpmResolver { }; if let Some(tag) = req.version_req.tag() { let initialized_file = - node_modules_deno_dir.join(&entry.name).join(".initialized"); - let Ok(contents) = self.fs.read_to_string_lossy(&initialized_file) + node_modules_deno_dir.join(&entry_name).join(".initialized"); + let Ok(contents) = self.sys.fs_read_to_string_lossy(&initialized_file) else { continue; }; @@ -336,19 +347,19 @@ impl ByonmNpmResolver { if tags.any(|t| t == tag) { if let Some((best_version_version, _)) = &best_version { if version > *best_version_version { - best_version = Some((version, entry.name)); + best_version = Some((version, entry_name)); } } else { - best_version = Some((version, entry.name)); + best_version = Some((version, entry_name)); } } } else if req.version_req.matches(&version) { if let Some((best_version_version, _)) = &best_version { if version > *best_version_version { - best_version = Some((version, entry.name)); + best_version = Some((version, entry_name)); } } else { - best_version = Some((version, entry.name)); + best_version = Some((version, entry_name)); } } } @@ -363,9 +374,14 @@ impl ByonmNpmResolver { } impl< - Fs: DenoResolverFs + Send + Sync + std::fmt::Debug, - TEnv: NodeResolverEnv, - > CliNpmReqResolver for ByonmNpmResolver + Sys: FsCanonicalize + + FsMetadata + + FsRead + + FsReadDir + + Send + + Sync + + std::fmt::Debug, + > CliNpmReqResolver for ByonmNpmResolver { fn resolve_pkg_folder_from_deno_module_req( &self, @@ -380,17 +396,22 @@ impl< } impl< - Fs: DenoResolverFs + Send + Sync + std::fmt::Debug, - TEnv: NodeResolverEnv, - > NpmPackageFolderResolver for ByonmNpmResolver + Sys: FsCanonicalize + + FsMetadata + + FsRead + + FsReadDir + + Send + + Sync + + std::fmt::Debug, + > NpmPackageFolderResolver for ByonmNpmResolver { fn resolve_package_folder_from_package( &self, name: &str, referrer: &Url, ) -> Result { - fn inner( - fs: &Fs, + fn inner( + sys: &TSys, name: &str, referrer: &Url, ) -> Result { @@ -407,7 +428,7 @@ impl< }; let sub_dir = join_package_name(&node_modules_folder, name); - if fs.is_dir_sync(&sub_dir) { + if sys.fs_is_dir_no_err(&sub_dir) { return Ok(sub_dir); } } @@ -423,8 +444,8 @@ impl< ) } - let path = inner(&self.fs, name, referrer)?; - self.fs.realpath_sync(&path).map_err(|err| { + let path = inner(&self.sys, name, referrer)?; + self.sys.fs_canonicalize(&path).map_err(|err| { PackageFolderResolveIoError { package_name: name.to_string(), referrer: referrer.clone(), diff --git a/resolvers/deno/npm/mod.rs b/resolvers/deno/npm/mod.rs index 64ec86fe3f6615..082940eb343733 100644 --- a/resolvers/deno/npm/mod.rs +++ b/resolvers/deno/npm/mod.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use boxed_error::Boxed; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; -use node_resolver::env::NodeResolverEnv; use node_resolver::errors::NodeResolveError; use node_resolver::errors::NodeResolveErrorKind; use node_resolver::errors::PackageFolderResolveErrorKind; @@ -15,15 +14,18 @@ use node_resolver::errors::PackageNotFoundError; use node_resolver::errors::PackageResolveErrorKind; use node_resolver::errors::PackageSubpathResolveError; use node_resolver::InNpmPackageCheckerRc; +use node_resolver::IsBuiltInNodeModuleChecker; use node_resolver::NodeResolution; use node_resolver::NodeResolutionKind; use node_resolver::NodeResolverRc; use node_resolver::ResolutionMode; +use sys_traits::FsCanonicalize; +use sys_traits::FsMetadata; +use sys_traits::FsRead; +use sys_traits::FsReadDir; use thiserror::Error; use url::Url; -use crate::fs::DenoResolverFs; - pub use byonm::ByonmInNpmPackageChecker; pub use byonm::ByonmNpmResolver; pub use byonm::ByonmNpmResolverCreateOptions; @@ -95,40 +97,46 @@ pub trait CliNpmReqResolver: Debug + Send + Sync { } pub struct NpmReqResolverOptions< - Fs: DenoResolverFs, - TNodeResolverEnv: NodeResolverEnv, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, > { /// The resolver when "bring your own node_modules" is enabled where Deno /// does not setup the node_modules directories automatically, but instead /// uses what already exists on the file system. - pub byonm_resolver: Option>, - pub fs: Fs, + pub byonm_resolver: Option>, pub in_npm_pkg_checker: InNpmPackageCheckerRc, - pub node_resolver: NodeResolverRc, + pub node_resolver: NodeResolverRc, pub npm_req_resolver: CliNpmReqResolverRc, + pub sys: TSys, } #[allow(clippy::disallowed_types)] -pub type NpmReqResolverRc = - crate::sync::MaybeArc>; +pub type NpmReqResolverRc = + crate::sync::MaybeArc>; #[derive(Debug)] -pub struct NpmReqResolver -{ - byonm_resolver: Option>, - fs: Fs, +pub struct NpmReqResolver< + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, +> { + byonm_resolver: Option>, + sys: TSys, in_npm_pkg_checker: InNpmPackageCheckerRc, - node_resolver: NodeResolverRc, + node_resolver: NodeResolverRc, npm_resolver: CliNpmReqResolverRc, } -impl - NpmReqResolver +impl< + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir, + > NpmReqResolver { - pub fn new(options: NpmReqResolverOptions) -> Self { + pub fn new( + options: NpmReqResolverOptions, + ) -> Self { Self { byonm_resolver: options.byonm_resolver, - fs: options.fs, + sys: options.sys, in_npm_pkg_checker: options.in_npm_pkg_checker, node_resolver: options.node_resolver, npm_resolver: options.npm_req_resolver, @@ -175,7 +183,7 @@ impl Err(err) => { if self.byonm_resolver.is_some() { let package_json_path = package_folder.join("package.json"); - if !self.fs.exists_sync(&package_json_path) { + if !self.sys.fs_exists_no_err(&package_json_path) { return Err( MissingPackageNodeModulesFolderError { package_json_path }.into(), ); diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml index 52aedbee9da55a..1e35c0a3555ab9 100644 --- a/resolvers/node/Cargo.toml +++ b/resolvers/node/Cargo.toml @@ -29,6 +29,7 @@ once_cell.workspace = true path-clean = "=0.1.0" regex.workspace = true serde_json.workspace = true +sys_traits.workspace = true thiserror.workspace = true tokio.workspace = true url.workspace = true diff --git a/resolvers/node/analyze.rs b/resolvers/node/analyze.rs index 9e6219b0822559..2024e6a1e89c24 100644 --- a/resolvers/node/analyze.rs +++ b/resolvers/node/analyze.rs @@ -16,11 +16,14 @@ use once_cell::sync::Lazy; use anyhow::Context; use anyhow::Error as AnyError; +use sys_traits::FsCanonicalize; +use sys_traits::FsMetadata; +use sys_traits::FsRead; use url::Url; -use crate::env::NodeResolverEnv; use crate::npm::InNpmPackageCheckerRc; use crate::resolution::NodeResolverRc; +use crate::IsBuiltInNodeModuleChecker; use crate::NodeResolutionKind; use crate::NpmPackageFolderResolverRc; use crate::PackageJsonResolverRc; @@ -60,34 +63,38 @@ pub trait CjsCodeAnalyzer { pub struct NodeCodeTranslator< TCjsCodeAnalyzer: CjsCodeAnalyzer, - TNodeResolverEnv: NodeResolverEnv, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TSys: FsCanonicalize + FsMetadata + FsRead, > { cjs_code_analyzer: TCjsCodeAnalyzer, - env: TNodeResolverEnv, in_npm_pkg_checker: InNpmPackageCheckerRc, - node_resolver: NodeResolverRc, + node_resolver: NodeResolverRc, npm_resolver: NpmPackageFolderResolverRc, - pkg_json_resolver: PackageJsonResolverRc, + pkg_json_resolver: PackageJsonResolverRc, + sys: TSys, } -impl - NodeCodeTranslator +impl< + TCjsCodeAnalyzer: CjsCodeAnalyzer, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TSys: FsCanonicalize + FsMetadata + FsRead, + > NodeCodeTranslator { pub fn new( cjs_code_analyzer: TCjsCodeAnalyzer, - env: TNodeResolverEnv, in_npm_pkg_checker: InNpmPackageCheckerRc, - node_resolver: NodeResolverRc, + node_resolver: NodeResolverRc, npm_resolver: NpmPackageFolderResolverRc, - pkg_json_resolver: PackageJsonResolverRc, + pkg_json_resolver: PackageJsonResolverRc, + sys: TSys, ) -> Self { Self { cjs_code_analyzer, - env, in_npm_pkg_checker, node_resolver, npm_resolver, pkg_json_resolver, + sys, } } @@ -366,7 +373,7 @@ impl // old school if package_subpath != "." { let d = module_dir.join(package_subpath); - if self.env.is_dir_sync(&d) { + if self.sys.fs_is_dir_no_err(&d) { // subdir might have a package.json that specifies the entrypoint let package_json_path = d.join("package.json"); let maybe_package_json = self @@ -423,13 +430,13 @@ impl referrer: &Path, ) -> Result { let p = p.clean(); - if self.env.exists_sync(&p) { + if self.sys.fs_exists_no_err(&p) { let file_name = p.file_name().unwrap(); let p_js = p.with_file_name(format!("{}.js", file_name.to_str().unwrap())); - if self.env.is_file_sync(&p_js) { + if self.sys.fs_is_file_no_err(&p_js) { return Ok(p_js); - } else if self.env.is_dir_sync(&p) { + } else if self.sys.fs_is_dir_no_err(&p) { return Ok(p.join("index.js")); } else { return Ok(p); @@ -438,14 +445,14 @@ impl { let p_js = p.with_file_name(format!("{}.js", file_name.to_str().unwrap())); - if self.env.is_file_sync(&p_js) { + if self.sys.fs_is_file_no_err(&p_js) { return Ok(p_js); } } { let p_json = p.with_file_name(format!("{}.json", file_name.to_str().unwrap())); - if self.env.is_file_sync(&p_json) { + if self.sys.fs_is_file_no_err(&p_json) { return Ok(p_json); } } diff --git a/resolvers/node/env.rs b/resolvers/node/env.rs deleted file mode 100644 index b520ece0f8d8dc..00000000000000 --- a/resolvers/node/env.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::path::Path; -use std::path::PathBuf; - -use crate::sync::MaybeSend; -use crate::sync::MaybeSync; - -pub struct NodeResolverFsStat { - pub is_file: bool, - pub is_dir: bool, - pub is_symlink: bool, -} - -pub trait NodeResolverEnv: std::fmt::Debug + MaybeSend + MaybeSync { - fn is_builtin_node_module(&self, specifier: &str) -> bool; - - fn realpath_sync(&self, path: &Path) -> std::io::Result; - - fn stat_sync(&self, path: &Path) -> std::io::Result; - - fn exists_sync(&self, path: &Path) -> bool; - - fn is_file_sync(&self, path: &Path) -> bool { - self - .stat_sync(path) - .map(|stat| stat.is_file) - .unwrap_or(false) - } - - fn is_dir_sync(&self, path: &Path) -> bool { - self - .stat_sync(path) - .map(|stat| stat.is_dir) - .unwrap_or(false) - } - - fn pkg_json_fs(&self) -> &dyn deno_package_json::fs::DenoPkgJsonFs; -} diff --git a/resolvers/node/lib.rs b/resolvers/node/lib.rs index c73c395dfccfb3..075f819ebbcf6e 100644 --- a/resolvers/node/lib.rs +++ b/resolvers/node/lib.rs @@ -4,7 +4,6 @@ #![deny(clippy::print_stdout)] pub mod analyze; -pub mod env; pub mod errors; mod npm; mod package_json; @@ -23,6 +22,7 @@ pub use package_json::PackageJsonThreadLocalCache; pub use path::PathClean; pub use resolution::parse_npm_pkg_name; pub use resolution::resolve_specifier_into_node_modules; +pub use resolution::IsBuiltInNodeModuleChecker; pub use resolution::NodeResolution; pub use resolution::NodeResolutionKind; pub use resolution::NodeResolver; diff --git a/resolvers/node/package_json.rs b/resolvers/node/package_json.rs index cb99e5a0aa0cbc..ebbe099014c043 100644 --- a/resolvers/node/package_json.rs +++ b/resolvers/node/package_json.rs @@ -1,15 +1,16 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_package_json::PackageJson; -use deno_package_json::PackageJsonRc; use std::cell::RefCell; use std::collections::HashMap; use std::io::ErrorKind; use std::path::Path; use std::path::PathBuf; + +use deno_package_json::PackageJson; +use deno_package_json::PackageJsonRc; +use sys_traits::FsRead; use url::Url; -use crate::env::NodeResolverEnv; use crate::errors::ClosestPkgJsonError; use crate::errors::PackageJsonLoadError; @@ -38,17 +39,17 @@ impl deno_package_json::PackageJsonCache for PackageJsonThreadLocalCache { } #[allow(clippy::disallowed_types)] -pub type PackageJsonResolverRc = - crate::sync::MaybeArc>; +pub type PackageJsonResolverRc = + crate::sync::MaybeArc>; #[derive(Debug)] -pub struct PackageJsonResolver { - env: TEnv, +pub struct PackageJsonResolver { + sys: TSys, } -impl PackageJsonResolver { - pub fn new(env: TEnv) -> Self { - Self { env } +impl PackageJsonResolver { + pub fn new(sys: TSys) -> Self { + Self { sys } } pub fn get_closest_package_json( @@ -81,9 +82,9 @@ impl PackageJsonResolver { path: &Path, ) -> Result, PackageJsonLoadError> { let result = PackageJson::load_from_path( - path, - self.env.pkg_json_fs(), + &self.sys, Some(&PackageJsonThreadLocalCache), + path, ); match result { Ok(pkg_json) => Ok(Some(pkg_json)), diff --git a/resolvers/node/resolution.rs b/resolvers/node/resolution.rs index 6b0bda57d7fde8..95631daf39565d 100644 --- a/resolvers/node/resolution.rs +++ b/resolvers/node/resolution.rs @@ -9,9 +9,13 @@ use anyhow::Error as AnyError; use deno_path_util::url_from_file_path; use serde_json::Map; use serde_json::Value; +use sys_traits::FileType; +use sys_traits::FsCanonicalize; +use sys_traits::FsMetadata; +use sys_traits::FsMetadataValue; +use sys_traits::FsRead; use url::Url; -use crate::env::NodeResolverEnv; use crate::errors; use crate::errors::DataUrlReferrerError; use crate::errors::FinalizeResolutionError; @@ -98,29 +102,44 @@ impl NodeResolution { } } +pub trait IsBuiltInNodeModuleChecker: std::fmt::Debug { + fn is_builtin_node_module(&self, specifier: &str) -> bool; +} + #[allow(clippy::disallowed_types)] -pub type NodeResolverRc = crate::sync::MaybeArc>; +pub type NodeResolverRc = + crate::sync::MaybeArc>; #[derive(Debug)] -pub struct NodeResolver { - env: TEnv, +pub struct NodeResolver< + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TSys: FsCanonicalize + FsMetadata + FsRead, +> { in_npm_pkg_checker: InNpmPackageCheckerRc, + is_built_in_node_module_checker: TIsBuiltInNodeModuleChecker, npm_pkg_folder_resolver: NpmPackageFolderResolverRc, - pkg_json_resolver: PackageJsonResolverRc, + pkg_json_resolver: PackageJsonResolverRc, + sys: TSys, } -impl NodeResolver { +impl< + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TSys: FsCanonicalize + FsMetadata + FsRead, + > NodeResolver +{ pub fn new( - env: TEnv, in_npm_pkg_checker: InNpmPackageCheckerRc, + is_built_in_node_module_checker: TIsBuiltInNodeModuleChecker, npm_pkg_folder_resolver: NpmPackageFolderResolverRc, - pkg_json_resolver: PackageJsonResolverRc, + pkg_json_resolver: PackageJsonResolverRc, + sys: TSys, ) -> Self { Self { - env, in_npm_pkg_checker, + is_built_in_node_module_checker, npm_pkg_folder_resolver, pkg_json_resolver, + sys, } } @@ -140,7 +159,10 @@ impl NodeResolver { // Note: if we are here, then the referrer is an esm module // TODO(bartlomieju): skipped "policy" part as we don't plan to support it - if self.env.is_builtin_node_module(specifier) { + if self + .is_built_in_node_module_checker + .is_builtin_node_module(specifier) + { return Ok(NodeResolution::BuiltIn(specifier.to_string())); } @@ -282,32 +304,25 @@ impl NodeResolver { p_str.to_string() }; - let (is_dir, is_file) = if let Ok(stats) = self.env.stat_sync(Path::new(&p)) - { - (stats.is_dir, stats.is_file) - } else { - (false, false) - }; - if is_dir { - return Err( + let maybe_file_type = self.sys.fs_metadata(p).map(|m| m.file_type()); + match maybe_file_type { + Ok(FileType::Dir) => Err( UnsupportedDirImportError { dir_url: resolved.clone(), maybe_referrer: maybe_referrer.map(ToOwned::to_owned), } .into(), - ); - } else if !is_file { - return Err( + ), + Ok(FileType::File) => Ok(resolved), + _ => Err( ModuleNotFoundError { specifier: resolved, maybe_referrer: maybe_referrer.map(ToOwned::to_owned), typ: "module", } .into(), - ); + ), } - - Ok(resolved) } pub fn resolve_package_subpath_from_deno_module( @@ -397,8 +412,8 @@ impl NodeResolver { maybe_referrer: Option<&Url>, resolution_mode: ResolutionMode, ) -> Result { - fn probe_extensions( - fs: &TEnv, + fn probe_extensions( + sys: &TSys, path: &Path, lowercase_path: &str, resolution_mode: ResolutionMode, @@ -407,20 +422,20 @@ impl NodeResolver { let mut searched_for_d_cts = false; if lowercase_path.ends_with(".mjs") { let d_mts_path = with_known_extension(path, "d.mts"); - if fs.exists_sync(&d_mts_path) { + if sys.fs_exists_no_err(&d_mts_path) { return Some(d_mts_path); } searched_for_d_mts = true; } else if lowercase_path.ends_with(".cjs") { let d_cts_path = with_known_extension(path, "d.cts"); - if fs.exists_sync(&d_cts_path) { + if sys.fs_exists_no_err(&d_cts_path) { return Some(d_cts_path); } searched_for_d_cts = true; } let dts_path = with_known_extension(path, "d.ts"); - if fs.exists_sync(&dts_path) { + if sys.fs_exists_no_err(&dts_path) { return Some(dts_path); } @@ -434,7 +449,7 @@ impl NodeResolver { _ => None, // already searched above }; if let Some(specific_dts_path) = specific_dts_path { - if fs.exists_sync(&specific_dts_path) { + if sys.fs_exists_no_err(&specific_dts_path) { return Some(specific_dts_path); } } @@ -449,11 +464,11 @@ impl NodeResolver { return Ok(url_from_file_path(path).unwrap()); } if let Some(path) = - probe_extensions(&self.env, path, &lowercase_path, resolution_mode) + probe_extensions(&self.sys, path, &lowercase_path, resolution_mode) { return Ok(url_from_file_path(&path).unwrap()); } - if self.env.is_dir_sync(path) { + if self.sys.fs_is_dir_no_err(path) { let resolution_result = self.resolve_package_dir_subpath( path, /* sub path */ ".", @@ -467,7 +482,7 @@ impl NodeResolver { } let index_path = path.join("index.js"); if let Some(path) = probe_extensions( - &self.env, + &self.sys, &index_path, &index_path.to_string_lossy().to_lowercase(), resolution_mode, @@ -671,7 +686,10 @@ impl NodeResolver { return match result { Ok(url) => Ok(url), Err(err) => { - if self.env.is_builtin_node_module(target) { + if self + .is_built_in_node_module_checker + .is_builtin_node_module(target) + { Ok(Url::parse(&format!("node:{}", target)).unwrap()) } else { Err(err) @@ -1353,7 +1371,7 @@ impl NodeResolver { if let Some(main) = maybe_main { let guess = package_json.path.parent().unwrap().join(main).clean(); - if self.env.is_file_sync(&guess) { + if self.sys.fs_is_file_no_err(&guess) { return Ok(url_from_file_path(&guess).unwrap()); } @@ -1382,7 +1400,7 @@ impl NodeResolver { .unwrap() .join(format!("{main}{ending}")) .clean(); - if self.env.is_file_sync(&guess) { + if self.sys.fs_is_file_no_err(&guess) { // TODO(bartlomieju): emitLegacyIndexDeprecation() return Ok(url_from_file_path(&guess).unwrap()); } @@ -1417,7 +1435,7 @@ impl NodeResolver { }; for index_file_name in index_file_names { let guess = directory.join(index_file_name).clean(); - if self.env.is_file_sync(&guess) { + if self.sys.fs_is_file_no_err(&guess) { // TODO(bartlomieju): emitLegacyIndexDeprecation() return Ok(url_from_file_path(&guess).unwrap()); } @@ -1454,9 +1472,7 @@ impl NodeResolver { { // Specifiers in the node_modules directory are canonicalized // so canoncalize then check if it's in the node_modules directory. - let specifier = resolve_specifier_into_node_modules(specifier, &|path| { - self.env.realpath_sync(path) - }); + let specifier = resolve_specifier_into_node_modules(&self.sys, specifier); return Some(specifier); } @@ -1717,16 +1733,15 @@ pub fn parse_npm_pkg_name( /// not be fully resolved at the time deno_graph is analyzing it /// because the node_modules folder might not exist at that time. pub fn resolve_specifier_into_node_modules( + sys: &impl FsCanonicalize, specifier: &Url, - canonicalize: &impl Fn(&Path) -> std::io::Result, ) -> Url { deno_path_util::url_to_file_path(specifier) .ok() // this path might not exist at the time the graph is being created // because the node_modules folder might not yet exist .and_then(|path| { - deno_path_util::canonicalize_path_maybe_not_exists(&path, canonicalize) - .ok() + deno_path_util::fs::canonicalize_path_maybe_not_exists(sys, &path).ok() }) .and_then(|path| deno_path_util::url_from_file_path(&path).ok()) .unwrap_or_else(|| specifier.clone()) diff --git a/resolvers/npm_cache/Cargo.toml b/resolvers/npm_cache/Cargo.toml index 010f8a436b4b87..48d0a32437449b 100644 --- a/resolvers/npm_cache/Cargo.toml +++ b/resolvers/npm_cache/Cargo.toml @@ -25,6 +25,7 @@ boxed_error.workspace = true deno_cache_dir.workspace = true deno_error.workspace = true deno_npm.workspace = true +deno_path_util.workspace = true deno_semver.workspace = true deno_unsync = { workspace = true, features = ["tokio"] } faster-hex.workspace = true @@ -37,6 +38,7 @@ percent-encoding.workspace = true rand.workspace = true ring.workspace = true serde_json.workspace = true +sys_traits.workspace = true tar.workspace = true tempfile = "3.4.0" thiserror.workspace = true diff --git a/resolvers/npm_cache/fs_util.rs b/resolvers/npm_cache/fs_util.rs new file mode 100644 index 00000000000000..ed123f085c1141 --- /dev/null +++ b/resolvers/npm_cache/fs_util.rs @@ -0,0 +1,99 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use anyhow::Context; +use anyhow::Error as AnyError; +use std::io::ErrorKind; +use std::path::Path; +use std::time::Duration; +use sys_traits::FsCreateDirAll; +use sys_traits::FsDirEntry; +use sys_traits::FsHardLink; +use sys_traits::FsReadDir; +use sys_traits::FsRemoveFile; +use sys_traits::ThreadSleep; + +/// Hardlinks the files in one directory to another directory. +/// +/// Note: Does not handle symlinks. +pub fn hard_link_dir_recursive< + TSys: FsCreateDirAll + FsHardLink + FsReadDir + FsRemoveFile + ThreadSleep, +>( + sys: &TSys, + from: &Path, + to: &Path, +) -> Result<(), AnyError> { + sys + .fs_create_dir_all(to) + .with_context(|| format!("Creating {}", to.display()))?; + let read_dir = sys + .fs_read_dir(from) + .with_context(|| format!("Reading {}", from.display()))?; + + for entry in read_dir { + let entry = entry?; + let file_type = entry.file_type()?; + let new_from = from.join(entry.file_name()); + let new_to = to.join(entry.file_name()); + + if file_type.is_dir() { + hard_link_dir_recursive(sys, &new_from, &new_to).with_context(|| { + format!("Dir {} to {}", new_from.display(), new_to.display()) + })?; + } else if file_type.is_file() { + // note: chance for race conditions here between attempting to create, + // then removing, then attempting to create. There doesn't seem to be + // a way to hard link with overwriting in Rust, but maybe there is some + // way with platform specific code. The workaround here is to handle + // scenarios where something else might create or remove files. + if let Err(err) = sys.fs_hard_link(&new_from, &new_to) { + if err.kind() == ErrorKind::AlreadyExists { + if let Err(err) = sys.fs_remove_file(&new_to) { + if err.kind() == ErrorKind::NotFound { + // Assume another process/thread created this hard link to the file we are wanting + // to remove then sleep a little bit to let the other process/thread move ahead + // faster to reduce contention. + sys.thread_sleep(Duration::from_millis(10)); + } else { + return Err(err).with_context(|| { + format!( + "Removing file to hard link {} to {}", + new_from.display(), + new_to.display() + ) + }); + } + } + + // Always attempt to recreate the hardlink. In contention scenarios, the other process + // might have been killed or exited after removing the file, but before creating the hardlink + if let Err(err) = sys.fs_hard_link(&new_from, &new_to) { + // Assume another process/thread created this hard link to the file we are wanting + // to now create then sleep a little bit to let the other process/thread move ahead + // faster to reduce contention. + if err.kind() == ErrorKind::AlreadyExists { + sys.thread_sleep(Duration::from_millis(10)); + } else { + return Err(err).with_context(|| { + format!( + "Hard linking {} to {}", + new_from.display(), + new_to.display() + ) + }); + } + } + } else { + return Err(err).with_context(|| { + format!( + "Hard linking {} to {}", + new_from.display(), + new_to.display() + ) + }); + } + } + } + } + + Ok(()) +} diff --git a/resolvers/npm_cache/lib.rs b/resolvers/npm_cache/lib.rs index dbd33d29eb773f..e681fa71ac0071 100644 --- a/resolvers/npm_cache/lib.rs +++ b/resolvers/npm_cache/lib.rs @@ -14,6 +14,7 @@ use deno_cache_dir::npm::NpmCacheDir; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; use deno_npm::NpmPackageCacheFolderId; +use deno_path_util::fs::atomic_write_file_with_retries; use deno_semver::package::PackageNv; use deno_semver::StackString; use deno_semver::Version; @@ -21,13 +22,24 @@ use http::HeaderName; use http::HeaderValue; use http::StatusCode; use parking_lot::Mutex; +use sys_traits::FsCreateDirAll; +use sys_traits::FsHardLink; +use sys_traits::FsMetadata; +use sys_traits::FsOpen; +use sys_traits::FsReadDir; +use sys_traits::FsRemoveFile; +use sys_traits::FsRename; +use sys_traits::SystemRandom; +use sys_traits::ThreadSleep; use url::Url; +mod fs_util; mod registry_info; mod remote; mod tarball; mod tarball_extract; +pub use fs_util::hard_link_dir_recursive; pub use registry_info::RegistryInfoProvider; pub use tarball::TarballCache; @@ -55,18 +67,7 @@ impl std::fmt::Display for DownloadError { } #[async_trait::async_trait(?Send)] -pub trait NpmCacheEnv: Send + Sync + 'static { - fn exists(&self, path: &Path) -> bool; - fn hard_link_dir_recursive( - &self, - from: &Path, - to: &Path, - ) -> Result<(), AnyError>; - fn atomic_write_file_with_retries( - &self, - file_path: &Path, - data: &[u8], - ) -> std::io::Result<()>; +pub trait NpmCacheHttpClient: Send + Sync + 'static { async fn download_with_retries_on_any_tokio_runtime( &self, url: Url, @@ -126,27 +127,48 @@ impl NpmCacheSetting { /// Stores a single copy of npm packages in a cache. #[derive(Debug)] -pub struct NpmCache { - env: Arc, +pub struct NpmCache< + TSys: FsCreateDirAll + + FsHardLink + + FsMetadata + + FsOpen + + FsReadDir + + FsRemoveFile + + FsRename + + ThreadSleep + + SystemRandom, +> { cache_dir: Arc, + sys: TSys, cache_setting: NpmCacheSetting, npmrc: Arc, previously_reloaded_packages: Mutex>, } -impl NpmCache { +impl< + TSys: FsCreateDirAll + + FsHardLink + + FsMetadata + + FsOpen + + FsReadDir + + FsRemoveFile + + FsRename + + ThreadSleep + + SystemRandom, + > NpmCache +{ pub fn new( cache_dir: Arc, + sys: TSys, cache_setting: NpmCacheSetting, - env: Arc, npmrc: Arc, ) -> Self { Self { cache_dir, + sys, cache_setting, - env, - previously_reloaded_packages: Default::default(), npmrc, + previously_reloaded_packages: Default::default(), } } @@ -211,9 +233,11 @@ impl NpmCache { // it seems Windows does an "AccessDenied" error when moving a // directory with hard links, so that's why this solution is done with_folder_sync_lock(&folder_id.nv, &package_folder, || { - self - .env - .hard_link_dir_recursive(&original_package_folder, &package_folder) + hard_link_dir_recursive( + &self.sys, + &original_package_folder, + &package_folder, + ) })?; Ok(()) } @@ -290,9 +314,12 @@ impl NpmCache { ) -> Result<(), AnyError> { let file_cache_path = self.get_registry_package_info_file_cache_path(name); let file_text = serde_json::to_string(&package_info)?; - self - .env - .atomic_write_file_with_retries(&file_cache_path, file_text.as_bytes())?; + atomic_write_file_with_retries( + &self.sys, + &file_cache_path, + file_text.as_bytes(), + 0o644, + )?; Ok(()) } @@ -304,6 +331,7 @@ impl NpmCache { const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock"; +// todo(dsherret): use `sys` here instead of `std::fs`. fn with_folder_sync_lock( package: &PackageNv, output_folder: &Path, diff --git a/resolvers/npm_cache/registry_info.rs b/resolvers/npm_cache/registry_info.rs index dd8a6395910f16..57e188200dd10d 100644 --- a/resolvers/npm_cache/registry_info.rs +++ b/resolvers/npm_cache/registry_info.rs @@ -18,12 +18,21 @@ use deno_unsync::sync::MultiRuntimeAsyncValueCreator; use futures::future::LocalBoxFuture; use futures::FutureExt; use parking_lot::Mutex; +use sys_traits::FsCreateDirAll; +use sys_traits::FsHardLink; +use sys_traits::FsMetadata; +use sys_traits::FsOpen; +use sys_traits::FsReadDir; +use sys_traits::FsRemoveFile; +use sys_traits::FsRename; +use sys_traits::SystemRandom; +use sys_traits::ThreadSleep; use thiserror::Error; use url::Url; use crate::remote::maybe_auth_header_for_npm_registry; use crate::NpmCache; -use crate::NpmCacheEnv; +use crate::NpmCacheHttpClient; use crate::NpmCacheSetting; type LoadResult = Result>; @@ -122,25 +131,54 @@ impl MemoryCache { /// /// This is shared amongst all the workers. #[derive(Debug)] -pub struct RegistryInfoProvider { +pub struct RegistryInfoProvider< + THttpClient: NpmCacheHttpClient, + TSys: FsCreateDirAll + + FsHardLink + + FsMetadata + + FsOpen + + FsReadDir + + FsRemoveFile + + FsRename + + ThreadSleep + + SystemRandom + + Send + + Sync + + 'static, +> { // todo(#27198): remove this - cache: Arc>, - env: Arc, + cache: Arc>, + http_client: Arc, npmrc: Arc, force_reload_flag: AtomicFlag, memory_cache: Mutex, previously_loaded_packages: Mutex>, } -impl RegistryInfoProvider { +impl< + THttpClient: NpmCacheHttpClient, + TSys: FsCreateDirAll + + FsHardLink + + FsMetadata + + FsOpen + + FsReadDir + + FsRemoveFile + + FsRename + + ThreadSleep + + SystemRandom + + Send + + Sync + + 'static, + > RegistryInfoProvider +{ pub fn new( - cache: Arc>, - env: Arc, + cache: Arc>, + http_client: Arc, npmrc: Arc, ) -> Self { Self { cache, - env, + http_client, npmrc, force_reload_flag: AtomicFlag::lowered(), memory_cache: Default::default(), @@ -170,7 +208,9 @@ impl RegistryInfoProvider { } } - pub fn as_npm_registry_api(self: &Arc) -> NpmRegistryApiAdapter { + pub fn as_npm_registry_api( + self: &Arc, + ) -> NpmRegistryApiAdapter { NpmRegistryApiAdapter(self.clone()) } @@ -341,7 +381,7 @@ impl RegistryInfoProvider { downloader.previously_loaded_packages.lock().insert(name.to_string()); let maybe_bytes = downloader - .env + .http_client .download_with_retries_on_any_tokio_runtime( package_url, maybe_auth_header, @@ -378,12 +418,39 @@ impl RegistryInfoProvider { } } -pub struct NpmRegistryApiAdapter( - Arc>, -); +pub struct NpmRegistryApiAdapter< + THttpClient: NpmCacheHttpClient, + TSys: FsCreateDirAll + + FsHardLink + + FsMetadata + + FsOpen + + FsReadDir + + FsRemoveFile + + FsRename + + ThreadSleep + + SystemRandom + + Send + + Sync + + 'static, +>(Arc>); #[async_trait(?Send)] -impl NpmRegistryApi for NpmRegistryApiAdapter { +impl< + THttpClient: NpmCacheHttpClient, + TSys: FsCreateDirAll + + FsHardLink + + FsMetadata + + FsOpen + + FsReadDir + + FsRemoveFile + + FsRename + + ThreadSleep + + SystemRandom + + Send + + Sync + + 'static, + > NpmRegistryApi for NpmRegistryApiAdapter +{ async fn package_info( &self, name: &str, diff --git a/resolvers/npm_cache/tarball.rs b/resolvers/npm_cache/tarball.rs index 5c8e460fd66b30..3a7e9df8a977e1 100644 --- a/resolvers/npm_cache/tarball.rs +++ b/resolvers/npm_cache/tarball.rs @@ -15,13 +15,22 @@ use futures::future::LocalBoxFuture; use futures::FutureExt; use http::StatusCode; use parking_lot::Mutex; +use sys_traits::FsCreateDirAll; +use sys_traits::FsHardLink; +use sys_traits::FsMetadata; +use sys_traits::FsOpen; +use sys_traits::FsReadDir; +use sys_traits::FsRemoveFile; +use sys_traits::FsRename; +use sys_traits::SystemRandom; +use sys_traits::ThreadSleep; use url::Url; use crate::remote::maybe_auth_header_for_npm_registry; use crate::tarball_extract::verify_and_extract_tarball; use crate::tarball_extract::TarballExtractionMode; use crate::NpmCache; -use crate::NpmCacheEnv; +use crate::NpmCacheHttpClient; use crate::NpmCacheSetting; type LoadResult = Result<(), Arc>; @@ -42,22 +51,54 @@ enum MemoryCacheItem { /// /// This is shared amongst all the workers. #[derive(Debug)] -pub struct TarballCache { - cache: Arc>, - env: Arc, +pub struct TarballCache< + THttpClient: NpmCacheHttpClient, + TSys: FsCreateDirAll + + FsHardLink + + FsMetadata + + FsOpen + + FsRemoveFile + + FsReadDir + + FsRename + + ThreadSleep + + SystemRandom + + Send + + Sync + + 'static, +> { + cache: Arc>, + http_client: Arc, + sys: TSys, npmrc: Arc, memory_cache: Mutex>, } -impl TarballCache { +impl< + THttpClient: NpmCacheHttpClient, + TSys: FsCreateDirAll + + FsHardLink + + FsMetadata + + FsOpen + + FsRemoveFile + + FsReadDir + + FsRename + + ThreadSleep + + SystemRandom + + Send + + Sync + + 'static, + > TarballCache +{ pub fn new( - cache: Arc>, - env: Arc, + cache: Arc>, + http_client: Arc, + sys: TSys, npmrc: Arc, ) -> Self { Self { cache, - env, + http_client, + sys, npmrc, memory_cache: Default::default(), } @@ -131,7 +172,7 @@ impl TarballCache { let package_folder = tarball_cache.cache.package_folder_for_nv_and_url(&package_nv, registry_url); let should_use_cache = tarball_cache.cache.should_use_cache_for_package(&package_nv); - let package_folder_exists = tarball_cache.env.exists(&package_folder); + let package_folder_exists = tarball_cache.sys.fs_exists_no_err(&package_folder); if should_use_cache && package_folder_exists { return Ok(()); } else if tarball_cache.cache.cache_setting() == &NpmCacheSetting::Only { @@ -156,7 +197,7 @@ impl TarballCache { tarball_cache.npmrc.tarball_config(&tarball_uri); let maybe_auth_header = maybe_registry_config.and_then(|c| maybe_auth_header_for_npm_registry(c).ok()?); - let result = tarball_cache.env + let result = tarball_cache.http_client .download_with_retries_on_any_tokio_runtime(tarball_uri, maybe_auth_header) .await; let maybe_bytes = match result { diff --git a/tests/Cargo.toml b/tests/Cargo.toml index fa51d7b77b35b6..1300066c64addc 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -60,6 +60,7 @@ pretty_assertions.workspace = true regex.workspace = true reqwest.workspace = true serde.workspace = true +sys_traits = { workspace = true, features = ["real", "getrandom", "libc", "winapi"] } test_util.workspace = true tokio.workspace = true tower-lsp.workspace = true diff --git a/tests/integration/jsr_tests.rs b/tests/integration/jsr_tests.rs index f438ebfd508745..d3fa5cd98f3131 100644 --- a/tests/integration/jsr_tests.rs +++ b/tests/integration/jsr_tests.rs @@ -191,8 +191,8 @@ fn reload_info_not_found_cache_but_exists_remote() { Url::parse(&format!("http://127.0.0.1:4250/{}/meta.json", package)) .unwrap(); let cache = deno_cache_dir::GlobalHttpCache::new( + sys_traits::impls::RealSys, deno_dir.path().join("remote").to_path_buf(), - deno_cache_dir::TestRealDenoCacheEnv, ); let entry = cache .get(&cache.cache_item_key(&specifier).unwrap(), None)