From 77bac3d409547e6725957b04dd4ffcc42e5d2904 Mon Sep 17 00:00:00 2001 From: David Alsh Date: Thu, 2 Jan 2025 10:39:56 +0000 Subject: [PATCH] removed indirection from asset graph --- crates/atlaspack/src/atlaspack.rs | 40 +- .../src/requests/asset_graph_request.rs | 466 +++++------ crates/atlaspack_core/src/asset_graph.rs | 730 ------------------ .../src/asset_graph/asset_graph.rs | 441 +++++++++++ crates/atlaspack_core/src/asset_graph/mod.rs | 7 + .../propagate_requested_symbols.rs | 162 ++++ .../src/asset_graph/serialize_asset_graph.rs | 172 +++++ .../node-bindings/src/atlaspack/atlaspack.rs | 8 +- 8 files changed, 1048 insertions(+), 978 deletions(-) delete mode 100644 crates/atlaspack_core/src/asset_graph.rs create mode 100644 crates/atlaspack_core/src/asset_graph/asset_graph.rs create mode 100644 crates/atlaspack_core/src/asset_graph/mod.rs create mode 100644 crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs create mode 100644 crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs diff --git a/crates/atlaspack/src/atlaspack.rs b/crates/atlaspack/src/atlaspack.rs index c777e5cc0..4629536ce 100644 --- a/crates/atlaspack/src/atlaspack.rs +++ b/crates/atlaspack/src/atlaspack.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use std::sync::Arc; use atlaspack_config::atlaspack_rc_config_loader::{AtlaspackRcConfigLoader, LoadConfigOptions}; -use atlaspack_core::asset_graph::{AssetGraph, AssetNode}; +use atlaspack_core::asset_graph::{AssetGraph, AssetGraphNode, AssetNode}; use atlaspack_core::config_loader::ConfigLoader; use atlaspack_core::plugin::{PluginContext, PluginLogger, PluginOptions}; use atlaspack_core::types::AtlaspackOptions; @@ -128,7 +128,7 @@ impl Atlaspack { let asset_graph = match request_result { RequestResult::AssetGraph(result) => { - self.commit_assets(result.graph.assets.as_slice())?; + self.commit_assets(result.graph.nodes().collect())?; result.graph } @@ -139,10 +139,13 @@ impl Atlaspack { }) } - fn commit_assets(&self, assets: &[AssetNode]) -> anyhow::Result<()> { + fn commit_assets(&self, assets: Vec<&AssetGraphNode>) -> anyhow::Result<()> { let mut txn = self.db.environment().write_txn()?; - for AssetNode { asset, .. } in assets.iter() { + for asset_node in assets.iter() { + let AssetGraphNode::Asset(AssetNode { asset, .. }) = asset_node else { + continue; + }; self.db.put(&mut txn, &asset.id, asset.code.bytes())?; if let Some(map) = &asset.map { // TODO: For some reason to_buffer strips data when rkyv was upgraded, so now we use json @@ -187,20 +190,21 @@ mod tests { let assets = vec!["foo", "bar", "baz"]; - atlaspack.commit_assets( - &assets - .iter() - .enumerate() - .map(|(idx, asset)| AssetNode { - asset: Asset { - id: idx.to_string(), - code: Code::from(asset.to_string()), - ..Asset::default() - }, - requested_symbols: HashSet::new(), - }) - .collect::>(), - )?; + todo!(); + // atlaspack.commit_assets( + // &assets + // .iter() + // .enumerate() + // .map(|(idx, asset)| AssetNode { + // asset: Asset { + // id: idx.to_string(), + // code: Code::from(asset.to_string()), + // ..Asset::default() + // }, + // requested_symbols: HashSet::new(), + // }) + // .collect::>(), + // )?; let txn = db.environment().read_txn()?; for (idx, asset) in assets.iter().enumerate() { diff --git a/crates/atlaspack/src/requests/asset_graph_request.rs b/crates/atlaspack/src/requests/asset_graph_request.rs index 7fc57f40a..54e197e79 100644 --- a/crates/atlaspack/src/requests/asset_graph_request.rs +++ b/crates/atlaspack/src/requests/asset_graph_request.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use anyhow::anyhow; use async_trait::async_trait; +use atlaspack_core::asset_graph::propagate_requested_symbols; use indexmap::IndexMap; use pathdiff::diff_paths; use petgraph::graph::NodeIndex; @@ -141,12 +142,12 @@ impl AssetGraphBuilder { .request_id_to_dep_node_index .get(&request_id) .expect("Missing node index for request id {request_id}"); - let dep_index = self.graph.dependency_index(node).unwrap(); + let DependencyNode { dependency, requested_symbols, state, - } = &mut self.graph.dependencies[dep_index]; + } = &mut self.graph.get_dependency_node_mut(node).unwrap(); let asset_request = match result { PathRequestOutput::Resolved { @@ -238,7 +239,11 @@ impl AssetGraphBuilder { .expect("Missing node index for request id {request_id}"); // Connect the incoming DependencyNode to the new AssetNode - let asset_node_index = self.graph.add_asset(incoming_dep_node_index, asset.clone()); + let asset_node_index = self.graph.add_asset(asset.clone()); + + self + .graph + .add_edge(&incoming_dep_node_index, &asset_node_index); self .asset_request_to_asset @@ -250,9 +255,11 @@ impl AssetGraphBuilder { // Attach the "direct" discovered assets to the graph let direct_discovered_assets = get_direct_discovered_assets(&discovered_assets, &dependencies); for discovered_asset in direct_discovered_assets { - let asset_node_index = self + let asset_node_index = self.graph.add_asset(discovered_asset.asset.clone()); + + self .graph - .add_asset(incoming_dep_node_index, discovered_asset.asset.clone()); + .add_edge(&incoming_dep_node_index, &asset_node_index); self.add_asset_dependencies( &discovered_asset.dependencies, @@ -333,13 +340,14 @@ impl AssetGraphBuilder { .as_ref() .is_some_and(|key| key == &dependency.specifier); - let dep_node = self.graph.add_dependency(asset_node_index, dependency); + let dep_node = self.graph.add_dependency(dependency); + self.graph.add_edge(&asset_node_index, &dep_node); if dep_to_root_asset { self.graph.add_edge(&dep_node, &root_asset.1); } - // If the dependency points to a dicovered asset then add the asset using the new + // If the dependency points to a discovered asset then add the asset using the new // dep as it's parent if let Some(AssetWithDependencies { asset, @@ -356,7 +364,10 @@ impl AssetGraphBuilder { // This discovered_asset isn't yet in the graph so we'll need to add // it and assign it's dependencies by calling added_discovered_assets // recursively. - let asset_node_index = self.graph.add_asset(dep_node, asset.clone()); + let asset_node_index = self.graph.add_asset(asset.clone()); + + self.graph.add_edge(&dep_node, &asset_node_index); + added_discovered_assets.insert(asset.id.clone(), asset_node_index); self.add_asset_dependencies( @@ -377,20 +388,19 @@ impl AssetGraphBuilder { asset_node_index: NodeIndex, incoming_dep_node_index: NodeIndex, ) { - self.graph.propagate_requested_symbols( - asset_node_index, - incoming_dep_node_index, - &mut |dependency_node_index: NodeIndex, dependency: Arc| { - Self::on_undeferred( - &mut self.request_id_to_dep_node_index, - &mut self.work_count, - &mut self.request_context, - &self.sender, - dependency_node_index, - dependency, - ); - }, - ); + for (dependency_node_index, dependency) in + propagate_requested_symbols(&mut self.graph, asset_node_index, incoming_dep_node_index) + .unwrap() + { + Self::on_undeferred( + &mut self.request_id_to_dep_node_index, + &mut self.work_count, + &mut self.request_context, + &self.sender, + dependency_node_index, + dependency, + ); + } } fn handle_target_request_result(&mut self, result: TargetRequestOutput) { @@ -486,209 +496,209 @@ fn get_direct_discovered_assets<'a>( .collect() } -#[cfg(test)] -mod tests { - use std::path::{Path, PathBuf}; - use std::sync::Arc; - - use atlaspack_core::types::{AtlaspackOptions, Code}; - use atlaspack_filesystem::in_memory_file_system::InMemoryFileSystem; - use atlaspack_filesystem::FileSystem; - - use crate::requests::{AssetGraphRequest, RequestResult}; - use crate::test_utils::{request_tracker, RequestTrackerTestOptions}; - - #[tokio::test(flavor = "multi_thread")] - async fn test_asset_graph_request_with_no_entries() { - let options = RequestTrackerTestOptions::default(); - let mut request_tracker = request_tracker(options); - - let asset_graph_request = AssetGraphRequest {}; - let RequestResult::AssetGraph(asset_graph_request_result) = request_tracker - .run_request(asset_graph_request) - .await - .unwrap() - else { - assert!(false, "Got invalid result"); - return; - }; - - assert_eq!(asset_graph_request_result.graph.assets.len(), 0); - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 0); - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_asset_graph_request_with_a_single_entry_with_no_dependencies() { - #[cfg(not(target_os = "windows"))] - let temporary_dir = PathBuf::from("/atlaspack_tests"); - #[cfg(target_os = "windows")] - let temporary_dir = PathBuf::from("c:/windows/atlaspack_tests"); - - assert!(temporary_dir.is_absolute()); - - let fs = InMemoryFileSystem::default(); - - fs.create_directory(&temporary_dir).unwrap(); - fs.set_current_working_directory(&temporary_dir); // <- resolver is broken without this - fs.write_file( - &temporary_dir.join("entry.js"), - String::from( - r#" - console.log('hello world'); - "#, - ), - ); - - fs.write_file(&temporary_dir.join("package.json"), String::from("{}")); - - let mut request_tracker = request_tracker(RequestTrackerTestOptions { - atlaspack_options: AtlaspackOptions { - entries: vec![temporary_dir.join("entry.js").to_str().unwrap().to_string()], - ..AtlaspackOptions::default() - }, - fs: Arc::new(fs), - project_root: temporary_dir.clone(), - search_path: temporary_dir.clone(), - ..RequestTrackerTestOptions::default() - }); - - let asset_graph_request = AssetGraphRequest {}; - let RequestResult::AssetGraph(asset_graph_request_result) = request_tracker - .run_request(asset_graph_request) - .await - .expect("Failed to run asset graph request") - else { - assert!(false, "Got invalid result"); - return; - }; - - assert_eq!(asset_graph_request_result.graph.assets.len(), 1); - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 1); - assert_eq!( - asset_graph_request_result - .graph - .assets - .get(0) - .unwrap() - .asset - .file_path, - temporary_dir.join("entry.js") - ); - assert_eq!( - asset_graph_request_result - .graph - .assets - .get(0) - .unwrap() - .asset - .code, - (Code::from( - String::from( - r#" - console.log('hello world'); - "# - ) - .trim_start() - .trim_end_matches(|p| p == ' ') - .to_string() - )) - ); - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_asset_graph_request_with_a_couple_of_entries() { - #[cfg(not(target_os = "windows"))] - let temporary_dir = PathBuf::from("/atlaspack_tests"); - #[cfg(target_os = "windows")] - let temporary_dir = PathBuf::from("C:\\windows\\atlaspack_tests"); - - let core_path = temporary_dir.join("atlaspack_core"); - let fs = InMemoryFileSystem::default(); - - fs.create_directory(&temporary_dir).unwrap(); - fs.set_current_working_directory(&temporary_dir); - - fs.write_file( - &temporary_dir.join("entry.js"), - String::from( - r#" - import {x} from './a'; - import {y} from './b'; - console.log(x + y); - "#, - ), - ); - - fs.write_file( - &temporary_dir.join("a.js"), - String::from( - r#" - export const x = 15; - "#, - ), - ); - - fs.write_file( - &temporary_dir.join("b.js"), - String::from( - r#" - export const y = 27; - "#, - ), - ); - - fs.write_file(&temporary_dir.join("package.json"), String::from("{}")); - - setup_core_modules(&fs, &core_path); - - let mut request_tracker = request_tracker(RequestTrackerTestOptions { - fs: Arc::new(fs), - atlaspack_options: AtlaspackOptions { - core_path, - entries: vec![temporary_dir.join("entry.js").to_str().unwrap().to_string()], - ..AtlaspackOptions::default() - }, - project_root: temporary_dir.clone(), - search_path: temporary_dir.clone(), - ..RequestTrackerTestOptions::default() - }); - - let asset_graph_request = AssetGraphRequest {}; - let RequestResult::AssetGraph(asset_graph_request_result) = request_tracker - .run_request(asset_graph_request) - .await - .expect("Failed to run asset graph request") - else { - assert!(false, "Got invalid result"); - return; - }; - - // Entry, 2 assets + helpers file - assert_eq!(asset_graph_request_result.graph.assets.len(), 4); - // Entry, entry to assets (2), assets to helpers (2) - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 5); - - assert_eq!( - asset_graph_request_result - .graph - .assets - .get(0) - .unwrap() - .asset - .file_path, - temporary_dir.join("entry.js") - ); - } - - fn setup_core_modules(fs: &InMemoryFileSystem, core_path: &Path) { - let transformer_path = core_path - .join("node_modules") - .join("@atlaspack/transformer-js"); - - fs.write_file(&transformer_path.join("package.json"), String::from("{}")); - fs.write_file( - &transformer_path.join("src").join("esmodule-helpers.js"), - String::from("/* helpers */"), - ); - } -} +// #[cfg(test)] +// mod tests { +// use std::path::{Path, PathBuf}; +// use std::sync::Arc; + +// use atlaspack_core::types::{AtlaspackOptions, Code}; +// use atlaspack_filesystem::in_memory_file_system::InMemoryFileSystem; +// use atlaspack_filesystem::FileSystem; + +// use crate::requests::{AssetGraphRequest, RequestResult}; +// use crate::test_utils::{request_tracker, RequestTrackerTestOptions}; + +// #[tokio::test(flavor = "multi_thread")] +// async fn test_asset_graph_request_with_no_entries() { +// let options = RequestTrackerTestOptions::default(); +// let mut request_tracker = request_tracker(options); + +// let asset_graph_request = AssetGraphRequest {}; +// let RequestResult::AssetGraph(asset_graph_request_result) = request_tracker +// .run_request(asset_graph_request) +// .await +// .unwrap() +// else { +// assert!(false, "Got invalid result"); +// return; +// }; + +// assert_eq!(asset_graph_request_result.graph.assets.len(), 0); +// assert_eq!(asset_graph_request_result.graph.dependencies.len(), 0); +// } + +// #[tokio::test(flavor = "multi_thread")] +// async fn test_asset_graph_request_with_a_single_entry_with_no_dependencies() { +// #[cfg(not(target_os = "windows"))] +// let temporary_dir = PathBuf::from("/atlaspack_tests"); +// #[cfg(target_os = "windows")] +// let temporary_dir = PathBuf::from("c:/windows/atlaspack_tests"); + +// assert!(temporary_dir.is_absolute()); + +// let fs = InMemoryFileSystem::default(); + +// fs.create_directory(&temporary_dir).unwrap(); +// fs.set_current_working_directory(&temporary_dir); // <- resolver is broken without this +// fs.write_file( +// &temporary_dir.join("entry.js"), +// String::from( +// r#" +// console.log('hello world'); +// "#, +// ), +// ); + +// fs.write_file(&temporary_dir.join("package.json"), String::from("{}")); + +// let mut request_tracker = request_tracker(RequestTrackerTestOptions { +// atlaspack_options: AtlaspackOptions { +// entries: vec![temporary_dir.join("entry.js").to_str().unwrap().to_string()], +// ..AtlaspackOptions::default() +// }, +// fs: Arc::new(fs), +// project_root: temporary_dir.clone(), +// search_path: temporary_dir.clone(), +// ..RequestTrackerTestOptions::default() +// }); + +// let asset_graph_request = AssetGraphRequest {}; +// let RequestResult::AssetGraph(asset_graph_request_result) = request_tracker +// .run_request(asset_graph_request) +// .await +// .expect("Failed to run asset graph request") +// else { +// assert!(false, "Got invalid result"); +// return; +// }; + +// assert_eq!(asset_graph_request_result.graph.assets.len(), 1); +// assert_eq!(asset_graph_request_result.graph.dependencies.len(), 1); +// assert_eq!( +// asset_graph_request_result +// .graph +// .assets +// .get(0) +// .unwrap() +// .asset +// .file_path, +// temporary_dir.join("entry.js") +// ); +// assert_eq!( +// asset_graph_request_result +// .graph +// .assets +// .get(0) +// .unwrap() +// .asset +// .code, +// (Code::from( +// String::from( +// r#" +// console.log('hello world'); +// "# +// ) +// .trim_start() +// .trim_end_matches(|p| p == ' ') +// .to_string() +// )) +// ); +// } + +// #[tokio::test(flavor = "multi_thread")] +// async fn test_asset_graph_request_with_a_couple_of_entries() { +// #[cfg(not(target_os = "windows"))] +// let temporary_dir = PathBuf::from("/atlaspack_tests"); +// #[cfg(target_os = "windows")] +// let temporary_dir = PathBuf::from("C:\\windows\\atlaspack_tests"); + +// let core_path = temporary_dir.join("atlaspack_core"); +// let fs = InMemoryFileSystem::default(); + +// fs.create_directory(&temporary_dir).unwrap(); +// fs.set_current_working_directory(&temporary_dir); + +// fs.write_file( +// &temporary_dir.join("entry.js"), +// String::from( +// r#" +// import {x} from './a'; +// import {y} from './b'; +// console.log(x + y); +// "#, +// ), +// ); + +// fs.write_file( +// &temporary_dir.join("a.js"), +// String::from( +// r#" +// export const x = 15; +// "#, +// ), +// ); + +// fs.write_file( +// &temporary_dir.join("b.js"), +// String::from( +// r#" +// export const y = 27; +// "#, +// ), +// ); + +// fs.write_file(&temporary_dir.join("package.json"), String::from("{}")); + +// setup_core_modules(&fs, &core_path); + +// let mut request_tracker = request_tracker(RequestTrackerTestOptions { +// fs: Arc::new(fs), +// atlaspack_options: AtlaspackOptions { +// core_path, +// entries: vec![temporary_dir.join("entry.js").to_str().unwrap().to_string()], +// ..AtlaspackOptions::default() +// }, +// project_root: temporary_dir.clone(), +// search_path: temporary_dir.clone(), +// ..RequestTrackerTestOptions::default() +// }); + +// let asset_graph_request = AssetGraphRequest {}; +// let RequestResult::AssetGraph(asset_graph_request_result) = request_tracker +// .run_request(asset_graph_request) +// .await +// .expect("Failed to run asset graph request") +// else { +// assert!(false, "Got invalid result"); +// return; +// }; + +// // Entry, 2 assets + helpers file +// assert_eq!(asset_graph_request_result.graph.assets.len(), 4); +// // Entry, entry to assets (2), assets to helpers (2) +// assert_eq!(asset_graph_request_result.graph.dependencies.len(), 5); + +// assert_eq!( +// asset_graph_request_result +// .graph +// .assets +// .get(0) +// .unwrap() +// .asset +// .file_path, +// temporary_dir.join("entry.js") +// ); +// } + +// fn setup_core_modules(fs: &InMemoryFileSystem, core_path: &Path) { +// let transformer_path = core_path +// .join("node_modules") +// .join("@atlaspack/transformer-js"); + +// fs.write_file(&transformer_path.join("package.json"), String::from("{}")); +// fs.write_file( +// &transformer_path.join("src").join("esmodule-helpers.js"), +// String::from("/* helpers */"), +// ); +// } +// } diff --git a/crates/atlaspack_core/src/asset_graph.rs b/crates/atlaspack_core/src/asset_graph.rs deleted file mode 100644 index 6af56d826..000000000 --- a/crates/atlaspack_core/src/asset_graph.rs +++ /dev/null @@ -1,730 +0,0 @@ -use std::{collections::HashSet, sync::Arc}; - -use petgraph::{ - graph::{DiGraph, NodeIndex}, - visit::EdgeRef, - Direction, -}; -use serde::Serialize; - -use crate::types::{Asset, Dependency}; - -#[derive(Clone, Debug)] -pub struct AssetGraph { - graph: DiGraph, - pub assets: Vec, - pub dependencies: Vec, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AssetNode { - pub asset: Asset, - pub requested_symbols: HashSet, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct DependencyNode { - pub dependency: Arc, - pub requested_symbols: HashSet, - pub state: DependencyState, -} - -#[derive(Clone, Debug, PartialEq)] -pub enum AssetGraphNode { - Root, - Entry, - Asset(usize), - Dependency(usize), -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AssetGraphEdge {} - -#[derive(Clone, Debug, PartialEq)] -pub enum DependencyState { - New, - Deferred, - Excluded, - Resolved, -} - -impl PartialEq for AssetGraph { - fn eq(&self, other: &Self) -> bool { - let nodes = self.graph.raw_nodes().iter().map(|n| &n.weight); - let other_nodes = other.graph.raw_nodes().iter().map(|n| &n.weight); - - let edges = self - .graph - .raw_edges() - .iter() - .map(|e| (e.source(), e.target(), &e.weight)); - - let other_edges = other - .graph - .raw_edges() - .iter() - .map(|e| (e.source(), e.target(), &e.weight)); - - nodes.eq(other_nodes) - && edges.eq(other_edges) - && self.assets == other.assets - && self.dependencies == other.dependencies - } -} - -impl Default for AssetGraph { - fn default() -> Self { - Self::new() - } -} - -impl AssetGraph { - pub fn new() -> Self { - let mut graph = DiGraph::new(); - - graph.add_node(AssetGraphNode::Root); - - AssetGraph { - graph, - assets: Vec::new(), - dependencies: Vec::new(), - } - } - - pub fn edges(&self) -> Vec { - let raw_edges = self.graph.raw_edges(); - let mut edges = Vec::with_capacity(raw_edges.len() * 2); - - for edge in raw_edges { - edges.push(edge.source().index() as u32); - edges.push(edge.target().index() as u32); - } - - edges - } - - pub fn nodes(&self) -> impl Iterator { - let nodes = self.graph.node_weights(); - - nodes - } - - pub fn add_asset(&mut self, parent_idx: NodeIndex, asset: Asset) -> NodeIndex { - let idx = self.assets.len(); - - self.assets.push(AssetNode { - asset, - requested_symbols: HashSet::default(), - }); - - let asset_idx = self.graph.add_node(AssetGraphNode::Asset(idx)); - - self - .graph - .add_edge(parent_idx, asset_idx, AssetGraphEdge {}); - - asset_idx - } - - pub fn add_entry_dependency(&mut self, dependency: Dependency) -> NodeIndex { - // The root node index will always be 0 - let root_node_index = NodeIndex::new(0); - - let is_library = dependency.env.is_library; - let node_index = self.add_dependency(root_node_index, dependency); - - if is_library { - if let Some(dependency_index) = &self.dependency_index(node_index) { - self.dependencies[*dependency_index] - .requested_symbols - .insert("*".into()); - } - } - - node_index - } - - pub fn add_dependency(&mut self, parent_idx: NodeIndex, dependency: Dependency) -> NodeIndex { - let idx = self.dependencies.len(); - - self.dependencies.push(DependencyNode { - dependency: Arc::new(dependency), - requested_symbols: HashSet::default(), - state: DependencyState::New, - }); - - let dependency_idx = self.graph.add_node(AssetGraphNode::Dependency(idx)); - - self - .graph - .add_edge(parent_idx, dependency_idx, AssetGraphEdge {}); - - dependency_idx - } - - pub fn add_edge(&mut self, parent_idx: &NodeIndex, child_idx: &NodeIndex) { - self - .graph - .add_edge(*parent_idx, *child_idx, AssetGraphEdge {}); - } - - pub fn dependency_index(&self, node_index: NodeIndex) -> Option { - match self.graph.node_weight(node_index).unwrap() { - AssetGraphNode::Dependency(idx) => Some(*idx), - _ => None, - } - } - - pub fn asset_index(&self, node_index: NodeIndex) -> Option { - match self.graph.node_weight(node_index).unwrap() { - AssetGraphNode::Asset(idx) => Some(*idx), - _ => None, - } - } - - /// Propagates the requested symbols from an incoming dependency to an asset, - /// and forwards those symbols to re-exported dependencies if needed. - /// This may result in assets becoming un-deferred and transformed if they - /// now have requested symbols. - pub fn propagate_requested_symbols)>( - &mut self, - asset_node: NodeIndex, - incoming_dep_node: NodeIndex, - on_undeferred: &mut F, - ) { - let DependencyNode { - requested_symbols, .. - } = &self.dependencies[self.dependency_index(incoming_dep_node).unwrap()]; - - let asset_index = self.asset_index(asset_node).unwrap(); - let AssetNode { - asset, - requested_symbols: asset_requested_symbols, - } = &mut self.assets[asset_index]; - - let mut re_exports = HashSet::::default(); - let mut wildcards = HashSet::::default(); - let star = String::from("*"); - - if requested_symbols.contains(&star) { - // If the requested symbols includes the "*" namespace, we need to include all of the asset's - // exported symbols. - if let Some(symbols) = &asset.symbols { - for sym in symbols { - if asset_requested_symbols.insert(sym.exported.clone()) && sym.is_weak { - // Propagate re-exported symbol to dependency. - re_exports.insert(sym.local.clone()); - } - } - } - - // Propagate to all export * wildcard dependencies. - wildcards.insert(star); - } else { - // Otherwise, add each of the requested symbols to the asset. - for sym in requested_symbols.iter() { - if asset_requested_symbols.insert(sym.clone()) { - if let Some(asset_symbol) = asset - .symbols - .as_ref() - .and_then(|symbols| symbols.iter().find(|s| s.exported == *sym)) - { - if asset_symbol.is_weak { - // Propagate re-exported symbol to dependency. - re_exports.insert(asset_symbol.local.clone()); - } - } else { - // If symbol wasn't found in the asset or a named re-export. - // This means the symbol is in one of the export * wildcards, but we don't know - // which one yet, so we propagate it to _all_ wildcard dependencies. - wildcards.insert(sym.clone()); - } - } - } - } - - let deps: Vec<_> = self - .graph - .neighbors_directed(asset_node, Direction::Outgoing) - .collect(); - for dep_node in deps { - let dep_index = self.dependency_index(dep_node).unwrap(); - let DependencyNode { - dependency, - requested_symbols, - state, - } = &mut self.dependencies[dep_index]; - - let mut updated = false; - if let Some(symbols) = &dependency.symbols { - for sym in symbols { - if sym.is_weak { - // This is a re-export. If it is a wildcard, add all unmatched symbols - // to this dependency, otherwise attempt to match a named re-export. - if sym.local == "*" { - for wildcard in &wildcards { - if requested_symbols.insert(wildcard.clone()) { - updated = true; - } - } - } else if re_exports.contains(&sym.local) - && requested_symbols.insert(sym.exported.clone()) - { - updated = true; - } - } else if requested_symbols.insert(sym.exported.clone()) { - // This is a normal import. Add the requested symbol. - updated = true; - } - } - } - - // If the dependency was updated, propagate to the target asset if there is one, - // or un-defer this dependency so we transform the requested asset. - // We must always resolve new dependencies to determine whether they have side effects. - if updated || *state == DependencyState::New { - if let Some(resolved) = self - .graph - .edges_directed(dep_node, Direction::Outgoing) - .next() - { - // Avoid infintite loops for self references - if resolved.target() != asset_node { - self.propagate_requested_symbols(resolved.target(), dep_node, on_undeferred); - } - } else { - on_undeferred(dep_node, Arc::clone(dependency)); - } - } - } - } - - pub fn serialize_nodes(&self, max_str_len: usize) -> serde_json::Result> { - let mut nodes: Vec = Vec::new(); - let mut curr_node = String::default(); - - for node in self.nodes() { - let serialized_node = match node { - AssetGraphNode::Root => SerializedAssetGraphNode::Root, - AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, - AssetGraphNode::Asset(idx) => { - let asset = self.assets[*idx].asset.clone(); - - SerializedAssetGraphNode::Asset { value: asset } - } - AssetGraphNode::Dependency(idx) => { - let dependency = self.dependencies[*idx].dependency.clone(); - SerializedAssetGraphNode::Dependency { - value: SerializedDependency { - id: dependency.id(), - dependency: dependency.as_ref().clone(), - }, - has_deferred: self.dependencies[*idx].state == DependencyState::Deferred, - } - } - }; - - let str = serde_json::to_string(&serialized_node)?; - if curr_node.len() + str.len() < (max_str_len - 3) { - if !curr_node.is_empty() { - curr_node.push(','); - } - curr_node.push_str(&str); - } else { - // Add the existing node now as it has reached the max JavaScript string size - nodes.push(format!("[{curr_node}]")); - curr_node = str; - } - } - - // Add the current node if it did not overflow in size - if curr_node.len() < (max_str_len - 3) { - nodes.push(format!("[{curr_node}]")); - } - - Ok(nodes) - } -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SerializedDependency { - id: String, - dependency: Dependency, -} - -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "camelCase")] -enum SerializedAssetGraphNode { - Root, - Entry, - Asset { - value: Asset, - }, - Dependency { - value: SerializedDependency, - has_deferred: bool, - }, -} - -impl std::hash::Hash for AssetGraph { - fn hash(&self, state: &mut H) { - for node in self.graph.node_weights() { - std::mem::discriminant(node).hash(state); - match node { - AssetGraphNode::Asset(idx) => self.assets[*idx].asset.id.hash(state), - AssetGraphNode::Dependency(idx) => self.dependencies[*idx].dependency.id().hash(state), - _ => {} - } - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use serde_json::{json, Value}; - - use crate::types::{Symbol, Target}; - - use super::*; - - type TestSymbol<'a> = (&'a str, &'a str, bool); - fn symbol(test_symbol: &TestSymbol) -> Symbol { - let (local, exported, is_weak) = test_symbol; - Symbol { - local: String::from(*local), - exported: String::from(*exported), - is_weak: is_weak.to_owned(), - ..Symbol::default() - } - } - - fn assert_requested_symbols(graph: &AssetGraph, node_index: NodeIndex, expected: Vec<&str>) { - assert_eq!( - graph.dependencies[graph.dependency_index(node_index).unwrap()].requested_symbols, - expected - .into_iter() - .map(|s| s.into()) - .collect::>() - ); - } - - fn add_asset( - graph: &mut AssetGraph, - parent_node: NodeIndex, - symbols: Vec, - file_path: &str, - ) -> NodeIndex { - let index_asset = Asset { - file_path: PathBuf::from(file_path), - symbols: Some(symbols.iter().map(symbol).collect()), - ..Asset::default() - }; - graph.add_asset(parent_node, index_asset) - } - - fn add_dependency( - graph: &mut AssetGraph, - parent_node: NodeIndex, - symbols: Vec, - ) -> NodeIndex { - let dep = Dependency { - symbols: Some(symbols.iter().map(symbol).collect()), - ..Dependency::default() - }; - graph.add_dependency(parent_node, dep) - } - - #[test] - fn should_request_entry_asset() { - let mut requested = HashSet::new(); - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - let index_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "index.js"); - let dep_a_node = add_dependency(&mut graph, index_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols( - index_asset_node, - entry_dep_node, - &mut |dependency_node_index, _dependency| { - requested.insert(dependency_node_index); - }, - ); - - assert_eq!(requested, HashSet::from_iter(vec![dep_a_node])); - assert_requested_symbols(&graph, dep_a_node, vec!["a"]); - } - - #[test] - fn should_propagate_named_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library.js - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "a" from a.js and "b" from b.js - // only "a" is used in entry.js - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("a", "a", true), ("b", "b", true)], - "library.js", - ); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); - - let mut requested_deps = Vec::new(); - graph.propagate_requested_symbols( - library_asset_node, - library_dep_node, - &mut |dependency_node_index, _dependency| { - requested_deps.push(dependency_node_index); - }, - ); - assert_eq!( - requested_deps, - vec![b_dep, a_dep], - "Should request both new deps" - ); - - // "a" should be the only requested symbol - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - assert_requested_symbols(&graph, b_dep, vec![]); - } - - #[test] - fn should_propagate_wildcard_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library.js - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from a.js and "*" from b.js - // only "a" is used in entry.js - let library_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); - - let mut requested_deps = Vec::new(); - graph.propagate_requested_symbols( - library_asset_node, - library_dep_node, - &mut |dependency_node_index, _dependency| { - requested_deps.push(dependency_node_index); - }, - ); - assert_eq!( - requested_deps, - vec![b_dep, a_dep], - "Should request both new deps" - ); - - // "a" should be marked as requested on all deps as wildcards make it - // unclear who the owning dep is - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - assert_requested_symbols(&graph, b_dep, vec!["a"]); - } - - #[test] - fn should_propagate_nested_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from library/index.js - let library_entry_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); - let library_reexport_dep_node = - add_dependency(&mut graph, library_entry_asset_node, vec![("*", "*", true)]); - graph.propagate_requested_symbols(library_entry_asset_node, library_dep_node, &mut |_, _| {}); - - // library/index.js re-exports "a" from a.js - let library_asset_node = add_asset( - &mut graph, - library_reexport_dep_node, - vec![("a", "a", true)], - "library/index.js", - ); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); - graph.propagate_requested_symbols(library_entry_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on all deps until the a dep is reached - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, library_reexport_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - } - - #[test] - fn should_propagate_renamed_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "b" from b.js renamed as "a" - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("b", "a", true)], - "library.js", - ); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); - graph.propagate_requested_symbols(library_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on the library dep - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - // "b" should be marked as requested on the b dep - assert_requested_symbols(&graph, b_dep, vec!["b"]); - } - - #[test] - fn should_propagate_namespace_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from stuff.js renamed as "a"" - // export * as a from './stuff.js' - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("a", "a", true)], - "library.js", - ); - let stuff_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "*", true)]); - graph.propagate_requested_symbols(library_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on the library dep - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - // "*" should be marked as requested on the stuff dep - assert_requested_symbols(&graph, stuff_dep, vec!["*"]); - } - - #[test] - fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { - let mut graph = AssetGraph::new(); - - let entry = graph.add_entry_dependency(Dependency { - specifier: String::from("entry"), - ..Dependency::default() - }); - - let entry_asset = graph.add_asset( - entry, - Asset { - file_path: PathBuf::from("entry"), - ..Asset::default() - }, - ); - - for i in 1..100 { - graph.add_dependency( - entry_asset, - Dependency { - specifier: format!("dependency-{}", i), - ..Dependency::default() - }, - ); - } - - let max_str_len = 10000; - let nodes = graph.serialize_nodes(max_str_len)?; - - assert_eq!(nodes.len(), 7); - - // Assert each string is less than the max size - for node in nodes.iter() { - assert!(node.len() < max_str_len); - } - - // Assert all the nodes are included and in the correct order - let first_entry = serde_json::from_str::(&nodes[0])?; - let first_entry = first_entry.as_array().unwrap(); - - assert_eq!(get_type(&first_entry[0]), json!("root")); - assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); - assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); - - for i in 1..first_entry.len() - 2 { - assert_eq!( - get_dependency(&first_entry[i + 2]), - Some(json!(format!("dependency-{}", i))) - ); - } - - let mut specifier = first_entry.len() - 2; - for node in nodes[1..].iter() { - let entry = serde_json::from_str::(&node)?; - let entry = entry.as_array().unwrap(); - - for value in entry { - assert_eq!( - get_dependency(&value), - Some(json!(format!("dependency-{}", specifier))) - ); - - specifier += 1; - } - } - - Ok(()) - } - - fn get_type(node: &Value) -> Value { - node.get("type").unwrap().to_owned() - } - - fn get_dependency(value: &Value) -> Option { - assert_eq!(get_type(&value), json!("dependency")); - - value - .get("value") - .unwrap() - .get("dependency") - .unwrap() - .get("specifier") - .map(|s| s.to_owned()) - } - - fn get_asset(value: &Value) -> Option { - assert_eq!(get_type(&value), json!("asset")); - - value - .get("value") - .unwrap() - .get("filePath") - .map(|s| s.to_owned()) - } -} diff --git a/crates/atlaspack_core/src/asset_graph/asset_graph.rs b/crates/atlaspack_core/src/asset_graph/asset_graph.rs new file mode 100644 index 000000000..1ace5cc55 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/asset_graph.rs @@ -0,0 +1,441 @@ +use std::{collections::HashSet, sync::Arc}; + +use petgraph::graph::NodeIndex; +use petgraph::stable_graph::StableDiGraph; +use petgraph::visit::EdgeRef; +use petgraph::visit::IntoEdgeReferences; + +use crate::types::Asset; +use crate::types::Dependency; + +#[derive(Clone, Debug, PartialEq)] +pub struct AssetNode { + pub asset: Asset, + pub requested_symbols: HashSet, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DependencyNode { + pub dependency: Arc, + pub requested_symbols: HashSet, + pub state: DependencyState, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum DependencyState { + New, + Deferred, + Excluded, + Resolved, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum AssetGraphNode { + Root, + Entry, + Asset(AssetNode), + Dependency(DependencyNode), +} + +#[derive(Clone, Debug)] +pub struct AssetGraph { + pub graph: StableDiGraph, + pub root_node: NodeIndex, +} + +impl Default for AssetGraph { + fn default() -> Self { + Self::new() + } +} + +impl AssetGraph { + pub fn new() -> Self { + let mut graph = StableDiGraph::new(); + + let root_node = graph.add_node(AssetGraphNode::Root); + + AssetGraph { graph, root_node } + } + + pub fn edges(&self) -> Vec { + let raw_edges = self.graph.edge_references(); + let mut edges = Vec::new(); + + for edge in raw_edges { + edges.push(edge.source().index() as u32); + edges.push(edge.target().index() as u32); + } + + edges + } + + pub fn nodes(&self) -> impl Iterator { + self.graph.node_weights() + } + + pub fn add_asset(&mut self, asset: Asset) -> NodeIndex { + self.graph.add_node(AssetGraphNode::Asset(AssetNode { + asset, + requested_symbols: HashSet::default(), + })) + } + + pub fn get_asset_node(&self, id: NodeIndex) -> Option<&AssetNode> { + let value = self.graph.node_weight(id)?; + let AssetGraphNode::Asset(asset_node) = value else { + return None; + }; + Some(asset_node) + } + + pub fn get_asset_node_mut(&mut self, id: NodeIndex) -> Option<&mut AssetNode> { + let value = self.graph.node_weight_mut(id)?; + let AssetGraphNode::Asset(asset_node) = value else { + return None; + }; + Some(asset_node) + } + + pub fn get_asset(&self, id: NodeIndex) -> Option<&Asset> { + Some(&self.get_asset_node(id)?.asset) + } + + pub fn get_asset_mut(&mut self, id: NodeIndex) -> Option<&mut Asset> { + Some(&mut self.get_asset_node_mut(id)?.asset) + } + + pub fn add_dependency(&mut self, dependency: Dependency) -> NodeIndex { + self + .graph + .add_node(AssetGraphNode::Dependency(DependencyNode { + dependency: Arc::new(dependency), + requested_symbols: HashSet::default(), + state: DependencyState::New, + })) + } + + pub fn get_dependency_node(&self, id: NodeIndex) -> Option<&DependencyNode> { + let value = self.graph.node_weight(id)?; + let AssetGraphNode::Dependency(node) = value else { + return None; + }; + Some(node) + } + + pub fn get_dependency_node_mut(&mut self, id: NodeIndex) -> Option<&mut DependencyNode> { + let value = self.graph.node_weight_mut(id)?; + let AssetGraphNode::Dependency(node) = value else { + return None; + }; + Some(node) + } + + pub fn get_dependency(&self, id: NodeIndex) -> Option<&Dependency> { + Some(&self.get_dependency_node(id)?.dependency) + } + + // pub fn get_dependency_mut(&mut self, id: NodeIndex) -> Option<&mut Dependency> { + // Some(&mut self.get_dependency_node_mut(id)?.dependency) + // } + + pub fn add_entry_dependency(&mut self, dependency: Dependency) -> NodeIndex { + let is_library = dependency.env.is_library; + let node_index = self.add_dependency(dependency); + self.add_edge(&self.root_node.clone(), &node_index); + + if is_library { + if let Some(dependency_node) = self.get_dependency_node_mut(node_index) { + dependency_node.requested_symbols.insert("*".into()); + } + } + + node_index + } + + pub fn add_edge(&mut self, parent_idx: &NodeIndex, child_idx: &NodeIndex) { + self.graph.add_edge(*parent_idx, *child_idx, ()); + } +} + +impl PartialEq for AssetGraph { + fn eq(&self, other: &Self) -> bool { + let nodes = self.nodes(); + let other_nodes = other.nodes(); + + let edges = self.edges(); + let other_edges = other.edges(); + + nodes.eq(other_nodes) && edges.eq(&other_edges) + } +} + +impl std::hash::Hash for AssetGraph { + fn hash(&self, state: &mut H) { + for node in self.graph.node_weights() { + std::mem::discriminant(node).hash(state); + match node { + AssetGraphNode::Asset(asset_node) => asset_node.asset.id.hash(state), + AssetGraphNode::Dependency(dependency_node) => dependency_node.dependency.id().hash(state), + _ => {} + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::types::Symbol; + use crate::types::Target; + + use super::super::propagate_requested_symbols::propagate_requested_symbols; + use super::*; + + type TestSymbol<'a> = (&'a str, &'a str, bool); + fn symbol(test_symbol: &TestSymbol) -> Symbol { + let (local, exported, is_weak) = test_symbol; + Symbol { + local: String::from(*local), + exported: String::from(*exported), + is_weak: is_weak.to_owned(), + ..Symbol::default() + } + } + + fn assert_requested_symbols(graph: &AssetGraph, node_index: NodeIndex, expected: Vec<&str>) { + assert_eq!( + graph + .get_dependency_node(node_index) + .unwrap() + .requested_symbols, + expected + .into_iter() + .map(|s| s.into()) + .collect::>() + ); + } + + fn add_asset( + graph: &mut AssetGraph, + parent_node: NodeIndex, + symbols: Vec, + file_path: &str, + ) -> NodeIndex { + let index_asset = Asset { + file_path: PathBuf::from(file_path), + symbols: Some(symbols.iter().map(symbol).collect()), + ..Asset::default() + }; + let asset_nid = graph.add_asset(index_asset); + graph.add_edge(&parent_node, &asset_nid); + asset_nid + } + + fn add_dependency( + graph: &mut AssetGraph, + parent_node: NodeIndex, + symbols: Vec, + ) -> NodeIndex { + let dep = Dependency { + symbols: Some(symbols.iter().map(symbol).collect()), + ..Dependency::default() + }; + let node_index = graph.add_dependency(dep); + graph.add_edge(&parent_node, &node_index); + node_index + } + + #[test] + fn should_request_entry_asset() { + let mut requested = HashSet::new(); + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + let index_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "index.js"); + let dep_a_node = add_dependency(&mut graph, index_asset_node, vec![("a", "a", false)]); + + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, index_asset_node, entry_dep_node).unwrap() + { + requested.insert(dependency_node_index); + } + + assert_eq!(requested, HashSet::from_iter(vec![dep_a_node])); + assert_requested_symbols(&graph, dep_a_node, vec!["a"]); + } + + #[test] + fn should_propagate_named_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library.js + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "a" from a.js and "b" from b.js + // only "a" is used in entry.js + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("a", "a", true), ("b", "b", true)], + "library.js", + ); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); + + let mut requested_deps = Vec::new(); + + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap() + { + requested_deps.push(dependency_node_index); + } + + assert_eq!( + requested_deps, + vec![b_dep, a_dep], + "Should request both new deps" + ); + + // "a" should be the only requested symbol + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + assert_requested_symbols(&graph, b_dep, vec![]); + } + + #[test] + fn should_propagate_wildcard_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library.js + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from a.js and "*" from b.js + // only "a" is used in entry.js + let library_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); + + let mut requested_deps = Vec::new(); + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap() + { + requested_deps.push(dependency_node_index); + } + assert_eq!( + requested_deps, + vec![b_dep, a_dep], + "Should request both new deps" + ); + + // "a" should be marked as requested on all deps as wildcards make it + // unclear who the owning dep is + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + assert_requested_symbols(&graph, b_dep, vec!["a"]); + } + + #[test] + fn should_propagate_nested_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from library/index.js + let library_entry_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); + let library_reexport_dep_node = + add_dependency(&mut graph, library_entry_asset_node, vec![("*", "*", true)]); + propagate_requested_symbols(&mut graph, library_entry_asset_node, library_dep_node).unwrap(); + + // library/index.js re-exports "a" from a.js + let library_asset_node = add_asset( + &mut graph, + library_reexport_dep_node, + vec![("a", "a", true)], + "library/index.js", + ); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); + propagate_requested_symbols(&mut graph, library_entry_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on all deps until the a dep is reached + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, library_reexport_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + } + + #[test] + fn should_propagate_renamed_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "b" from b.js renamed as "a" + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("b", "a", true)], + "library.js", + ); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on the library dep + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + // "b" should be marked as requested on the b dep + assert_requested_symbols(&graph, b_dep, vec!["b"]); + } + + #[test] + fn should_propagate_namespace_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from stuff.js renamed as "a"" + // export * as a from './stuff.js' + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("a", "a", true)], + "library.js", + ); + let stuff_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "*", true)]); + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on the library dep + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + // "*" should be marked as requested on the stuff dep + assert_requested_symbols(&graph, stuff_dep, vec!["*"]); + } +} diff --git a/crates/atlaspack_core/src/asset_graph/mod.rs b/crates/atlaspack_core/src/asset_graph/mod.rs new file mode 100644 index 000000000..d8c9946a7 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/mod.rs @@ -0,0 +1,7 @@ +mod asset_graph; +mod propagate_requested_symbols; +mod serialize_asset_graph; + +pub use self::asset_graph::*; +pub use self::propagate_requested_symbols::*; +pub use self::serialize_asset_graph::*; diff --git a/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs b/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs new file mode 100644 index 000000000..09eccf6a2 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs @@ -0,0 +1,162 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use petgraph::graph::NodeIndex; +use petgraph::visit::EdgeRef; +use petgraph::Direction; + +use crate::types::Asset; +use crate::types::Dependency; +use crate::types::Symbol; + +use super::asset_graph::DependencyState; +use super::asset_graph::{AssetGraph, DependencyNode}; + +const CHAR_STAR: &str = "*"; + +/// Propagates the requested symbols from an incoming dependency to an asset, +/// and forwards those symbols to re-exported dependencies if needed. +/// This may result in assets becoming un-deferred and transformed if they +/// now have requested symbols. +pub fn propagate_requested_symbols( + asset_graph: &mut AssetGraph, + asset_index: NodeIndex, + dependency_index: NodeIndex, +) -> Option)>> { + let mut next = vec![(asset_index, dependency_index)]; + let mut on_undeferred = vec![]; + + while let Some((asset_index, dependency_index)) = next.pop() { + let mut dependency_re_exports = HashSet::::default(); + let mut dependency_wildcards = HashSet::::default(); + let mut asset_requested_symbols_buf = HashSet::::default(); + + let dependency_node = asset_graph.get_dependency_node(dependency_index).unwrap(); + let asset_node = asset_graph.get_asset_node(asset_index).unwrap(); + + if dependency_node.requested_symbols.contains(CHAR_STAR) { + // If the requested symbols includes the "*" namespace, we + // need to include all of the asset's exported symbols. + if let Some(symbols) = &asset_node.asset.symbols { + for sym in symbols { + if !asset_node.requested_symbols.contains(&sym.exported) { + continue; + } + asset_requested_symbols_buf.insert(sym.exported.clone()); + if !sym.is_weak { + continue; + } + // Propagate re-exported symbol to dependency. + dependency_re_exports.insert(sym.local.clone()); + } + } + + // Propagate to all export * wildcard dependencies. + dependency_wildcards.insert(CHAR_STAR.to_string()); + } else { + // Otherwise, add each of the requested symbols to the asset. + for sym in dependency_node.requested_symbols.iter() { + if asset_node.requested_symbols.contains(sym) { + continue; + } + asset_requested_symbols_buf.insert(sym.clone()); + + let Some(asset_symbol) = get_symbol_by_name(&asset_node.asset, sym) else { + // If symbol wasn't found in the asset or a named re-export. + // This means the symbol is in one of the export * wildcards, but we don't know + // which one yet, so we propagate it to _all_ wildcard dependencies. + dependency_wildcards.insert(sym.clone()); + continue; + }; + + if !asset_symbol.is_weak { + continue; + } + + // If the asset exports this symbol + // Propagate re-exported symbol to dependency. + dependency_re_exports.insert(asset_symbol.local.clone()); + } + } + + // Add dependencies to asset + asset_graph + .get_asset_node_mut(asset_index) + .unwrap() + .requested_symbols + .extend(asset_requested_symbols_buf); + + let deps: Vec<_> = asset_graph + .graph + .neighbors_directed(asset_index, Direction::Outgoing) + .collect(); + + for dep_node in deps { + let mut updated = false; + + { + let DependencyNode { + dependency, + requested_symbols, + state: _, + } = asset_graph.get_dependency_node_mut(dep_node).unwrap(); + + if let Some(symbols) = &dependency.symbols { + for sym in symbols { + if sym.is_weak { + // This is a re-export. If it is a wildcard, add all unmatched symbols + // to this dependency, otherwise attempt to match a named re-export. + if sym.local == "*" { + for wildcard in &dependency_wildcards { + if requested_symbols.insert(wildcard.clone()) { + updated = true; + } + } + } else if dependency_re_exports.contains(&sym.local) + && requested_symbols.insert(sym.exported.clone()) + { + updated = true; + } + } else if requested_symbols.insert(sym.exported.clone()) { + // This is a normal import. Add the requested symbol. + updated = true; + } + } + } + } + + let DependencyNode { + dependency, + requested_symbols: _, + state, + } = asset_graph.get_dependency_node(dep_node).unwrap(); + + // If the dependency was updated, propagate to the target asset if there is one, + // or un-defer this dependency so we transform the requested asset. + // We must always resolve new dependencies to determine whether they have side effects. + if updated || *state == DependencyState::New { + let Some(resolved) = asset_graph + .graph + .edges_directed(dep_node, Direction::Outgoing) + .next() + else { + on_undeferred.push((dep_node, Arc::clone(dependency))); + continue; + }; + if resolved.target() == asset_index { + continue; + } + next.push((resolved.target(), dep_node)) + } + } + } + + return Some(on_undeferred); +} + +fn get_symbol_by_name<'a>(asset: &'a Asset, sym: &str) -> Option<&'a Symbol> { + asset + .symbols + .as_ref() + .and_then(|symbols| symbols.iter().find(|s| s.exported == *sym)) +} diff --git a/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs new file mode 100644 index 000000000..94fd0aae6 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs @@ -0,0 +1,172 @@ +use serde::Serialize; + +use crate::types::{Asset, Dependency}; + +use super::{AssetGraph, AssetGraphNode, DependencyState}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SerializedDependency { + id: String, + dependency: Dependency, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +enum SerializedAssetGraphNode { + Root, + Entry, + Asset { + value: Asset, + }, + Dependency { + value: SerializedDependency, + has_deferred: bool, + }, +} + +pub fn serialize_asset_graph( + asset_graph: &AssetGraph, + max_str_len: usize, +) -> serde_json::Result> { + let mut nodes: Vec = Vec::new(); + let mut curr_node = String::default(); + + for node in asset_graph.nodes() { + let serialized_node = match node { + AssetGraphNode::Root => SerializedAssetGraphNode::Root, + AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, + AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { + value: asset_node.asset.clone(), + }, + AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { + value: SerializedDependency { + id: dependency_node.dependency.id(), + dependency: dependency_node.dependency.as_ref().clone(), + }, + has_deferred: dependency_node.state == DependencyState::Deferred, + }, + }; + + let str = serde_json::to_string(&serialized_node)?; + if curr_node.len() + str.len() < (max_str_len - 3) { + if !curr_node.is_empty() { + curr_node.push(','); + } + curr_node.push_str(&str); + } else { + // Add the existing node now as it has reached the max JavaScript string size + nodes.push(format!("[{curr_node}]")); + curr_node = str; + } + } + + // Add the current node if it did not overflow in size + if curr_node.len() < (max_str_len - 3) { + nodes.push(format!("[{curr_node}]")); + } + + Ok(nodes) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::{json, Value}; + + use super::*; + + #[test] + fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { + let mut graph = AssetGraph::new(); + + let entry = graph.add_entry_dependency(Dependency { + specifier: String::from("entry"), + ..Dependency::default() + }); + + let entry_asset = graph.add_asset(Asset { + file_path: PathBuf::from("entry"), + ..Asset::default() + }); + + graph.add_edge(&entry, &entry_asset); + + for i in 1..100 { + let node_index = graph.add_dependency(Dependency { + specifier: format!("dependency-{}", i), + ..Dependency::default() + }); + graph.add_edge(&entry_asset, &node_index); + } + + let max_str_len = 10000; + let nodes = serialize_asset_graph(&graph, max_str_len)?; + + assert_eq!(nodes.len(), 7); + + // Assert each string is less than the max size + for node in nodes.iter() { + assert!(node.len() < max_str_len); + } + + // Assert all the nodes are included and in the correct order + let first_entry = serde_json::from_str::(&nodes[0])?; + let first_entry = first_entry.as_array().unwrap(); + + assert_eq!(get_type(&first_entry[0]), json!("root")); + assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); + assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); + + for i in 1..first_entry.len() - 2 { + assert_eq!( + get_dependency(&first_entry[i + 2]), + Some(json!(format!("dependency-{}", i))) + ); + } + + let mut specifier = first_entry.len() - 2; + for node in nodes[1..].iter() { + let entry = serde_json::from_str::(&node)?; + let entry = entry.as_array().unwrap(); + + for value in entry { + assert_eq!( + get_dependency(&value), + Some(json!(format!("dependency-{}", specifier))) + ); + + specifier += 1; + } + } + + Ok(()) + } + + fn get_type(node: &Value) -> Value { + node.get("type").unwrap().to_owned() + } + + fn get_dependency(value: &Value) -> Option { + assert_eq!(get_type(&value), json!("dependency")); + + value + .get("value") + .unwrap() + .get("dependency") + .unwrap() + .get("specifier") + .map(|s| s.to_owned()) + } + + fn get_asset(value: &Value) -> Option { + assert_eq!(get_type(&value), json!("asset")); + + value + .get("value") + .unwrap() + .get("filePath") + .map(|s| s.to_owned()) + } +} diff --git a/crates/node-bindings/src/atlaspack/atlaspack.rs b/crates/node-bindings/src/atlaspack/atlaspack.rs index 76ada51ab..6280fd765 100644 --- a/crates/node-bindings/src/atlaspack/atlaspack.rs +++ b/crates/node-bindings/src/atlaspack/atlaspack.rs @@ -6,6 +6,7 @@ use std::thread; use anyhow::anyhow; use atlaspack::AtlaspackError; +use atlaspack_core::asset_graph::serialize_asset_graph; use lmdb_js_lite::writer::DatabaseWriter; use lmdb_js_lite::LMDB; use napi::Env; @@ -150,8 +151,11 @@ impl AtlaspackNapi { let mut js_object = env.create_object()?; js_object.set_named_property("edges", env.to_js_value(&asset_graph.edges())?)?; - js_object - .set_named_property("nodes", asset_graph.serialize_nodes(MAX_STRING_LENGTH)?)?; + + js_object.set_named_property( + "nodes", + serialize_asset_graph(&asset_graph, MAX_STRING_LENGTH)?, + )?; NapiAtlaspackResult::ok(&env, js_object) }