feat: concatenate plugin part one (#5237)

* chore: 🤖 ck point * chore: 🤖 ck point * chore: 🤖 ck point * chore: 🤖 ck point * chore: 🤖 replace all atoms as JsWord * chore: 🤖 concatenation scope * chore: 🤖 ck point * chore: 🤖 remove tmp file * chore: 🤖 ck point * chore: 🤖 fix compilation err * chore: 🤖 reserved_names * chore: 🤖 build * chore: 🤖 lint * chore: 🤖 move concatenation scope * chore: 🤖 pass concate scope * chore: 🤖 make it can pass concatenation scope * chore: 🤖 reset concatenation scope * chore: 🤖 push error * chore: 🤖 parse js source file * chore: 🤖 update module after analyze module * chore: 🤖 create concatenation list * chore: 🤖 get module with info * chore: 🤖 collect all ident * chore: 🤖 ck point * chore: 🤖 ck point * chore: 🤖 get final binding * chore: 🤖 remove unnecessary branch * chore: 🤖 update doc * chore: 🤖 generate all used names * chore: 🤖 remove temp file * chore: 🤖 rename in concatenate module * chore: 🤖 basic return * feat: 🎸 create concate module * feat: 🎸 concatenate module build * feat: 🎸 clone attrs * feat: 🎸 copy outgoing module * feat: 🎸 chunk graph replace module * chore: 🤖 remove redundant main * feat: 🎸 add module * feat: 🎸 add config * chore: 🤖 ckpoint * feat: 🎸 ck point * chore: 🤖 remove temp file * feat: 🎸 incomming from modules * chore: 🤖 no harmony connection * chore: 🤖 basic work * chore: 🤖 recover * chore: 🤖 make it right * feat: 🎸 scope for harmony import specifier * feat: 🎸 export specifier * feat: 🎸 finish codegen * fix: 🐛 update connection * fix: 🐛 compile error * chore: 🤖 lint * chore: 🤖 lint * chore: 🤖 ck point * chore: 🤖 ck point * chore: 🤖 lint * chore: 🤖 ck point * feat: 🎸 update concate source * chore: 🤖 replace in chunk * feat: 🎸 update snap * fix: 🐛 depth deduplicated * chore: 🤖 update depth * chore: 🤖 update compatibility dep * fix: 🐛 compile err * fix: 🐛 typos * chore: 🤖 remove temp snapshot * chore: 🤖 clean up * chore: 🤖 lint * chore: 🤖 add concatenateModules option * chore: 🤖 update config * chore: 🤖 fix test * fix: 🐛 fix js test * fix: 🐛 js test case * fix: 🐛 finish todo!
web-infra-dev · Jan 23, 2024 · 5cb3d9b · 5cb3d9b · rspack-bot · Jan 23, 2024
1 parent 800542b
commit 5cb3d9b
Show file tree

Hide file tree

Showing 72 changed files with 4,841 additions and 164 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/node_binding/binding.d.ts b/crates/node_binding/binding.d.ts
@@ -1004,6 +1004,7 @@ export interface RawOptimizationOptions {
   providedExports: boolean
   innerGraph: boolean
   mangleExports: string
+  concatenateModules: boolean
 }
 
 export interface RawOptions {

diff --git a/crates/rspack/tests/fixtures.rs b/crates/rspack/tests/fixtures.rs
@@ -6,7 +6,7 @@ use rspack_core::{
 };
 use rspack_plugin_javascript::{
   FlagDependencyExportsPlugin, FlagDependencyUsagePlugin, MangleExportsPlugin,
-  SideEffectsFlagPlugin,
+  ModuleConcatenationPlugin, SideEffectsFlagPlugin,
 };
 use rspack_testing::test_fixture;
 use testing_macros::fixture;
@@ -31,6 +31,9 @@ fn samples(fixture_path: PathBuf) {
         options.optimization.used_exports = UsedExportsOption::Global;
         plugins.push(Box::<FlagDependencyExportsPlugin>::default());
         plugins.push(FlagDependencyUsagePlugin::new(is_used_exports_global(options)).boxed());
+        if options.optimization.side_effects.is_enable() {
+          plugins.push(Box::<SideEffectsFlagPlugin>::default());
+        }
         if options.optimization.mangle_exports.is_enable() {
           plugins.push(
             MangleExportsPlugin::new(!matches!(
@@ -40,6 +43,9 @@ fn samples(fixture_path: PathBuf) {
             .boxed(),
           );
         }
+        if options.optimization.concatenate_modules {
+          plugins.push(Box::new(ModuleConcatenationPlugin));
+        }
       },
     ),
     None,

diff --git a/crates/rspack/tests/samples/concatenate-modules-demo/answer.js b/crates/rspack/tests/samples/concatenate-modules-demo/answer.js
@@ -0,0 +1 @@
+export const answer = 103330
diff --git a/crates/rspack/tests/samples/concatenate-modules-demo/index.js b/crates/rspack/tests/samples/concatenate-modules-demo/index.js
@@ -0,0 +1,14 @@
+import { answer as some } from "./lib";
+some
+
+function answer() {
+
+}
+
+answer()
+
+function test() {
+
+}
+
+
diff --git a/crates/rspack/tests/samples/concatenate-modules-demo/lib.js b/crates/rspack/tests/samples/concatenate-modules-demo/lib.js
@@ -0,0 +1 @@
+export {answer} from './answer.js'
diff --git a/crates/rspack/tests/samples/concatenate-modules-demo/snapshot/output.snap b/crates/rspack/tests/samples/concatenate-modules-demo/snapshot/output.snap
@@ -0,0 +1,25 @@
+---
+source: crates/rspack_testing/src/run_fixture.rs
+---
+```js title=main.js
+(self['webpackChunkwebpack'] = self['webpackChunkwebpack'] || []).push([["main"], {
+"./index.js": (function () {
+
+;// CONCATENATED MODULE: ./answer.js
+const answer = 103330;
+
+;// CONCATENATED MODULE: ./index.js
+
+answer;
+function indexjs_answerindexjs_answer_0() {}
+indexjs_answerindexjs_answer_0();
+function test() {}
+}),
+
+},function(__webpack_require__) {
+var __webpack_exec__ = function(moduleId) { return __webpack_require__(__webpack_require__.s = moduleId) }
+var __webpack_exports__ = (__webpack_exec__("./index.js"));
+
+}
+]);
+```
diff --git a/crates/rspack/tests/samples/concatenate-modules-demo/test.config.json b/crates/rspack/tests/samples/concatenate-modules-demo/test.config.json
@@ -0,0 +1,11 @@
+{
+	"optimization": {
+    "concatenateModules": true,
+		"sideEffects": "true"
+	},
+	"builtins": {
+		"define": {
+			"process.env.NODE_ENV": "'development'"
+		}
+	}
+}
diff --git a/crates/rspack_binding_options/src/options/raw_optimization.rs b/crates/rspack_binding_options/src/options/raw_optimization.rs
@@ -15,6 +15,7 @@ pub struct RawOptimizationOptions {
   pub provided_exports: bool,
   pub inner_graph: bool,
   pub mangle_exports: String,
+  pub concatenate_modules: bool,
 }
 
 impl TryFrom<RawOptimizationOptions> for Optimization {
@@ -28,6 +29,7 @@ impl TryFrom<RawOptimizationOptions> for Optimization {
       used_exports: UsedExportsOption::from(value.used_exports.as_str()),
       inner_graph: value.inner_graph,
       mangle_exports: MangleExportsOption::from(value.mangle_exports.as_str()),
+      concatenate_modules: value.concatenate_modules,
     })
   }
 }
diff --git a/crates/rspack_core/Cargo.toml b/crates/rspack_core/Cargo.toml
@@ -17,6 +17,7 @@ either = "1"
 futures = { workspace = true }
 glob-match = "0.2.1"
 hashlink = { workspace = true }
+hex = "0.4.3"
 indexmap = { workspace = true }
 itertools = { workspace = true }
 json = { workspace = true }

diff --git a/crates/rspack_core/src/build_chunk_graph/code_splitter.rs b/crates/rspack_core/src/build_chunk_graph/code_splitter.rs
@@ -7,10 +7,10 @@ use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
 use super::remove_parent_modules::RemoveParentModulesContext;
 use crate::dependencies_block::AsyncDependenciesToInitialChunkError;
 use crate::{
-  get_entry_runtime, AsyncDependenciesBlockId, BoxDependency, ChunkGroup, ChunkGroupInfo,
-  ChunkGroupKind, ChunkGroupOptions, ChunkGroupUkey, ChunkLoading, ChunkUkey, Compilation,
-  ConnectionState, DependenciesBlock, Dependency, GroupOptions, Logger, ModuleGraphConnection,
-  ModuleIdentifier, RuntimeSpec,
+  assign_depth, assign_depths, get_entry_runtime, AsyncDependenciesBlockId, BoxDependency,
+  ChunkGroup, ChunkGroupInfo, ChunkGroupKind, ChunkGroupOptions, ChunkGroupUkey, ChunkLoading,
+  ChunkUkey, Compilation, ConnectionState, DependenciesBlock, Dependency, GroupOptions, Logger,
+  ModuleGraphConnection, ModuleIdentifier, RuntimeSpec,
 };
 
 #[derive(Debug, Clone, Eq, PartialEq, Hash)]
@@ -82,10 +82,10 @@ impl<'me> CodeSplitter<'me> {
     &mut self,
   ) -> Result<HashMap<ChunkGroupUkey, Vec<ModuleIdentifier>>> {
     let compilation = &mut self.compilation;
-    let module_graph = &compilation.module_graph;
 
     let mut input_entrypoints_and_modules: HashMap<ChunkGroupUkey, Vec<ModuleIdentifier>> =
       HashMap::default();
+    let mut assign_depths_map = HashMap::default();
 
     for (name, entry_data) in &compilation.entries {
       let options = &entry_data.options;
@@ -96,7 +96,11 @@ impl<'me> CodeSplitter<'me> {
       .concat();
       let module_identifiers = dependencies
         .iter()
-        .filter_map(|dep| module_graph.module_identifier_by_dependency_id(dep))
+        .filter_map(|dep| {
+          compilation
+            .module_graph
+            .module_identifier_by_dependency_id(dep)
+        })
         .collect::<Vec<_>>();
 
       let chunk_ukey = Compilation::add_named_chunk(
@@ -151,7 +155,7 @@ impl<'me> CodeSplitter<'me> {
         compilation.chunk_group_by_ukey.expect_get(&ukey)
       };
 
-      for module_identifier in module_identifiers {
+      for &module_identifier in module_identifiers.iter() {
         compilation.chunk_graph.add_module(*module_identifier);
 
         input_entrypoints_and_modules
@@ -165,6 +169,11 @@ impl<'me> CodeSplitter<'me> {
           entrypoint.ukey,
         );
       }
+      assign_depths(
+        &mut assign_depths_map,
+        &compilation.module_graph,
+        module_identifiers,
+      );
 
       let global_included_modules = compilation
         .global_entry
@@ -188,6 +197,13 @@ impl<'me> CodeSplitter<'me> {
         .copied()
         .sorted_unstable();
       let included_modules = global_included_modules.chain(included_modules);
+      for included_module in included_modules.clone() {
+        assign_depth(
+          &mut assign_depths_map,
+          &compilation.module_graph,
+          included_module,
+        );
+      }
       input_entrypoints_and_modules
         .entry(entrypoint.ukey)
         .or_default()
@@ -200,6 +216,11 @@ impl<'me> CodeSplitter<'me> {
       }
     }
 
+    // Using this defer insertion strategies to workaround rustc borrow rules
+    for (k, v) in assign_depths_map {
+      compilation.module_graph.set_depth(k, v);
+    }
+
     let mut runtime_chunks = HashSet::default();
     let mut runtime_error = None;
     for (name, entry_data) in &compilation.entries {

diff --git a/crates/rspack_core/src/chunk_graph/chunk_graph_chunk.rs b/crates/rspack_core/src/chunk_graph/chunk_graph_chunk.rs
@@ -1,13 +1,15 @@
 //!  There are methods whose verb is `ChunkGraphChunk`
 
+use hashlink::LinkedHashMap;
 use indexmap::IndexSet;
 use rspack_database::Database;
 use rspack_identifier::{IdentifierLinkedMap, IdentifierSet};
 use rustc_hash::FxHashMap as HashMap;
 
 use crate::{
-  find_graph_roots, merge_runtime, BoxModule, Chunk, ChunkByUkey, ChunkGroup, ChunkGroupByUkey,
-  ChunkGroupUkey, ChunkUkey, Module, ModuleGraph, ModuleIdentifier, RuntimeGlobals, SourceType,
+  find_graph_roots, merge_runtime, BoxModule, Chunk, ChunkByUkey, ChunkGraphModule, ChunkGroup,
+  ChunkGroupByUkey, ChunkGroupUkey, ChunkUkey, Module, ModuleGraph, ModuleIdentifier,
+  RuntimeGlobals, SourceType,
 };
 use crate::{ChunkGraph, Compilation};
 
@@ -65,6 +67,88 @@ impl ChunkGraph {
     self.chunk_graph_chunk_by_chunk_ukey.insert(chunk_ukey, cgc);
   }
 
+  pub fn replace_module(
+    &mut self,
+    old_module_id: &ModuleIdentifier,
+    new_module_id: &ModuleIdentifier,
+  ) {
+    if self
+      .chunk_graph_module_by_module_identifier
+      .get(new_module_id)
+      .is_none()
+    {
+      let new_chunk_graph_module = ChunkGraphModule::new();
+      self
+        .chunk_graph_module_by_module_identifier
+        .insert(*new_module_id, new_chunk_graph_module);
+    }
+
+    let old_cgm = self.get_chunk_graph_module(*old_module_id);
+    // Using clone to avoid using mutable borrow and immutable borrow at the same time.
+    for chunk in old_cgm.chunks.clone().into_iter() {
+      let cgc = self.get_chunk_graph_chunk_mut(chunk);
+      cgc.modules.remove(old_module_id);
+      cgc.modules.insert(*new_module_id);
+      let new_cgm = self.get_chunk_graph_module_mut(*new_module_id);
+      new_cgm.chunks.insert(chunk);
+    }
+
+    // shadowing the mut ref to avoid violating rustc borrow rules
+    let old_cgm = self.get_chunk_graph_module_mut(*old_module_id);
+    old_cgm.chunks.clear();
+
+    for chunk in old_cgm.entry_in_chunks.clone().into_iter() {
+      let cgc = self.get_chunk_graph_chunk_mut(chunk);
+      if let Some(old) = cgc.entry_modules.get(old_module_id).cloned() {
+        let mut new_entry_modules = LinkedHashMap::default();
+        for (m, cg) in cgc.entry_modules.iter() {
+          if m == old_module_id {
+            new_entry_modules.insert(*new_module_id, old);
+          } else {
+            new_entry_modules.insert(*m, *cg);
+          }
+        }
+        cgc.entry_modules = new_entry_modules;
+      }
+
+      let new_cgm = self.get_chunk_graph_module_mut(*new_module_id);
+      new_cgm.entry_in_chunks.insert(chunk);
+    }
+
+    let old_cgm = self.get_chunk_graph_module_mut(*old_module_id);
+    old_cgm.entry_in_chunks.clear();
+    let old_cgm = self.get_chunk_graph_module(*old_module_id);
+
+    for chunk in old_cgm.runtime_in_chunks.clone().into_iter() {
+      let cgc = self.get_chunk_graph_chunk_mut(chunk);
+      // delete old module
+      cgc.runtime_modules = std::mem::take(&mut cgc.runtime_modules)
+        .into_iter()
+        .filter(|id| old_module_id != id)
+        .collect::<Vec<_>>();
+      cgc.runtime_modules.push(*new_module_id);
+      let new_cgm = self.get_chunk_graph_module_mut(*new_module_id);
+      new_cgm.runtime_in_chunks.insert(chunk);
+
+      // TODO: full_hash_modules and dependent_hash_modules, we don't have now https://github.com/webpack/webpack/blob/1f99ad6367f2b8a6ef17cce0e058f7a67fb7db18/lib/ChunkGraph.js#L445-L462
+      // if let Some(full_hash_modules) = cgc.full_hash_modules.as_mut() {
+      //   if full_hash_modules.contains(old_module as &RuntimeModule) {
+      //     full_hash_modules.remove(old_module as &RuntimeModule);
+      //     full_hash_modules.insert(new_module as &RuntimeModule);
+      //   }
+      // }
+      // if let Some(dependent_hash_modules) = cgc.dependent_hash_modules.as_mut() {
+      //   if dependent_hash_modules.contains(old_module as &RuntimeModule) {
+      //     dependent_hash_modules.remove(old_module as &RuntimeModule);
+      //     dependent_hash_modules.insert(new_module as &RuntimeModule);
+      //   }
+      // }
+    }
+
+    let old_cgm = self.get_chunk_graph_module_mut(*old_module_id);
+    old_cgm.runtime_in_chunks.clear();
+  }
+
   pub fn get_chunk_entry_modules(&self, chunk_ukey: &ChunkUkey) -> Vec<ModuleIdentifier> {
     let chunk_graph_chunk = self.get_chunk_graph_chunk(chunk_ukey);
 

diff --git a/crates/rspack_core/src/chunk_graph/mod.rs b/crates/rspack_core/src/chunk_graph/mod.rs
@@ -1,7 +1,7 @@
 use rspack_identifier::IdentifierMap;
 use rustc_hash::FxHashMap as HashMap;
 
-use crate::AsyncDependenciesBlockId;
+use crate::{AsyncDependenciesBlockId, ModuleIdentifier};
 use crate::{ChunkGroupUkey, ChunkUkey};
 
 pub mod chunk_graph_chunk;
@@ -22,3 +22,10 @@ pub struct ChunkGraph {
 
   runtime_ids: HashMap<String, Option<String>>,
 }
+
+impl ChunkGraph {
+  pub fn is_entry_module(&self, module_id: &ModuleIdentifier) -> bool {
+    let cgm = self.get_chunk_graph_module(*module_id);
+    !cgm.entry_in_chunks.is_empty()
+  }
+}
diff --git a/crates/rspack_core/src/code_generation_results.rs b/crates/rspack_core/src/code_generation_results.rs
@@ -11,8 +11,8 @@ use rustc_hash::FxHashMap as HashMap;
 use serde::Serialize;
 
 use crate::{
-  AssetInfo, ChunkInitFragments, ModuleIdentifier, PublicPath, RuntimeGlobals, RuntimeMode,
-  RuntimeSpec, RuntimeSpecMap, SourceType,
+  AssetInfo, ChunkInitFragments, ConcatenationScope, ModuleIdentifier, PublicPath, RuntimeGlobals,
+  RuntimeMode, RuntimeSpec, RuntimeSpecMap, SourceType,
 };
 
 #[derive(Clone, Debug)]
@@ -96,6 +96,7 @@ pub struct CodeGenerationResult {
   pub runtime_requirements: RuntimeGlobals,
   pub hash: Option<RspackHashDigest>,
   pub id: CodeGenResultId,
+  pub concatenation_scope: Option<ConcatenationScope>,
 }
 
 impl CodeGenerationResult {
name	base	current	%
10000_development-mode + exec	1.71 s ± 35 ms	1.7 s ± 16 ms	-0.80%
10000_development-mode_hmr + exec	1 s ± 11 ms	1.03 s ± 10 ms	+2.58%
10000_production-mode + exec	2.74 s ± 57 ms	2.7 s ± 50 ms	-1.57%
threejs_development-mode_10x + exec	2.02 s ± 18 ms	2.02 s ± 24 ms	-0.12%
threejs_development-mode_10x_hmr + exec	1.33 s ± 5.1 ms	1.34 s ± 6.2 ms	+0.66%
threejs_production-mode_10x + exec	5.94 s ± 39 ms	5.96 s ± 34 ms	+0.38%