[Turbopack] Reduce memory (vercel#75723)

### What?

* avoid cloning SourceMap for ignore list
* shrink vec before converting to Rope
* shrink Strings too
* shrink source map

Cargo.toml

@@ -133,6 +133,7 @@ clap = { version = "4.5.2", features = ["derive"] }
 concurrent-queue = "2.5.0"
 console = "0.15.5"
 console-subscriber = "0.4.1"
+const_format = "0.2.30"
 criterion = "0.5.1"
 crossbeam-channel = "0.5.8"
 dashmap = "6.1.0"

turbopack/crates/turbo-tasks-fs/src/rope.rs

@@ -118,8 +118,22 @@ impl Rope {
     }
 }
 
+impl From<Vec<u8>> for Rope {
+    fn from(mut bytes: Vec<u8>) -> Self {
+        bytes.shrink_to_fit();
+        Rope::from(Bytes::from(bytes))
+    }
+}
+
+impl From<String> for Rope {
+    fn from(mut bytes: String) -> Self {
+        bytes.shrink_to_fit();
+        Rope::from(Bytes::from(bytes))
+    }
+}
+
 impl<T: Into<Bytes>> From<T> for Rope {
-    fn from(bytes: T) -> Self {
+    default fn from(bytes: T) -> Self {
         let bytes = bytes.into();
         // We can't have an InnerRope which contains an empty Local section.
         if bytes.is_empty() {
@@ -309,7 +323,10 @@ impl Uncommitted {
         match mem::take(self) {
             Self::None => None,
             Self::Static(s) => Some(s.into()),
-            Self::Owned(v) => Some(v.into()),
+            Self::Owned(mut v) => {
+                v.shrink_to_fit();
+                Some(v.into())
+            }
         }
     }
 }

turbopack/crates/turbopack-core/Cargo.toml

@@ -17,6 +17,7 @@ anyhow = { workspace = true }
 async-trait = { workspace = true }
 auto-hash-map = { workspace = true }
 browserslist-rs = { workspace = true }
+const_format = { workspace = true }
 either = { workspace = true }
 futures = { workspace = true }
 indexmap = { workspace = true }

turbopack/crates/turbopack-core/src/code_builder.rs

@@ -5,7 +5,7 @@ use std::{
 };
 
 use anyhow::{Context, Result};
-use indexmap::{IndexMap, IndexSet};
+use indexmap::IndexMap;
 use turbo_tasks::{ResolvedVc, Vc};
 use turbo_tasks_fs::{
     rope::{Rope, RopeBuilder},
@@ -181,38 +181,14 @@ impl GenerateSourceMap for Code {
             last_byte_pos = *byte_pos;
 
             let encoded = match map {
-                None => SourceMap::empty(),
+                None => SourceMap::empty().to_resolved().await?,
                 Some(map) => match *map.generate_source_map().await? {
-                    None => SourceMap::empty(),
-                    Some(map) => {
-                        let map = &*map.await?;
-                        let map = map.to_source_map().await?;
-                        match map.as_regular_source_map() {
-                            None => SourceMap::empty(),
-                            Some(map) => {
-                                let mut map = map.into_owned();
-                                let mut ignored_ids = IndexSet::new();
-                                for (src_id, src) in map.sources().enumerate() {
-                                    if src.starts_with("turbopack://[next]")
-                                        || src.starts_with("turbopack://[turbopack]")
-                                        || src.contains("/node_modules/")
-                                    {
-                                        ignored_ids.insert(src_id);
-                                    }
-                                }
-
-                                for ignored_id in ignored_ids {
-                                    map.add_to_ignore_list(ignored_id as _);
-                                }
-
-                                SourceMap::new_decoded(sourcemap::DecodedMap::Regular(map)).cell()
-                            }
-                        }
-                    }
+                    None => SourceMap::empty().to_resolved().await?,
+                    Some(map) => map,
                 },
             };
 
-            sections.push(SourceMapSection::new(pos, encoded.to_resolved().await?))
+            sections.push(SourceMapSection::new(pos, encoded))
         }
 
         Ok(Vc::cell(Some(

turbopack/crates/turbopack-core/src/source_map/mod.rs

@@ -1,7 +1,6 @@
 use std::{borrow::Cow, io::Write, ops::Deref, sync::Arc};
 
 use anyhow::Result;
-use indexmap::IndexSet;
 use once_cell::sync::Lazy;
 use ref_cast::RefCast;
 use regex::Regex;
@@ -15,11 +14,12 @@ use turbo_tasks_fs::{
 };
 
 use crate::{
-    asset::AssetContent, source::Source, source_pos::SourcePos, virtual_source::VirtualSource,
-    SOURCE_MAP_PREFIX,
+    asset::AssetContent, source::Source, source_map::utils::add_default_ignore_list,
+    source_pos::SourcePos, virtual_source::VirtualSource, SOURCE_MAP_PREFIX,
 };
 
 pub(crate) mod source_map_asset;
+pub mod utils;
 
 pub use source_map_asset::SourceMapAsset;
 
@@ -416,28 +416,15 @@ impl SourceMap {
                 .collect::<Vec<_>>();
             let mut new_sources = Vec::with_capacity(count);
             let mut new_source_contents = Vec::with_capacity(count);
-            let mut ignored_sources = IndexSet::new();
-            for (src_id, (source, source_content)) in sources
-                .into_iter()
-                .zip(source_contents.into_iter())
-                .enumerate()
-            {
+            for (source, source_content) in sources.into_iter().zip(source_contents.into_iter()) {
                 let (source, name) = resolve_source(source, source_content, origin).await?;
-                if source.starts_with("turbopack://[next]")
-                    || source.starts_with("turbopack://[turbopack]")
-                    || source.contains("/node_modules/")
-                {
-                    ignored_sources.insert(src_id);
-                }
                 new_sources.push(source);
                 new_source_contents.push(Some(name));
             }
             let mut map =
                 RegularMap::new(file, tokens, names, new_sources, Some(new_source_contents));
 
-            for ignored_source in ignored_sources {
-                map.add_to_ignore_list(ignored_source as _);
-            }
+            add_default_ignore_list(&mut map);
 
             Ok(map)
         }

turbopack/crates/turbopack-core/src/source_map/utils.rs

@@ -0,0 +1,23 @@
+use std::collections::HashSet;
+
+use const_format::concatcp;
+use sourcemap::SourceMap;
+
+use crate::SOURCE_MAP_PREFIX;
+
+pub fn add_default_ignore_list(map: &mut SourceMap) {
+    let mut ignored_ids = HashSet::new();
+
+    for (source_id, source) in map.sources().enumerate() {
+        if source.starts_with(concatcp!(SOURCE_MAP_PREFIX, "[next]"))
+            || source.starts_with(concatcp!(SOURCE_MAP_PREFIX, "[turbopack]"))
+            || source.contains("/node_modules/")
+        {
+            ignored_ids.insert(source_id);
+        }
+    }
+
+    for ignored_id in ignored_ids {
+        map.add_to_ignore_list(ignored_id as _);
+    }
+}

turbopack/crates/turbopack-css/src/process.rs

@@ -31,7 +31,7 @@ use turbopack_core::{
     reference_type::ImportContext,
     resolve::origin::ResolveOrigin,
     source::Source,
-    source_map::{GenerateSourceMap, OptionSourceMap},
+    source_map::{utils::add_default_ignore_list, GenerateSourceMap, OptionSourceMap},
     source_pos::SourcePos,
     SOURCE_MAP_PREFIX,
 };
@@ -647,20 +647,22 @@ impl GenerateSourceMap for ParseCssResultSourceMap {
                     );
                 }
 
+                let mut map = builder.into_sourcemap();
+                add_default_ignore_list(&mut map);
                 Vc::cell(Some(
-                    turbopack_core::source_map::SourceMap::new_regular(builder.into_sourcemap())
-                        .resolved_cell(),
+                    turbopack_core::source_map::SourceMap::new_regular(map).resolved_cell(),
                 ))
             }
             ParseCssResultSourceMap::Swc {
                 source_map,
                 mappings,
             } => {
-                let map = source_map.build_source_map_with_config(
+                let mut map = source_map.build_source_map_with_config(
                     mappings,
                     None,
                     InlineSourcesContentConfig {},
                 );
+                add_default_ignore_list(&mut map);
                 Vc::cell(Some(
                     turbopack_core::source_map::SourceMap::new_regular(map).resolved_cell(),
                 ))

turbopack/crates/turbopack-ecmascript/src/minify.rs

@@ -39,7 +39,7 @@ pub async fn minify(
     let code = code.await?;
 
     let cm = Arc::new(SwcSourceMap::new(FilePathMapping::empty()));
-    let (src, src_map_buf) = {
+    let (src, mut src_map_buf) = {
         let compiler = Arc::new(Compiler::new(cm.clone()));
         let fm = compiler.cm.new_source_file(
             FileName::Custom(path.path.to_string()).into(),
@@ -116,6 +116,7 @@ pub async fn minify(
 
     let mut builder = CodeBuilder::default();
     if let Some(original_map) = source_maps {
+        src_map_buf.shrink_to_fit();
         builder.push_source(
             &src.into(),
             Some(ResolvedVc::upcast(

turbopack/crates/turbopack-ecmascript/src/parse.rs

@@ -30,7 +30,7 @@ use turbopack_core::{
     error::PrettyPrintError,
     issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
     source::Source,
-    source_map::{GenerateSourceMap, OptionSourceMap, SourceMap},
+    source_map::{utils::add_default_ignore_list, GenerateSourceMap, OptionSourceMap, SourceMap},
     SOURCE_MAP_PREFIX,
 };
 use turbopack_swc_utils::emitter::IssueEmitter;
@@ -124,11 +124,12 @@ impl GenerateSourceMap for ParseResultSourceMap {
         } else {
             None
         };
-        let map = self.files_map.build_source_map_with_config(
+        let mut map = self.files_map.build_source_map_with_config(
             &self.mappings,
             input_map.as_deref(),
             InlineSourcesContentConfig {},
         );
+        add_default_ignore_list(&mut map);
         Ok(Vc::cell(Some(SourceMap::new_regular(map).resolved_cell())))
     }
 }

turbopack/crates/turbopack-node/Cargo.toml

@@ -20,7 +20,7 @@ workspace = true
 anyhow = { workspace = true }
 async-stream = "0.3.4"
 async-trait = { workspace = true }
-const_format = "0.2.30"
+const_format = { workspace = true }
 either = { workspace = true, features = ["serde"] }
 futures = { workspace = true }
 futures-retry = { workspace = true }