Merge pull request #1700 from GitoxideLabs/reduce-memory

reduce memory consumption during clone

Author: Byron

gix-pack/src/cache/delta/tree.rs

@@ -1,4 +1,5 @@
 use super::{traverse, Error};
+use crate::exact_vec;
 /// An item stored within the [`Tree`] whose data is stored in a pack file, identified by
 /// the offset of its first (`offset`) and last (`next_offset`) bytes.
 ///
@@ -61,8 +62,8 @@ impl<T> Tree<T> {
     /// Instantiate a empty tree capable of storing `num_objects` amounts of items.
     pub fn with_capacity(num_objects: usize) -> Result<Self, Error> {
         Ok(Tree {
-            root_items: Vec::with_capacity(num_objects / 2),
-            child_items: Vec::with_capacity(num_objects / 2),
+            root_items: exact_vec(num_objects / 2),
+            child_items: exact_vec(num_objects / 2),
             last_seen: None,
             future_child_offsets: Vec::new(),
         })

gix-pack/src/data/output/bytes.rs

@@ -3,6 +3,7 @@ use std::io::Write;
 use gix_features::hash;
 
 use crate::data::output;
+use crate::exact_vec;
 
 /// The error returned by `next()` in the [`FromEntriesIter`] iterator.
 #[allow(missing_docs)]
@@ -73,7 +74,7 @@ where
             output: hash::Write::new(output, object_hash),
             trailer: None,
             entry_version: version,
-            pack_offsets_and_validity: Vec::with_capacity(num_entries as usize),
+            pack_offsets_and_validity: exact_vec(num_entries as usize),
             written: 0,
             header_info: Some((version, num_entries)),
             is_done: false,

gix-pack/src/index/traverse/with_lookup.rs

@@ -9,7 +9,7 @@ use gix_features::{
 
 use super::{Error, Reducer};
 use crate::{
-    data, index,
+    data, exact_vec, index,
     index::{traverse::Outcome, util},
 };
 
@@ -152,7 +152,7 @@ impl index::File {
                             Some(entries.len()),
                             gix_features::progress::count_with_decimals("objects", 2),
                         );
-                        let mut stats = Vec::with_capacity(entries.len());
+                        let mut stats = exact_vec(entries.len());
                         progress.set(0);
                         for index_entry in entries.iter() {
                             let result = self.decode_and_process_entry(

gix-pack/src/index/util.rs

@@ -1,5 +1,6 @@
 use std::time::Instant;
 
+use crate::exact_vec;
 use gix_features::progress::{self, Progress};
 
 pub(crate) fn index_entries_sorted_by_offset_ascending(
@@ -9,7 +10,7 @@ pub(crate) fn index_entries_sorted_by_offset_ascending(
     progress.init(Some(idx.num_objects as usize), progress::count("entries"));
     let start = Instant::now();
 
-    let mut v = Vec::with_capacity(idx.num_objects as usize);
+    let mut v = exact_vec(idx.num_objects as usize);
     for entry in idx.iter() {
         v.push(entry);
         progress.inc();

gix-pack/src/lib.rs

@@ -69,3 +69,9 @@ fn read_u32(b: &[u8]) -> u32 {
 fn read_u64(b: &[u8]) -> u64 {
     u64::from_be_bytes(b.try_into().unwrap())
 }
+
+fn exact_vec<T>(capacity: usize) -> Vec<T> {
+    let mut v = Vec::new();
+    v.reserve_exact(capacity);
+    v
+}

gix-pack/src/multi_index/verify.rs

@@ -2,7 +2,7 @@ use std::{cmp::Ordering, sync::atomic::AtomicBool, time::Instant};
 
 use gix_features::progress::{Count, DynNestedProgress, Progress};
 
-use crate::{index, multi_index::File};
+use crate::{exact_vec, index, multi_index::File};
 
 ///
 pub mod integrity {
@@ -166,7 +166,7 @@ impl File {
 
         let operation_start = Instant::now();
         let mut total_objects_checked = 0;
-        let mut pack_ids_and_offsets = Vec::with_capacity(self.num_objects as usize);
+        let mut pack_ids_and_offsets = exact_vec(self.num_objects as usize);
         {
             let order_start = Instant::now();
             let mut progress = progress.add_child_with_id("checking oid order".into(), gix_features::progress::UNKNOWN);