Merge pull request #625 from gilbertchen/memory_optimization

Rewrite the backup procedure to reduce memory usage

duplicacy/duplicacy_main.go

@@ -785,7 +785,10 @@ func backupRepository(context *cli.Context) {
 
 	backupManager.SetupSnapshotCache(preference.Name)
 	backupManager.SetDryRun(dryRun)
-	backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly)
+
+	metadataChunkSize := context.Int("metadata-chunk-size")
+	maximumInMemoryEntries := context.Int("max-in-memory-entries")
+	backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly, metadataChunkSize, maximumInMemoryEntries)
 
 	runScript(context, preference.Name, "post")
 }
@@ -1510,6 +1513,19 @@ func main() {
 					Name:  "enum-only",
 					Usage: "enumerate the repository recursively and then exit",
 				},
+				cli.IntFlag{
+					Name:     "metadata-chunk-size",
+					Value:    1024 * 1024,
+					Usage:    "the average size of metadata chunks (defaults to 1M)",
+					Argument: "<size>",
+				},
+				cli.IntFlag{
+					Name:     "max-in-memory-entries",
+					Value:    1024 * 1024,
+					Usage:    "the maximum number of entries kept in memory (defaults to 1M)",
+					Argument: "<number>",
+				},
+
 			},
 			Usage:     "Save a snapshot of the repository to the storage",
 			ArgsUsage: " ",