Skip to content

Commit

Permalink
Merge branch 'master' into memory_optimization
Browse files Browse the repository at this point in the history
  • Loading branch information
@gilbertchen
gilbertchen committed Apr 8, 2022
2 parents f52dcf7 + a953c4e commit fede9c7
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 61 deletions.
4 changes: 2 additions & 2 deletions src/duplicacy_backupmanager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,12 +235,12 @@ func TestBackupManager(t *testing.T) {

dataShards := 0
parityShards := 0
if testErasureCoding {
if *testErasureCoding {
dataShards = 5
parityShards = 2
}

if testFixedChunkSize {
if *testFixedChunkSize {
if !ConfigStorage(storage, 16384, 100, 64*1024, 64*1024, 64*1024, password, nil, false, "", dataShards, parityShards) {
t.Errorf("Failed to initialize the storage")
}
Expand Down
6 changes: 3 additions & 3 deletions src/duplicacy_chunk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func TestChunkBasic(t *testing.T) {
config.CompressionLevel = DEFAULT_COMPRESSION_LEVEL
maxSize := 1000000

if testRSAEncryption {
if *testRSAEncryption {
privateKey, err := rsa.GenerateKey(crypto_rand.Reader, 2048)
if err != nil {
t.Errorf("Failed to generate a random private key: %v", err)
Expand All @@ -71,7 +71,7 @@ func TestChunkBasic(t *testing.T) {
config.rsaPublicKey = privateKey.Public().(*rsa.PublicKey)
}

if testErasureCoding {
if *testErasureCoding {
config.DataShards = 5
config.ParityShards = 2
}
Expand All @@ -98,7 +98,7 @@ func TestChunkBasic(t *testing.T) {
encryptedData := make([]byte, chunk.GetLength())
copy(encryptedData, chunk.GetBytes())

if testErasureCoding {
if *testErasureCoding {
offset := 24 + 32 * 7
start := rand.Int() % (len(encryptedData) - offset) + offset
length := (len(encryptedData) - offset) / 7
Expand Down
8 changes: 4 additions & 4 deletions src/duplicacy_chunkoperator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@ func TestChunkOperator(t *testing.T) {
os.RemoveAll(testDir)
os.MkdirAll(testDir, 0700)

t.Logf("storage: %s", testStorageName)
t.Logf("storage: %s", *testStorageName)

storage, err := loadStorage(testDir, 1)
if err != nil {
t.Errorf("Failed to create storage: %v", err)
return
}
storage.EnableTestMode()
storage.SetRateLimits(testRateLimit, testRateLimit)
storage.SetRateLimits(*testRateLimit, *testRateLimit)

for _, dir := range []string{"chunks", "snapshots"} {
err = storage.CreateDirectory(0, dir)
Expand All @@ -59,7 +59,7 @@ func TestChunkOperator(t *testing.T) {
numberOfChunks := 100
maxChunkSize := 64 * 1024

if testQuickMode {
if *testQuickMode {
numberOfChunks = 10
}

Expand Down Expand Up @@ -87,7 +87,7 @@ func TestChunkOperator(t *testing.T) {
totalFileSize += chunk.GetLength()
}

chunkOperator := CreateChunkOperator(config, storage, nil, false, testThreads, false)
chunkOperator := CreateChunkOperator(config, storage, nil, false, *testThreads, false)
chunkOperator.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
t.Logf("Chunk %s size %d (%d/%d) uploaded", chunk.GetID(), chunkSize, chunkIndex, len(chunks))
}
Expand Down
93 changes: 41 additions & 52 deletions src/duplicacy_storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,17 @@ import (
"math/rand"
)

var testStorageName string
var testRateLimit int
var testQuickMode bool
var testThreads int
var testFixedChunkSize bool
var testRSAEncryption bool
var testErasureCoding bool

func init() {
flag.StringVar(&testStorageName, "storage", "", "the test storage to use")
flag.IntVar(&testRateLimit, "limit-rate", 0, "maximum transfer speed in kbytes/sec")
flag.BoolVar(&testQuickMode, "quick", false, "quick test")
flag.IntVar(&testThreads, "threads", 1, "number of downloading/uploading threads")
flag.BoolVar(&testFixedChunkSize, "fixed-chunk-size", false, "fixed chunk size")
flag.BoolVar(&testRSAEncryption, "rsa", false, "enable RSA encryption")
flag.BoolVar(&testErasureCoding, "erasure-coding", false, "enable Erasure Coding")
flag.Parse()
}
var testStorageName = flag.String("storage", "", "the test storage to use")
var testRateLimit = flag.Int("limit-rate", 0, "maximum transfer speed in kbytes/sec")
var testQuickMode = flag.Bool("quick", false, "quick test")
var testThreads = flag.Int("threads", 1, "number of downloading/uploading threads")
var testFixedChunkSize = flag.Bool("fixed-chunk-size", false, "fixed chunk size")
var testRSAEncryption = flag.Bool("rsa", false, "enable RSA encryption")
var testErasureCoding = flag.Bool("erasure-coding", false, "enable Erasure Coding")

func loadStorage(localStoragePath string, threads int) (Storage, error) {

if testStorageName == "" || testStorageName == "file" {
if *testStorageName == "" || *testStorageName == "file" {
storage, err := CreateFileStorage(localStoragePath, false, threads)
if storage != nil {
// Use a read level of at least 2 because this will catch more errors than a read level of 1.
Expand All @@ -64,116 +53,116 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
return nil, err
}

config, found := configs[testStorageName]
config, found := configs[*testStorageName]
if !found {
return nil, fmt.Errorf("No storage named '%s' found", testStorageName)
return nil, fmt.Errorf("No storage named '%s' found", *testStorageName)
}

if testStorageName == "flat" {
if *testStorageName == "flat" {
storage, err := CreateFileStorage(localStoragePath, false, threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "samba" {
} else if *testStorageName == "samba" {
storage, err := CreateFileStorage(localStoragePath, true, threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "sftp" {
} else if *testStorageName == "sftp" {
port, _ := strconv.Atoi(config["port"])
storage, err := CreateSFTPStorageWithPassword(config["server"], port, config["username"], config["directory"], 2, config["password"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "s3" {
} else if *testStorageName == "s3" {
storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "wasabi" {
} else if *testStorageName == "wasabi" {
storage, err := CreateWasabiStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "s3c" {
} else if *testStorageName == "s3c" {
storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "digitalocean" {
} else if *testStorageName == "digitalocean" {
storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "minio" {
} else if *testStorageName == "minio" {
storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, false, true)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "minios" {
} else if *testStorageName == "minios" {
storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, true)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "dropbox" {
} else if *testStorageName == "dropbox" {
storage, err := CreateDropboxStorage(config["token"], config["directory"], 1, threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "b2" {
} else if *testStorageName == "b2" {
storage, err := CreateB2Storage(config["account"], config["key"], "", config["bucket"], config["directory"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "gcs-s3" {
} else if *testStorageName == "gcs-s3" {
storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "gcs" {
} else if *testStorageName == "gcs" {
storage, err := CreateGCSStorage(config["token_file"], config["bucket"], config["directory"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "gcs-sa" {
} else if *testStorageName == "gcs-sa" {
storage, err := CreateGCSStorage(config["token_file"], config["bucket"], config["directory"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "azure" {
} else if *testStorageName == "azure" {
storage, err := CreateAzureStorage(config["account"], config["key"], config["container"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "acd" {
} else if *testStorageName == "acd" {
storage, err := CreateACDStorage(config["token_file"], config["storage_path"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "gcd" {
} else if *testStorageName == "gcd" {
storage, err := CreateGCDStorage(config["token_file"], "", config["storage_path"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "gcd-shared" {
} else if *testStorageName == "gcd-shared" {
storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "gcd-impersonate" {
} else if *testStorageName == "gcd-impersonate" {
storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "one" {
} else if *testStorageName == "one" {
storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "odb" {
} else if *testStorageName == "odb" {
storage, err := CreateOneDriveStorage(config["token_file"], true, config["storage_path"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "one" {
} else if *testStorageName == "one" {
storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "hubic" {
} else if *testStorageName == "hubic" {
storage, err := CreateHubicStorage(config["token_file"], config["storage_path"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "memset" {
} else if *testStorageName == "memset" {
storage, err := CreateSwiftStorage(config["storage_url"], config["key"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "pcloud" || testStorageName == "box" {
} else if *testStorageName == "pcloud" || *testStorageName == "box" {
storage, err := CreateWebDAVStorage(config["host"], 0, config["username"], config["password"], config["storage_path"], false, threads)
if err != nil {
return nil, err
}
storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err
} else if testStorageName == "fabric" {
} else if *testStorageName == "fabric" {
storage, err := CreateFileFabricStorage(config["endpoint"], config["token"], config["storage_path"], threads)
if err != nil {
return nil, err
Expand All @@ -182,7 +171,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
return storage, err
}

return nil, fmt.Errorf("Invalid storage named: %s", testStorageName)
return nil, fmt.Errorf("Invalid storage named: %s", *testStorageName)
}

func cleanStorage(storage Storage) {
Expand Down Expand Up @@ -322,7 +311,7 @@ func TestStorage(t *testing.T) {
os.RemoveAll(testDir)
os.MkdirAll(testDir, 0700)

LOG_INFO("STORAGE_TEST", "storage: %s", testStorageName)
LOG_INFO("STORAGE_TEST", "storage: %s", *testStorageName)

threads := 8
storage, err := loadStorage(testDir, threads)
Expand All @@ -331,7 +320,7 @@ func TestStorage(t *testing.T) {
return
}
storage.EnableTestMode()
storage.SetRateLimits(testRateLimit, testRateLimit)
storage.SetRateLimits(*testRateLimit, *testRateLimit)

delay := 0
if _, ok := storage.(*ACDStorage); ok {
Expand Down Expand Up @@ -453,7 +442,7 @@ func TestStorage(t *testing.T) {
numberOfFiles := 10
maxFileSize := 64 * 1024

if testQuickMode {
if *testQuickMode {
numberOfFiles = 2
}

Expand Down Expand Up @@ -588,7 +577,7 @@ func TestCleanStorage(t *testing.T) {
os.RemoveAll(testDir)
os.MkdirAll(testDir, 0700)

LOG_INFO("STORAGE_TEST", "storage: %s", testStorageName)
LOG_INFO("STORAGE_TEST", "storage: %s", *testStorageName)

storage, err := loadStorage(testDir, 1)
if err != nil {
Expand Down

0 comments on commit fede9c7

Please sign in to comment.