Merge pull request #459 from fairDataSociety/staging

Staging

.github/workflows/coverge.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.19
+          go-version: 1.20
       - name: Checkout
         uses: actions/checkout@v2
         with:

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go: [1.19]
+        go: [1.20]
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
     - name: Setup Go

.github/workflows/release.yaml

@@ -18,7 +18,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19
+          go-version: 1.20
       - name: Checkout
         uses: actions/checkout@v2
         with:
@@ -45,7 +45,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19
+          go-version: 1.20
       - name: Checkout
         uses: actions/checkout@v2
         with:
@@ -85,7 +85,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19
+          go-version: 1.20
       - name: Checkout
         uses: actions/checkout@v2
         with:

Makefile

@@ -1,6 +1,6 @@
 GO ?= go
 GOLANGCI_LINT ?= $$($(GO) env GOPATH)/bin/golangci-lint
-GOLANGCI_LINT_VERSION ?= v1.50.0
+GOLANGCI_LINT_VERSION ?= v1.52.2
 GOGOPROTOBUF ?= protoc-gen-gogofaster
 GOGOPROTOBUF_VERSION ?= v1.3.1
 

cmd/dfs-cli/cmd/filesystem.go

@@ -52,18 +52,16 @@ func isDirectoryPresent(podName, dirNameWithpath string) bool {
 	return resp.Present
 }
 
-func listFileAndDirectories(podName, dirNameWithpath string) {
+func listFileAndDirectories(podName, dirNameWithpath string) (*api.ListFileResponse, error) {
 	args := fmt.Sprintf("podName=%s&dirPath=%s", podName, dirNameWithpath)
 	data, err := fdfsAPI.getReq(apiDirLs, args)
 	if err != nil {
-		fmt.Println("ls failed: ", err)
-		return
+		return nil, err
 	}
 	var resp api.ListFileResponse
 	err = json.Unmarshal(data, &resp)
 	if err != nil {
-		fmt.Println("dir ls: ", err)
-		return
+		return nil, err
 	}
 	if resp.Directories == nil && resp.Files == nil {
 		fmt.Println("empty directory")
@@ -74,6 +72,7 @@ func listFileAndDirectories(podName, dirNameWithpath string) {
 	for _, entry := range resp.Files {
 		fmt.Println("<File>: ", entry.Name)
 	}
+	return &resp, nil
 }
 
 func statFileOrDirectory(podName, statElement string) {
@@ -213,6 +212,7 @@ func uploadFile(fileName, podName, localFileWithPath, podDir, blockSize, compres
 	args["podName"] = podName
 	args["dirPath"] = podDir
 	args["blockSize"] = blockSize
+	args["overwrite"] = "true"
 	data, err := fdfsAPI.uploadMultipartFile(apiFileUpload, fileName, fi.Size(), fd, args, "files", compression)
 	if err != nil {
 		fmt.Println("upload failed: ", err)

cmd/dfs-cli/cmd/prompt.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -228,6 +229,8 @@ var suggestions = []prompt.Suggest{
 	{Text: "cd", Description: "change path"},
 	{Text: "download", Description: "download file from dfs to local machine"},
 	{Text: "upload", Description: "upload file from local machine to dfs"},
+	{Text: "uploadDir", Description: "upload a dir from local machine to dfs"},
+	{Text: "downloadDir", Description: "download dir from dfs to local machine"},
 	{Text: "share", Description: "share file with another user"},
 	{Text: "receive", Description: "receive a shared file"},
 	{Text: "exit", Description: "exit dfs-prompt"},
@@ -297,7 +300,6 @@ func executor(in string) {
 				mnemonic = strings.TrimPrefix(mnemonic, " ")
 			}
 			userNew(userName, mnemonic)
-			currentUser = userName
 			currentPod = ""
 			currentDirectory = ""
 			currentPrompt = getCurrentPrompt()
@@ -750,7 +752,10 @@ func executor(in string) {
 		if !isPodOpened() {
 			return
 		}
-		listFileAndDirectories(currentPod, currentDirectory)
+		_, err = listFileAndDirectories(currentPod, currentDirectory)
+		if err != nil {
+			fmt.Println("ls failed: ", err)
+		}
 		currentPrompt = getCurrentPrompt()
 	case "mkdir":
 		if !isPodOpened() {
@@ -797,6 +802,81 @@ func executor(in string) {
 		}
 		rmDir(currentPod, dirToRm)
 		currentPrompt = getCurrentPrompt()
+	case "uploadDir":
+		if !isPodOpened() {
+			return
+		}
+		if len(blocks) < 4 {
+			fmt.Println("invalid command. Missing one or more arguments")
+			return
+		}
+		dirName := blocks[1]
+		podDir := blocks[2]
+		if podDir == "." {
+			podDir = currentDirectory
+		}
+		blockSize := blocks[3]
+		compression := ""
+		if len(blocks) >= 5 {
+			compression = blocks[4]
+			if compression != "snappy" && compression != "gzip" {
+				fmt.Println("invalid value for \"compression\", should either be \"snappy\" or \"gzip\"")
+				return
+			}
+		}
+		toUpload, err := findFilesToUpload(dirName)
+		if err != nil {
+			fmt.Println("Failed to list files to upload at: ", dirName, err)
+			return
+		}
+		for _, item := range toUpload.filesToUpload {
+			if toUpload.rootDirectory != item {
+				itemStats, err := os.Stat(item)
+				if err != nil {
+					fmt.Println("Failed to read stats for: ", item, err)
+					continue
+				}
+				fmt.Println("Handling item: ", removeParentDirectory(toUpload.rootDirectory, item))
+				if itemStats.IsDir() {
+					dirToMk := filepath.ToSlash(filepath.Join(podDir, removeParentDirectory(toUpload.rootDirectory, item)))
+					mkdir(currentPod, dirToMk)
+				} else {
+					filePath := removeParentDirectory(toUpload.rootDirectory, item)
+					uploadFile(filepath.Base(filePath), currentPod, item, filepath.ToSlash(filepath.Join(podDir, filepath.Dir(filePath))), blockSize, compression)
+				}
+			}
+		}
+		currentPrompt = getCurrentPrompt()
+	case "downloadDir":
+		if !isPodOpened() {
+			return
+		}
+		if len(blocks) < 3 {
+			fmt.Println("invalid command. Missing one or more arguments")
+			return
+		}
+		localDir := blocks[1]
+		dirStat, err := os.Stat(localDir)
+		if err != nil {
+			fmt.Println("local path is not a present: ", err)
+			return
+		}
+
+		if !dirStat.IsDir() {
+			fmt.Println("local path is not a directory")
+			return
+		}
+
+		podPath := blocks[2]
+		if !strings.HasPrefix(podPath, utils.PathSeparator) {
+			if currentDirectory == utils.PathSeparator {
+				podPath = currentDirectory + podPath
+			} else {
+				podPath = currentDirectory + utils.PathSeparator + podPath
+			}
+		}
+		downloadDir(currentPod, localDir, podPath)
+		currentPrompt = getCurrentPrompt()
 	case "upload":
 		if !isPodOpened() {
 			return
@@ -988,6 +1068,8 @@ func help() {
 	fmt.Println(" - ls ")
 	fmt.Println(" - download <destination dir in local fs> <relative path of source file in pod>")
 	fmt.Println(" - upload <source file in local fs> <destination directory in pod> <block size (ex: 1Mb, 64Mb)>, <compression (snappy/gzip)>")
+	fmt.Println(" - uploadDir <source location in local fs> <destination directory in pod> <block size (ex: 1Mb, 64Mb)>, <compression (snappy/gzip)>")
+	fmt.Println(" - downloadDir <destination location in local fs> <source directory in pod>")
 	fmt.Println(" - share <file name> -  shares a file with another user")
 	fmt.Println(" - receive <sharing reference> <pod dir> - receives a file from another user")
 	fmt.Println(" - receiveinfo <sharing reference> - shows the received file info before accepting the receive")
@@ -1050,3 +1132,77 @@ func getPassword() (password string) {
 	password = strings.TrimSpace(passwd)
 	return password
 }
+
+type files struct {
+	filesToUpload []string
+	rootDirectory string
+}
+
+func findFilesToUpload(searchPath string) (*files, error) {
+	searchResults := []string{}
+	rawSearchResults, err := filepath.Glob(searchPath)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, searchResult := range rawSearchResults {
+		searchResults = append(searchResults, searchResult)
+		fileStats, err := os.Stat(searchResult)
+		if err != nil {
+			return nil, err
+		}
+
+		if fileStats.IsDir() {
+			toUpload, err := findFilesToUpload(filepath.Join(searchResult, "*"))
+			if err == nil {
+				searchResults = append(searchResults, toUpload.filesToUpload...)
+			}
+		}
+	}
+
+	if len(searchResults) == 1 && searchPath == searchResults[0] {
+		return &files{
+			filesToUpload: searchResults,
+			rootDirectory: filepath.Dir(searchResults[0]),
+		}, nil
+	}
+
+	return &files{
+		filesToUpload: searchResults,
+		rootDirectory: searchPath,
+	}, nil
+}
+
+func removeParentDirectory(parentPath, childPath string) string {
+	relativePath, err := filepath.Rel(parentPath, childPath)
+	if err != nil {
+		panic(err)
+	}
+	if relativePath == "" {
+		return "/"
+	}
+	if strings.HasPrefix(relativePath, "..") {
+		return ""
+	}
+	return "/" + relativePath
+}
+
+func downloadDir(currentPod, localDir, podPath string) {
+	stat, err := listFileAndDirectories(currentPod, podPath)
+	if err != nil {
+		fmt.Println("failed to get contents of dir: ", podPath, err)
+		return
+	}
+	for _, dir := range stat.Directories {
+		err := os.Mkdir(filepath.Join(localDir, dir.Name), os.FileMode(dir.Mode))
+		if err != nil && !errors.Is(err, os.ErrExist) {
+			fmt.Println("failed to create local dir: ", filepath.Join(localDir, dir.Name), err)
+			continue
+		}
+		downloadDir(currentPod, filepath.Join(localDir, dir.Name), filepath.ToSlash(filepath.Join(podPath, dir.Name)))
+	}
+	for _, file := range stat.Files {
+		loalFile := filepath.Join(localDir, file.Name)
+		downloadFile(currentPod, loalFile, filepath.ToSlash(filepath.Join(podPath, file.Name)))
+	}
+}

cmd/dfs-cli/cmd/user.go

@@ -53,17 +53,20 @@ func userNew(userName, mnemonic string) {
 		return
 	}
 	if resp.Message == eth.ErrInsufficientBalance.Error() {
-		fmt.Println(resp.Message)
+		fmt.Println("Failed to create new user")
+		fmt.Println("Please fund your account with some eth and try again with the following command")
+		fmt.Printf(">>> user new %s %s\n", userName, resp.Mnemonic)
 		fmt.Println("address :", resp.Address)
 		fmt.Println("=============== Mnemonic ==========================")
 		fmt.Println(resp.Mnemonic)
 		fmt.Println("=============== Mnemonic ==========================")
-	} else {
-		fmt.Println("user created with address ", resp.Address)
-		fmt.Println("Please store the 12 words mnemonic safely")
-		fmt.Println("if you loose that, you cannot recover the data in-case of an emergency.")
-		fmt.Println("you can also use that mnemonic to access the data in-case this device is lost")
+		return
 	}
+	fmt.Println("user created with address ", resp.Address)
+	fmt.Println("Please store the 12 words mnemonic safely")
+	fmt.Println("if you loose that, you cannot recover the data in-case of an emergency.")
+	fmt.Println("you can also use that mnemonic to access the data in-case this device is lost")
+	currentUser = userName
 }
 
 func userLogin(userName, apiEndpoint string) {

cmd/dfs/cmd/server_test.go

@@ -430,7 +430,7 @@ func TestApis(t *testing.T) {
 				if err != nil {
 					t.Fatal(err)
 				}
-				err = writer.WriteField("blockSize", "4kb")
+				err = writer.WriteField("blockSize", "1Mb")
 				if err != nil {
 					t.Fatal(err)
 				}

go.mod

@@ -1,13 +1,13 @@
 module github.com/fairdatasociety/fairOS-dfs
 
-go 1.18
+go 1.20
 
 require (
 	github.com/btcsuite/btcd v0.22.3
 	github.com/c-bata/go-prompt v0.2.6
 	github.com/dustin/go-humanize v1.0.1
-	github.com/ethereum/go-ethereum v1.11.4
-	github.com/ethersphere/bee v1.12.0
+	github.com/ethereum/go-ethereum v1.11.5
+	github.com/ethersphere/bee v1.13.0
 	github.com/ethersphere/bmt v0.1.4
 	github.com/fairdatasociety/fairOS-dfs-utils v0.0.0-20221230123929-aec4ed8b854d
 	github.com/golang/snappy v0.0.4
@@ -26,12 +26,12 @@ require (
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.2
 	github.com/swaggo/http-swagger v1.3.4
-	github.com/swaggo/swag v1.8.10
+	github.com/swaggo/swag v1.8.12
 	github.com/tinygrasshopper/bettercsv v0.0.1
 	github.com/tyler-smith/go-bip39 v1.1.0
 	github.com/wealdtech/go-ens/v3 v3.5.5
 	go.uber.org/goleak v1.2.1
-	golang.org/x/crypto v0.4.0
+	golang.org/x/crypto v0.7.0
 	golang.org/x/term v0.6.0
 	gopkg.in/yaml.v2 v2.4.0
 	resenje.org/jsonhttp v0.2.0
@@ -54,6 +54,7 @@ require (
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/holiman/uint256 v1.2.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/ipfs/go-cid v0.3.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -91,12 +92,14 @@ require (
 	github.com/wealdtech/go-multicodec v1.4.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	golang.org/x/tools v0.3.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 )
+
+replace github.com/codahale/hdrhistogram => github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1