COMPUTE-15 Revert /listFolder changes for v1.4.0 release (#87)

* Revert "DEVEX-2122 List folder content in a single `/listFolder` request (#84)"

* gofmt

* Bump version to v1.4.0 for release

cli/main.go

@@ -104,7 +104,7 @@ func getUser() user.User {
 }
 
 // Mount the filesystem:
-//  - setup the debug log to the FUSE kernel log (I think)
+//   - setup the debug log to the FUSE kernel log (I think)
 func fsDaemon(
 	mountpoint string,
 	dxEnv dxda.DXEnvironment,
@@ -419,7 +419,6 @@ func buildDaemonCommandLine(cfg Config, fullManifestPath string) []string {
 }
 
 // We are in the parent process.
-//
 func startDaemonAndWaitForInitializationToComplete(cfg Config, logFile string) {
 	manifest, err := parseManifest(cfg)
 	if err != nil {

cmd_client.go

@@ -10,7 +10,6 @@ type CmdClient struct {
 }
 
 // Sending commands with a client
-//
 func NewCmdClient() *CmdClient {
 	return &CmdClient{}
 }

dx_describe.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"net/http"
 	"strings"
-	"time"
 
 	// The dxda package has the get-environment code
 	"github.com/dnanexus/dxda"
@@ -34,24 +33,6 @@ type DxDescribeDataObject struct {
 	Properties    map[string]string
 }
 
-func ConvertDescribeRawToDataObject(
-	projectId string,
-	descRaw DxDescribeRaw) DxDescribeDataObject {
-	return DxDescribeDataObject{
-		Id:            descRaw.Id,
-		ProjId:        projectId,
-		Name:          descRaw.Name,
-		State:         descRaw.State,
-		ArchivalState: descRaw.ArchivalState,
-		Folder:        descRaw.Folder,
-		Size:          descRaw.Size,
-		CtimeSeconds:  descRaw.CreatedMillisec / 1000,
-		MtimeSeconds:  descRaw.ModifiedMillisec / 1000,
-		Tags:          descRaw.Tags,
-		Properties:    descRaw.Properties,
-	}
-}
-
 // https://documentation.dnanexus.com/developer/api/data-containers/projects#api-method-project-xxxx-describe
 type FileUploadParameters struct {
 	MinimumPartSize      int64 `json:"minimumPartSize"`
@@ -184,7 +165,21 @@ func submit(
 
 	var files = make(map[string]DxDescribeDataObject)
 	for _, descRawTop := range reply.Results {
-		desc := ConvertDescribeRawToDataObject(descRawTop.Describe.ProjId, descRawTop.Describe)
+		descRaw := descRawTop.Describe
+
+		desc := DxDescribeDataObject{
+			Id:            descRaw.Id,
+			ProjId:        descRaw.ProjId,
+			Name:          descRaw.Name,
+			State:         descRaw.State,
+			ArchivalState: descRaw.ArchivalState,
+			Folder:        descRaw.Folder,
+			Size:          descRaw.Size,
+			CtimeSeconds:  descRaw.CreatedMillisec / 1000,
+			MtimeSeconds:  descRaw.ModifiedMillisec / 1000,
+			Tags:          descRaw.Tags,
+			Properties:    descRaw.Properties,
+		}
 		//fmt.Printf("%v\n", desc)
 		files[desc.Id] = desc
 	}
@@ -229,10 +224,9 @@ func DxDescribeBulkObjects(
 }
 
 type ListFolderRequest struct {
-	Folder        string          `json:"folder"`
-	Only          string          `json:"only"`
-	IncludeHidden bool            `json:"includeHidden"`
-	Describe      map[string]bool `json:"describe"`
+	Folder        string `json:"folder"`
+	Only          string `json:"only"`
+	IncludeHidden bool   `json:"includeHidden"`
 }
 
 type ListFolderResponse struct {
@@ -241,76 +235,87 @@ type ListFolderResponse struct {
 }
 
 type ObjInfo struct {
-	Id       string        `json:"id"`
-	Describe DxDescribeRaw `json:"describe"`
+	Id string `json:"id"`
 }
 
-func DxDescribeFolder(
+type DxListFolder struct {
+	objIds  []string
+	subdirs []string
+}
+
+// Issue a /project-xxxx/listFolder API call. Get
+// back a list of object-ids and sub-directories.
+func listFolder(
 	ctx context.Context,
 	httpClient *http.Client,
-	dxOptions *Options,
 	dxEnv *dxda.DXEnvironment,
 	projectId string,
-	folder string) (*DxFolder, error) {
+	dir string) (*DxListFolder, error) {
+
 	request := ListFolderRequest{
-		Folder:        folder,
+		Folder:        dir,
 		Only:          "all",
 		IncludeHidden: false,
 	}
-	request.Describe = map[string]bool{
-		"id":            true,
-		"name":          true,
-		"state":         true,
-		"archivalState": true,
-		"folder":        true,
-		"created":       true,
-		"modified":      true,
-		"size":          true,
-		"tags":          true,
-		"properties":    true,
-	}
-
 	var payload []byte
 	payload, err := json.Marshal(request)
 	if err != nil {
-		log.Printf("listFolder(%s) payload marshalling error %s", folder, err.Error())
 		return nil, err
 	}
 	dxRequest := fmt.Sprintf("%s/listFolder", projectId)
-	reqStartTime := time.Now()
 	repJs, err := dxda.DxAPI(ctx, httpClient, NumRetriesDefault, dxEnv, dxRequest, string(payload))
-	if dxOptions.VerboseLevel > 1 {
-		log.Printf("listFolder(%s) API call duration %s", folder, time.Now().Sub(reqStartTime))
-	}
 	if err != nil {
-		log.Printf("listFolder(%s) request error %s", folder, err.Error())
 		return nil, err
 	}
 	var reply ListFolderResponse
 	if err := json.Unmarshal(repJs, &reply); err != nil {
-		log.Printf("listFolder(%s) response unmarshalling error %s", folder, err.Error())
 		return nil, err
 	}
-	dataObjects := make(map[string]DxDescribeDataObject)
-	for _, oDesc := range reply.Objects {
-		dataObjects[oDesc.Id] = ConvertDescribeRawToDataObject(projectId, oDesc.Describe)
+	var objIds []string
+	for _, objInfo := range reply.Objects {
+		objIds = append(objIds, objInfo.Id)
 	}
-
-	var folderInfo *DxFolder
-	folderInfo = &DxFolder{
-		path:        folder,
-		dataObjects: dataObjects,
-		subdirs:     reply.Folders,
+	retval := DxListFolder{
+		objIds:  objIds,
+		subdirs: reply.Folders,
 	}
+	return &retval, nil
+}
 
+func DxDescribeFolder(
+	ctx context.Context,
+	httpClient *http.Client,
+	dxEnv *dxda.DXEnvironment,
+	projectId string,
+	folder string) (*DxFolder, error) {
+	// The listFolder API call returns a list of object ids and folders.
+	// We could describe the objects right here, but we do that separately.
+	folderInfo, err := listFolder(ctx, httpClient, dxEnv, projectId, folder)
+	if err != nil {
+		log.Printf("listFolder(%s) error %s", folder, err.Error())
+		return nil, err
+	}
 	// limit the number of directory elements
-	numElementsInDir := len(folderInfo.dataObjects)
+	numElementsInDir := len(folderInfo.objIds)
 	if numElementsInDir > MaxDirSize {
 		return nil, fmt.Errorf(
 			"Too many elements (%d) in a directory, the limit is %d",
 			numElementsInDir, MaxDirSize)
 	}
-	return folderInfo, nil
+	dxObjs, err := DxDescribeBulkObjects(ctx, httpClient, dxEnv, projectId, folderInfo.objIds)
+	if err != nil {
+		log.Printf("describeBulkObjects(%v) error %s", folderInfo.objIds, err.Error())
+		return nil, err
+	}
+	dataObjects := make(map[string]DxDescribeDataObject)
+	for _, oDesc := range dxObjs {
+		dataObjects[oDesc.Id] = oDesc
+	}
+	return &DxFolder{
+		path:        folder,
+		dataObjects: dataObjects,
+		subdirs:     folderInfo.subdirs,
+	}, nil
 }
 
 type RequestDescribeProject struct {

dx_ops.go

@@ -357,9 +357,9 @@ type ReplyRename struct {
 	Id string `json:"id"`
 }
 
-//  API method: /class-xxxx/rename
+// API method: /class-xxxx/rename
 //
-//  rename a data object
+// rename a data object
 func (ops *DxOps) DxRename(
 	ctx context.Context,
 	httpClient *http.Client,
@@ -404,7 +404,7 @@ type ReplyMove struct {
 	Id string `json:"id"`
 }
 
-//  API method: /class-xxxx/move
+//	API method: /class-xxxx/move
 //
 // Moves the specified data objects and folders to a destination folder in the same container.
 func (ops *DxOps) DxMove(

manifest.go

@@ -198,8 +198,9 @@ func MakeManifestFromProjectIds(
 
 // return all the parents of a directory.
 // For example:
-//   "/A/B/C"      ["/", "/A", "/A/B"]
-//   "/foo/bar"   ["/", "/foo"]
+//
+//	"/A/B/C"      ["/", "/A", "/A/B"]
+//	"/foo/bar"   ["/", "/foo"]
 func ancestors(p string) []string {
 	if p == "" || p == "/" {
 		return []string{"/"}
@@ -233,9 +234,12 @@ func (d Dirs) Less(i, j int) bool {
 
 // Figure out the directory structure needed to support
 // the leaf nodes. For example, if we need to create:
-//     ["/A/B/C", "/D", "/D/E"]
+//
+//	["/A/B/C", "/D", "/D/E"]
+//
 // then the skeleton is:
-//     ["/A", "/A/B", "/D"]
+//
+//	["/A", "/A/B", "/D"]
 //
 // The root directory is not reported in the skeleton.
 func (m *Manifest) DirSkeleton() ([]string, error) {

metadata_db.go

@@ -973,7 +973,7 @@ func (mdb *MetadataDb) directoryReadFromDNAx(
 	}
 
 	// describe all (closed) files
-	dxDir, err := DxDescribeFolder(ctx, oph.httpClient, &mdb.options, &mdb.dxEnv, projId, projFolder)
+	dxDir, err := DxDescribeFolder(ctx, oph.httpClient, &mdb.dxEnv, projId, projFolder)
 	if err != nil {
 		fmt.Printf(err.Error())
 		fmt.Printf("reading directory frmo DNAx error")

nonce.go

@@ -34,7 +34,6 @@ func NewNonce() *Nonce {
 }
 
 // Create a random nonce, no longer than 128 bytes
-//
 func (n *Nonce) String() string {
 	b := make([]byte, nonceLen)
 

posix.go

@@ -10,27 +10,26 @@ import (
 
 // Try to fix a DNAx directory, so it will adhere to POSIX.
 //
-// 1. If several files share the same name, make them unique by moving into an
-//    extra subdirectory. For example:
+//  1. If several files share the same name, make them unique by moving into an
+//     extra subdirectory. For example:
 //
-//    src name file-id      new name
-//    X.txt    file-0001    X.txt
-//    X.txt    file-0005    1/X.txt
-//    X.txt    file-0012    2/X.txt
+//     src name file-id      new name
+//     X.txt    file-0001    X.txt
+//     X.txt    file-0005    1/X.txt
+//     X.txt    file-0012    2/X.txt
 //
 // 2. DNAx files can include slashes. Drop these files, with a put note in the log.
 //
-// 3. A directory and a file can have the same name. For example:
-//    ROOT/
-//         zoo/      sub-directory
-//         zoo       regular file
-//
-//    Is converted into:
-//    ROOT
-//         zoo/      sub-directory
-//         1/        faux sub-directory
-//           zoo     regular file
+//  3. A directory and a file can have the same name. For example:
+//     ROOT/
+//     zoo/      sub-directory
+//     zoo       regular file
 //
+//     Is converted into:
+//     ROOT
+//     zoo/      sub-directory
+//     1/        faux sub-directory
+//     zoo     regular file
 type PosixDir struct {
 	path        string // entire directory path
 	dataObjects []DxDescribeDataObject
@@ -103,7 +102,6 @@ func (px *Posix) pickFauxDirNames(subdirs []string, uniqueFileNames []string, nu
 }
 
 // Find all the unique file names.
-//
 func (px *Posix) uniqueFileNames(dxObjs []DxDescribeDataObject) []string {
 	usedNames := make(map[string]bool)
 

test/local/mm.go

@@ -1,16 +1,16 @@
 package main
 
 import (
-	"golang.org/x/exp/mmap"
 	"flag"
 	"fmt"
+	"golang.org/x/exp/mmap"
 	"os"
 	"path/filepath"
 )
 
 type Config struct {
 	filename string
-	verbose bool
+	verbose  bool
 }
 
 var progName = filepath.Base(os.Args[0])
@@ -22,11 +22,10 @@ func usage() {
 }
 
 var (
-	help = flag.Bool("help", false, "display program options")
+	help    = flag.Bool("help", false, "display program options")
 	verbose = flag.Bool("verbose", false, "Enable verbose debugging")
 )
 
-
 func parseCmdLineArgs() Config {
 	if *help {
 		usage()
@@ -40,8 +39,8 @@ func parseCmdLineArgs() Config {
 	}
 
 	return Config{
-		filename : flag.Arg(0),
-		verbose : *verbose,
+		filename: flag.Arg(0),
+		verbose:  *verbose,
 	}
 }
 

util.go

@@ -30,7 +30,7 @@ const (
 	NumRetriesDefault         = 10
 	InitialUploadPartSize     = 16 * MiB
 	MaxUploadPartSize         = 700 * MiB
-	Version                   = "v1.3.0"
+	Version                   = "v1.4.0"
 )
 const (
 	InodeInvalid = 0