WIP

.github/workflows/master.yml

@@ -12,6 +12,7 @@ on:
 jobs:
   applet-test:
     name: Applet dxfuse tests
+    needs: local-test
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2

README.md

@@ -157,21 +157,22 @@ as DNAnexus files left in open state are eventually removed by the DNAnexus clea
 
 ## Upload benchmarks
 
-Upload benchmarks are from an AWS m5n.xlarge instance running Ubuntu 18.04 with kernel 5.4.0-1048-aws.
+Upload benchmarks are from an Ubuntu 20.04 DNAnexus worker mem2_ssd1_v2_x32 (AWS m5d.8xlarge) instance running kernel 5.4.0-1055-aws.
 `dx` and `dxfuse` benchmark commands were run like so. These benchmarks are not exact because they include the wait time until the uploaded file is transitioned to the `closed` state.
 
 `time dd if=/dev/zero bs=1M count=$SIZE | dx upload --wait -`
 
 `time dd if=/dev/zero bs=1M count=$SIZE of=MNT/project/$SIZE`
 | dx upload --wait (seconds) | dxfuse upload(seconds) | file size |
 | ---                        |  ----                  | ----      |
-|	4.4 |	5.6| 100M |
-|	9  |	6.1 | 200M |
-|	8.8 |	6.7 | 400M |
-|	8.8 |	12.8 | 800M |
-|	18 |	19 | 1600M |
-|	32 |	24 | 3200M |
-|	79  |	65 | 10000M |
+|	4.4 |	3.5 | 100MiB |
+|	4.8  |	4.2 | 200MiB |
+|	5.9 |	4.8 | 400MiB |
+|	5.9 |	6.8 | 800MiB |
+|	7 |	10 | 1GiB |
+|	22.5 |	19.2 | 2GiB |
+|	37.8  |	87 | 10GiB |
+|	254  |	495 | 100GiB |
 
 
 # Building

dxfuse.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 	"sync"
@@ -137,6 +138,11 @@ func NewDxfuse(
 	options Options) (*Filesys, error) {
 
 	// initialize a pool of http-clients.
+	HttpClientPoolSize := MinHttpClientPoolSize
+	if runtime.NumCPU()*3 > HttpClientPoolSize {
+		HttpClientPoolSize = runtime.NumCPU() * 3
+	}
+
 	httpIoPool := make(chan *http.Client, HttpClientPoolSize)
 	for i := 0; i < HttpClientPoolSize; i++ {
 		httpIoPool <- dxda.NewHttpClient()
@@ -156,6 +162,9 @@ func NewDxfuse(
 		tmpFileCounter: 0,
 		shutdownCalled: false,
 	}
+	if options.Verbose {
+		fsys.log("Http client pool size: %d", HttpClientPoolSize)
+	}
 
 	// Create a fresh SQL database
 	databaseFile := dxfuseBaseDir + "/" + DatabaseFile
@@ -207,7 +216,6 @@ func NewDxfuse(
 	}
 
 	fsys.uploader = NewFileUploader(options.VerboseLevel, options, dxEnv)
-
 	// initialize sync daemon
 	//fsys.sybx = NewSyncDbDx(options, dxEnv, projId2Desc, mdb, fsys.mutex)
 
@@ -595,6 +603,7 @@ func (fsys *Filesys) MkDir(ctx context.Context, op *fuseops.MkDirOp) error {
 		oph.RecordError(err)
 		return fsys.translateError(err)
 	}
+	fsys.log("Mkdir %s:%s", parentDir.ProjId, folderFullPath)
 
 	// Add the directory to the database
 	now := time.Now()

test/correctness/code.sh

@@ -35,7 +35,7 @@ function teardown {
 
     if [[ $DX_JOB_ID != "" && $verbose != "" ]]; then
         mkdir -p out/filesystem_log
-        cp /root/.dxfuse/dxfuse.log out/filesystem_log/
+        cp /root/.dxfuse/dxfuse.log out/filesystem_log/dxfuse_correctness.log
         dx-upload-all-outputs
     fi
 }
@@ -547,7 +547,6 @@ main() {
     fi
     set -x
     $dxfuse $flags $mountpoint dxfuse_test_data dxfuse_test_read_only ArchivedStuff
-    set +x
 
     echo "can write to a small file"
     check_file_write_content $mountpoint/$projName $target_dir

test/local/fs_test_cases.sh

@@ -26,6 +26,8 @@ function teardown {
     cd $HOME
     fusermount -u $mountpoint
 
+    cat /root/.dxfuse/dxfuse.log
+
     for d in ${writeable_dirs[@]}; do
         dx rm -r $projName:/$d >& /dev/null || true
     done

test/local/local.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 
 CRNT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 echo "current dir=$CRNT_DIR"

test/local/xattr_test.sh

@@ -20,6 +20,7 @@ function teardown {
     rm -f cmd_results.txt
 
     echo "unmounting dxfuse"
+    cat /root/.dxfuse/dxfuse.log
     cd $HOME
     fusermount -u $mountpoint
 
@@ -195,7 +196,7 @@ function xattr_test {
     dx mkdir $projName:/$base_dir
 
     setup $base_dir
-
+    set -x
     # Start the dxfuse daemon in the background, and wait for it to initilize.
     echo "Mounting dxfuse"
     flags="-limitedWrite"

util.go

@@ -23,7 +23,7 @@ const (
 	LogFile      = "dxfuse.log"
 )
 const (
-	HttpClientPoolSize        = 25
+	MinHttpClientPoolSize     = 8
 	FileWriteInactivityThresh = 5 * time.Minute
 	MaxDirSize                = 255 * 1000
 	MaxNumFileHandles         = 1000 * 1000