diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 7c06be3..da0eb88 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -12,6 +12,7 @@ on: jobs: applet-test: name: Applet dxfuse tests + needs: local-test runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 522289b..bcb7c4a 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ as DNAnexus files left in open state are eventually removed by the DNAnexus clea ## Upload benchmarks -Upload benchmarks are from an AWS m5n.xlarge instance running Ubuntu 18.04 with kernel 5.4.0-1048-aws. +Upload benchmarks are from an Ubuntu 20.04 DNAnexus worker mem2_ssd1_v2_x32 (AWS m5d.8xlarge) instance running kernel 5.4.0-1055-aws. `dx` and `dxfuse` benchmark commands were run like so. These benchmarks are not exact because they include the wait time until the uploaded file is transitioned to the `closed` state. `time dd if=/dev/zero bs=1M count=$SIZE | dx upload --wait -` @@ -165,13 +165,14 @@ Upload benchmarks are from an AWS m5n.xlarge instance running Ubuntu 18.04 with `time dd if=/dev/zero bs=1M count=$SIZE of=MNT/project/$SIZE` | dx upload --wait (seconds) | dxfuse upload(seconds) | file size | | --- | ---- | ---- | -| 4.4 | 5.6| 100M | -| 9 | 6.1 | 200M | -| 8.8 | 6.7 | 400M | -| 8.8 | 12.8 | 800M | -| 18 | 19 | 1600M | -| 32 | 24 | 3200M | -| 79 | 65 | 10000M | +| 4.4 | 3.5 | 100MiB | +| 4.8 | 4.2 | 200MiB | +| 5.9 | 4.8 | 400MiB | +| 5.9 | 6.8 | 800MiB | +| 7 | 10 | 1GiB | +| 22.5 | 19.2 | 2GiB | +| 37.8 | 87 | 10GiB | +| 254 | 495 | 100GiB | # Building diff --git a/dxfuse.go b/dxfuse.go index 2ef97ac..f89c1e2 100644 --- a/dxfuse.go +++ b/dxfuse.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -137,6 +138,11 @@ func NewDxfuse( options Options) (*Filesys, error) { // initialize a pool of http-clients. + HttpClientPoolSize := MinHttpClientPoolSize + if runtime.NumCPU()*3 > HttpClientPoolSize { + HttpClientPoolSize = runtime.NumCPU() * 3 + } + httpIoPool := make(chan *http.Client, HttpClientPoolSize) for i := 0; i < HttpClientPoolSize; i++ { httpIoPool <- dxda.NewHttpClient() @@ -156,6 +162,9 @@ func NewDxfuse( tmpFileCounter: 0, shutdownCalled: false, } + if options.Verbose { + fsys.log("Http client pool size: %d", HttpClientPoolSize) + } // Create a fresh SQL database databaseFile := dxfuseBaseDir + "/" + DatabaseFile @@ -207,7 +216,6 @@ func NewDxfuse( } fsys.uploader = NewFileUploader(options.VerboseLevel, options, dxEnv) - // initialize sync daemon //fsys.sybx = NewSyncDbDx(options, dxEnv, projId2Desc, mdb, fsys.mutex) @@ -595,6 +603,7 @@ func (fsys *Filesys) MkDir(ctx context.Context, op *fuseops.MkDirOp) error { oph.RecordError(err) return fsys.translateError(err) } + fsys.log("Mkdir %s:%s", parentDir.ProjId, folderFullPath) // Add the directory to the database now := time.Now() diff --git a/test/correctness/code.sh b/test/correctness/code.sh index 8e7b7b9..09a5fdb 100644 --- a/test/correctness/code.sh +++ b/test/correctness/code.sh @@ -35,7 +35,7 @@ function teardown { if [[ $DX_JOB_ID != "" && $verbose != "" ]]; then mkdir -p out/filesystem_log - cp /root/.dxfuse/dxfuse.log out/filesystem_log/ + cp /root/.dxfuse/dxfuse.log out/filesystem_log/dxfuse_correctness.log dx-upload-all-outputs fi } @@ -547,7 +547,6 @@ main() { fi set -x $dxfuse $flags $mountpoint dxfuse_test_data dxfuse_test_read_only ArchivedStuff - set +x echo "can write to a small file" check_file_write_content $mountpoint/$projName $target_dir diff --git a/test/local/fs_test_cases.sh b/test/local/fs_test_cases.sh index 9401944..98e0198 100644 --- a/test/local/fs_test_cases.sh +++ b/test/local/fs_test_cases.sh @@ -26,6 +26,8 @@ function teardown { cd $HOME fusermount -u $mountpoint + cat /root/.dxfuse/dxfuse.log + for d in ${writeable_dirs[@]}; do dx rm -r $projName:/$d >& /dev/null || true done diff --git a/test/local/local.sh b/test/local/local.sh index c0d94e6..9201068 100755 --- a/test/local/local.sh +++ b/test/local/local.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash -ex CRNT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "current dir=$CRNT_DIR" diff --git a/test/local/xattr_test.sh b/test/local/xattr_test.sh index e894ea9..16c624b 100755 --- a/test/local/xattr_test.sh +++ b/test/local/xattr_test.sh @@ -20,6 +20,7 @@ function teardown { rm -f cmd_results.txt echo "unmounting dxfuse" + cat /root/.dxfuse/dxfuse.log cd $HOME fusermount -u $mountpoint @@ -195,7 +196,7 @@ function xattr_test { dx mkdir $projName:/$base_dir setup $base_dir - + set -x # Start the dxfuse daemon in the background, and wait for it to initilize. echo "Mounting dxfuse" flags="-limitedWrite" diff --git a/util.go b/util.go index a87a29f..5c91fda 100644 --- a/util.go +++ b/util.go @@ -23,7 +23,7 @@ const ( LogFile = "dxfuse.log" ) const ( - HttpClientPoolSize = 25 + MinHttpClientPoolSize = 8 FileWriteInactivityThresh = 5 * time.Minute MaxDirSize = 255 * 1000 MaxNumFileHandles = 1000 * 1000