diff --git a/CHANGELOG.md b/CHANGELOG.md index 2611695..6180908 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ * Added option to sleep between test phases. (See "--phasedelay".) * Added option to rotate hosts list for service instances between phases to avoid caching effects. (See "--rotatehosts".) * Added support for S3 object and bucket ACL PUT/GET. (See "--s3aclput", "--s3aclputinl", "--s3baclput".) -* Added new ops log to log all IO operations (open, read, ...). (See "--opslog"). +* Added new ops log to log all IO operations (open, read, ...). (See "--opslog".) +* Added support for shared secret to protect services from unauthorized requests. (See "--svcpwfile".) * Added new ops for S3 bucket and object tagging and object locking. (See "--s3btag", "--s3otag", "--s3olockcfg"). [STILL WORK-IN-PROGRESS] ### General Changes diff --git a/Makefile b/Makefile index c1731db..03954a6 100644 --- a/Makefile +++ b/Makefile @@ -46,8 +46,8 @@ CXXFLAGS_COMMON = -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 $(CXXFLAGS_BOOS CXXFLAGS_RELEASE = -O3 -Wuninitialized CXXFLAGS_DEBUG = -O0 -D_FORTIFY_SOURCE=2 -DBUILD_DEBUG -LDFLAGS_COMMON = -rdynamic -pthread -lrt -lstdc++fs $(LDFLAGS_NUMA) $(LDFLAGS_AIO) \ - $(LDFLAGS_BOOST) +LDFLAGS_COMMON = -rdynamic -pthread -l crypto -l rt -l ssl -l stdc++fs $(LDFLAGS_NUMA) \ + $(LDFLAGS_AIO) $(LDFLAGS_BOOST) LDFLAGS_RELASE = -O3 LDFLAGS_DEBUG = -O0 @@ -76,10 +76,12 @@ endif ifeq ($(BUILD_STATIC), 1) LDFLAGS += -static # NOTE: Alpine v3.20+ requires additional "-l cares -l zstd" +# NOTE: "-l ssl -l crypto" intetionally appear again here although they are in LDFLAGS_COMMON. This +# is because the link order matters for static libs. LDFLAGS_S3_STATIC += -l curl -l ssl -l crypto -l tls -l z -l nghttp2 -l brotlidec -l brotlicommon \ -l idn2 -l unistring -l psl -l dl else # dynamic linking -LDFLAGS_S3_DYNAMIC += -l curl -l ssl -l crypto -l z -l dl +LDFLAGS_S3_DYNAMIC += -l curl -l z -l dl endif # Compiler and linker flags for S3 support diff --git a/README.md b/README.md index cf35e4d..e587520 100644 --- a/README.md +++ b/README.md @@ -51,13 +51,13 @@ Building elbencho requires a C++17 compatible compiler, such as gcc version 7.x ### Dependencies for Debian/Ubuntu ```bash -sudo apt install build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev lintian +sudo apt install build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev lintian libssl-dev ``` ### Dependencies for RHEL/CentOS ```bash -sudo yum install boost-devel gcc-c++ git libaio-devel make ncurses-devel numactl-devel rpm-build +sudo yum install boost-devel gcc-c++ git libaio-devel make ncurses-devel numactl-devel openssl-devel rpm-build ``` #### On RHEL / CentOS 7.x: Prepare Environment with newer gcc Version @@ -142,13 +142,13 @@ Enabling S3 Object Storage support will automatically download a AWS SDK git rep ##### S3 Dependencies for RHEL/CentOS 8.0 or newer ```bash -sudo yum install cmake libarchive libcurl-devel openssl-devel libuuid-devel zlib zlib-devel +sudo yum install cmake libarchive libcurl-devel libuuid-devel zlib zlib-devel ``` ##### S3 Dependencies for Ubuntu 20.04 or newer ```bash -sudo apt install cmake libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev +sudo apt install cmake libcurl4-openssl-dev uuid-dev zlib1g-dev ``` ##### Build elbencho with S3 Support diff --git a/build_helpers/docker/Dockerfile.ubuntu1804 b/build_helpers/docker/Dockerfile.ubuntu1804 index 56eeb88..85c8601 100644 --- a/build_helpers/docker/Dockerfile.ubuntu1804 +++ b/build_helpers/docker/Dockerfile.ubuntu1804 @@ -5,7 +5,7 @@ FROM ubuntu:18.04 as builder RUN export DEBIAN_FRONTEND=noninteractive && \ apt update && \ apt -y upgrade && \ - apt install -y build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev lintian && \ + apt install -y build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev libssl-dev lintian && \ cd /root && git clone https://github.com/breuner/elbencho.git && \ cd elbencho && \ make -j "$(nproc)" && \ diff --git a/build_helpers/docker/Dockerfile.ubuntu1804.local b/build_helpers/docker/Dockerfile.ubuntu1804.local index 3bc2e8e..2555191 100644 --- a/build_helpers/docker/Dockerfile.ubuntu1804.local +++ b/build_helpers/docker/Dockerfile.ubuntu1804.local @@ -10,7 +10,7 @@ COPY ./ /root/elbencho RUN export DEBIAN_FRONTEND=noninteractive && \ apt update && \ apt -y upgrade && \ - apt install -y build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev lintian && \ + apt install -y build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev libssl-dev lintian && \ cd /root/elbencho && \ make -j "$(nproc)" && \ make deb diff --git a/build_helpers/docker/Dockerfile.ubuntu2204.local b/build_helpers/docker/Dockerfile.ubuntu2204.local index 60266d3..52b7ea0 100644 --- a/build_helpers/docker/Dockerfile.ubuntu2204.local +++ b/build_helpers/docker/Dockerfile.ubuntu2204.local @@ -1,7 +1,7 @@ # Full elbencho deb install of latest github master on Ubuntu 22.04 # # Run docker build from elbencho repository root dir like this: -# docker build -t elbencho-local -f build_helpers/docker/Dockerfile.ubuntu2004.local . +# docker build -t elbencho-local -f build_helpers/docker/Dockerfile.ubuntu2204.local . FROM ubuntu:22.04 as builder diff --git a/build_helpers/docker/build_all_local.sh b/build_helpers/docker/build_all_local.sh index 9cc1d4a..5ea2139 100755 --- a/build_helpers/docker/build_all_local.sh +++ b/build_helpers/docker/build_all_local.sh @@ -3,12 +3,16 @@ # Build all Dockerfile.*.local from local repository and prune containers/images after each build. # Call this script from the repository root dir. +NUM_DOCKERFILES_TOTAL=$(ls build_helpers/docker/Dockerfile.*.local | wc -l) +current_file_idx=1 + + for dockerfile in $(ls build_helpers/docker/Dockerfile.*.local); do echo - echo "Building: $dockerfile" + echo "*** Building $(( current_file_idx++ ))/$NUM_DOCKERFILES_TOTAL: $dockerfile" echo - echo "Cleaning up build artifacts..." + echo "*** Cleaning up build artifacts..." make clean-all echo @@ -19,9 +23,9 @@ for dockerfile in $(ls build_helpers/docker/Dockerfile.*.local); do exit 1 fi - echo "Pruning docker containers and images..." + echo "*** Pruning docker containers and images..." docker container prune -f && docker image prune -fa && docker builder prune -af done echo -echo "All done." +echo "*** All done." diff --git a/dist/etc/bash_completion.d/elbencho b/dist/etc/bash_completion.d/elbencho index 06cdf0b..5034be1 100644 --- a/dist/etc/bash_completion.d/elbencho +++ b/dist/etc/bash_completion.d/elbencho @@ -146,6 +146,7 @@ _elbencho_opts() --size --start --stat + --svcpwfile --svcupint --sync --threads @@ -303,6 +304,8 @@ _elbencho() ;& --serversfile) ;& + --svcpwfile) + ;& --treefile) compopt -o filenames 2>/dev/null COMPREPLY=( $(compgen -f ${cur}) ) diff --git a/source/Common.h b/source/Common.h index e7fe3d7..36e6569 100644 --- a/source/Common.h +++ b/source/Common.h @@ -172,6 +172,7 @@ typedef std::vector BenchPathInfoVec; #define XFER_PREP_ERRORHISTORY "ErrorHistory" #define XFER_PREP_NUMBENCHPATHS "NumBenchPaths" #define XFER_PREP_FILENAME "FileName" +#define XFER_PREP_AUTHORIZATION "PwHash" #define XFER_STATS_BENCHID "BenchID" #define XFER_STATS_BENCHPHASENAME "PhaseName" diff --git a/source/HTTPServiceSWS.cpp b/source/HTTPServiceSWS.cpp index cb6a798..45dcaf2 100644 --- a/source/HTTPServiceSWS.cpp +++ b/source/HTTPServiceSWS.cpp @@ -103,7 +103,9 @@ void HTTPServiceSWS::startServer() "Port: " + std::to_string(progArgs.getServicePort() ) ); } - std::cout << "Elbencho service now listening. Port: " << serverPortFutureValue << std::endl; + std::cout << "Elbencho service now listening. " << + (progArgs.getSvcPasswordHash().empty() ? "" : "Protected by shared secret. ") << + "Port: " << serverPortFutureValue << std::endl; serverThread.join(); @@ -230,6 +232,16 @@ void HTTPServiceSWS::defineServerResources(HttpServer& server) "Service version: " HTTP_PROTOCOLVERSION "; " "Received master version: " + masterProtoVer); + // check authorization hash + + iter = query_fields.find(XFER_PREP_AUTHORIZATION); + if(iter == query_fields.end() ) + throw ProgException("Missing parameter: " XFER_PREP_AUTHORIZATION); + + std::string masterAuthHash = iter->second; + if(masterAuthHash != progArgs.getSvcPasswordHash() ) + throw ProgException("Invalid authorization code."); + // get and prepare filename iter = query_fields.find(XFER_PREP_FILENAME); @@ -304,6 +316,8 @@ void HTTPServiceSWS::defineServerResources(HttpServer& server) Logger(Log_VERBOSE) << "HTTP: " << request->path << "?" << request->query_string << std::endl; + bool resetWorkersOnError = true; + try { // check protocol version for compatibility @@ -316,9 +330,27 @@ void HTTPServiceSWS::defineServerResources(HttpServer& server) std::string masterProtoVer = iter->second; if(masterProtoVer != HTTP_PROTOCOLVERSION) + { + resetWorkersOnError = false; + throw ProgException("Protocol version mismatch. " "Service version: " HTTP_PROTOCOLVERSION "; " "Received master version: " + masterProtoVer); + } + + // check authorization hash + + iter = query_fields.find(XFER_PREP_AUTHORIZATION); + if(iter == query_fields.end() ) + throw ProgException("Missing parameter: " XFER_PREP_AUTHORIZATION); + + std::string masterAuthHash = iter->second; + if(masterAuthHash != progArgs.getSvcPasswordHash() ) + { + resetWorkersOnError = false; + + throw ProgException("Invalid authorization code."); + } // print prep phase to log @@ -375,11 +407,14 @@ void HTTPServiceSWS::defineServerResources(HttpServer& server) corresponding RemoteWorker on master terminates on prep error reply, so we need to clean up and release everything here before replying. */ - workerManager.interruptAndNotifyWorkers(); - workerManager.joinAllThreads(); - workerManager.cleanupWorkersAfterPhaseDone(); + if(resetWorkersOnError) + { + workerManager.interruptAndNotifyWorkers(); + workerManager.joinAllThreads(); + workerManager.cleanupWorkersAfterPhaseDone(); - progArgs.resetBenchPath(); + progArgs.resetBenchPath(); + } std::stringstream stream; diff --git a/source/HTTPServiceUWS.cpp b/source/HTTPServiceUWS.cpp index da07db9..8ad674f 100644 --- a/source/HTTPServiceUWS.cpp +++ b/source/HTTPServiceUWS.cpp @@ -45,7 +45,9 @@ void HTTPServiceUWS::startServer() if(listenSocket) { globalListenSocket = listenSocket; - std::cout << "Elbencho alternative service now listening. Port: " << listenPort + std::cout << "Elbencho alternative service now listening. " << + (progArgs.getSvcPasswordHash().empty() ? "" : "Protected by shared secret. ") << + "Port: " << listenPort << std::endl; } else @@ -171,6 +173,13 @@ void HTTPServiceUWS::defineServerResources(uWS::App& uWSApp) "Service version: " HTTP_PROTOCOLVERSION "; " "Received master version: " + std::string(masterProtoVer) ); + // check authorization hash + + std::string_view masterAuthHash = req->getQuery(XFER_PREP_AUTHORIZATION); + + if(masterAuthHash != progArgs.getSvcPasswordHash() ) + throw ProgException("Invalid authorization code."); + // get and prepare filename std::string_view clientFilenameVal = req->getQuery(XFER_PREP_FILENAME); @@ -286,6 +295,8 @@ void HTTPServiceUWS::defineServerResources(uWS::App& uWSApp) { logReqAndError(res, std::string(req->getUrl() ), std::string(req->getQuery() ) ); + bool resetWorkersOnError = true; + try { // check protocol version for compatibility @@ -295,9 +306,24 @@ void HTTPServiceUWS::defineServerResources(uWS::App& uWSApp) throw ProgException("Missing parameter: " XFER_PREP_PROTCOLVERSION); if(masterProtoVer != HTTP_PROTOCOLVERSION) + { + resetWorkersOnError = false; + throw ProgException(std::string("Protocol version mismatch. ") + "Service version: " HTTP_PROTOCOLVERSION "; " "Received master version: " + std::string(masterProtoVer) ); + } + + // check authorization hash + + std::string_view masterAuthHash = req->getQuery(XFER_PREP_AUTHORIZATION); + + if(masterAuthHash != progArgs.getSvcPasswordHash() ) + { + resetWorkersOnError = false; + + throw ProgException("Invalid authorization code."); + } // print prep phase to log @@ -403,11 +429,14 @@ void HTTPServiceUWS::defineServerResources(uWS::App& uWSApp) corresponding RemoteWorker on master terminates on prep error reply, so we need to clean up and release everything here before replying. */ - workerManager.interruptAndNotifyWorkers(); - workerManager.joinAllThreads(); - workerManager.cleanupWorkersAfterPhaseDone(); + if(resetWorkersOnError) + { + workerManager.interruptAndNotifyWorkers(); + workerManager.joinAllThreads(); + workerManager.cleanupWorkersAfterPhaseDone(); - progArgs.resetBenchPath(); + progArgs.resetBenchPath(); + } std::stringstream stream; diff --git a/source/ProgArgs.cpp b/source/ProgArgs.cpp index 2a03192..d09a8a2 100644 --- a/source/ProgArgs.cpp +++ b/source/ProgArgs.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -593,6 +594,9 @@ void ProgArgs::defineAllowedArgs() "(Hint: Try 'date +%s' to get seconds since the epoch.)") /*sv*/ (ARG_SHOWSVCELAPSED_LONG, bpo::bool_switch(&this->showServicesElapsed), "Show elapsed time to completion of each service instance ordered by slowest thread.") +/*sv*/ (ARG_SVCPASSWORDFILE_LONG, bpo::value(&this->svcPasswordFile), + "Path to a text file containing a single line of text as shared secret between service " + "instances and master. This is to prevent unauthorized requests to service instances.") /*sv*/ (ARG_SVCUPDATEINTERVAL_LONG, bpo::value(&this->svcUpdateIntervalMS), "Update retrieval interval for service hosts in milliseconds. (Default: 500)") /*sy*/ (ARG_SYNCPHASE_LONG, bpo::bool_switch(&this->runSyncPhase), @@ -847,6 +851,8 @@ void ProgArgs::initImplicitValues() } useS3ObjectPrefixRand = (s3ObjectPrefix.find(RAND_PREFIX_MARKS_SUBSTR) != std::string::npos); + + loadServicePasswordFile(); // sets svcPasswordHash } /** @@ -2248,6 +2254,43 @@ void ProgArgs::loadCustomTreeFile() } } +/** + * Load svcPasswordFile and set svcPasswordHash. svcPasswordHash will be left empty if no password + * file is given. + * + * @throw ProgException on error, e.g. file open failed or file empty. + */ +void ProgArgs::loadServicePasswordFile() +{ + if(svcPasswordFile.empty() ) + return; // nothing to do + + std::string lineStr; + + std::ifstream fileStream(svcPasswordFile); + if(!fileStream) + throw ProgException("Opening service password file failed: " + svcPasswordFile); + + // read first line + std::getline(fileStream, lineStr); + + if(lineStr.empty() ) + throw ProgException("First line in service password file is empty: " + svcPasswordFile); + + unsigned char messageDigestBuf[SHA_DIGEST_LENGTH]; + + SHA1( (const unsigned char*)lineStr.c_str(), lineStr.length(), messageDigestBuf); + + std::stringstream hexStream; + + hexStream << std::hex; + + for(unsigned i=0; i < SHA_DIGEST_LENGTH; i++) + hexStream << std::setw(2) << std::setfill('0') << (unsigned)messageDigestBuf[i]; + + svcPasswordHash = hexStream.str(); +} + /** * Turn given path into an absolute path. If given path was absolute before, it is returned * unmodified. Otherwise the path to the current work dir is prepended. diff --git a/source/ProgArgs.h b/source/ProgArgs.h index 139e2c5..5100467 100644 --- a/source/ProgArgs.h +++ b/source/ProgArgs.h @@ -177,6 +177,7 @@ namespace bpt = boost::property_tree; #define ARG_STARTTIME_LONG "start" #define ARG_STATFILES_LONG "stat" #define ARG_STATFILESINLINE_LONG "statinline" +#define ARG_SVCPASSWORDFILE_LONG "svcpwfile" #define ARG_SVCUPDATEINTERVAL_LONG "svcupint" #define ARG_SYNCPHASE_LONG "sync" #define ARG_TIMELIMITSECS_LONG "timelimit" @@ -448,13 +449,6 @@ class ProgArgs unsigned short servicePort; // HTTP/TCP port for service std::string serversFilePath; // path to file for preprended service hosts std::string serversStr; // prepended to hostsStr in netbench mode - size_t svcUpdateIntervalMS; // update retrieval interval for service hosts in milliseconds - int sockRecvBufSize; // custom netbench socket recv buf size (0 means no change) - int sockSendBufSize; // custom netbench socket send buf size (0 means no change) - std::string sockRecvBufSizeOrigStr; // original sockRecvBufSize str from user with unit - std::string sockSendBufSizeOrigStr; // original sockSendBufSize str from user with unit - time_t startTime; /* UTC start time to coordinate multiple benchmarks, in seconds since the - epoch. 0 means immediate start. */ bool showAllElapsed; // print elapsed time of each I/O worker thread bool showCPUUtilization; // show cpu utilization in phase stats results bool showDirStats; // show processed dirs stats in file write/read phase of dir mode @@ -462,6 +456,15 @@ class ProgArgs bool showLatencyHistogram; // show latency histogram bool showLatencyPercentiles; // show latency percentiles bool showServicesElapsed; // print elapsed time of each service by slowest thread + int sockRecvBufSize; // custom netbench socket recv buf size (0 means no change) + int sockSendBufSize; // custom netbench socket send buf size (0 means no change) + std::string sockRecvBufSizeOrigStr; // original sockRecvBufSize str from user with unit + std::string sockSendBufSizeOrigStr; // original sockSendBufSize str from user with unit + time_t startTime; /* UTC start time to coordinate multiple benchmarks, in seconds since the + epoch. 0 means immediate start. */ + std::string svcPasswordFile; // protect against unauthorized service commands + std::string svcPasswordHash; // implicitly set if svcPasswordFile is given, empty otherwise + size_t svcUpdateIntervalMS; // update retrieval interval for service hosts in milliseconds std::string treeFilePath; // path to file containing custom tree (list of dirs and files) uint64_t treeRoundUpSize; /* in treefile, round up file sizes to multiple of given size. (useful for directIO with its alignment reqs on some file systems. 0 disables this.) */ @@ -516,6 +519,7 @@ class ProgArgs void parseS3Endpoints(); void parseNetDevs(); void loadCustomTreeFile(); + void loadServicePasswordFile(); void splitCustomTreeForSharedS3Upload(); std::string absolutePath(std::string pathStr); BenchPathType findBenchPathType(std::string pathStr); @@ -667,9 +671,14 @@ class ProgArgs std::string getS3EndpointsServiceOverride() const { return s3EndpointsServiceOverrideStr; } std::string getS3EndpointsStr() const { return s3EndpointsStr; } const StringVec& getS3EndpointsVec() const { return s3EndpointsVec; } + uint64_t getS3ListObjNum() const { return runS3ListObjNum; } + unsigned short getS3LogLevel() const { return s3LogLevel; } std::string getS3LogfilePrefix() const { return s3LogfilePrefix; } + uint64_t getS3MultiDelObjNum() const { return runS3MultiDelObjNum; } const std::string& getS3ObjectPrefix() const { return s3ObjectPrefix; } std::string getS3Region() const { return s3Region; } + unsigned short getServicePort() const { return servicePort; } + unsigned short getS3SignPolicy() const { return s3SignPolicy; } bool getShowAllElapsed() const { return showAllElapsed; } bool getShowCPUUtilization() const { return showCPUUtilization; } bool getShowDirStats() const { return showDirStats; } @@ -679,13 +688,10 @@ class ProgArgs bool getShowServicesElapsed() const { return showServicesElapsed; } int getSockRecvBufSize() const { return sockRecvBufSize; } int getSockSendBufSize() const { return sockSendBufSize; } - unsigned short getServicePort() const { return servicePort; } - uint64_t getS3ListObjNum() const { return runS3ListObjNum; } - unsigned short getS3LogLevel() const { return s3LogLevel; } - uint64_t getS3MultiDelObjNum() const { return runS3MultiDelObjNum; } - unsigned short getS3SignPolicy() const { return s3SignPolicy; } time_t getStartTime() const { return startTime; } size_t getSvcUpdateIntervalMS() const { return svcUpdateIntervalMS; } + std::string getSvcPasswordFile() const { return svcPasswordFile; } + std::string getSvcPasswordHash() const { return svcPasswordHash; } bool getUseAlternativeHTTPService() const { return useAlternativeHTTPService; } bool getUseBriefLiveStats() const { return useBriefLiveStats; } bool getUseBriefLiveStatsNewLine() const { return useBriefLiveStatsNewLine; } diff --git a/source/workers/RemoteWorker.cpp b/source/workers/RemoteWorker.cpp index 55d15ed..4307512 100644 --- a/source/workers/RemoteWorker.cpp +++ b/source/workers/RemoteWorker.cpp @@ -295,8 +295,9 @@ void RemoteWorker::prepareRemoteFile() try { std::string requestPath = HTTPCLIENTPATH_PREPAREFILE "?" - XFER_PREP_PROTCOLVERSION "=" HTTP_PROTOCOLVERSION - "&" XFER_PREP_FILENAME "=" SERVICE_UPLOAD_TREEFILE; + XFER_PREP_PROTCOLVERSION "=" HTTP_PROTOCOLVERSION "&" + XFER_PREP_FILENAME "=" SERVICE_UPLOAD_TREEFILE "&" + XFER_PREP_AUTHORIZATION "=" + progArgs->getSvcPasswordHash(); auto response = httpClient.request("POST", requestPath, treeFileStream); @@ -340,7 +341,8 @@ void RemoteWorker::preparePhase() try { std::string requestPath = HTTPCLIENTPATH_PREPAREPHASE "?" - XFER_PREP_PROTCOLVERSION "=" HTTP_PROTOCOLVERSION; + XFER_PREP_PROTCOLVERSION "=" HTTP_PROTOCOLVERSION "&" + XFER_PREP_AUTHORIZATION "=" + progArgs->getSvcPasswordHash(); auto response = httpClient.request("POST", requestPath, treeStream);