service: new option to protect services based on shared secret

CHANGELOG.md

@@ -7,7 +7,8 @@
 * Added option to sleep between test phases. (See "--phasedelay".)
 * Added option to rotate hosts list for service instances between phases to avoid caching effects. (See "--rotatehosts".)
 * Added support for S3 object and bucket ACL PUT/GET. (See "--s3aclput", "--s3aclputinl", "--s3baclput".)
-* Added new ops log to log all IO operations (open, read, ...). (See "--opslog").
+* Added new ops log to log all IO operations (open, read, ...). (See "--opslog".)
+* Added support for shared secret to protect services from unauthorized requests. (See "--svcpwfile".)
 * Added new ops for S3 bucket and object tagging and object locking. (See "--s3btag", "--s3otag", "--s3olockcfg"). [STILL WORK-IN-PROGRESS]
 
 ### General Changes

Makefile

@@ -46,8 +46,8 @@ CXXFLAGS_COMMON   = -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 $(CXXFLAGS_BOOS
 CXXFLAGS_RELEASE  = -O3 -Wuninitialized
 CXXFLAGS_DEBUG    = -O0 -D_FORTIFY_SOURCE=2 -DBUILD_DEBUG
 
-LDFLAGS_COMMON    = -rdynamic -pthread -lrt -lstdc++fs $(LDFLAGS_NUMA) $(LDFLAGS_AIO) \
-	$(LDFLAGS_BOOST)
+LDFLAGS_COMMON    = -rdynamic -pthread -l crypto -l rt -l ssl -l stdc++fs $(LDFLAGS_NUMA) \
+	$(LDFLAGS_AIO) $(LDFLAGS_BOOST)
 LDFLAGS_RELASE    = -O3
 LDFLAGS_DEBUG     = -O0
 
@@ -76,10 +76,12 @@ endif
 ifeq ($(BUILD_STATIC), 1)
 LDFLAGS            += -static
 # NOTE: Alpine v3.20+ requires additional "-l cares -l zstd"
+# NOTE: "-l ssl -l crypto" intetionally appear again here although they are in LDFLAGS_COMMON. This
+#       is because the link order matters for static libs.
 LDFLAGS_S3_STATIC  += -l curl -l ssl -l crypto -l tls -l z -l nghttp2 -l brotlidec -l brotlicommon \
 	-l idn2 -l unistring -l psl -l dl
 else # dynamic linking
-LDFLAGS_S3_DYNAMIC += -l curl -l ssl -l crypto -l z -l dl
+LDFLAGS_S3_DYNAMIC += -l curl -l z -l dl
 endif
 
 # Compiler and linker flags for S3 support

README.md

@@ -51,13 +51,13 @@ Building elbencho requires a C++17 compatible compiler, such as gcc version 7.x
 ### Dependencies for Debian/Ubuntu
 
 ```bash
-sudo apt install build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev lintian
+sudo apt install build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev lintian libssl-dev
 ```
 
 ### Dependencies for RHEL/CentOS 
 
 ```bash
-sudo yum install boost-devel gcc-c++ git libaio-devel make ncurses-devel numactl-devel rpm-build
+sudo yum install boost-devel gcc-c++ git libaio-devel make ncurses-devel numactl-devel openssl-devel rpm-build
 ```
 
 #### On RHEL / CentOS 7.x: Prepare Environment with newer gcc Version
@@ -142,13 +142,13 @@ Enabling S3 Object Storage support will automatically download a AWS SDK git rep
 ##### S3 Dependencies for RHEL/CentOS 8.0 or newer
 
 ```bash
-sudo yum install cmake libarchive libcurl-devel openssl-devel libuuid-devel zlib zlib-devel
+sudo yum install cmake libarchive libcurl-devel libuuid-devel zlib zlib-devel
 ```
 
 ##### S3 Dependencies for Ubuntu 20.04 or newer
 
 ```bash
-sudo apt install cmake libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev
+sudo apt install cmake libcurl4-openssl-dev uuid-dev zlib1g-dev
 ```
 
 ##### Build elbencho with S3 Support

build_helpers/docker/Dockerfile.ubuntu1804

@@ -5,7 +5,7 @@ FROM ubuntu:18.04 as builder
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt update && \
     apt -y upgrade && \
-    apt install -y build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev lintian && \
+    apt install -y build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev libssl-dev lintian && \
     cd /root && git clone https://github.com/breuner/elbencho.git && \
     cd elbencho && \
     make -j "$(nproc)" && \

build_helpers/docker/Dockerfile.ubuntu1804.local

@@ -10,7 +10,7 @@ COPY ./ /root/elbencho
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt update && \
     apt -y upgrade && \
-    apt install -y build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev lintian && \
+    apt install -y build-essential debhelper devscripts fakeroot git libaio-dev libboost-filesystem-dev libboost-program-options-dev libboost-thread-dev libncurses-dev libnuma-dev libssl-dev lintian && \
     cd /root/elbencho && \
     make -j "$(nproc)" && \
     make deb

build_helpers/docker/Dockerfile.ubuntu2204.local

@@ -1,7 +1,7 @@
 # Full elbencho deb install of latest github master on Ubuntu 22.04
 #
 # Run docker build from elbencho repository root dir like this:
-# docker build -t elbencho-local -f build_helpers/docker/Dockerfile.ubuntu2004.local .
+# docker build -t elbencho-local -f build_helpers/docker/Dockerfile.ubuntu2204.local .
 
 FROM ubuntu:22.04 as builder
 

build_helpers/docker/build_all_local.sh

@@ -3,12 +3,16 @@
 # Build all Dockerfile.*.local from local repository and prune containers/images after each build.
 # Call this script from the repository root dir.
 
+NUM_DOCKERFILES_TOTAL=$(ls build_helpers/docker/Dockerfile.*.local | wc -l)
+current_file_idx=1
+
+
 for dockerfile in $(ls build_helpers/docker/Dockerfile.*.local); do
     echo
-    echo "Building: $dockerfile"
+    echo "*** Building $(( current_file_idx++ ))/$NUM_DOCKERFILES_TOTAL: $dockerfile"
     echo
 
-    echo "Cleaning up build artifacts..."
+    echo "*** Cleaning up build artifacts..."
     make clean-all
 
     echo
@@ -19,9 +23,9 @@ for dockerfile in $(ls build_helpers/docker/Dockerfile.*.local); do
         exit 1
     fi
 
-    echo "Pruning docker containers and images..."
+    echo "*** Pruning docker containers and images..."
     docker container prune -f && docker image prune -fa && docker builder prune -af
 done
 
 echo
-echo "All done."
+echo "*** All done."

dist/etc/bash_completion.d/elbencho

@@ -146,6 +146,7 @@ _elbencho_opts()
         --size
         --start
         --stat
+        --svcpwfile
         --svcupint
         --sync
         --threads
@@ -303,6 +304,8 @@ _elbencho()
         ;&
         --serversfile)
         ;&
+        --svcpwfile)
+        ;&
         --treefile)
             compopt -o filenames 2>/dev/null
             COMPREPLY=( $(compgen -f ${cur}) )

source/Common.h

@@ -172,6 +172,7 @@ typedef std::vector<BenchPathInfo> BenchPathInfoVec;
 #define XFER_PREP_ERRORHISTORY					"ErrorHistory"
 #define XFER_PREP_NUMBENCHPATHS					"NumBenchPaths"
 #define XFER_PREP_FILENAME						"FileName"
+#define XFER_PREP_AUTHORIZATION                 "PwHash"
 
 #define XFER_STATS_BENCHID 						"BenchID"
 #define XFER_STATS_BENCHPHASENAME 				"PhaseName"

source/HTTPServiceSWS.cpp

@@ -103,7 +103,9 @@ void HTTPServiceSWS::startServer()
 			"Port: " + std::to_string(progArgs.getServicePort() ) );
 	}
 
-	std::cout << "Elbencho service now listening. Port: " << serverPortFutureValue << std::endl;
+	std::cout << "Elbencho service now listening. " <<
+	    (progArgs.getSvcPasswordHash().empty() ? "" : "Protected by shared secret. ") <<
+	    "Port: " << serverPortFutureValue << std::endl;
 
 	serverThread.join();
 
@@ -230,6 +232,16 @@ void HTTPServiceSWS::defineServerResources(HttpServer& server)
 					"Service version: " HTTP_PROTOCOLVERSION "; "
 					"Received master version: " + masterProtoVer);
 
+            // check authorization hash
+
+            iter = query_fields.find(XFER_PREP_AUTHORIZATION);
+            if(iter == query_fields.end() )
+                throw ProgException("Missing parameter: " XFER_PREP_AUTHORIZATION);
+
+            std::string masterAuthHash = iter->second;
+            if(masterAuthHash != progArgs.getSvcPasswordHash() )
+                throw ProgException("Invalid authorization code.");
+
 			// get and prepare filename
 
 			iter = query_fields.find(XFER_PREP_FILENAME);
@@ -304,6 +316,8 @@ void HTTPServiceSWS::defineServerResources(HttpServer& server)
 		Logger(Log_VERBOSE) << "HTTP: " << request->path << "?" <<
 			request->query_string << std::endl;
 
+		bool resetWorkersOnError = true;
+
 		try
 		{
 			// check protocol version for compatibility
@@ -316,9 +330,27 @@ void HTTPServiceSWS::defineServerResources(HttpServer& server)
 
 			std::string masterProtoVer = iter->second;
 			if(masterProtoVer != HTTP_PROTOCOLVERSION)
+			{
+			    resetWorkersOnError = false;
+
 				throw ProgException("Protocol version mismatch. "
 					"Service version: " HTTP_PROTOCOLVERSION "; "
 					"Received master version: " + masterProtoVer);
+			}
+
+			// check authorization hash
+
+            iter = query_fields.find(XFER_PREP_AUTHORIZATION);
+            if(iter == query_fields.end() )
+                throw ProgException("Missing parameter: " XFER_PREP_AUTHORIZATION);
+
+            std::string masterAuthHash = iter->second;
+            if(masterAuthHash != progArgs.getSvcPasswordHash() )
+            {
+                resetWorkersOnError = false;
+
+                throw ProgException("Invalid authorization code.");
+            }
 
 			// print prep phase to log
 
@@ -375,11 +407,14 @@ void HTTPServiceSWS::defineServerResources(HttpServer& server)
 				corresponding RemoteWorker on master terminates on prep error reply, so we need to
 				clean up and release everything here before replying. */
 
-			workerManager.interruptAndNotifyWorkers();
-			workerManager.joinAllThreads();
-			workerManager.cleanupWorkersAfterPhaseDone();
+		    if(resetWorkersOnError)
+		    {
+                workerManager.interruptAndNotifyWorkers();
+                workerManager.joinAllThreads();
+                workerManager.cleanupWorkersAfterPhaseDone();
 
-			progArgs.resetBenchPath();
+                progArgs.resetBenchPath();
+		    }
 
 			std::stringstream stream;
 

source/HTTPServiceUWS.cpp

@@ -45,7 +45,9 @@ void HTTPServiceUWS::startServer()
 			if(listenSocket)
 			{
 				globalListenSocket = listenSocket;
-				std::cout << "Elbencho alternative service now listening. Port: " << listenPort
+				std::cout << "Elbencho alternative service now listening. " <<
+			        (progArgs.getSvcPasswordHash().empty() ? "" : "Protected by shared secret. ") <<
+				    "Port: " << listenPort
 					<< std::endl;
 			}
 			else
@@ -171,6 +173,13 @@ void HTTPServiceUWS::defineServerResources(uWS::App& uWSApp)
 					"Service version: " HTTP_PROTOCOLVERSION "; "
 					"Received master version: " + std::string(masterProtoVer) );
 
+            // check authorization hash
+
+            std::string_view masterAuthHash = req->getQuery(XFER_PREP_AUTHORIZATION);
+
+            if(masterAuthHash != progArgs.getSvcPasswordHash() )
+                throw ProgException("Invalid authorization code.");
+
 			// get and prepare filename
 
 			std::string_view clientFilenameVal = req->getQuery(XFER_PREP_FILENAME);
@@ -286,6 +295,8 @@ void HTTPServiceUWS::defineServerResources(uWS::App& uWSApp)
 	{
 		logReqAndError(res, std::string(req->getUrl() ), std::string(req->getQuery() ) );
 
+		bool resetWorkersOnError = true;
+
 		try
 		{
 			// check protocol version for compatibility
@@ -295,9 +306,24 @@ void HTTPServiceUWS::defineServerResources(uWS::App& uWSApp)
 				throw ProgException("Missing parameter: " XFER_PREP_PROTCOLVERSION);
 
 			if(masterProtoVer != HTTP_PROTOCOLVERSION)
+            {
+                resetWorkersOnError = false;
+
 				throw ProgException(std::string("Protocol version mismatch. ") +
 					"Service version: " HTTP_PROTOCOLVERSION "; "
 					"Received master version: " + std::string(masterProtoVer) );
+            }
+
+			// check authorization hash
+
+			std::string_view masterAuthHash = req->getQuery(XFER_PREP_AUTHORIZATION);
+
+            if(masterAuthHash != progArgs.getSvcPasswordHash() )
+            {
+                resetWorkersOnError = false;
+
+                throw ProgException("Invalid authorization code.");
+            }
 
 			// print prep phase to log
 
@@ -403,11 +429,14 @@ void HTTPServiceUWS::defineServerResources(uWS::App& uWSApp)
 				corresponding RemoteWorker on master terminates on prep error reply, so we need to
 				clean up and release everything here before replying. */
 
-			workerManager.interruptAndNotifyWorkers();
-			workerManager.joinAllThreads();
-			workerManager.cleanupWorkersAfterPhaseDone();
+		    if(resetWorkersOnError)
+		    {
+                workerManager.interruptAndNotifyWorkers();
+                workerManager.joinAllThreads();
+                workerManager.cleanupWorkersAfterPhaseDone();
 
-			progArgs.resetBenchPath();
+                progArgs.resetBenchPath();
+		    }
 
 			std::stringstream stream;
 

source/ProgArgs.cpp

@@ -4,6 +4,7 @@
 #include <iostream>
 #include <iterator>
 #include <libgen.h>
+#include <openssl/sha.h>
 #include <string>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
@@ -593,6 +594,9 @@ void ProgArgs::defineAllowedArgs()
 			"(Hint: Try 'date +%s' to get seconds since the epoch.)")
 /*sv*/	(ARG_SHOWSVCELAPSED_LONG, bpo::bool_switch(&this->showServicesElapsed),
 			"Show elapsed time to completion of each service instance ordered by slowest thread.")
+/*sv*/  (ARG_SVCPASSWORDFILE_LONG, bpo::value(&this->svcPasswordFile),
+            "Path to a text file containing a single line of text as shared secret between service "
+            "instances and master. This is to prevent unauthorized requests to service instances.")
 /*sv*/	(ARG_SVCUPDATEINTERVAL_LONG, bpo::value(&this->svcUpdateIntervalMS),
 			"Update retrieval interval for service hosts in milliseconds. (Default: 500)")
 /*sy*/	(ARG_SYNCPHASE_LONG, bpo::bool_switch(&this->runSyncPhase),
@@ -847,6 +851,8 @@ void ProgArgs::initImplicitValues()
 	}
 
 	useS3ObjectPrefixRand = (s3ObjectPrefix.find(RAND_PREFIX_MARKS_SUBSTR) != std::string::npos);
+
+	loadServicePasswordFile(); // sets svcPasswordHash
 }
 
 /**
@@ -2248,6 +2254,43 @@ void ProgArgs::loadCustomTreeFile()
 	}
 }
 
+/**
+ * Load svcPasswordFile and set svcPasswordHash. svcPasswordHash will be left empty if no password
+ * file is given.
+ *
+ * @throw ProgException on error, e.g. file open failed or file empty.
+ */
+void ProgArgs::loadServicePasswordFile()
+{
+    if(svcPasswordFile.empty() )
+        return; // nothing to do
+
+    std::string lineStr;
+
+    std::ifstream fileStream(svcPasswordFile);
+    if(!fileStream)
+        throw ProgException("Opening service password file failed: " + svcPasswordFile);
+
+    // read first line
+    std::getline(fileStream, lineStr);
+
+    if(lineStr.empty() )
+        throw ProgException("First line in service password file is empty: " + svcPasswordFile);
+
+    unsigned char messageDigestBuf[SHA_DIGEST_LENGTH];
+
+    SHA1( (const unsigned char*)lineStr.c_str(), lineStr.length(), messageDigestBuf);
+
+    std::stringstream hexStream;
+
+    hexStream << std::hex;
+
+    for(unsigned i=0; i < SHA_DIGEST_LENGTH; i++)
+        hexStream << std::setw(2) << std::setfill('0') << (unsigned)messageDigestBuf[i];
+
+    svcPasswordHash = hexStream.str();
+}
+
 /**
  * Turn given path into an absolute path. If given path was absolute before, it is returned
  * unmodified. Otherwise the path to the current work dir is prepended.