s3: new option to set AWS SDK log file prefix

CHANGELOG.md

@@ -5,9 +5,10 @@
 ### New Features & Enancements
 * Added new multi-arch (ARM64 & x86_64) docker container with CUDA/GDS support.
 * If an S3 object prefix contains a sequence of three or more '%' chars, this sequence will now get replaced by a random hex string of the same length.
+* New option to specify a custom path and prefix for AWS S3 SDK log file: "--s3logprefix".
 
 ### Contributors
-* Thanks to Phil Canman, Erez Binia and Ohad Shamir for helpful comments and suggestions.
+* Thanks to Phil Canman, Erez Binia, Ohad Shamir and Erez Horev for helpful comments and suggestions.
 
 ## v3.0.5 (Jan 05, 2024)
 

source/ProgArgs.cpp

@@ -46,6 +46,9 @@
 #define CPUCORES_ALL_ARG			"all" // shortcut for user to select all cpu cores
 #define GPUIDS_ALL_ARG				"all" // shortcut for user to select all gpus
 
+#define AWS_SDK_LOGPREFIX_DEFAULT	"aws_sdk_"
+
+
 /**
  * Constructor.
  *
@@ -460,8 +463,12 @@ void ProgArgs::defineAllowedArgs()
 			"\"--" ARG_S3LISTOBJPARALLEL_LONG "\". This requires the dataset to be created with "
 			"the same values for \"-" ARG_NUMDIRS_SHORT "\" and \"-" ARG_NUMFILES_SHORT "\".")
 /*s3l*/	(ARG_S3LOGLEVEL_LONG, bpo::value(&this->s3LogLevel),
-			"Log level of AWS S3 SDK. This will create a log file named \"aws_sdk_DATE.log\" in "
-			"the current working directory. (Default: 0=disabled; Max: 6)")
+			"Log level of AWS S3 SDK. See \"--" ARG_S3LOGFILEPREFIX_LONG "\" for filename. "
+			"(Default: 0=disabled; Max: 6)")
+/*s3l*/	(ARG_S3LOGFILEPREFIX_LONG, bpo::value(&this->s3LogfilePrefix),
+			"Path and filename prefix of AWS S3 SDK log file. \"DATE.log\" will get appended to "
+			"the given filename. "
+			"(Default: \"" AWS_SDK_LOGPREFIX_DEFAULT "\" in current working directory)")
 /*s3m*/	(ARG_S3MULTIDELETE_LONG, bpo::value(&this->runS3MultiDelObjNum),
 			"Delete multiple objects in a single DeleteObjects request. This loops on retrieving "
 			"a chunk of objects from a listing request and then deleting the retrieved set of "
@@ -651,6 +658,7 @@ void ProgArgs::defineDefaults()
 	this->useS3FastRead = false;
 	this->useS3TransferManager = false;
 	this->s3LogLevel = 0;
+	this->s3LogfilePrefix = AWS_SDK_LOGPREFIX_DEFAULT;
 	this->noDirectIOCheck = false;
 	this->runS3ListObjNum = 0;
 	this->runS3ListObjParallel = false;

source/ProgArgs.h

@@ -124,6 +124,7 @@ namespace bpt = boost::property_tree;
 #define ARG_S3FASTGET_LONG			"s3fastget"
 #define ARG_S3TRANSMAN_LONG			"s3transman"
 #define ARG_S3LOGLEVEL_LONG			"s3log"
+#define ARG_S3LOGFILEPREFIX_LONG	"s3logprefix"
 #define ARG_NODIRECTIOCHECK_LONG	"nodiocheck"
 #define ARG_S3OBJECTPREFIX_LONG		"s3objprefix"
 #define ARG_DRYRUN_LONG				"dryrun"
@@ -374,6 +375,7 @@ class ProgArgs
 									via buffer possible, such as GPU copy or data verification) */
 		bool useS3TransferManager; // use AWS SDK TransferManager for object downloads
 		unsigned short s3LogLevel; // log level for AWS SDK
+		std::string s3LogfilePrefix; // dir and name prefix of aws sdk log file
 		bool noDirectIOCheck; // ignore directIO alignment and sanity checks
 		std::string s3ObjectPrefix; // object name/path prefix for s3 "directory mode"
 		bool useS3ObjectPrefixRand; // implicit based on RAND_PREFIX_MARKS_SUBSTR in s3ObjectPrefix
@@ -563,6 +565,7 @@ class ProgArgs
 		bool getUseS3FastRead() const { return useS3FastRead; }
 		bool getUseS3TransferManager() const { return useS3TransferManager; }
 		unsigned short getS3LogLevel() const { return s3LogLevel; }
+		std::string getS3LogfilePrefix() const { return s3LogfilePrefix; }
 		bool getNoDirectIOCheck() const { return noDirectIOCheck; }
 		const std::string& getS3ObjectPrefix() const { return s3ObjectPrefix; }
 		bool getUseS3ObjectPrefixRand() const { return useS3ObjectPrefixRand; }

source/toolkits/S3Tk.cpp

@@ -36,7 +36,8 @@ void S3Tk::initS3Global(const ProgArgs& progArgs)
 	if(progArgs.getS3LogLevel() > 0)
 		Aws::Utils::Logging::InitializeAWSLogging(
 			Aws::MakeShared<Aws::Utils::Logging::DefaultLogSystem>("DebugLogging",
-				(Aws::Utils::Logging::LogLevel)progArgs.getS3LogLevel(), "aws_sdk_"));
+				(Aws::Utils::Logging::LogLevel)progArgs.getS3LogLevel(),
+				progArgs.getS3LogfilePrefix() ) );
 
 	s3SDKOptions = new Aws::SDKOptions;
 	Aws::InitAPI(*s3SDKOptions);