diff --git a/CHANGELOG.md b/CHANGELOG.md index 6180908..65e78fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog of elbencho -## v3.0.8 (work in progress) +## v3.0.9 (June 04, 2024) ### New Features & Enhancements * Added options to read and stat files immediately after creation while they are still open. (See "--readinline" and "--statinline".) @@ -17,7 +17,7 @@ * Added new script in `build_helpers/chroot` to build static executable via Alpine chroot. ### Contributors -Thanks to Casey Peel, Michael Shustin, Erez Horev and Github user russki for code contributions. Thanks to Andy Black and Github user mhanafi1970 for helpful comments and suggestions. +Thanks to Casey Peel, Michael Shustin, Erez Horev and Github user russki for code contributions. Thanks to Andy Black, Jean-Baptiste Denis and Github user mhanafi1970 for helpful comments and suggestions. ## v3.0.7 (March 21, 2024) diff --git a/Makefile b/Makefile index 03954a6..5f0b86d 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ EXE_NAME ?= elbencho EXE_VER_MAJOR ?= 3 EXE_VER_MINOR ?= 0 -EXE_VER_PATCHLEVEL ?= 8 +EXE_VER_PATCHLEVEL ?= 9 EXE_VERSION ?= $(EXE_VER_MAJOR).$(EXE_VER_MINOR)-$(EXE_VER_PATCHLEVEL) EXE ?= $(BIN_PATH)/$(EXE_NAME) EXE_UNSTRIPPED ?= $(EXE)-unstripped diff --git a/source/Common.h b/source/Common.h index 36e6569..3ddd870 100644 --- a/source/Common.h +++ b/source/Common.h @@ -43,13 +43,13 @@ typedef std::vector UInt64Vec; #define PHASENAME_PUTBUCKETACL "PUTBACL" #define PHASENAME_GETBUCKETACL "GETBACL" -// S3-only phases -#define PHASENAME_GET_S3_OBJECT_METADATA "GET_S3_OBJECT_META" -#define PHASENAME_PUT_S3_OBJECT_METADATA "PUT_S3_OBJECT_META" -#define PHASENAME_DEL_S3_OBJECT_METADATA "DEL_S3_OBJECT_META" -#define PHASENAME_GET_S3_BUCKET_METADATA "GET_S3_BUCKET_META" -#define PHASENAME_PUT_S3_BUCKET_METADATA "PUT_S3_BUCKET_META" -#define PHASENAME_DEL_S3_BUCKET_METADATA "DEL_S3_BUCKET_META" +// special S3 metadata phases for multiple metadata operation types +#define PHASENAME_GETOBJECTMETADATA "GETOBJMD" +#define PHASENAME_PUTOBJECTMETADATA "PUTOBJMD" +#define PHASENAME_DELOBJECTMETADATA "DELOBJMD" +#define PHASENAME_GETBUCKETMETADATA "GETBUCKETMD" +#define PHASENAME_PUTBUCKETMETADATA "PUTBUCKETMD" +#define PHASENAME_DELBUCKETMETADATA "DELBUCKETMD" // human-readable entry type in current benchmark phase @@ -62,7 +62,7 @@ typedef std::vector UInt64Vec; * (Only exact matches are assumed to be compatible, that's why this can differ from the program * version.) */ -#define HTTP_PROTOCOLVERSION "3.0.4" +#define HTTP_PROTOCOLVERSION "3.0.9" /** * Default access mode bits for new files. diff --git a/source/ProgArgs.cpp b/source/ProgArgs.cpp index d09a8a2..f22485a 100644 --- a/source/ProgArgs.cpp +++ b/source/ProgArgs.cpp @@ -489,12 +489,12 @@ void ProgArgs::defineAllowedArgs() /*s3b*/ (ARG_S3BUCKETACLPUT_LONG, bpo::bool_switch(&this->runS3BucketAclPut), "Put S3 bucket ACLs. This requires definition of grantee, grantee type and " "permissions.") -/*s3b*/ (ARG_S3_BUCKET_TAG, bpo::bool_switch(&this->doS3BucketTag), +/*s3b*/ (ARG_S3BUCKETTAG_LONG, bpo::bool_switch(&this->doS3BucketTag), "Activate bucket tagging operations during other benchmarking phases: " "Adding of bucket tags in create bucket phase, " "reading of tags in stat bucket phase and deletion of tags in delete bucket phase.") -/*s3b*/ (ARG_S3_BUCKET_TAG_VERIFY, bpo::bool_switch(&this->doS3BucketTagVerify), - "Verify the correctness of S3 bucket tagging results (requires \"--" ARG_S3_BUCKET_TAG "\")") +/*s3b*/ (ARG_S3BUCKETTAGVERIFY_LONG, bpo::bool_switch(&this->doS3BucketTagVerify), + "Verify the correctness of S3 bucket tagging results (requires \"--" ARG_S3BUCKETTAG_LONG "\")") /*s3e*/ (ARG_S3ENDPOINTS_LONG, bpo::value(&this->s3EndpointsStr), "Comma-separated list of S3 endpoints. When this argument is used, the given " "benchmark paths are used as bucket names. Also see \"--" ARG_S3ACCESSKEY_LONG "\" & " @@ -541,13 +541,13 @@ void ProgArgs::defineAllowedArgs() "S3 object prefix. This will be prepended to all object names when the benchmark path " "is a bucket. (A sequence of 3 to 16 \"" RAND_PREFIX_MARKS_SUBSTR "\" chars will be " "replaced by a random hex string of the same length.)") -/*s3o*/ (ARG_S3_OBJECT_LOCK_CFG, bpo::bool_switch(&this->doS3ObjectLockCfg), +/*s3o*/ (ARG_S3OBJLOCKCFG_LONG, bpo::bool_switch(&this->doS3ObjectLockCfg), "Activate object lock configuration creation") -/*s3o*/ (ARG_S3_OBJECT_LOCK_CFG_VERIFY, bpo::bool_switch(&this->doS3ObjectLockCfgVerify), +/*s3o*/ (ARG_S3OBJLOCKCFGVERIFY_LONG, bpo::bool_switch(&this->doS3ObjectLockCfgVerify), "Verify the correctness of object lock configurations") -/*s3o*/ (ARG_S3_OBJECT_TAG, bpo::bool_switch(&this->doS3ObjectTag), +/*s3o*/ (ARG_S3OBJTAG_LONG, bpo::bool_switch(&this->doS3ObjectTag), "Activate S3 object tagging") -/*s3o*/ (ARG_S3_OBJECT_TAG_VERIFY, bpo::bool_switch(&this->doS3ObjectTagVerify), +/*s3o*/ (ARG_S3OBJTAGVERIFY_LONG, bpo::bool_switch(&this->doS3ObjectTagVerify), "Verify the correctness of created S3 object tags") /*s3r*/ (ARG_S3RANDOBJ_LONG, bpo::bool_switch(&this->useS3RandObjSelect), "Read at random offsets and randomly select a new object for each S3 block read. Only " @@ -3021,12 +3021,12 @@ void ProgArgs::setFromPropertyTreeForService(bpt::ptree& tree) doS3AclVerify = tree.get(ARG_S3ACLVERIFY_LONG); doS3ListObjVerify = tree.get(ARG_S3LISTOBJVERIFY_LONG); doStatInline = tree.get(ARG_STATFILESINLINE_LONG); - doS3BucketTag = tree.get(ARG_S3_BUCKET_TAG); - doS3BucketTagVerify = tree.get(ARG_S3_BUCKET_TAG_VERIFY); - doS3ObjectTag = tree.get(ARG_S3_OBJECT_TAG); - doS3ObjectTagVerify = tree.get(ARG_S3_OBJECT_TAG_VERIFY); - doS3ObjectLockCfg = tree.get(ARG_S3_OBJECT_LOCK_CFG); - doS3ObjectLockCfgVerify = tree.get(ARG_S3_OBJECT_LOCK_CFG_VERIFY); + doS3BucketTag = tree.get(ARG_S3BUCKETTAG_LONG); + doS3BucketTagVerify = tree.get(ARG_S3BUCKETTAGVERIFY_LONG); + doS3ObjectTag = tree.get(ARG_S3OBJTAG_LONG); + doS3ObjectTagVerify = tree.get(ARG_S3OBJTAGVERIFY_LONG); + doS3ObjectLockCfg = tree.get(ARG_S3OBJLOCKCFG_LONG); + doS3ObjectLockCfgVerify = tree.get(ARG_S3OBJLOCKCFGVERIFY_LONG); doTruncate = tree.get(ARG_TRUNCATE_LONG); doTruncToSize = tree.get(ARG_TRUNCTOSIZE_LONG); fadviseFlags = tree.get(ARG_FADVISE_LONG); @@ -3228,12 +3228,12 @@ void ProgArgs::getAsPropertyTreeForService(bpt::ptree& outTree, size_t serviceRa outTree.put(ARG_SENDBUFSIZE_LONG, sockSendBufSize); outTree.put(ARG_STATFILES_LONG, runStatFilesPhase); outTree.put(ARG_STATFILESINLINE_LONG, doStatInline); - outTree.put(ARG_S3_BUCKET_TAG, doS3BucketTag); - outTree.put(ARG_S3_BUCKET_TAG_VERIFY, doS3BucketTagVerify); - outTree.put(ARG_S3_OBJECT_TAG, doS3ObjectTag); - outTree.put(ARG_S3_OBJECT_TAG_VERIFY, doS3ObjectTagVerify); - outTree.put(ARG_S3_OBJECT_LOCK_CFG, doS3ObjectLockCfg); - outTree.put(ARG_S3_OBJECT_LOCK_CFG_VERIFY, doS3ObjectLockCfgVerify); + outTree.put(ARG_S3BUCKETTAG_LONG, doS3BucketTag); + outTree.put(ARG_S3BUCKETTAGVERIFY_LONG, doS3BucketTagVerify); + outTree.put(ARG_S3OBJTAG_LONG, doS3ObjectTag); + outTree.put(ARG_S3OBJTAGVERIFY_LONG, doS3ObjectTagVerify); + outTree.put(ARG_S3OBJLOCKCFG_LONG, doS3ObjectLockCfg); + outTree.put(ARG_S3OBJLOCKCFGVERIFY_LONG, doS3ObjectLockCfgVerify); outTree.put(ARG_SYNCPHASE_LONG, runSyncPhase); outTree.put(ARG_TRUNCATE_LONG, doTruncate); outTree.put(ARG_TRUNCTOSIZE_LONG, doTruncToSize); diff --git a/source/ProgArgs.h b/source/ProgArgs.h index 5100467..174b672 100644 --- a/source/ProgArgs.h +++ b/source/ProgArgs.h @@ -16,7 +16,7 @@ namespace bpo = boost::program_options; namespace bpt = boost::property_tree; -/* command line args and config file options (sorted by "ARG_..." column). +/* command line args and config file options (sorted alphabetically by "ARG_..." column). note: keep length of "_LONG" argument names without parameters within a max length of 16 chars and "_LONG" arguments with parameters to a max of 12 chars, as the description column in the help output otherwise gets too small. */ @@ -144,6 +144,8 @@ namespace bpt = boost::property_tree; #define ARG_S3ACLVERIFY_LONG "s3aclverify" #define ARG_S3BUCKETACLGET_LONG "s3baclget" #define ARG_S3BUCKETACLPUT_LONG "s3baclput" +#define ARG_S3BUCKETTAG_LONG "s3btag" +#define ARG_S3BUCKETTAGVERIFY_LONG "s3btagverify" #define ARG_S3ENDPOINTS_LONG "s3endpoints" #define ARG_S3FASTGET_LONG "s3fastget" #define ARG_S3IGNOREERRORS_LONG "s3ignoreerrors" @@ -155,19 +157,15 @@ namespace bpt = boost::property_tree; #define ARG_S3MULTIDELETE_LONG "s3multidel" #define ARG_S3NOMPCHECK_LONG "s3nompcheck" #define ARG_S3OBJECTPREFIX_LONG "s3objprefix" +#define ARG_S3OBJLOCKCFG_LONG "s3olockcfg" +#define ARG_S3OBJLOCKCFGVERIFY_LONG "s3olockcfgverify" +#define ARG_S3OBJTAG_LONG "s3otag" +#define ARG_S3OBJTAGVERIFY_LONG "s3otagverify" #define ARG_S3RANDOBJ_LONG "s3randobj" #define ARG_S3REGION_LONG "s3region" #define ARG_S3SIGNPAYLOAD_LONG "s3sign" -#define ARG_S3TRANSMAN_LONG "s3transman" #define ARG_S3STATDIRS_LONG "s3statdirs" - -#define ARG_S3_BUCKET_TAG "s3btag" -#define ARG_S3_BUCKET_TAG_VERIFY "s3btagverify" -#define ARG_S3_OBJECT_TAG "s3otag" -#define ARG_S3_OBJECT_TAG_VERIFY "s3otagverify" -#define ARG_S3_OBJECT_LOCK_CFG "s3olockcfg" -#define ARG_S3_OBJECT_LOCK_CFG_VERIFY "s3olockcfgverify" - +#define ARG_S3TRANSMAN_LONG "s3transman" #define ARG_SENDBUFSIZE_LONG "sendbuf" #define ARG_SERVERS_LONG "servers" #define ARG_SERVERSFILE_LONG "serversfile" @@ -547,6 +545,18 @@ class ProgArgs const IntVec& getBenchPathFDs() const { return benchPathFDsVec; } BenchPathType getBenchPathType() const { return benchPathType; } + bool getS3BucketMetadataRequested() const { return doS3BucketTag || doS3ObjectLockCfg; } + bool getS3ObjectMetadataRequested() const { return doS3ObjectTag; } + bool getRunS3GetObjectMetadata() const { return getS3ObjectMetadataRequested(); } + bool getRunS3PutObjectMetadata() const + { return getS3ObjectMetadataRequested() && runCreateFilesPhase; } + bool getRunS3DelObjectMetadata() const + { return getS3ObjectMetadataRequested() && runDeleteFilesPhase; } + bool getRunS3GetBucketMetadata() const { return getS3BucketMetadataRequested(); } + bool getRunS3PutBucketMetadata() const + { return getS3BucketMetadataRequested() && runCreateDirsPhase; } + bool getRunS3DelBucketMetadata() const + { return getS3BucketMetadataRequested() && runDeleteDirsPhase; } // getters for config options in alphabetic order... @@ -649,16 +659,6 @@ class ProgArgs bool getRunS3BucketAclPut() const { return runS3BucketAclPut; } bool getRunS3BucketAclGet() const { return runS3BucketAclGet; } bool getRunS3StatDirs() const { return runS3StatDirs; } - - [[nodiscard]] bool s3BucketMetadataRequested() const noexcept { return doS3BucketTag || doS3ObjectLockCfg; } - [[nodiscard]] bool s3ObjectMetadataRequested() const noexcept { return doS3ObjectTag; } - [[nodiscard]] bool getRunS3GetObjectMetadata() const noexcept { return s3ObjectMetadataRequested(); } - [[nodiscard]] bool getRunS3PutObjectMetadata() const noexcept { return s3ObjectMetadataRequested() && runCreateFilesPhase; } - [[nodiscard]] bool getRunS3DelObjectMetadata() const noexcept { return s3ObjectMetadataRequested() && runDeleteFilesPhase; } - [[nodiscard]] bool getRunS3GetBucketMetadata() const noexcept { return s3BucketMetadataRequested(); } - [[nodiscard]] bool getRunS3PutBucketMetadata() const noexcept { return s3BucketMetadataRequested() && runCreateDirsPhase; } - [[nodiscard]] bool getRunS3DelBucketMetadata() const noexcept { return s3BucketMetadataRequested() && runDeleteDirsPhase; } - bool getRunServiceInForeground() const { return runServiceInForeground; } bool getRunStatFilesPhase() const { return runStatFilesPhase; } bool getRunSyncPhase() const { return runSyncPhase; } diff --git a/source/Statistics.cpp b/source/Statistics.cpp index 565260b..1fdf583 100644 --- a/source/Statistics.cpp +++ b/source/Statistics.cpp @@ -1125,7 +1125,7 @@ void Statistics::printPhaseResultsTableHeaderToStream(std::ostream& outStream) % "LAST DONE" << std::endl; outStream << boost::format(phaseResultsFormatStr) - % "====================" + % "===========" % "================" % "" % "==========" diff --git a/source/Statistics.h b/source/Statistics.h index 3d914bc..93461fd 100644 --- a/source/Statistics.h +++ b/source/Statistics.h @@ -91,7 +91,7 @@ class Statistics WorkersSharedData& workersSharedData; WorkerVec& workerVec; bool consoleBufferingDisabled{false}; - const std::string phaseResultsFormatStr{"%|-20| %|-17|%|1| %|11| %|11|"}; + const std::string phaseResultsFormatStr{"%|-11| %|-17|%|1| %|11| %|11|"}; const std::string phaseResultsLeftFormatStr{"%|-9| %|-17|%|1| "}; // left side format str const std::string phaseResultsFooterStr = std::string(3, '-'); CPUUtil liveCpuUtil; // updated by live stats loop or through http service live stat calls diff --git a/source/toolkits/TranslatorTk.cpp b/source/toolkits/TranslatorTk.cpp index de4fe77..10e58bb 100644 --- a/source/toolkits/TranslatorTk.cpp +++ b/source/toolkits/TranslatorTk.cpp @@ -75,12 +75,12 @@ std::string TranslatorTk::benchPhaseToPhaseName(BenchPhase benchPhase, const Pro case BenchPhase_LISTOBJECTS: return PHASENAME_LISTOBJECTS; case BenchPhase_LISTOBJPARALLEL: return PHASENAME_LISTOBJPAR; case BenchPhase_MULTIDELOBJ: return PHASENAME_MULTIDELOBJ; - case BenchPhase_GET_S3_OBJECT_MD: return PHASENAME_GET_S3_OBJECT_METADATA; - case BenchPhase_PUT_S3_OBJECT_MD: return PHASENAME_PUT_S3_OBJECT_METADATA; - case BenchPhase_DEL_S3_OBJECT_MD: return PHASENAME_DEL_S3_OBJECT_METADATA; - case BenchPhase_GET_S3_BUCKET_MD: return PHASENAME_GET_S3_BUCKET_METADATA; - case BenchPhase_PUT_S3_BUCKET_MD: return PHASENAME_PUT_S3_BUCKET_METADATA; - case BenchPhase_DEL_S3_BUCKET_MD: return PHASENAME_DEL_S3_BUCKET_METADATA; + case BenchPhase_GET_S3_OBJECT_MD: return PHASENAME_GETOBJECTMETADATA; + case BenchPhase_PUT_S3_OBJECT_MD: return PHASENAME_PUTOBJECTMETADATA; + case BenchPhase_DEL_S3_OBJECT_MD: return PHASENAME_DELOBJECTMETADATA; + case BenchPhase_GET_S3_BUCKET_MD: return PHASENAME_GETBUCKETMETADATA; + case BenchPhase_PUT_S3_BUCKET_MD: return PHASENAME_PUTBUCKETMETADATA; + case BenchPhase_DEL_S3_BUCKET_MD: return PHASENAME_DELBUCKETMETADATA; default: { // should never happen throw ProgException("Phase name requested for unknown/invalid phase type: " + diff --git a/source/workers/LocalWorker.cpp b/source/workers/LocalWorker.cpp index ef8d036..4b1c385 100644 --- a/source/workers/LocalWorker.cpp +++ b/source/workers/LocalWorker.cpp @@ -3931,48 +3931,38 @@ void LocalWorker::s3ModeIterateCustomObjects() } /** - * Throw an informative WorkerException if the outcome has an error + * Throw an informative WorkerException if the s3 request outcome has the error flag set. * + * @outcome s3 request outcome. + * @failMessage human-friendly error message, e.g. "Object upload failed." + * @objectName name of object to which this error applies, can be empty. * @throw WorkerException on error. */ -template -inline void throwOnError(const OutcomeType& outcome, const std::string& operation, - const std::string& bucketName) +template +void LocalWorker::s3ModeThrowOnError(const OUTCOMETYPE& outcome, const std::string& failMessage, + const std::string& bucketName, const std::string& objectName) { - IF_UNLIKELY(!outcome.IsSuccess()) - { - const auto& err = outcome.GetError(); +#ifndef S3_SUPPORT + throw WorkerException(std::string(__func__) + " called, but this was built without S3 support"); +#else - std::stringstream errStr; - errStr << "Unable to " << operation << ": " << err.GetMessage() << std::endl - << "HTTP Error Code: " << static_cast(err.GetResponseCode()) << "; " - << "Exception name: " << err.GetExceptionName() << "\n" - << "Bucket: " << bucketName << std::endl; - throw WorkerException(errStr.str()); - } -} + IF_LIKELY(outcome.IsSuccess() ) + return; -/** - * Throw an informative WorkerException if the outcome has an error - * - * @throw WorkerException on error. - */ -template -inline void throwOnError(const OutcomeType& outcome, const std::string& operation, - const std::string& bucketName, const std::string& objectName) -{ - IF_UNLIKELY(!outcome.IsSuccess()) - { - const auto& err = outcome.GetError(); + const auto& s3Error = outcome.GetError(); - std::stringstream errStr; - errStr << "Unable to " << operation << ": " << err.GetMessage() << std::endl - << "HTTP Error Code: " << static_cast(err.GetResponseCode()) << "; " - << "Exception name: " << err.GetExceptionName() << "\n" - << "Bucket: " << bucketName << "; " - << "Key: " << objectName << std::endl; - throw WorkerException(errStr.str()); - } + std::stringstream errStr; + errStr << failMessage << " " << + "Endpoint: " << s3EndpointStr << "; " << + "Bucket: " << bucketName << "; " << + (objectName.empty() ? std::string("") : ("Object: " + objectName + "; ") ) << + "Exception: " << s3Error.GetExceptionName() << "; " << + "Message: " << s3Error.GetMessage() << "; " << + "HTTP Error Code: " << (int)s3Error.GetResponseCode(); + + throw WorkerException(errStr.str() ); + +#endif // S3_SUPPORT } /** @@ -4028,7 +4018,7 @@ void LocalWorker::s3ModeHeadBucket(std::string bucketName) OPLOG_POST_OP("HeadBucket", bucketName, 0, 0, !outcome.IsSuccess()); - throwOnError(outcome, "head bucket", bucketName); + s3ModeThrowOnError(outcome, "Head bucket request failed.", bucketName); #endif // S3_SUPPORT } @@ -4060,7 +4050,7 @@ void LocalWorker::s3ModeCreateBucketTagging(const std::string& bucketName) OPLOG_POST_OP("PutBucketTagging", bucketName, 0, 0, !taggingOutcome.IsSuccess()); - throwOnError(taggingOutcome, "put bucket tagging", bucketName); + s3ModeThrowOnError(taggingOutcome, "Put bucket tagging failed.", bucketName); #endif // S3_SUPPORT } @@ -4081,7 +4071,7 @@ void LocalWorker::s3ModeGetBucketTagging(const std::string& bucketName) OPLOG_POST_OP("GetBucketTagging", bucketName, 0, 0, !outcome.IsSuccess()); - throwOnError(outcome, "get bucket tagging", bucketName); + s3ModeThrowOnError(outcome, "Get bucket tagging failed.", bucketName); if (!progArgs->getDoS3BucketTaggingVerify()) return; @@ -4120,7 +4110,7 @@ void LocalWorker::s3ModeDeleteBucketTagging(const std::string& bucketName) OPLOG_POST_OP("DeleteBucketTagging", bucketName, 0, 0, !outcome.IsSuccess()); - throwOnError(outcome, "delete bucket tagging", bucketName); + s3ModeThrowOnError(outcome, "Delete bucket tagging failed.", bucketName); #endif // S3_SUPPORT } @@ -5717,6 +5707,170 @@ void LocalWorker::s3ModeGetObjectAcl(std::string bucketName, std::string objectN #endif // S3_SUPPORT } + +void LocalWorker::s3ModeGetObjectTags(const std::string& bucketName, const std::string& objectName) +{ +#ifndef S3_SUPPORT + throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); +#else + OPLOG_PRE_OP("GetObjectTagging", bucketName + "/" + objectName, 0, 0); + + const auto& getTagOutcome = s3Client->GetObjectTagging( + S3::GetObjectTaggingRequest() + .WithBucket(bucketName) + .WithKey(objectName) + ); + + OPLOG_POST_OP("GetObjectTagging", bucketName + "/" + objectName, 0, 0, !getTagOutcome.IsSuccess()); + + s3ModeThrowOnError(getTagOutcome, "Get object tagging failed.", bucketName, objectName); + + // Continue only if we need to verify + if (!progArgs->getDoS3ObjectTaggingVerify()) + return; + + const auto& firstTag = getTagOutcome.GetResult().GetTagSet().front(); + + IF_UNLIKELY(!StringTk::verifyRandomS3TagValue(firstTag.GetValue(), objectName)) + { + std::stringstream errStr; + errStr << "Random tag value is corrupted (invalid checksum). " + << "Bucket: " << bucketName << "; " + << "Key: " << objectName << std::endl + << "Tag: " << firstTag.GetKey() << "=" << firstTag.GetValue() << std::endl; + throw WorkerException(errStr.str()); + } +#endif // S3_SUPPORT +} + +void LocalWorker::s3ModePutObjectTags(const std::string& bucketName, const std::string& objectName) +{ +#ifndef S3_SUPPORT + throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); +#else + const auto tag = S3::Tag() + .WithKey(TAG_KEY_MEDIUM_NAME) + .WithValue(StringTk::generateRandomS3TagValue(objectName, TAG_VALUE_MEDIUM_LEN)); + + OPLOG_PRE_OP("PutObjectTagging", bucketName + "/" + objectName, 0, TAG_VALUE_MEDIUM_LEN); + + const auto& putTagOutcome = s3Client->PutObjectTagging( + S3::PutObjectTaggingRequest() + .WithBucket(bucketName) + .WithKey(objectName) + .WithTagging(S3::Tagging().AddTagSet(tag)) + ); + + OPLOG_POST_OP("PutObjectTagging", bucketName + "/" + objectName, + 0, TAG_VALUE_MEDIUM_LEN, !putTagOutcome.IsSuccess()); + + s3ModeThrowOnError(putTagOutcome, "Put object tagging failed.", bucketName, objectName); + +#endif // S3_SUPPORT +} + +void LocalWorker::s3ModeDeleteObjectTags(const std::string& bucketName, const std::string& objectName) +{ +#ifndef S3_SUPPORT + throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); +#else + + OPLOG_PRE_OP("DeleteObjectTagging", bucketName + "/" + objectName, 0, 0); + + const auto& delTagOutcome = s3Client->DeleteObjectTagging( + S3::DeleteObjectTaggingRequest() + .WithBucket(bucketName) + .WithKey(objectName) + ); + + OPLOG_POST_OP("DeleteObjectTagging", bucketName + "/" + objectName, 0, 0, !delTagOutcome.IsSuccess()); + + s3ModeThrowOnError(delTagOutcome, "Delete object tagging failed.", bucketName, objectName); + +#endif // S3_SUPPORT +} + +void LocalWorker::s3ModeGetObjectLockConfiguration(const std::string &bucketName) +{ +#ifndef S3_SUPPORT + throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); +#else + OPLOG_PRE_OP("GetObjectLockConfiguration", bucketName, 0, 0); + + const auto& getObjLockOutcome = s3Client->GetObjectLockConfiguration( + S3::GetObjectLockConfigurationRequest().WithBucket(bucketName) + ); + + OPLOG_POST_OP("GetObjectLockConfiguration", bucketName, 0, 0, !getObjLockOutcome.IsSuccess()); + + s3ModeThrowOnError(getObjLockOutcome, "Get object lock configuration failed.", bucketName); + + // Continue only if we need to verify + if (!progArgs->getDoS3ObjectLockConfigurationVerify()) + return; + + const auto& objLockCfg = getObjLockOutcome.GetResult().GetObjectLockConfiguration(); + + IF_UNLIKELY(!objLockCfg.ObjectLockEnabledHasBeenSet()) + throw WorkerException("Object lock has not been enabled for bucket: " + bucketName); + + IF_UNLIKELY(!objLockCfg.RuleHasBeenSet()) + throw WorkerException("Object lock rule has not been set for bucket: " + bucketName); + + const auto& objRetentionRule = objLockCfg.GetRule(); + + IF_UNLIKELY(!objRetentionRule.DefaultRetentionHasBeenSet()) + throw WorkerException("Object lock default retention has not been set for bucket: " + bucketName); + + const auto& objDefaultRetention = objRetentionRule.GetDefaultRetention(); + + IF_UNLIKELY(!objDefaultRetention.DaysHasBeenSet()) + throw WorkerException("Object lock retention days have not been set for bucket: " + bucketName); + + const auto& retentionDays = objDefaultRetention.GetDays(); + + IF_UNLIKELY(retentionDays != RETENTION_PERIOD_DAYS) + { + std::stringstream errStr; + errStr << "Object lock default retention was set to '" << retentionDays << " days' in bucket '" + << bucketName << "', but the expected value was '" << RETENTION_PERIOD_DAYS << " days'"; + throw WorkerException(errStr.str()); + } +#endif // S3_SUPPORT +} + +void LocalWorker::s3ModePutObjectLockConfiguration(const std::string &bucketName, bool unset) +{ +#ifndef S3_SUPPORT + throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); +#else + S3::ObjectLockConfiguration objectLockCfg; + + if (unset) + objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::NOT_SET); + else + { + objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::Enabled); + objectLockCfg.SetRule( + S3::ObjectLockRule().WithDefaultRetention( + S3::DefaultRetention().WithMode(S3::ObjectLockRetentionMode::COMPLIANCE).WithDays(1) + ) + ); + } + + OPLOG_PRE_OP("PutObjectLockConfiguration", bucketName, 0, 0); + + const auto& putObjLockOutcome = s3Client->PutObjectLockConfiguration( + S3::PutObjectLockConfigurationRequest() + .WithBucket(bucketName) + .WithObjectLockConfiguration(objectLockCfg)); + + OPLOG_POST_OP("PutObjectLockConfiguration", bucketName, 0, 0, !putObjLockOutcome.IsSuccess()); + + s3ModeThrowOnError(putObjLockOutcome, "Put object lock configuration failed.", bucketName); +#endif // S3_SUPPORT +} + /** * In S3 mode, decide if we do fallback to reverse upload. This would be the case if this is a write * phase and user selected random offsets. @@ -6557,165 +6711,3 @@ void LocalWorker::anyModeDropCaches() "Path: " + dropCachesPath + "; " "SysErr: " + strerror(errno) ); } - -void LocalWorker::s3ModeGetObjectTags(const std::string& bucketName, const std::string& objectName) -{ -#ifndef S3_SUPPORT - throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); -#else - OPLOG_PRE_OP("GetObjectTagging", bucketName + "/" + objectName, 0, 0); - - const auto& getTagOutcome = s3Client->GetObjectTagging( - S3::GetObjectTaggingRequest() - .WithBucket(bucketName) - .WithKey(objectName) - ); - - OPLOG_POST_OP("GetObjectTagging", bucketName + "/" + objectName, 0, 0, !getTagOutcome.IsSuccess()); - - throwOnError(getTagOutcome, "get tagging", bucketName, objectName); - - // Continue only if we need to verify - if (!progArgs->getDoS3ObjectTaggingVerify()) - return; - - const auto& firstTag = getTagOutcome.GetResult().GetTagSet().front(); - - IF_UNLIKELY(!StringTk::verifyRandomS3TagValue(firstTag.GetValue(), objectName)) - { - std::stringstream errStr; - errStr << "Random tag value is corrupted (invalid checksum). " - << "Bucket: " << bucketName << "; " - << "Key: " << objectName << std::endl - << "Tag: " << firstTag.GetKey() << "=" << firstTag.GetValue() << std::endl; - throw WorkerException(errStr.str()); - } -#endif // S3_SUPPORT -} - -void LocalWorker::s3ModePutObjectTags(const std::string& bucketName, const std::string& objectName) -{ -#ifndef S3_SUPPORT - throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); -#else - const auto tag = S3::Tag() - .WithKey(TAG_KEY_MEDIUM_NAME) - .WithValue(StringTk::generateRandomS3TagValue(objectName, TAG_VALUE_MEDIUM_LEN)); - - OPLOG_PRE_OP("PutObjectTagging", bucketName + "/" + objectName, 0, TAG_VALUE_MEDIUM_LEN); - - const auto& putTagOutcome = s3Client->PutObjectTagging( - S3::PutObjectTaggingRequest() - .WithBucket(bucketName) - .WithKey(objectName) - .WithTagging(S3::Tagging().AddTagSet(tag)) - ); - - OPLOG_POST_OP("PutObjectTagging", bucketName + "/" + objectName, - 0, TAG_VALUE_MEDIUM_LEN, !putTagOutcome.IsSuccess()); - - throwOnError(putTagOutcome, "put tagging", bucketName, objectName); - -#endif // S3_SUPPORT -} -void LocalWorker::s3ModeDeleteObjectTags(const std::string& bucketName, const std::string& objectName) -{ -#ifndef S3_SUPPORT - throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); -#else - - OPLOG_PRE_OP("DeleteObjectTagging", bucketName + "/" + objectName, 0, 0); - - const auto& delTagOutcome = s3Client->DeleteObjectTagging( - S3::DeleteObjectTaggingRequest() - .WithBucket(bucketName) - .WithKey(objectName) - ); - - OPLOG_POST_OP("DeleteObjectTagging", bucketName + "/" + objectName, 0, 0, !delTagOutcome.IsSuccess()); - - throwOnError(delTagOutcome, "delete tagging", bucketName, objectName); - -#endif // S3_SUPPORT -} - -void LocalWorker::s3ModeGetObjectLockConfiguration(const std::string &bucketName) -{ -#ifndef S3_SUPPORT - throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); -#else - OPLOG_PRE_OP("GetObjectLockConfiguration", bucketName, 0, 0); - - const auto& getObjLockOutcome = s3Client->GetObjectLockConfiguration( - S3::GetObjectLockConfigurationRequest().WithBucket(bucketName) - ); - - OPLOG_POST_OP("GetObjectLockConfiguration", bucketName, 0, 0, !getObjLockOutcome.IsSuccess()); - - throwOnError(getObjLockOutcome, "get object lock configuration", bucketName); - - // Continue only if we need to verify - if (!progArgs->getDoS3ObjectLockConfigurationVerify()) - return; - - const auto& objLockCfg = getObjLockOutcome.GetResult().GetObjectLockConfiguration(); - - IF_UNLIKELY(!objLockCfg.ObjectLockEnabledHasBeenSet()) - throw WorkerException("Object lock has not been enabled for bucket: " + bucketName); - - IF_UNLIKELY(!objLockCfg.RuleHasBeenSet()) - throw WorkerException("Object lock rule has not been set for bucket: " + bucketName); - - const auto& objRetentionRule = objLockCfg.GetRule(); - - IF_UNLIKELY(!objRetentionRule.DefaultRetentionHasBeenSet()) - throw WorkerException("Object lock default retention has not been set for bucket: " + bucketName); - - const auto& objDefaultRetention = objRetentionRule.GetDefaultRetention(); - - IF_UNLIKELY(!objDefaultRetention.DaysHasBeenSet()) - throw WorkerException("Object lock retention days have not been set for bucket: " + bucketName); - - const auto& retentionDays = objDefaultRetention.GetDays(); - - IF_UNLIKELY(retentionDays != RETENTION_PERIOD_DAYS) - { - std::stringstream errStr; - errStr << "Object lock default retention was set to '" << retentionDays << " days' in bucket '" - << bucketName << "', but the expected value was '" << RETENTION_PERIOD_DAYS << " days'"; - throw WorkerException(errStr.str()); - } -#endif // S3_SUPPORT -} - -void LocalWorker::s3ModePutObjectLockConfiguration(const std::string &bucketName, bool unset) -{ -#ifndef S3_SUPPORT - throw WorkerException(std::string(__func__) + "called, but this was built without S3 support"); -#else - S3::ObjectLockConfiguration objectLockCfg; - - if (unset) - objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::NOT_SET); - else - { - objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::Enabled); - objectLockCfg.SetRule( - S3::ObjectLockRule().WithDefaultRetention( - S3::DefaultRetention().WithMode(S3::ObjectLockRetentionMode::COMPLIANCE).WithDays(1) - ) - ); - } - - OPLOG_PRE_OP("PutObjectLockConfiguration", bucketName, 0, 0); - - const auto& putObjLockOutcome = s3Client->PutObjectLockConfiguration( - S3::PutObjectLockConfigurationRequest() - .WithBucket(bucketName) - .WithObjectLockConfiguration(objectLockCfg)); - - OPLOG_POST_OP("PutObjectLockConfiguration", bucketName, 0, 0, !putObjLockOutcome.IsSuccess()); - - throwOnError(putObjLockOutcome, "put object lock configuration", bucketName); -#endif // S3_SUPPORT -} diff --git a/source/workers/LocalWorker.h b/source/workers/LocalWorker.h index ffb2438..1abf2b7 100644 --- a/source/workers/LocalWorker.h +++ b/source/workers/LocalWorker.h @@ -205,6 +205,9 @@ class LocalWorker : public Worker void s3ModeIterateObjects(); void s3ModeIterateObjectsRand(); void s3ModeIterateCustomObjects(); + template + void s3ModeThrowOnError(const OUTCOMETYPE& outcome, const std::string& failMessage, + const std::string& bucketName, const std::string& objectName=""); void s3ModeCreateBucket(std::string bucketName); void s3ModeHeadBucket(std::string bucketName); void s3ModeCreateBucketTagging(const std::string& bucketName); @@ -225,11 +228,6 @@ class LocalWorker : public Worker void s3ModeDownloadObjectTransMan(std::string bucketName, std::string objectName, const bool isRWMixedReader); void s3ModeStatObject(std::string bucketName, std::string objectName); - void s3ModeGetObjectTags(const std::string& bucketName, const std::string& objectName); - void s3ModePutObjectTags(const std::string& bucketName, const std::string& objectName); - void s3ModeDeleteObjectTags(const std::string& bucketName, const std::string& objectName); - void s3ModeGetObjectLockConfiguration(const std::string& bucketName); - void s3ModePutObjectLockConfiguration(const std::string& bucketName, bool unset = false); void s3ModeDeleteObject(std::string bucketName, std::string objectName); void s3ModeListObjects(); void s3ModeListObjParallel(); @@ -238,6 +236,11 @@ class LocalWorker : public Worker void s3ModeListAndMultiDeleteObjects(); void s3ModePutObjectAcl(std::string bucketName, std::string objectName); void s3ModeGetObjectAcl(std::string bucketName, std::string objectName); + void s3ModeGetObjectTags(const std::string& bucketName, const std::string& objectName); + void s3ModePutObjectTags(const std::string& bucketName, const std::string& objectName); + void s3ModeDeleteObjectTags(const std::string& bucketName, const std::string& objectName); + void s3ModeGetObjectLockConfiguration(const std::string& bucketName); + void s3ModePutObjectLockConfiguration(const std::string& bucketName, bool unset = false); bool getS3ModeDoReverseSeqFallback(); std::string getS3RandObjectPrefix(size_t workerRank, size_t dirIdx, size_t fileIdx, const std::string& objectPrefix);