diff --git a/src/libraries/JANA/Engine/JMailbox.h b/src/libraries/JANA/Engine/JMailbox.h index e2a13030f..2ca6432e1 100644 --- a/src/libraries/JANA/Engine/JMailbox.h +++ b/src/libraries/JANA/Engine/JMailbox.h @@ -8,6 +8,7 @@ #include #include +#include #include /// JMailbox is a threadsafe event queue designed for communication between Arrows. @@ -32,11 +33,6 @@ /// 3. Triple mutex trick to give push() priority? -#ifndef CACHE_LINE_BYTES -#define CACHE_LINE_BYTES 64 -#endif - - template class JMailbox { diff --git a/src/libraries/JANA/Engine/JSubeventMailbox.h b/src/libraries/JANA/Engine/JSubeventMailbox.h index b82e783ad..fe9a2f6ec 100644 --- a/src/libraries/JANA/Engine/JSubeventMailbox.h +++ b/src/libraries/JANA/Engine/JSubeventMailbox.h @@ -10,11 +10,7 @@ #include #include #include - -#ifndef CACHE_LINE_BYTES -#define CACHE_LINE_BYTES 64 -#endif - +#include template struct SubeventWrapper { @@ -37,7 +33,7 @@ class JSubeventMailbox { private: - struct alignas(CACHE_LINE_BYTES) LocalMailbox { + struct alignas(JANA2_CACHE_LINE_BYTES) LocalMailbox { std::mutex mutex; std::deque> ready; std::map, size_t> in_progress; diff --git a/src/libraries/JANA/JApplication.h b/src/libraries/JANA/JApplication.h index 1e37a2467..ccdc3e016 100644 --- a/src/libraries/JANA/JApplication.h +++ b/src/libraries/JANA/JApplication.h @@ -8,6 +8,7 @@ #include #include #include +#include class JApplication; class JEventProcessor; @@ -25,12 +26,11 @@ extern JApplication* japp; #include #include #include -#include #include ////////////////////////////////////////////////////////////////////////////////////////////////// -/// JANA application class (singleton). +/// JANA application class /// /// The JApplication class serves as a central access point for getting to most things /// in the JANA application. It owns the JThreadManager, JParameterManager, etc. diff --git a/src/libraries/JANA/Utils/JCpuInfo.h b/src/libraries/JANA/Utils/JCpuInfo.h index c47de2a7e..004f0067f 100644 --- a/src/libraries/JANA/Utils/JCpuInfo.h +++ b/src/libraries/JANA/Utils/JCpuInfo.h @@ -1,9 +1,19 @@ // Copyright 2020, Jefferson Science Associates, LLC. // Subject to the terms in the LICENSE file found in the top-level directory. - #pragma once +#ifndef JANA2_CACHE_LINE_BYTES +#define JANA2_CACHE_LINE_BYTES 64 +#endif +// The cache line size is 64 for ifarm1402. gcc won't allow larger than 128 +// You can find the cache line size in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size +/*! The cache line size is useful for creating a buffer to make sure that a variable accessed by multiple threads does not share the cache line it is on. + This is useful for variables that may be written-to by one of the threads, because the thread will acquire locked access to the entire cache line. + This blocks other threads from operating on the other data stored on the cache line. Note that it is also important to align the shared data as well. + See http://www.drdobbs.com/parallel/eliminate-false-sharing/217500206?pgno=4 for more details. */ + + #include namespace JCpuInfo { @@ -20,4 +30,4 @@ namespace JCpuInfo { bool PinThreadToCpu(std::thread *thread, size_t cpu_id); -} \ No newline at end of file +} diff --git a/src/libraries/JANA/Utils/JEventPool.h b/src/libraries/JANA/Utils/JEventPool.h index df81cced8..890d2968c 100644 --- a/src/libraries/JANA/Utils/JEventPool.h +++ b/src/libraries/JANA/Utils/JEventPool.h @@ -7,13 +7,13 @@ #define JANA2_JEVENTPOOL_H #include +#include #include #include - class JEventPool { private: - struct alignas(64) LocalPool { + struct alignas(JANA2_CACHE_LINE_BYTES) LocalPool { std::mutex mutex; std::vector> events; }; diff --git a/src/libraries/JANA/Utils/JResourcePool.h b/src/libraries/JANA/Utils/JResourcePool.h index 4023c2c4b..06d8a8e1a 100644 --- a/src/libraries/JANA/Utils/JResourcePool.h +++ b/src/libraries/JANA/Utils/JResourcePool.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -88,18 +89,6 @@ template class JResourcePool std::size_t Get_PoolSize(void) const; std::size_t Get_NumObjectsAllThreads(void) const{return dObjectCounter;} - static constexpr unsigned int Get_CacheLineSize(void) - { - /// Returns the cache line size for the processor of the target platform. - /*! The cache line size is useful for creating a buffer to make sure that a variable accessed by multiple threads does not share the cache line it is on. - This is useful for variables that may be written-to by one of the threads, because the thread will acquire locked access to the entire cache line. - This blocks other threads from operating on the other data stored on the cache line. Note that it is also important to align the shared data as well. - See http://www.drdobbs.com/parallel/eliminate-false-sharing/217500206?pgno=4 for more details. */ - - //cache line size is 64 for ifarm1402, gcc won't allow larger than 128 - //the cache line size is in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size - return 64; //units are in bytes - } private: @@ -119,10 +108,10 @@ template class JResourcePool void Recycle_Resources_StaticPool(std::vector& sResources); void Recycle_Resource_StaticPool(DType* sResource); - alignas(Get_CacheLineSize()) std::size_t dDebugLevel = 0; + alignas(JANA2_CACHE_LINE_BYTES) std::size_t dDebugLevel = 0; //static class members have external linkage: same instance shared between every translation unit (would be globally, put only private access) - alignas(Get_CacheLineSize()) static std::atomic dPoolLock; + alignas(JANA2_CACHE_LINE_BYTES) static std::atomic dPoolLock; alignas(Get_CacheLineSize()) static std::vector mResourcePool; alignas(Get_CacheLineSize()) static std::size_t dMaxPoolSize; alignas(Get_CacheLineSize()) static std::size_t dPoolCounter; //must be accessed within a lock due to how it's used in destructor: freeing all resources @@ -147,11 +136,11 @@ template class JSharedPtrRecycler //STATIC MEMBER DEFINITIONS //Since these are part of a template, these statics will only be defined once, no matter how much this header is included -template std::atomic JResourcePool::dPoolLock{0}; -template std::vector JResourcePool::mResourcePool = {}; -template std::size_t JResourcePool::dMaxPoolSize{10}; -template std::size_t JResourcePool::dPoolCounter{0}; -template std::atomic JResourcePool::dObjectCounter{0}; +template alignas(JANA2_CACHE_LINE_BYTES) std::atomic JResourcePool::dPoolLock{0}; +template alignas(JANA2_CACHE_LINE_BYTES) std::vector JResourcePool::mResourcePool = {}; +template alignas(JANA2_CACHE_LINE_BYTES) std::size_t JResourcePool::dMaxPoolSize{10}; +template alignas(JANA2_CACHE_LINE_BYTES) std::size_t JResourcePool::dPoolCounter{0}; +template alignas(JANA2_CACHE_LINE_BYTES) std::atomic JResourcePool::dObjectCounter{0}; //CONSTRUCTORS template JResourcePool::JResourcePool(std::size_t sMaxPoolSize, std::size_t sDebugLevel) : JResourcePool()