Merge pull request #253 from JeffersonLab/AyanRoy16_issue_245

Handle cache line size consistently (Addresses issue #245)

src/libraries/JANA/Engine/JMailbox.h

@@ -8,6 +8,7 @@
 
 #include <queue>
 #include <mutex>
+#include <JANA/Utils/JCpuInfo.h>
 #include <JANA/Services/JLoggingService.h>
 
 /// JMailbox is a threadsafe event queue designed for communication between Arrows.
@@ -32,11 +33,6 @@
 ///   3. Triple mutex trick to give push() priority?
 
 
-#ifndef CACHE_LINE_BYTES
-#define CACHE_LINE_BYTES 64
-#endif
-
-
 template <typename T>
 class JMailbox {
 

src/libraries/JANA/Engine/JSubeventMailbox.h

@@ -10,11 +10,7 @@
 #include <mutex>
 #include <JANA/Services/JLoggingService.h>
 #include <JANA/JEvent.h>
-
-#ifndef CACHE_LINE_BYTES
-#define CACHE_LINE_BYTES 64
-#endif
-
+#include <JANA/Utils/JCpuInfo.h>
 
 template <typename SubeventT>
 struct SubeventWrapper {
@@ -37,7 +33,7 @@ class JSubeventMailbox {
 
 private:
 
-    struct alignas(CACHE_LINE_BYTES) LocalMailbox {
+    struct alignas(JANA2_CACHE_LINE_BYTES) LocalMailbox {
         std::mutex mutex;
         std::deque<std::shared_ptr<JEvent>> ready;
         std::map<std::shared_ptr<JEvent>, size_t> in_progress;

src/libraries/JANA/JApplication.h

@@ -8,6 +8,7 @@
 #include <vector>
 #include <string>
 #include <iostream>
+#include <atomic>
 
 class JApplication;
 class JEventProcessor;
@@ -25,12 +26,11 @@ extern JApplication* japp;
 #include <JANA/Services/JParameterManager.h>
 #include <JANA/Services/JLoggingService.h>
 #include <JANA/Status/JComponentSummary.h>
-#include <JANA/Utils/JResourcePool.h>
 #include <JANA/Status/JPerfSummary.h>
 
 
 //////////////////////////////////////////////////////////////////////////////////////////////////
-/// JANA application class (singleton).
+/// JANA application class
 ///
 /// The JApplication class serves as a central access point for getting to most things
 /// in the JANA application. It owns the JThreadManager, JParameterManager, etc.

src/libraries/JANA/Utils/JCpuInfo.h

@@ -1,9 +1,19 @@
 
 // Copyright 2020, Jefferson Science Associates, LLC.
 // Subject to the terms in the LICENSE file found in the top-level directory.
-
 #pragma once
 
+#ifndef JANA2_CACHE_LINE_BYTES
+#define JANA2_CACHE_LINE_BYTES 64
+#endif
+// The cache line size is 64 for ifarm1402. gcc won't allow larger than 128
+// You can find the cache line size in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size
+/*! The cache line size is useful for creating a buffer to make sure that a variable accessed by multiple threads does not share the cache line it is on.
+    This is useful for variables that may be written-to by one of the threads, because the thread will acquire locked access to the entire cache line.
+    This blocks other threads from operating on the other data stored on the cache line. Note that it is also important to align the shared data as well.
+    See http://www.drdobbs.com/parallel/eliminate-false-sharing/217500206?pgno=4 for more details. */
+
+
 #include <thread>
 
 namespace JCpuInfo {
@@ -20,4 +30,4 @@ namespace JCpuInfo {
 
     bool PinThreadToCpu(std::thread *thread, size_t cpu_id);
 
-}
+}

src/libraries/JANA/Utils/JEventPool.h

@@ -7,13 +7,13 @@
 #define JANA2_JEVENTPOOL_H
 
 #include <JANA/JEvent.h>
+#include <JANA/Utils/JCpuInfo.h>
 #include <JANA/JFactoryGenerator.h>
 #include <JANA/Services/JComponentManager.h>
-
 class JEventPool {
 private:
 
-    struct alignas(64) LocalPool {
+    struct alignas(JANA2_CACHE_LINE_BYTES) LocalPool {
         std::mutex mutex;
         std::vector<std::shared_ptr<JEvent>> events;
     };

src/libraries/JANA/Utils/JResourcePool.h

@@ -13,6 +13,7 @@
 #include <iostream>
 #include <type_traits>
 #include <memory>
+#include <JANA/Utils/JCpuInfo.h>
 #include <algorithm>
 #include <iterator>
 
@@ -88,18 +89,6 @@ template <typename DType> class JResourcePool
         std::size_t Get_PoolSize(void) const;
         std::size_t Get_NumObjectsAllThreads(void) const{return dObjectCounter;}
 
-        static constexpr unsigned int Get_CacheLineSize(void)
-        {
-            /// Returns the cache line size for the processor of the target platform.
-            /*! The cache line size is useful for creating a buffer to make sure that a variable accessed by multiple threads does not share the cache line it is on.
-                This is useful for variables that may be written-to by one of the threads, because the thread will acquire locked access to the entire cache line.
-                This blocks other threads from operating on the other data stored on the cache line. Note that it is also important to align the shared data as well.
-                See http://www.drdobbs.com/parallel/eliminate-false-sharing/217500206?pgno=4 for more details. */
-
-            //cache line size is 64 for ifarm1402, gcc won't allow larger than 128
-            //the cache line size is in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size
-            return 64; //units are in bytes
-        }
 
     private:
 
@@ -119,10 +108,10 @@ template <typename DType> class JResourcePool
         void Recycle_Resources_StaticPool(std::vector<DType*>& sResources);
         void Recycle_Resource_StaticPool(DType* sResource);
 
-        alignas(Get_CacheLineSize()) std::size_t dDebugLevel = 0;
+        alignas(JANA2_CACHE_LINE_BYTES) std::size_t dDebugLevel = 0;
 
         //static class members have external linkage: same instance shared between every translation unit (would be globally, put only private access)
-        alignas(Get_CacheLineSize()) static std::atomic<bool> dPoolLock;
+        alignas(JANA2_CACHE_LINE_BYTES) static std::atomic<bool> dPoolLock;
         alignas(Get_CacheLineSize()) static std::vector<DType*> mResourcePool;
         alignas(Get_CacheLineSize()) static std::size_t dMaxPoolSize;
         alignas(Get_CacheLineSize()) static std::size_t dPoolCounter; //must be accessed within a lock due to how it's used in destructor: freeing all resources
@@ -147,11 +136,11 @@ template <typename DType> class JSharedPtrRecycler
 
 //STATIC MEMBER DEFINITIONS
 //Since these are part of a template, these statics will only be defined once, no matter how much this header is included
-template <typename DType> std::atomic<bool> JResourcePool<DType>::dPoolLock{0};
-template <typename DType> std::vector<DType*> JResourcePool<DType>::mResourcePool = {};
-template <typename DType> std::size_t JResourcePool<DType>::dMaxPoolSize{10};
-template <typename DType> std::size_t JResourcePool<DType>::dPoolCounter{0};
-template <typename DType> std::atomic<std::size_t> JResourcePool<DType>::dObjectCounter{0};
+template <typename DType> alignas(JANA2_CACHE_LINE_BYTES) std::atomic<bool> JResourcePool<DType>::dPoolLock{0};
+template <typename DType> alignas(JANA2_CACHE_LINE_BYTES) std::vector<DType*> JResourcePool<DType>::mResourcePool = {};
+template <typename DType> alignas(JANA2_CACHE_LINE_BYTES) std::size_t JResourcePool<DType>::dMaxPoolSize{10};
+template <typename DType> alignas(JANA2_CACHE_LINE_BYTES) std::size_t JResourcePool<DType>::dPoolCounter{0};
+template <typename DType> alignas(JANA2_CACHE_LINE_BYTES) std::atomic<std::size_t> JResourcePool<DType>::dObjectCounter{0};
 
 //CONSTRUCTORS
 template <typename DType> JResourcePool<DType>::JResourcePool(std::size_t sMaxPoolSize, std::size_t sDebugLevel) : JResourcePool()