add support for cpu binding with srun

Greasy reads how many CPUs avail in reservation
then automatically creates strides and binds tasks
to CPUs if variable usecpubinding is set to 1. Cannot
be used if partial node is reserved, also changed
default num workers to cpus in greasy reservation/2.
Details of this process added to logs, (level 3 and 4).

src/abstractengine.cpp

@@ -111,6 +111,8 @@ bool AbstractEngine::isReady() {
 }
 
 void AbstractEngine::init() {
+  n_node_cpus = sysconf(_SC_NPROCESSORS_ONLN);
+  n_slurm_reservation_cpus = fromString(n_slurm_reservation_cpus, getenv("SLURM_NPROCS"));
 
   log->record(GreasyLog::devel, "AbstractEngine::init", "Entering...");
 
@@ -134,6 +136,23 @@ void AbstractEngine::init() {
       log->record(GreasyLog::warning, "Consider setting environment variable GREASY_NWORKERS to the desired cpus to use");
     }
   }
+
+  log->record(GreasyLog::debug, "Configuration: \n" + config->printContents());
+
+  // Determine whether to use CPU binding (bool) expect 1 or 0 in config file
+  if (config->keyExists("UseCPUBinding")) {
+    usecpubinding = fromString(usecpubinding, config->getValue("UseCPUBinding"));
+    if (usecpubinding) {
+      if (n_node_cpus != n_slurm_reservation_cpus) {
+        usecpubinding = 0;
+        log->record(GreasyLog::warning, "CPU binding is not available for partial node SLURM reservations, CPU binding deactivated.");
+      }
+    }
+  } else {
+    log->record(GreasyLog::warning, "CPU binding falling back to default (deactivated)");
+    log->record(GreasyLog::warning, "Consider setting environment variable GREASY_USECPUBINDING to 0 or 1.");
+    usecpubinding = 0;
+  }
 
   if (!fileErrors) {
     if(nworkers>0) {  

src/abstractengine.h

@@ -137,7 +137,10 @@ class AbstractEngine
   string restartFile; /**< Path to the file where the restart will be written. */
   int nworkers; /**< Number of greasy workers (possibly the number of cpus available). */
   bool ready; /**< Flag to know if the engine is ready to run. */
-
+  bool usecpubinding;
+  int n_node_cpus;
+  int n_slurm_reservation_cpus;
+
   map<int,GreasyTask*> taskMap; ///< Main task map linking the taskId (the line in the file)
 				///< with the actual GreasyTask object.
   set<int> validTasks; /**< Set containing the valid tasks read in the file. */

src/abstractschedulerengine.cpp

@@ -30,14 +30,26 @@ AbstractSchedulerEngine::AbstractSchedulerEngine ( const string& filename) : Abs
 }
 
 void AbstractSchedulerEngine::init() {
-    
+
   log->record(GreasyLog::devel, "AbstractSchedulerEngine::init", "Entering...");
 
   AbstractEngine::init();
 
+  if (usecpubinding) {
+    log->record(GreasyLog::info, "Creating " + toString(nworkers) + " CPU binding workers with strides for " + toString(n_node_cpus) + " CPUs." );
+  }
+
   // Fill the freeWorkers queue
-  for (int i=1;i<=nworkers; i++) {
-    freeWorkers.push(i);
+  // If there is cpu binding it will create strides. This is only desirable if
+  // CPU numbering is done sequentially by socket. If sockets are numbered even/odd
+  // then this will create the worst possible load splitting. 
+  for (int i=0; i<nworkers; i++) {
+    if (usecpubinding) {
+      worker_id = i*n_node_cpus/nworkers; // will create evenly spaced strides
+    } else {
+      worker_id = i;
+    }
+    freeWorkers.push(worker_id);
   }
 
   log->record(GreasyLog::devel, "AbstractSchedulerEngine::init", "Exiting...");
@@ -200,8 +212,8 @@ void AbstractSchedulerEngine::taskEpilogue(GreasyTask *task) {
 
 void AbstractSchedulerEngine::getDefaultNWorkers() {
 
-  nworkers = sysconf(_SC_NPROCESSORS_ONLN);
-  if ( nworkers>4 ) nworkers=4;
+  nworkers = n_slurm_reservation_cpus;
+  if ( nworkers>4 ) nworkers/=2;
 
   log->record(GreasyLog::devel, "AbstractSchedulerEngine::getDefaultNWorkers", "Default nworkers: " + toString(nworkers));
 

src/abstractschedulerengine.h

@@ -102,6 +102,7 @@ class AbstractSchedulerEngine : public AbstractEngine
    */  
   virtual void taskEpilogue(GreasyTask *task);
 
+  int worker_id;
 
   map <int,int> taskAssignation; ///<  Map that holds the task assignation to workers.
 				 ///< worker -> taskId

src/basicengine.cpp

@@ -47,6 +47,7 @@ void BasicEngine::init() {
 
   gethostname(hostname, sizeof(hostname));
   masterHostname=hostname;
+  slurm_partition = getenv("SLURM_JOB_PARTITION");
 
   // Setup the nodelist
   if (config->keyExists("NodeList")) {
@@ -200,8 +201,13 @@ int BasicEngine::executeTask(GreasyTask *task, int worker) {
 	} 
     }
   }
+
+  if (usecpubinding) {
+    command = "srun -n 1 -N 1 --partition=" + slurm_partition + " -w " + node + " --cpu-bind=map_cpu:" + toString(worker) + " ";
+  }
 
   command += task->getCommand();
+  log->record(GreasyLog::debug, "Worker " + toString(worker) + " executing command=" + command);
   log->record(GreasyLog::devel,  "BasicEngine::executeTask[" + toString(worker) +"]", "Task " 
 		      + toString(task->getTaskId()) + " on node " + node + " command: " + command);
   ret =  system(command.c_str());

src/basicengine.h

@@ -81,6 +81,7 @@ class BasicEngine : public AbstractSchedulerEngine
    * @return True if local node, false otherwise.
    */  
   bool isLocalNode(string node);
+  string slurm_partition;
 
   /**
    * Get the name of the node where the worker is asigned.