#8 make memory usage more effective and adjustable

 #6 it may happen that finish log is triggered too early
 first steps for #7 better handling of connectionTimeout especially on commit

jminer.default.properties

@@ -163,4 +163,7 @@ byteUnitDecimal=true
 # (default:false) shows info about every drive on finish reading it, useful to find the slow ones ...
 showDriveInfo=false
 
+# (default:0) 0 -> number of plotPaths is taken (one thread per drive) this is recommend, choosing a lower number can be useful on memory issues. (a higher number is useless)
+readerThreads=0
+
 

pom.xml

@@ -104,6 +104,7 @@
                 <directory>src/main/resources</directory>
                 <includes>
                     <include>**/*.txt</include>
+                    <include>**/*.properties</include>
                 </includes>
             </resource>
         </resources>

src/main/java/burstcoin/jminer/CommandLineRunner.java

@@ -32,6 +32,7 @@
 import burstcoin.jminer.core.network.model.DevPoolResult;
 import burstcoin.jminer.core.reader.event.ReaderCorruptFileEvent;
 import burstcoin.jminer.core.reader.event.ReaderDriveFinishEvent;
+import burstcoin.jminer.core.reader.event.ReaderDriveInterruptedEvent;
 import burstcoin.jminer.core.reader.event.ReaderProgressChangedEvent;
 import burstcoin.jminer.core.round.Round;
 import burstcoin.jminer.core.round.event.RoundFinishedEvent;
@@ -57,11 +58,12 @@ public class CommandLineRunner
   private static final Logger LOG = LoggerFactory.getLogger(CommandLineRunner.class);
 
   private static final int NUMBER_OF_PROGRESS_LOGS_PER_ROUND = CoreProperties.getReadProgressPerRound();
+
   private static final int SIZE_DIVISOR = CoreProperties.isByteUnitDecimal() ? 1000 : 1024;
   private static final String T_UNIT = CoreProperties.isByteUnitDecimal() ? "TB" : "TiB";
   private static final String G_UNIT = CoreProperties.isByteUnitDecimal() ? "GB" : "GiB";
-
   private static final String M_UNIT = CoreProperties.isByteUnitDecimal() ? "MB" : "MiB";
+
   private static ConfigurableApplicationContext context;
   private static boolean roundFinished = true;
   private static long blockNumber;
@@ -209,6 +211,9 @@ public void onApplicationEvent(ReaderProgressChangedEvent event)
             {
               progressLogStep--;
 
+              // trigger garbage collection on every progress step
+              System.gc();
+
               BigDecimal totalCapacity = new BigDecimal(event.getCapacity());
               BigDecimal factor = BigDecimal.ONE.divide(totalCapacity, MathContext.DECIMAL32);
               BigDecimal progress = factor.multiply(new BigDecimal(event.getCapacity() - event.getRemainingCapacity()));
@@ -325,6 +330,15 @@ public void onApplicationEvent(ReaderDriveFinishEvent event)
           }
         });
 
+        context.addApplicationListener(new ApplicationListener<ReaderDriveInterruptedEvent>()
+        {
+          @Override
+          public void onApplicationEvent(ReaderDriveInterruptedEvent event)
+          {
+            LOG.info("stopped '" + event.getDirectory() + "' for block '" + event.getBlockNumber() + "'.");
+          }
+        });
+
         LOG.info("");
         LOG.info("                            Burstcoin (BURST)");
         LOG.info("            __         __   GPU assisted PoC-Miner");

src/main/java/burstcoin/jminer/core/CoreConfig.java

@@ -50,8 +50,8 @@ public ThreadPoolTaskExecutor readerPool()
   {
     ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();
     pool.setThreadPriority(Thread.NORM_PRIORITY);
-    // tasks will interrupt them self and stop fast.
-    pool.setWaitForTasksToCompleteOnShutdown(true);
+    // false-> triggers interrupt exception at shutdown
+    pool.setWaitForTasksToCompleteOnShutdown(false);
     return pool;
   }
 
@@ -66,7 +66,6 @@ public ThreadPoolTaskExecutor networkPool()
     ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor();
     pool.setMaxPoolSize(2);
     pool.setThreadPriority(Thread.NORM_PRIORITY + 1);
-    pool.setWaitForTasksToCompleteOnShutdown(true);
     return pool;
   }
 

src/main/java/burstcoin/jminer/core/CoreProperties.java

@@ -62,6 +62,7 @@ public class CoreProperties
   private static final boolean DEFAULT_BYTE_UNIT_DECIMAL = true;
   private static final boolean DEFAULT_LIST_PLOT_FILES = false;
   private static final boolean DEFAULT_SHOW_DRIVE_INFO = false;
+  private static final int DEFAULT_READER_THREADS = 0;
   // there seams to be a issue on checker
   private static final boolean DEFAULT_OPT_DEV_POOL = false;
 
@@ -102,6 +103,7 @@ public class CoreProperties
   private static Boolean byteUnitDecimal;
   private static Boolean listPlotFiles;
   private static Boolean showDriveInfo;
+  private static Integer readerThreads;
 
   private CoreProperties()
   {
@@ -399,6 +401,15 @@ public static int getRestartInterval()
     return restartInterval;
   }
 
+  public static int getReaderThreads()
+  {
+    if(readerThreads == null)
+    {
+      readerThreads = asInteger("readerThreads", DEFAULT_READER_THREADS);
+    }
+    return readerThreads;
+  }
+
   /**
    * Gets plot paths.
    *

src/main/java/burstcoin/jminer/core/checker/Checker.java

@@ -113,7 +113,7 @@ public void handleMessage(ReaderLoadedPartEvent event)
     }
     else
     {
-      LOG.debug("skipped check scoop ... old block ...");
+      LOG.trace("skipped check scoop ... old block ...");
     }
   }
 }

src/main/java/burstcoin/jminer/core/checker/util/OCLChecker.java

@@ -131,9 +131,9 @@ public void initChecker(int platformId, int deviceId)
     for(int i = 0; i < 3; i++)
     {
       clGetKernelWorkGroupInfo(kernel[i], devices[deviceId], CL_KERNEL_WORK_GROUP_SIZE, 8, Pointer.to(maxWorkGroupSize), null);
-      LOG.debug("Max work group size: " + maxWorkGroupSize[0]);
       workgroupSize[i] = maxWorkGroupSize[0];
     }
+    LOG.debug("Max work group size: " + maxWorkGroupSize[0]);
 
     long[] maxComputeUnits = new long[1];
     clGetDeviceInfo(devices[deviceId], CL_DEVICE_MAX_COMPUTE_UNITS, 8, Pointer.to(maxComputeUnits), null);

src/main/java/burstcoin/jminer/core/network/Network.java

@@ -153,14 +153,14 @@ public void checkLastWinner(long blockNumber)
   }
 
   /**
-   * Check result.
+   * Commit result.
    *
    * @param blockNumber the block number
    * @param calculatedDeadline the calculated deadline
    * @param nonce the nonce
    * @param chunkPartStartNonce the chunk part start nonce
    */
-  public void checkResult(long blockNumber, long calculatedDeadline, long nonce, long chunkPartStartNonce)
+  public void commitResult(long blockNumber, long calculatedDeadline, long nonce, long chunkPartStartNonce)
   {
     if(poolMining)
     {
@@ -189,12 +189,12 @@ public void checkResult(long blockNumber, long calculatedDeadline, long nonce, l
   }
 
   /**
-   * Check dev result.
+   * Commit dev result.
    *
    * @param blockNumber the block number
    * @param devPoolResults the dev pool results
    */
-  public void checkDevResult(long blockNumber, List<DevPoolResult> devPoolResults)
+  public void commitDevResult(long blockNumber, List<DevPoolResult> devPoolResults)
   {
     NetworkSubmitDevPoolNoncesTask networkSubmitDevPoolNoncesTask = context.getBean(NetworkSubmitDevPoolNoncesTask.class);
     networkSubmitDevPoolNoncesTask.init(blockNumber, numericAccountId, poolServer, connectionTimeout, devPoolResults);

src/main/java/burstcoin/jminer/core/network/task/NetworkSubmitDevPoolNoncesTask.java

@@ -37,6 +37,7 @@
 
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 /**
  * The type Network submit dev pool nonces task.
@@ -107,6 +108,10 @@ public void run()
         LOG.error("Error: could not commit nonces to devPool: " + submitResult);
       }
     }
+    catch(TimeoutException timeoutException)
+    {
+      LOG.warn("Unable to commit nonces to devPool, caused by connectionTimeout, currently '" + (connectionTimeout/1000) + " sec.' try increasing it!");
+    }
     catch(Exception e)
     {
       LOG.error("Error: Failed to submit nonce to devPool: " + e.getMessage());

src/main/java/burstcoin/jminer/core/network/task/NetworkSubmitPoolNonceTask.java

@@ -38,6 +38,7 @@
 import org.springframework.stereotype.Component;
 
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 /**
  * The type Network submit pool nonce task.
@@ -131,6 +132,10 @@ public void run()
         }
       }
     }
+    catch(TimeoutException timeoutException)
+    {
+      LOG.warn("Unable to commit nonce to pool, caused by connectionTimeout, currently '" + (connectionTimeout/1000) + " sec.' try increasing it!");
+    }
     catch(Exception e)
     {
       LOG.warn("Error: Failed to submit nonce to pool: " + e.getMessage());

src/main/java/burstcoin/jminer/core/network/task/NetworkSubmitSoloNonceTask.java

@@ -37,6 +37,7 @@
 import org.springframework.stereotype.Component;
 
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 /**
  * The type Network submit solo nonce task.
@@ -121,6 +122,10 @@ public void run()
         LOG.warn("Error: Submit solo nonce not successful: " + response.getContentAsString());
       }
     }
+    catch(TimeoutException timeoutException)
+    {
+      LOG.warn("Unable to solo nonce, caused by connectionTimeout, currently '" + (connectionTimeout/1000) + " sec.' try increasing it!");
+    }
     catch(Exception e)
     {
       LOG.warn("Error: Failed to submit solo nonce: " + e.getMessage());

src/main/java/burstcoin/jminer/core/reader/Reader.java

@@ -53,8 +53,6 @@
 import java.util.List;
 import java.util.Map;
 
-//import org.springframework.context.event.EventListener;
-
 /**
  * The type Reader.
  */
@@ -88,6 +86,7 @@ public class Reader
   private long remainingCapacity;
   private long capacity;
   private long readerStartTime;
+  private int readerThreads;
 
   /**
    * Post construct.
@@ -123,15 +122,12 @@ protected void postConstruct()
     directories = CoreProperties.getPlotPaths();
     chunkPartNonces = CoreProperties.getChunkPartNonces();
     scanPathsEveryRound = CoreProperties.isScanPathsEveryRound();
+    readerThreads = CoreProperties.getReaderThreads();
     capacityLookup = new HashMap<>();
 
-    if(!scanPathsEveryRound)
+    if(CoreProperties.isListPlotFiles())
     {
-      plots = new Plots(directories, numericAccountId, chunkPartNonces);
-      if(CoreProperties.isListPlotFiles())
-      {
-        plots.printPlotFiles();
-      }
+      getPlots().printPlotFiles();
     }
   }
 
@@ -140,17 +136,13 @@ protected void postConstruct()
    *
    * @param blockNumber the block number
    * @param scoopNumber the scoop number
-   * @return chunkPartStartNonces that will be read and their capacity
    */
-  public Plots read(long previousBlockNumber, long blockNumber, int scoopNumber, long lastBestCommittedDeadline)
+  public void read(long previousBlockNumber, long blockNumber, int scoopNumber, long lastBestCommittedDeadline)
   {
     this.blockNumber = blockNumber;
 
-    // re-scan drives each round
-    if(scanPathsEveryRound)
-    {
-      plots = new Plots(directories, numericAccountId, chunkPartNonces);
-    }
+    // ensure plots are initialized
+    plots = plots == null ? getPlots() : plots;
 
     if(readerPool.getActiveCount() > 0)
     {
@@ -161,8 +153,9 @@ public Plots read(long previousBlockNumber, long blockNumber, int scoopNumber, l
     }
 
     // update reader thread count
-    readerPool.setCorePoolSize(directories.size());
-    readerPool.setMaxPoolSize(directories.size());
+    int poolSize = readerThreads <= 0 ? directories.size() : readerThreads;
+    readerPool.setCorePoolSize(poolSize);
+    readerPool.setMaxPoolSize(poolSize);
 
     // we use the startnonce of loaded (startnonce+chunk+part) as unique job identifier
     capacityLookup.clear();
@@ -173,15 +166,30 @@ public Plots read(long previousBlockNumber, long blockNumber, int scoopNumber, l
 
     readerStartTime = new Date().getTime();
 
+    // todo why use a threadPool in case we have one thread per drive anyway
     for(PlotDrive plotDrive : plots.getPlotDrives())
     {
       ReaderLoadDriveTask readerLoadDriveTask = context.getBean(ReaderLoadDriveTask.class);
       readerLoadDriveTask.init(scoopNumber, blockNumber, plotDrive);
       readerPool.execute(readerLoadDriveTask);
     }
+  }
+
+  public Plots getPlots()
+  {
+    if(scanPathsEveryRound || plots == null)
+    {
+      plots = new Plots(directories, numericAccountId, chunkPartNonces);
+    }
     return plots;
   }
 
+  public void freeResources()
+  {
+    readerPool.setCorePoolSize(1);
+    readerPool.setMaxPoolSize(1);
+  }
+
   @Override
   @EventListener
   public void handleMessage(ReaderLoadedPartEvent event)
@@ -204,7 +212,7 @@ public void handleMessage(ReaderLoadedPartEvent event)
     }
     else
     {
-      LOG.debug("update reader progress skipped ... old block ...");
+      LOG.trace("update reader progress skipped ... old block ...");
     }
   }
 

src/main/java/burstcoin/jminer/core/reader/event/ReaderDriveInterruptedEvent.java

@@ -0,0 +1,53 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 by luxe - https://github.com/de-luxe -  BURST-LUXE-RED2-G6JW-H4HG5
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+ * and associated documentation files (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge, publish, distribute,
+ * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software
+ * is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies
+ * or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+package burstcoin.jminer.core.reader.event;
+
+import org.springframework.context.ApplicationEvent;
+
+/**
+ * fired if read is interrupted on shutdown threadPool
+ */
+public class ReaderDriveInterruptedEvent
+  extends ApplicationEvent
+{
+  private long blockNumber;
+  private String directory;
+
+  public ReaderDriveInterruptedEvent(long blockNumber, String directory)
+  {
+    super(blockNumber);
+
+    this.blockNumber = blockNumber;
+    this.directory = directory;
+  }
+
+  public long getBlockNumber()
+  {
+    return blockNumber;
+  }
+
+  public String getDirectory()
+  {
+    return directory;
+  }
+}