From 7231a05a945a2fcd101b1542668a0c6d6eeab044 Mon Sep 17 00:00:00 2001 From: luav Date: Thu, 21 Jun 2018 20:44:57 +0200 Subject: [PATCH] Description and comments refined --- README.md | 1 - mpepool.py | 7 +++---- setup.py | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7d584c3..c4d48cd 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ Large executing jobs can be postponed for the later execution with less number o Demo of the *scheduling with memory constraints* for the worker processes: ![mpepool_memory](images/mpepool_mem.png) -> Zombie processes still may appear on abnormal termination of worker processes during rescheduling due to some bug in the Python interpreter. Anyway all resources are clearer up and the zombies disappear on the execution pool completion. Demo of the *scheduling with cache L1 maximization* for single-threaded processes on the server with cross-node CPUs enumeration. Whole physical CPU core consisting of two hardware threads assigned to each worker process, so the L1 cache is dedicated (not shared), but the maximal loading over all CPUs is 50%: ![mpepool_cacheL1_1](images/mpepool_cacheL1_1.png) diff --git a/mpepool.py b/mpepool.py index ef162cf..31d54e4 100755 --- a/mpepool.py +++ b/mpepool.py @@ -2227,7 +2227,7 @@ def __start(self, job, concur=True): # Kill non-terminated process if active: if job.proc.poll() is None: - print(' Killing ~zombie "{}" #{} ...'.format(job.name, job.proc.pid), file=sys.stderr) + print(' Killing ~hanged "{}" #{} ...'.format(job.name, job.proc.pid), file=sys.stderr) job.proc.kill() self.__complete(job, False) # ATTENTION: re-raise exception for the BaseException but not Exception sub-classes @@ -2523,7 +2523,6 @@ def __reviseWorkers(self): memall = self.memlimit + memov # Note: memov is negative here # Process completed (and terminated) jobs: execute callbacks and remove the workers - #cterminated = False # Completed terminated procs processed for job in completed: # Note: check for the termination in all cycles if not self.alive: @@ -2533,7 +2532,6 @@ def __reviseWorkers(self): exectime = job.tstop - job.tstart # Restart the job if it was terminated and should be restarted if not job.terminates: - #cterminated = True continue print('WARNING, "{}" #{} is terminated because of the {} violation' ', chtermtime: {}, consumes {:.4f} / {:.4f} GB, timeout {:.4f} sec, executed: {:.4f} sec ({} h {} m {:.4f} s)' @@ -2575,7 +2573,8 @@ def __reviseWorkers(self): self.__postpone(job, True) # Note: the number of workers is not reduced to less than 1 - # Note: active_children() does not impact on the existence of zombie procs + # Note: active_children() does not impact on the existence of zombie procs, + # proc table clearup implemented in complete() using wait() #if cterminated: # # Note: required to join terminated child procs and avoid zombies # # Return list of all live children of the current process,joining any processes which have already finished diff --git a/setup.py b/setup.py index fb170c4..606f806 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ setup( name=pkgname, # This is the name of your PyPI-package. - # version='2.2.0', # Update the version number for new releases version='3.1.0', # Update the version number for new releases description=('A lightweight multi-process Execution Pool with load balancing' ' and customizable resource consumption constraints.'), # Required, "Summary" metadata field