properly close curses UI, skip hwaccel on comskip with csv file (#59)

.github/workflows/publish.yaml

@@ -31,13 +31,13 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           cache: 'pip'
 
       - uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: '11'
+          java-version: '17'
 
       - name: Install hunspell
         run:  sudo apt-get install -y hunspell libhunspell-dev hunspell-en-us

dvrprocess/comchap.py

@@ -367,16 +367,6 @@ def do_comchap(infile, outfile, edlfile=None, delete_edl=True, delete_meta=True,
                                          input_info=input_info, workdir=workdir,
                                          keep=not delete_ini, log_file=not delete_log)
 
-        comskip_command = []
-
-        if debug:
-            comskip_invoker = tools.comskip_gui
-            comskip_command.append("-w")
-        else:
-            comskip_invoker = tools.comskip
-            # TODO: only include hwassist when csv isn't available
-            comskip_command.extend(get_comskip_hwassist_options())
-
         # check for csv and logo file which makes the process much faster
         if not os.path.exists(csvfile):
             csvfile = common.replace_extension(infile, 'csv')
@@ -386,6 +376,14 @@ def do_comchap(infile, outfile, edlfile=None, delete_edl=True, delete_meta=True,
         if not os.path.isfile(logofile):
             logofile = common.replace_extension(os.path.join(workdir, os.path.basename(infile)), 'logo.txt')
 
+        comskip_command = []
+
+        if debug:
+            comskip_invoker = tools.comskip_gui
+            comskip_command.append("-w")
+        else:
+            comskip_invoker = tools.comskip
+
         comskip_command.extend([f"--output={workdir}", f"--output-filename={outfile_base}",
                                 f"--ini={comskip_temp}"])
 
@@ -395,6 +393,7 @@ def do_comchap(infile, outfile, edlfile=None, delete_edl=True, delete_meta=True,
         if use_csv and os.path.isfile(csvfile):
             comskip_command.append(csvfile)
         else:
+            comskip_command.extend(get_comskip_hwassist_options())
             comskip_command.append(infile)
 
         logger.debug(' '.join(comskip_command))

dvrprocess/common/__init__.py

@@ -143,11 +143,11 @@ def finish():
     global TEMPFILENAMES
     while TEMPFILENAMES:
         fn = TEMPFILENAMES.pop()
-        if os.path.isfile(fn):
-            try:
+        try:
+            if os.path.isfile(fn):
                 os.remove(fn)
-            except FileNotFoundError:
-                pass
+        except FileNotFoundError:
+            pass
 
 
 def get_plex_url():
@@ -1018,40 +1018,52 @@ def setup_cli(level=logging.INFO, start_gauges=True):
         progress.start_compute_gauges()
 
 
+_CLI_WRAPPED: bool = False
+
+
 def cli_wrapper(func, *args, **kwargs):
-    if not args and not kwargs:
-        cli = True
-        args = sys.argv[1:]
-    else:
-        cli = False
-
-    no_curses_opt = False
-    if '--no-curses' in args:
-        no_curses_opt = True
-        args = list(filter(lambda e: e != '--no-curses', args))
-    if kwargs:
-        no_curses_opt = kwargs.get('no_curses', False) or kwargs.get('no-curses', False)
-        kwargs = kwargs.copy()
-        kwargs.pop('no_curses', '')
-        kwargs.pop('no-curses', '')
-
-    def wrapped_func() -> int:
-        try:
-            atexit.register(finish)
-            procprofile.memory_monitor_start()
-            if cli:
-                return func(args)
-            else:
-                return func(*args, **kwargs)
-        finally:
-            procprofile.memory_monitor_stop()
+    global _CLI_WRAPPED
+    try:
+        if _CLI_WRAPPED:
+            return func(*args, **kwargs)
+        _CLI_WRAPPED = True
 
-    use_curses = sys.stdout.isatty() and not no_curses_opt
-    if use_curses:
-        sys.exit(terminalui_wrapper(wrapped_func))
-    else:
-        setup_cli()
-        sys.exit(wrapped_func())
+        if not args and not kwargs:
+            cli = True
+            args = sys.argv[1:]
+        else:
+            cli = False
+
+        no_curses_opt = False
+        if '--no-curses' in args:
+            no_curses_opt = True
+            args = list(filter(lambda e: e != '--no-curses', args))
+        if kwargs:
+            no_curses_opt = kwargs.get('no_curses', False) or kwargs.get('no-curses', False)
+            kwargs = kwargs.copy()
+            kwargs.pop('no_curses', '')
+            kwargs.pop('no-curses', '')
+
+        def wrapped_func() -> int:
+            try:
+                atexit.register(finish)
+                procprofile.memory_monitor_start()
+                if cli:
+                    return func(args)
+                else:
+                    return func(*args, **kwargs)
+            finally:
+                procprofile.memory_monitor_stop()
+
+        use_curses = sys.stdout.isatty() and not no_curses_opt
+        if use_curses:
+            sys.exit(terminalui_wrapper(wrapped_func))
+        else:
+            setup_cli()
+            sys.exit(wrapped_func())
+
+    finally:
+        _CLI_WRAPPED = False
 
 
 class PoolApplyWrapper:
@@ -1382,7 +1394,7 @@ def match_owner_and_perm(target_path: str, source_path: str) -> bool:
         st_mode = source_stat.st_mode
         # if source is dir and has suid or guid and target is a file, mask suid/guid
         if os.path.isfile(target_path):
-            st_mode &= ~(stat.S_ISUID | stat.S_ISGID)
+            st_mode &= ~(stat.S_ISUID | stat.S_ISGID | 0o111)
         os.chmod(target_path, st_mode)
     except OSError:
         logger.warning(f"Changing permission of {target_path} failed, continuing")

dvrprocess/common/progress.py

@@ -298,9 +298,12 @@ def gauge(self, name: str, low: float = None, high: float = None) -> Gauge:
 _progress_reporter = ProgressReporter()
 
 
-def set_progress_reporter(new_reporter: ProgressReporter):
+def set_progress_reporter(new_reporter: Union[ProgressReporter, None]):
     global _progress_reporter
-    _progress_reporter = new_reporter
+    if new_reporter is None:
+        _progress_reporter = ProgressReporter()
+    else:
+        _progress_reporter = new_reporter
 
 
 def progress(task: str, start: int, end: int, msg: Union[None, str] = None) -> Progress:

dvrprocess/common/terminalui.py

@@ -113,6 +113,9 @@ def progress(self, task, position: int, msg: str):
     def stop(self, task):
         pass
 
+    def close(self):
+        pass
+
 
 class ProgressCurses(progress.Progress):
 
@@ -141,13 +144,23 @@ def progress(self, position: int, msg: Union[str, None] = None, start: Union[int
 class ProgressWindow(CursesProgressListener):
 
     def __init__(self, window):
+        self._closed = False
         self.window = window
         self.tasks: Dict[str, ProgressCurses] = dict()
 
+    def close(self):
+        self._closed = True
+
     def refresh(self):
+        if self._closed:
+            return
+
         self.window.refresh()
 
     def resize(self):
+        if self._closed:
+            return
+
         try:
             _curses_ui_lock.acquire()
             lines, cols = self.window.getmaxyx()
@@ -222,6 +235,9 @@ def _draw(self, task: ProgressCurses, position: int = 0, msg: str = None):
         self.window.clrtoeol()
 
     def start(self, task: ProgressCurses):
+        if self._closed:
+            return
+
         _check_resize()
         try:
             _curses_ui_lock.acquire()
@@ -238,6 +254,9 @@ def start(self, task: ProgressCurses):
             _curses_ui_lock.release()
 
     def progress(self, task: ProgressCurses, position: int, msg: str):
+        if self._closed:
+            return
+
         _check_resize()
         try:
             _curses_ui_lock.acquire()
@@ -251,6 +270,9 @@ def progress(self, task: ProgressCurses, position: int, msg: str):
             _curses_ui_lock.release()
 
     def stop(self, task: ProgressCurses):
+        if self._closed:
+            return
+
         _check_resize()
         try:
             _curses_ui_lock.acquire()
@@ -270,6 +292,9 @@ def create(self, gauge):
     def value(self, gauge, value: float):
         pass
 
+    def close(self):
+        pass
+
 
 class GaugeCurses(progress.Gauge):
     def __init__(self, name: str, low: float, high: float, listener: CursesGaugeListener):
@@ -285,10 +310,17 @@ def value(self, value: float):
 
 class GaugeWindow(CursesGaugeListener):
     def __init__(self, window):
+        self._closed = False
         self.window = window
         self.gauges: Dict[str, GaugeCurses] = dict()
 
+    def close(self):
+        self._closed = True
+
     def resize(self):
+        if self._closed:
+            return
+
         self.window.erase()
         self.window.move(self.window.getbegyx()[0], self.window.getbegyx()[1])
         try:
@@ -311,10 +343,16 @@ def resize(self):
         self.window.refresh()
 
     def create(self, gauge):
+        if self._closed:
+            return
+
         self.gauges[gauge.name] = gauge
         self.resize()
 
     def value(self, gauge, value: float):
+        if self._closed:
+            return
+
         self.resize()
 
 
@@ -333,6 +371,8 @@ def _create_gauge(self, name: str, low: float = None, high: float = None) -> pro
 
 class CursesUI(object):
     def __init__(self, screen):
+        self._closed = False
+
         # hide the cursor
         try:
             curses.curs_set(0)
@@ -362,6 +402,23 @@ def __init__(self, screen):
         self.gauge_win.resize()
         self.log_handler.resize()
 
+    def close(self):
+        if self._closed:
+            return
+
+        self._closed = True
+        logging.root.removeHandler(self.log_handler)
+        self.log_handler.close()
+        progress.set_progress_reporter(None)
+        self.progress_win.close()
+        self.gauge_win.close()
+
+        # show the cursor
+        try:
+            curses.curs_set(1)
+        except Exception:
+            pass
+
     def _compute_window_dims(self) -> list[tuple[int, int, int, int]]:
         """
         :return: tuples of (lines, columns, y, x): [ status, progress, log ]
@@ -373,6 +430,9 @@ def _compute_window_dims(self) -> list[tuple[int, int, int, int]]:
         return [status_win, progress_win, log_win]
 
     def resize(self):
+        if self._closed:
+            return
+
         self.screen.refresh()
         window_dims = self._compute_window_dims()
         self._resize_window(self.gauge_win.window, window_dims[0])
@@ -407,10 +467,8 @@ def main(screen) -> int:
             return func(*args, **kwargs)
         finally:
             # show the cursor
-            try:
-                curses.curs_set(1)
-            except Exception:
-                pass
+            _CURSESUI.close()
+            _CURSESUI = None
 
     stderr_capture = StreamCapture('stderr')
     stdout_capture = StreamCapture('stdout')

dvrprocess/comtune.py

@@ -966,7 +966,7 @@ def tune_show(season_dir, process_pool: Pool, files, workdir, dry_run, force, ex
     target_comskip_ini = os.path.join(season_dir, 'comskip.ini')
 
     # https://pygad.readthedocs.io/en/latest/README_pygad_ReadTheDocs.html#pygad-ga-class
-    num_generations = 50
+    num_generations = 100
     sol_per_pop = 500
     num_parents_mating = ceil(sol_per_pop / 2)
     keep_elitism = 5
@@ -978,7 +978,8 @@ def tune_show(season_dir, process_pool: Pool, files, workdir, dry_run, force, ex
             force=force, comskip_defaults=comskip_defaults,
             expensive_genes=expensive_genes, check_compute=check_compute, num_generations=num_generations,
             experimental=experimental,
-            file_sample_size=10)
+            file_sample_size=15,
+        )
     except UserWarning as e:
         logger.warning(e.args[0])
         thread_pool.shutdown(cancel_futures=True)
@@ -1166,8 +1167,6 @@ def comtune_cli_run(media_paths: list[str], verbose: bool, workdir, force: int,
                     check_compute: bool, time_limit: int, processes: int) -> int:
     logger.debug("work_dir is %s, processes is %s", workdir, processes)
 
-    atexit.register(common.finish)
-
     time_start = time.time()
     time_progress = progress.progress("time limit", 0, time_limit)
     time_progress.renderer = common.s_to_ts

dvrprocess/profanity-filter-apply.py

@@ -47,6 +47,13 @@ def usage():
     Find files to process from a Plex Media Server. Specify the URL such as http://127.0.0.1:32400 or '.' for {common.get_plex_url()}
 --verbose
     Verbose information about the process
+
+Environment:
+    LANGUAGE_TOOL_HOST=127.0.0.1
+    LANGUAGE_TOOL_PORT=8100
+    KALDI_EN_HOST=kaldi-en
+    KALDI_EN_PORT=2700
+      (duplicate KALDI_* above for other languages use 2-letter code)
 """, file=sys.stderr)