code quality

.github/workflows/publish.yaml

@@ -32,6 +32,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
+          cache: 'pip'
 
       - uses: actions/setup-java@v4
         with:
@@ -41,13 +42,30 @@ jobs:
       - name: Install hunspell
         run:  sudo apt-get install -y hunspell libhunspell-dev hunspell-en-us
 
+      - name: Lint with Ruff
+        run: |
+          pip install ruff
+          ruff check --output-format=github .
+        continue-on-error: true
+
       - name: test
         run: |
           pip install -r requirements.txt
           pip install -r dev_requirements.txt
           python3 -c "import language_tool_python; tool = language_tool_python.LanguageTool('en')"
           cd dvrprocess
-          pytest --cov . --cov-branch --cov-report term-missing
+          pytest . --junitxml=../junit/test-results.xml --cov --cov-branch --cov-report=xml:../coverage.xml --cov-report=html:../htmlcov --cov-report=term-missing
+
+      - name: Upload pytest test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: pytest-results
+          path: |
+            junit/test-results.xml
+            coverage.xml
+            htmlcov
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ always() }}
 
   build-and-push-image:
     runs-on: ubuntu-22.04

.gitignore

@@ -39,13 +39,14 @@ pip-delete-this-directory.txt
 
 # Unit test / coverage reports
 htmlcov/
+junit/
 .tox/
 .nox/
 .coverage
 .coverage.*
 .cache
-nosetests.xml
-coverage.xml
+nosetests.*xml
+coverage*.xml
 *.cover
 *.py,cover
 .hypothesis/

Dockerfile-test

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:23.10
 
 ENV DEBIAN_FRONTEND=noninteractive
 

dev_requirements.txt

@@ -1,2 +1,2 @@
-pytest==7.4.2
-pytest-cov==4.1.0
+pytest==8.1.1
+pytest-cov==5.0.0

dvrprocess/comchap-apply.py

@@ -129,7 +129,7 @@ def comchap_apply(media_paths, dry_run=False, comskip_ini=None, workdir=None, fo
                             return_code = result[1].get()
                         except CalledProcessError as e:
                             return_code = e.returncode
-                        except:
+                        except Exception:
                             pool.terminate()
                             return 255
 
@@ -167,7 +167,7 @@ def comchap_apply(media_paths, dry_run=False, comskip_ini=None, workdir=None, fo
                         return 0
 
                 if check_compute and common.should_stop_processing():
-                    logger.info(f"INFO: not enough compute available")
+                    logger.info("INFO: not enough compute available")
                     return 0
 
     bytes_progress.stop()

dvrprocess/comchap.py

@@ -458,7 +458,7 @@ def do_comchap(infile, outfile, edlfile=None, delete_edl=True, delete_meta=True,
     with open(metafile, "w") as metafd:
         metafd.write(";FFMETADATA1\n")
         with open(mkvchapterfile, "w") as mkvchapterfd:
-            mkvchapterfd.write(f"""<?xml version="1.0" encoding="ISO-8859-1"?>
+            mkvchapterfd.write("""<?xml version="1.0" encoding="ISO-8859-1"?>
 <!DOCTYPE Chapters SYSTEM "matroskachapters.dtd">
 <Chapters><EditionEntry>\n""")
 
@@ -485,7 +485,7 @@ def do_comchap(infile, outfile, edlfile=None, delete_edl=True, delete_meta=True,
                     write_chapter_metadata(metafd, start, duration, f"Chapter {chapter_num}")
                     write_chapter_atom(mkvchapterfd, start, duration, f"Chapter {chapter_num}")
 
-            mkvchapterfd.write(f"""</EditionEntry></Chapters>\n""")
+            mkvchapterfd.write("""</EditionEntry></Chapters>\n""")
 
     # need this outside of metafd and mkvchapterfd blocks to ensure it's flushed and closed
     if hascommercials:

dvrprocess/comcut.py

@@ -126,7 +126,7 @@ def comcut(infile, outfile, delete_edl=True, force_clear_edl=False, delete_meta=
     input_duration = float(input_info[constants.K_FORMAT]['duration'])
     if len(edl_events) > 0:
         if edl_events[-1].end > input_duration + 3:
-            logger.fatal(f"edl cuts past end of file")
+            logger.fatal("edl cuts past end of file")
             return 255
 
     keyframes = common.load_keyframes_by_seconds(infile)
@@ -468,7 +468,7 @@ def comcut(infile, outfile, delete_edl=True, force_clear_edl=False, delete_meta=
             output_mapping = f"[v{output_stream_idx}]"
             video_filter_str += output_mapping
 
-            ffmpeg_command.extend([f"-filter_complex", video_filter_str])
+            ffmpeg_command.extend(["-filter_complex", video_filter_str])
             ffmpeg_command.extend(["-map", output_mapping])
 
             ffmpeg_command.extend(encoding_options)
@@ -539,8 +539,8 @@ def comcut(infile, outfile, delete_edl=True, force_clear_edl=False, delete_meta=
     # verify video is valid
     try:
         common.find_input_info(temp_outfile or outfile)
-    except:
-        logger.error(f"Cut file is not readable by ffmpeg, skipping")
+    except Exception:
+        logger.error("Cut file is not readable by ffmpeg, skipping")
         os.remove(temp_outfile or outfile)
         return 255
 

dvrprocess/common/__init__.py

@@ -22,7 +22,7 @@
 from psutil import AccessDenied, NoSuchProcess
 
 from . import hwaccel, tools, config, constants, progress, procprofile
-from .proc_invoker import StreamCapture
+from .proc_invoker import StreamCapture, pre_flight_check
 from .terminalui import terminalui_wrapper
 
 _allocate_lock = _thread.allocate_lock
@@ -804,7 +804,7 @@ def map_opus_audio_stream(arguments: list[str], audio_info: dict, audio_stream_i
                     if i > 0:
                         mute_filter_complex += '+'
                     mute_filter_complex += f"c{in_ch_idx[0]}"
-        mute_filter_complex += f'[afiltered]'
+        mute_filter_complex += '[afiltered]'
         arguments.extend(["-filter_complex", mute_filter_complex, "-map", "[afiltered]"])
     else:
         arguments.extend(["-map", f"{audio_stream_idx}:{audio_info[constants.K_STREAM_INDEX]}"])
@@ -888,8 +888,8 @@ def filepath_is_mkv(filepath):
     return filename.endswith(".mkv") and not filename.startswith('.')
 
 
-def filter_for_mkv(l):
-    return filter(lambda f: filepath_is_mkv(f), l)
+def filter_for_mkv(file_list):
+    return filter(lambda f: filepath_is_mkv(f), file_list)
 
 
 def split_every(n, iterable):
@@ -994,7 +994,7 @@ def setup_logging(level=logging.INFO):
 
 
 def setup_cli(level=logging.INFO, start_gauges=True):
-    proc_invoker.pre_flight_check()
+    pre_flight_check()
     setup_logging(level)
     setup_debugging()
     if start_gauges:

dvrprocess/common/edl_util.py

@@ -1,5 +1,5 @@
 import logging
-from copy import copy, deepcopy
+from copy import copy
 from enum import Enum
 from typing import Union
 

dvrprocess/common/hwaccel.py

@@ -364,7 +364,7 @@ def _nvenc_encoding(output_stream: str, codec: str, output_type: str, tune: str,
 
     options = [f"-c:{output_stream}"]
     if codec in ['h265']:
-        options.extend([f"hevc_nvenc"])
+        options.extend(["hevc_nvenc"])
     else:
         options.extend([f"{codec}_nvenc"])
 

dvrprocess/common/proc_invoker.py

@@ -21,13 +21,13 @@ def __init__(self, command_basename: str, version=None):
         _all_invokers.append(self)
 
     def run(self, arguments: list[str], **kwargs) -> int:
-        raise NotImplemented()
+        raise NotImplementedError()
 
     def check_output(self, arguments: list[str], **kwargs) -> str:
-        raise NotImplemented()
+        raise NotImplementedError()
 
     def Popen(self, arguments: list[str], **kwargs) -> subprocess.Popen[str]:
-        raise NotImplemented()
+        raise NotImplementedError()
 
     def present(self) -> bool:
         """
@@ -87,12 +87,12 @@ def _run(self, arguments: list[str], kwargs) -> int:
             if result.stdout:
                 try:
                     logger.debug(str(result.stdout))
-                except:
+                except Exception:
                     pass
             if result.stderr:
                 try:
                     logger.debug(str(result.stderr))
-                except:
+                except Exception:
                     pass
 
         if result.returncode == 130:

dvrprocess/common/progress.py

@@ -544,7 +544,8 @@ def __init__(self):
         self.cpu_percent.renderer = _percent_renderer
         self.cpu_percent.critical_range = (90, 101)
 
-        loadavg_renderer = lambda v: f"{v: 2.2f}"
+        def loadavg_renderer(value):
+            return f"{value: 2.2f}"
         self.loadavg1 = gauge('load1')
         self.loadavg1.renderer = loadavg_renderer
         # self.loadavg5 = gauge('load5')

dvrprocess/common/subtitle.py

@@ -148,8 +148,8 @@ def normalized_texts(self) -> list[str]:
 
     def set_normalized_texts(self, new_values: list[str]):
         self._normalized_texts = new_values
-        l = list(map(lambda e: e[0:2], map(lambda f: f.split(), self._normalized_texts)))
-        self._normalized_start_words = set([item for sublist in l for item in sublist])
+        words = list(map(lambda e: e[0:2], map(lambda f: f.split(), self._normalized_texts)))
+        self._normalized_start_words = set([item for sublist in words for item in sublist])
 
     def normalized_start_words(self) -> set[str]:
         self._check_normalized_texts()

dvrprocess/common/terminalui.py

@@ -73,7 +73,7 @@ def emit(self, record: logging.LogRecord) -> None:
             self.y = min(self.y + 1, max_y)
 
             self.refresh()
-        except:
+        except Exception:
             # don't break the application because of a logging error
             pass
         finally:
@@ -336,7 +336,7 @@ def __init__(self, screen):
         # hide the cursor
         try:
             curses.curs_set(0)
-        except:
+        except Exception:
             pass
 
         self.gauge_win: GaugeWindow = None
@@ -409,7 +409,7 @@ def main(screen) -> int:
             # show the cursor
             try:
                 curses.curs_set(1)
-            except:
+            except Exception:
                 pass
 
     stderr_capture = StreamCapture('stderr')

dvrprocess/common/tools.py

@@ -58,13 +58,13 @@ def _run(self, arguments: list[str], kwargs) -> int:
         duration = None
         ffmpeg_progress = None
         stream_gen = ProcessStreamGenerator(proc)
-        for _, l in stream_gen.generator():
-            duration_match = self.duration_matcher.search(l)
+        for _, err_line in stream_gen.generator():
+            duration_match = self.duration_matcher.search(err_line)
             if duration_match:
                 duration = ceil(edl_util.parse_edl_ts(duration_match.group(1)))
                 if ffmpeg_progress is None:
                     ffmpeg_progress = progress.progress(task_name, 0, duration)
-            time_match = self.time_matcher.search(l)
+            time_match = self.time_matcher.search(err_line)
             if time_match and ffmpeg_progress:
                 ffmpeg_progress.progress(ceil(edl_util.parse_edl_ts(time_match.group(1))), end=duration)
         proc.wait()
@@ -125,8 +125,8 @@ def _run(self, arguments: list[str], kwargs) -> int:
         proc = subprocess.Popen(arguments, **kwargs2)
         se_progress = None
         stream_gen = ProcessStreamGenerator(proc)
-        for l, _ in stream_gen.generator():
-            pct_match = self.pct_matcher.search(l)
+        for output_line, _ in stream_gen.generator():
+            pct_match = self.pct_matcher.search(output_line)
             if pct_match:
                 if se_progress is None:
                     se_progress = progress.progress(task_name, 0, 100)
@@ -172,8 +172,8 @@ def _run(self, arguments: list[str], kwargs) -> int:
         proc = subprocess.Popen(arguments, **kwargs2)
         se_progress = None
         stream_gen = ProcessStreamGenerator(proc)
-        for l, _ in stream_gen.generator():
-            pct_match = self.pct_matcher.search(l)
+        for output_line, _ in stream_gen.generator():
+            pct_match = self.pct_matcher.search(output_line)
             if pct_match:
                 if se_progress is None:
                     se_progress = progress.progress(task_name, 0, 100)

dvrprocess/common/vosk.py

@@ -133,7 +133,7 @@ def kaldi_recognizer(language: str, freq: int, num_channels: int) -> BaseKaldiRe
             logger.info(f"Using remote transcriber at {rec.remote_url}")
         else:
             rec = LocalKaldiRecognizer(_vosk_language, freq, num_channels)
-    except BaseException as e:
+    except BaseException:
         logger.info("Remote transcriber not available")
         REMOTE_KALDI_SERVER[_vosk_language] = False
         rec = LocalKaldiRecognizer(_vosk_language, freq, num_channels)

dvrprocess/comtune.py

@@ -588,7 +588,7 @@
                 try:
                     result.result()
                     video_stats_progress.progress(result_idx)
                 except subprocess.CalledProcessError as e:
                     # generate with the files we have
                     pass
                 except KeyboardInterrupt as e:
@@ -850,7 +850,7 @@
     script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
     return [f"{os.environ['HOME']}/.comskip-starter.ini",
             f"{script_dir}/comskip-starter.ini",
-            f"/etc/comskip-starter.ini"]
+            "/etc/comskip-starter.ini"]
 
 
 @lru_cache(maxsize=None)
@@ -1035,7 +1035,7 @@
             if config_default in [None, val, str(genes[idx].default_value)] and genes[idx].exclude_if_default:
                 logger.info(f"{config} removing because the default value is part of the solution space")
                 solution[idx] = None
             if type(genes[idx].space) is list and len(genes[idx].space) == len(val):
                 logger.info(f"{config} removing because all gene space is part of the solution space")
                 solution[idx] = None
         else:
@@ -1158,7 +1158,7 @@
                 return True
 
         if check_compute and common.should_stop_processing():
-            logger.info(f"INFO: not enough compute available")
+            logger.info("INFO: not enough compute available")
             return True
 
         return False

dvrprocess/conflict-resolve.py

@@ -35,7 +35,7 @@ def dryrun(self):
 
 def resolution_generator(paths: list[str]):
     latest_filter_hash = profanity_filter.compute_filter_hash()
-    latest_filter_version = profanity_filter.FILTER_VERSION
+    # latest_filter_version = profanity_filter.FILTER_VERSION
     for conflict_path, _ in common.generate_video_files(paths, fail_on_missing=False):
         if 'conflicted copy' not in conflict_path:
             continue

dvrprocess/find_need_transcode.py

@@ -427,7 +427,7 @@ def _plex_host_name_to_local(file_name: str, media_roots: list[str]) -> Tuple[st
         if i >= 0:
             for root in media_roots:
                 f = os.path.join(root, file_name[i:])
-                logger.debug(f"Checking if %s is a valid path for %s", f, file_name)
+                logger.debug("Checking if %s is a valid path for %s", f, file_name)
                 if os.path.isfile(f):
                     return f, file_name[i:]
     return None, None

dvrprocess/profanity-filter-apply.py

@@ -226,7 +226,7 @@ def profanity_filter_apply(media_paths, plex_url=None, dry_run=False, workdir=No
                     return 0
 
                 if check_compute and common.should_stop_processing():
-                    logger.info(f"not enough compute available")
+                    logger.info("not enough compute available")
                     return 0
 
                 try: