From f5f5de6031ac9abd9dcfe93c25a99faa58c69d32 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Mon, 24 Feb 2025 10:55:23 +0100 Subject: [PATCH 1/8] give kinda helpful message if too many open files --- garak/probes/base.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/garak/probes/base.py b/garak/probes/base.py index b3fbdb025..4bca1b8b2 100644 --- a/garak/probes/base.py +++ b/garak/probes/base.py @@ -17,7 +17,7 @@ from garak import _config from garak.configurable import Configurable -from garak.exception import PluginConfigurationError +from garak.exception import GarakException import garak.attempt import garak.resources.theme @@ -178,17 +178,25 @@ def _execute_all(self, attempts) -> Iterable[garak.attempt.Attempt]: attempt_bar = tqdm.tqdm(total=len(attempts), leave=False) attempt_bar.set_description(self.probename.replace("garak.", "")) - with Pool(_config.system.parallel_attempts) as attempt_pool: - for result in attempt_pool.imap_unordered( - self._execute_attempt, attempts - ): - _config.transient.reportfile.write( - json.dumps(result.as_dict()) + "\n" - ) - attempts_completed.append( - result - ) # these will be out of original order - attempt_bar.update(1) + try: + with Pool(_config.system.parallel_attempts) as attempt_pool: + for result in attempt_pool.imap_unordered( + self._execute_attempt, attempts + ): + _config.transient.reportfile.write( + json.dumps(result.as_dict()) + "\n" + ) + attempts_completed.append( + result + ) # these will be out of original order + attempt_bar.update(1) + except OSError as o: + if o.errno == 24: + msg = "Parallelisation limit hit. Try reducing parallel_attempts or raising limit (e.g. ulimit -n 4096)" + logging.critical(msg) + raise GarakException(msg) from o + else: + raise (o) else: attempt_iterator = tqdm.tqdm(attempts, leave=False) From 49ce62dba7d4b8781623d041cefbfbf71e10618b Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 26 Feb 2025 09:52:02 +0100 Subject: [PATCH 2/8] also give advice for parallel_requests too high --- garak/generators/base.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/garak/generators/base.py b/garak/generators/base.py index e09d4f303..3cca02f35 100644 --- a/garak/generators/base.py +++ b/garak/generators/base.py @@ -12,6 +12,7 @@ from garak import _config from garak.configurable import Configurable +from garak.exception import GarakException import garak.resources.theme @@ -162,13 +163,21 @@ def generate( ) multi_generator_bar.set_description(self.fullname[:55]) - with Pool(_config.system.parallel_requests) as pool: - for result in pool.imap_unordered( - self._call_model, [prompt] * generations_this_call - ): - self._verify_model_result(result) - outputs.append(result[0]) - multi_generator_bar.update(1) + try: + with Pool(_config.system.parallel_requests) as pool: + for result in pool.imap_unordered( + self._call_model, [prompt] * generations_this_call + ): + self._verify_model_result(result) + outputs.append(result[0]) + multi_generator_bar.update(1) + except OSError as o: + if o.errno == 24: + msg = "Parallelisation limit hit. Try reducing parallel_attempts or raising limit (e.g. ulimit -n 4096)" + logging.critical(msg) + raise GarakException(msg) from o + else: + raise (o) else: generation_iterator = tqdm.tqdm( From 2289c9ad5737753247f187a0700bb2be07379bb9 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 26 Feb 2025 09:55:08 +0100 Subject: [PATCH 3/8] cap worker pool size to the volume of work requested --- garak/generators/base.py | 4 +++- garak/probes/base.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/garak/generators/base.py b/garak/generators/base.py index 3cca02f35..f1a4f5205 100644 --- a/garak/generators/base.py +++ b/garak/generators/base.py @@ -163,8 +163,10 @@ def generate( ) multi_generator_bar.set_description(self.fullname[:55]) + pool_size = min(generations_this_call, _config.system.parallel_requests) + try: - with Pool(_config.system.parallel_requests) as pool: + with Pool(pool_size) as pool: for result in pool.imap_unordered( self._call_model, [prompt] * generations_this_call ): diff --git a/garak/probes/base.py b/garak/probes/base.py index 4bca1b8b2..4446940ea 100644 --- a/garak/probes/base.py +++ b/garak/probes/base.py @@ -178,8 +178,10 @@ def _execute_all(self, attempts) -> Iterable[garak.attempt.Attempt]: attempt_bar = tqdm.tqdm(total=len(attempts), leave=False) attempt_bar.set_description(self.probename.replace("garak.", "")) + pool_size = min(_config.system.parallel_attempts, len(attempts)) + try: - with Pool(_config.system.parallel_attempts) as attempt_pool: + with Pool(pool_size) as attempt_pool: for result in attempt_pool.imap_unordered( self._execute_attempt, attempts ): From 354566a17c318963c1997773f4f697935348cf20 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 26 Feb 2025 10:02:07 +0100 Subject: [PATCH 4/8] get the error message right --- garak/generators/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/garak/generators/base.py b/garak/generators/base.py index f1a4f5205..6a25b7bfb 100644 --- a/garak/generators/base.py +++ b/garak/generators/base.py @@ -175,7 +175,7 @@ def generate( multi_generator_bar.update(1) except OSError as o: if o.errno == 24: - msg = "Parallelisation limit hit. Try reducing parallel_attempts or raising limit (e.g. ulimit -n 4096)" + msg = "Parallelisation limit hit. Try reducing parallel_requests or raising limit (e.g. ulimit -n 4096)" logging.critical(msg) raise GarakException(msg) from o else: From 98a745420651153462c434b09427b291d54c5e5c Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 26 Feb 2025 10:02:29 +0100 Subject: [PATCH 5/8] add configurable cap on max # of workers to spawn --- garak/cli.py | 20 ++++++++++++++++---- garak/resources/garak.core.yaml | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/garak/cli.py b/garak/cli.py index e0e37df18..453119991 100644 --- a/garak/cli.py +++ b/garak/cli.py @@ -64,6 +64,16 @@ def main(arguments=None) -> None: import argparse + def worker_count_validation(workers): + iworkers = int(workers) + if iworkers <= 0: + raise argparse.ArgumentTypeError("Need >0 workers (int)" % workers) + if iworkers > _config.system.max_workers: + raise argparse.ArgumentTypeError( + "Parallel worker count capped at %s (config.system.max_workers)" % _config.system.max_workers + ) + return iworkers + parser = argparse.ArgumentParser( prog="python -m garak", description="LLM safety & security scanning tool", @@ -92,15 +102,15 @@ def main(arguments=None) -> None: ) parser.add_argument( "--parallel_requests", - type=int, + type=worker_count_validation, default=_config.system.parallel_requests, help="How many generator requests to launch in parallel for a given prompt. Ignored for models that support multiple generations per call.", ) parser.add_argument( "--parallel_attempts", - type=int, + type=worker_count_validation, default=_config.system.parallel_attempts, - help="How many probe attempts to launch in parallel.", + help="How many probe attempts to launch in parallel. Raise this for faster runs when using non-local models.", ) parser.add_argument( "--skip_unknown", @@ -484,7 +494,9 @@ def main(arguments=None) -> None: if has_changes: exit(1) # exit with error code to denote changes else: - print("No revisions applied. Please verify options provided for `--fix`") + print( + "No revisions applied. Please verify options provided for `--fix`" + ) elif args.report: from garak.report import Report diff --git a/garak/resources/garak.core.yaml b/garak/resources/garak.core.yaml index 72f7caa8d..881d21813 100644 --- a/garak/resources/garak.core.yaml +++ b/garak/resources/garak.core.yaml @@ -7,6 +7,7 @@ system: lite: true show_z: false enable_experimental: false + max_workers: 1000 run: seed: From e5efe4112ee14adea38dec8fca59ba34f36bf9d6 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 26 Feb 2025 10:04:55 +0100 Subject: [PATCH 6/8] also cap pool sizes with max_workers --- garak/generators/base.py | 6 +++++- garak/probes/base.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/garak/generators/base.py b/garak/generators/base.py index 6a25b7bfb..fd2c269cd 100644 --- a/garak/generators/base.py +++ b/garak/generators/base.py @@ -163,7 +163,11 @@ def generate( ) multi_generator_bar.set_description(self.fullname[:55]) - pool_size = min(generations_this_call, _config.system.parallel_requests) + pool_size = min( + generations_this_call, + _config.system.parallel_requests, + _config.system.max_workers, + ) try: with Pool(pool_size) as pool: diff --git a/garak/probes/base.py b/garak/probes/base.py index 4446940ea..dbf880f02 100644 --- a/garak/probes/base.py +++ b/garak/probes/base.py @@ -178,7 +178,11 @@ def _execute_all(self, attempts) -> Iterable[garak.attempt.Attempt]: attempt_bar = tqdm.tqdm(total=len(attempts), leave=False) attempt_bar.set_description(self.probename.replace("garak.", "")) - pool_size = min(_config.system.parallel_attempts, len(attempts)) + pool_size = min( + len(attempts), + _config.system.parallel_attempts, + _config.system.max_workers, + ) try: with Pool(pool_size) as attempt_pool: From 06ecb96d35232294b1af26464052045d21689fa5 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 26 Feb 2025 10:11:08 +0100 Subject: [PATCH 7/8] reduce max_workers to something well below default soft ulimit --- garak/resources/garak.core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/garak/resources/garak.core.yaml b/garak/resources/garak.core.yaml index 881d21813..00e7e9c34 100644 --- a/garak/resources/garak.core.yaml +++ b/garak/resources/garak.core.yaml @@ -7,7 +7,7 @@ system: lite: true show_z: false enable_experimental: false - max_workers: 1000 + max_workers: 500 run: seed: From f7280077b1f8dfb6e4a07e60fcf5d2fdc73b4a0d Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 26 Feb 2025 10:29:00 +0100 Subject: [PATCH 8/8] document max_workers --- docs/source/configurable.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/configurable.rst b/docs/source/configurable.rst index f58fda94c..90d2d5de2 100644 --- a/docs/source/configurable.rst +++ b/docs/source/configurable.rst @@ -46,6 +46,8 @@ Let's take a look at the core config. parallel_attempts: false lite: true show_z: false + enable_experimental: false + max_workers: 500 run: seed: @@ -93,6 +95,7 @@ such as ``show_100_pass_modules``. * ``narrow_output`` - Support output on narrower CLIs * ``show_z`` - Display Z-scores and visual indicators on CLI. It's good, but may be too much info until one has seen garak run a couple of times * ``enable_experimental`` - Enable experimental function CLI flags. Disabled by default. Experimental functions may disrupt your installation and provide unusual/unstable results. Can only be set by editing core config, so a git checkout of garak is recommended for this. +* ``max_workers`` - Cap on how many parallel workers can be requested. When raising this in order to use higher parallelisation, keep an eye on system resources (e.g. `ulimit -n 4026` on Linux) ``run`` config items """"""""""""""""""""