Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add resume functionality #1107

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add resume functionality #1107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion garak/_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from garak import __version__ as version

system_params = (
"verbose narrow_output parallel_requests parallel_attempts skip_unknown".split()
"verbose narrow_output parallel_requests parallel_attempts skip_unknown resume".split()
)
run_params = "seed deprefix eval_threshold generations probe_tags interactive".split()
plugins_params = "model_type model_name extended_detectors".split()
Expand Down
17 changes: 17 additions & 0 deletions garak/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,13 @@ def main(arguments=None) -> None:
action="store_true",
help="allow skip of unknown probes, detectors, or buffs",
)
parser.add_argument(
"--resume",
"-R",
type=str,
default=None,
help="resume previous unfinnished scan",
)

## RUN
parser.add_argument(
Expand Down Expand Up @@ -367,6 +374,16 @@ def main(arguments=None) -> None:
if "buffs" in args:
_config.plugins.buff_spec = args.buffs

# Parse existing attempts
if _config.system.resume:
import json
_config.system.previous_attempts = []
with open(_config.system.resume, 'r') as fin:
for line in fin:
attempt_json = json.loads(line.strip())
if attempt_json['entry_type'] == 'attempt':
_config.system.previous_attempts.append((attempt_json['seq'], attempt_json['prompt']))

# base config complete

if hasattr(_config.run, "seed") and isinstance(_config.run.seed, int):
Expand Down
22 changes: 14 additions & 8 deletions garak/command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,15 +74,21 @@ def start_run():
f"Can't create reporting directory {report_path}, quitting"
) from e

filename = f"garak.{_config.transient.run_id}.report.jsonl"
if not _config.reporting.report_prefix:
filename = f"garak.{_config.transient.run_id}.report.jsonl"
if _config.system.resume:
_config.transient.report_filename = _config.system.resume
_config.transient.reportfile = open(
_config.transient.report_filename, "a", buffering=1, encoding="utf-8"
)
else:
filename = _config.reporting.report_prefix + ".report.jsonl"
_config.transient.report_filename = str(report_path / filename)
_config.transient.reportfile = open(
_config.transient.report_filename, "w", buffering=1, encoding="utf-8"
)
filename = f"garak.{_config.transient.run_id}.report.jsonl"
if not _config.reporting.report_prefix:
filename = f"garak.{_config.transient.run_id}.report.jsonl"
else:
filename = _config.reporting.report_prefix + ".report.jsonl"
_config.transient.report_filename = str(report_path / filename)
_config.transient.reportfile = open(
_config.transient.report_filename, "w", buffering=1, encoding="utf-8"
)
setup_dict = {"entry_type": "start_run setup"}
for k, v in _config.__dict__.items():
if k[:2] != "__" and type(v) in (
Expand Down
5 changes: 4 additions & 1 deletion garak/probes/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,10 @@ def probe(self, generator) -> Iterable[garak.attempt.Attempt]:
attempts_todo: Iterable[garak.attempt.Attempt] = []
prompts = list(self.prompts)
for seq, prompt in enumerate(prompts):
attempts_todo.append(self._mint_attempt(prompt, seq))
if hasattr(_config.system, 'previous_attempts') and (seq, prompt) in _config.system.previous_attempts:
continue
Comment on lines +212 to +213
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The probe() method should not access _config directly, previously completed state should to be injected in some way.

Some probes also randomize selection of a subset of prompts this would not properly match state to resume.

else:
attempts_todo.append(self._mint_attempt(prompt, seq))

# buff hook
if len(_config.buffmanager.buffs) > 0:
Expand Down
Loading