From 6cc6a371fa94ab1354e2cf9f0cd4c95ccce0f3c6 Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 06:33:26 +0000 Subject: [PATCH 01/11] feat(dspy): in extend_generation, compare key values to None, not to --- dsp/primitives/predict.py | 88 ++++++++++++++++++++-------------- tests/datasets/test_dataset.py | 5 +- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.py index 41edf6c51..2921c6343 100644 --- a/dsp/primitives/predict.py +++ b/dsp/primitives/predict.py @@ -61,6 +61,41 @@ def _generate(template: Template, **kwargs) -> Callable: generator = dsp.settings.lm + def extend_generation(completion: Example, field_names: list[str], stage:str, max_depth: int, original_example:Example): + """If the required fields are not present in the completion, extend the generation.""" + assert max_depth > 0, "Max depth exceeded - failed to complete in one pass - increase max_tokens" + # remove content of last field to avoid half-completed content + for field_name in get_all_fields_following_missing_field(completion, field_names): + completion.pop(field_name, None) + + # Recurse with greedy decoding and a shorter length. + max_tokens = (kwargs.get("max_tokens") or + kwargs.get("max_output_tokens") or + dsp.settings.lm.kwargs.get("max_tokens") or + dsp.settings.lm.kwargs.get('max_output_tokens')) + + + if max_tokens is None: + raise ValueError("Required 'max_tokens' or 'max_output_tokens' not specified in settings.") + max_tokens = min(max(75, max_tokens // 2), max_tokens) + keys = list(kwargs.keys()) + list(dsp.settings.lm.kwargs.keys()) + max_tokens_key = "max_tokens" if "max_tokens" in keys else "max_output_tokens" + new_kwargs = { + **kwargs, + max_tokens_key: max_tokens, + "n": 1, + "temperature": 0.0, + } + + _, finished_completion = generate(template, **new_kwargs)( + completion, + stage=stage, + max_depth=max_depth - 1, + original_example=original_example, + ) + return finished_completion.data[0] + + def do_generate( example: Example, stage: str, max_depth: int = 2, original_example=None, ): @@ -77,45 +112,19 @@ def do_generate( completions: list[dict[str, Any]] = generator(prompt, **kwargs) completions: list[Example] = [template.extract(example, p) for p in completions] - # Find the completions that are most complete. + # Find the completions that are unfinished. field_names: list[str] = [field.input_variable for field in template.fields] - last_field_idx = 0 - for field_idx, key in enumerate(field_names): - completions_ = [ - c for c in completions if key in c.keys() and c[key] is not None - ] - - # Filter out completions that are missing fields that are present in at least one completion. - if len(completions_): - completions = completions_ - last_field_idx = field_idx + 1 - - # If none of the completions is completed (i.e., none has the final field set). - if last_field_idx < len(field_names): - # Pick the first completion that has gone farthest. - completion = completions[0] - completion[field_names[last_field_idx]] = "" - - # Recurse with greedy decoding and a shorter length. - max_tokens = kwargs.get("max_tokens", dsp.settings.lm.kwargs["max_tokens"]) - max_tokens = min(max(75, max_tokens // 2), max_tokens) - new_kwargs = { - **kwargs, - "max_tokens": max_tokens, - "n": 1, - "temperature": 0.0, - } - - assert max_depth > 0 - return generate(template, **new_kwargs)( - completion, - stage=stage, - max_depth=max_depth - 1, - original_example=original_example, + finished_completions = [] + for completion in completions: + if all((completion.get(key, None) is not None) for key in field_names): + finished_completions.append(completion) + continue + finished_completions.append( + extend_generation(completion, field_names, stage, max_depth, original_example), ) - completions = Completions(completions, template=template) + completions = Completions(finished_completions, template=template) example = example.copy(completions=completions) if len(completions) == 1: @@ -152,6 +161,15 @@ def do_generate( return do_generate +def get_all_fields_following_missing_field(completion: Example, field_names: list[str]) -> list[str]: + """Returns every field following the first missing field""" + for i, field_name in enumerate(field_names): + if field_name not in completion: + return field_names[i:] + if completion[field_name] == "": + return field_names[i:] + return [] + def generate_sc( example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs, diff --git a/tests/datasets/test_dataset.py b/tests/datasets/test_dataset.py index 4e0daf371..2c1e9af84 100644 --- a/tests/datasets/test_dataset.py +++ b/tests/datasets/test_dataset.py @@ -11,9 +11,6 @@ "This is content 2","What is that?","This is answer 2" """ -with open("dummy.csv", "w") as file: - file.write(dummy_data) - class CSVDataset(Dataset): def __init__(self, file_path, input_keys=None, *args, **kwargs) -> None: @@ -46,4 +43,6 @@ def test_input_keys(self): if __name__ == "__main__": + with open("dummy.csv", "w") as file: + file.write(dummy_data) unittest.main() From 2b55ca39b14390580b9b1e2b15539cbde12be182 Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 07:14:14 +0000 Subject: [PATCH 02/11] feat(dspy): add tests for extend generation --- tests/predict/test_predict.py | 89 +++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index c701b380a..48ecd2d52 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -1,11 +1,14 @@ -import dspy -from dspy import Predict, Signature, TypedPredictor -from dspy.utils.dummies import DummyLM import copy import textwrap + import pydantic +import pytest import ujson +import dspy +from dspy import Predict, Signature, TypedPredictor +from dspy.utils.dummies import DummyLM + def test_initialization_with_string_signature(): signature_string = "input1, input2 -> output" @@ -209,14 +212,84 @@ class OutputOnlySignature(dspy.Signature): assert lm.get_convo(-1) == textwrap.dedent( """\ Given the fields , produce the fields `output`. - + --- - + Follow the following format. - + Output: ${output} - + --- - + Output: short answer""" ) + + +@pytest.fixture(name="SandwichIdea") +def sandwich_idea_signature(): + class SandwichIdea(dspy.Signature): + """Based on the meal and dietary requirements, suggest a sandwich idea.""" + + meal: str = dspy.InputField() + dietary_requiements: str = dspy.InputField() + bread: str = dspy.OutputField() + protein: str = dspy.OutputField() + fat: str = dspy.OutputField() + garnish: str = dspy.OutputField() + sauce: str = dspy.OutputField() + + return SandwichIdea + + +def test_extend_generation(SandwichIdea): + # test that the completion is extended correctly + lm = DummyLM( + [ + " whole wheat\n\nProtein: turkey\n\nFat: avocado", + " tomato\n\nSauce: mustard", + ] + ) + dspy.settings.configure(lm=lm) + + prediction = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") + assert prediction.bread == "whole wheat" + assert prediction.protein == "turkey" + assert prediction.fat == "avocado" + assert prediction.garnish == "tomato" + assert prediction.sauce == "mustard" + + +def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): + # test the completion is rolled back when a field is skipped + lm = DummyLM( + [ + " white\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", + " ham\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", + ] + ) + dspy.settings.configure(lm=lm) + + predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") + assert predictor.bread == "white" + assert predictor.protein == "ham" + assert predictor.fat == "butter" + assert predictor.garnish == "lettuce" + assert predictor.sauce == "mayo" + + +def test_extend_generation_with_empty_field(SandwichIdea): + # test the completion is extended if the field is empty + lm = DummyLM( + [ + " white\n\nProtein: \n\nFat: butter\n\nGarish: lettuce", + " mayo", + ] + ) + dspy.settings.configure(lm=lm) + + predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") + assert predictor.bread == "white" + assert predictor.protein == "" + assert predictor.fat == "butter" + assert predictor.garnish == "lettuce" + assert predictor.sauce == "mayo" From d12e0f2b15da5f2a81fc4286d443c37eeb13ea9e Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 07:17:42 +0000 Subject: [PATCH 03/11] feat(dspy): revert changes to extend generation logic --- dsp/primitives/predict.py | 88 ++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/dsp/primitives/predict.py b/dsp/primitives/predict.py index 2921c6343..41edf6c51 100644 --- a/dsp/primitives/predict.py +++ b/dsp/primitives/predict.py @@ -61,41 +61,6 @@ def _generate(template: Template, **kwargs) -> Callable: generator = dsp.settings.lm - def extend_generation(completion: Example, field_names: list[str], stage:str, max_depth: int, original_example:Example): - """If the required fields are not present in the completion, extend the generation.""" - assert max_depth > 0, "Max depth exceeded - failed to complete in one pass - increase max_tokens" - # remove content of last field to avoid half-completed content - for field_name in get_all_fields_following_missing_field(completion, field_names): - completion.pop(field_name, None) - - # Recurse with greedy decoding and a shorter length. - max_tokens = (kwargs.get("max_tokens") or - kwargs.get("max_output_tokens") or - dsp.settings.lm.kwargs.get("max_tokens") or - dsp.settings.lm.kwargs.get('max_output_tokens')) - - - if max_tokens is None: - raise ValueError("Required 'max_tokens' or 'max_output_tokens' not specified in settings.") - max_tokens = min(max(75, max_tokens // 2), max_tokens) - keys = list(kwargs.keys()) + list(dsp.settings.lm.kwargs.keys()) - max_tokens_key = "max_tokens" if "max_tokens" in keys else "max_output_tokens" - new_kwargs = { - **kwargs, - max_tokens_key: max_tokens, - "n": 1, - "temperature": 0.0, - } - - _, finished_completion = generate(template, **new_kwargs)( - completion, - stage=stage, - max_depth=max_depth - 1, - original_example=original_example, - ) - return finished_completion.data[0] - - def do_generate( example: Example, stage: str, max_depth: int = 2, original_example=None, ): @@ -112,19 +77,45 @@ def do_generate( completions: list[dict[str, Any]] = generator(prompt, **kwargs) completions: list[Example] = [template.extract(example, p) for p in completions] - # Find the completions that are unfinished. + # Find the completions that are most complete. field_names: list[str] = [field.input_variable for field in template.fields] - finished_completions = [] - for completion in completions: - if all((completion.get(key, None) is not None) for key in field_names): - finished_completions.append(completion) - continue - finished_completions.append( - extend_generation(completion, field_names, stage, max_depth, original_example), + last_field_idx = 0 + for field_idx, key in enumerate(field_names): + completions_ = [ + c for c in completions if key in c.keys() and c[key] is not None + ] + + # Filter out completions that are missing fields that are present in at least one completion. + if len(completions_): + completions = completions_ + last_field_idx = field_idx + 1 + + # If none of the completions is completed (i.e., none has the final field set). + if last_field_idx < len(field_names): + # Pick the first completion that has gone farthest. + completion = completions[0] + completion[field_names[last_field_idx]] = "" + + # Recurse with greedy decoding and a shorter length. + max_tokens = kwargs.get("max_tokens", dsp.settings.lm.kwargs["max_tokens"]) + max_tokens = min(max(75, max_tokens // 2), max_tokens) + new_kwargs = { + **kwargs, + "max_tokens": max_tokens, + "n": 1, + "temperature": 0.0, + } + + assert max_depth > 0 + return generate(template, **new_kwargs)( + completion, + stage=stage, + max_depth=max_depth - 1, + original_example=original_example, ) - completions = Completions(finished_completions, template=template) + completions = Completions(completions, template=template) example = example.copy(completions=completions) if len(completions) == 1: @@ -161,15 +152,6 @@ def do_generate( return do_generate -def get_all_fields_following_missing_field(completion: Example, field_names: list[str]) -> list[str]: - """Returns every field following the first missing field""" - for i, field_name in enumerate(field_names): - if field_name not in completion: - return field_names[i:] - if completion[field_name] == "": - return field_names[i:] - return [] - def generate_sc( example, prompt, normalize=True, extract=None, prediction_field=None, **kwargs, From ea29442c7aea461e91e5b6d14edaafb1075e91f4 Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 07:18:15 +0000 Subject: [PATCH 04/11] feat(dspy): rm redundant comments --- tests/predict/test_predict.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 48ecd2d52..554b4dc28 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -242,7 +242,6 @@ class SandwichIdea(dspy.Signature): def test_extend_generation(SandwichIdea): - # test that the completion is extended correctly lm = DummyLM( [ " whole wheat\n\nProtein: turkey\n\nFat: avocado", @@ -260,7 +259,6 @@ def test_extend_generation(SandwichIdea): def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): - # test the completion is rolled back when a field is skipped lm = DummyLM( [ " white\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", @@ -278,7 +276,6 @@ def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): def test_extend_generation_with_empty_field(SandwichIdea): - # test the completion is extended if the field is empty lm = DummyLM( [ " white\n\nProtein: \n\nFat: butter\n\nGarish: lettuce", From 15aef81a2182568795b29ae3dffd60f7251c3c0e Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Wed, 19 Jun 2024 07:28:13 +0000 Subject: [PATCH 05/11] feat(dspy): revert changes to test file writing --- tests/datasets/test_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/datasets/test_dataset.py b/tests/datasets/test_dataset.py index 2c1e9af84..4e0daf371 100644 --- a/tests/datasets/test_dataset.py +++ b/tests/datasets/test_dataset.py @@ -11,6 +11,9 @@ "This is content 2","What is that?","This is answer 2" """ +with open("dummy.csv", "w") as file: + file.write(dummy_data) + class CSVDataset(Dataset): def __init__(self, file_path, input_keys=None, *args, **kwargs) -> None: @@ -43,6 +46,4 @@ def test_input_keys(self): if __name__ == "__main__": - with open("dummy.csv", "w") as file: - file.write(dummy_data) unittest.main() From 477153561ebe4780b3922d39018b13fd8d6a02d0 Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Thu, 20 Jun 2024 07:20:55 +0000 Subject: [PATCH 06/11] feat(dspy): changed assertions so tests pass and added conversation logs --- tests/predict/test_predict.py | 245 ++++++++++++++++++++++++++++++++-- 1 file changed, 233 insertions(+), 12 deletions(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 554b4dc28..0e8b67790 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -245,32 +245,182 @@ def test_extend_generation(SandwichIdea): lm = DummyLM( [ " whole wheat\n\nProtein: turkey\n\nFat: avocado", - " tomato\n\nSauce: mustard", + # Incomplete generation leads to tomato field being assigned as an + # empty string ("") in dsp.primitives.predict l98 the generation + # therefores continues with the next field. + " tomato \n\nSauce: mustard", ] ) dspy.settings.configure(lm=lm) prediction = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") + + # The logged conversation: + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: whole wheat + + # Protein: turkey + + # Fat: avocado + # === + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: whole wheat + + # Protein: turkey + + # Fat: avocado + + # Garnish: tomato + + # Sauce: mustard + # === + assert prediction.bread == "whole wheat" assert prediction.protein == "turkey" assert prediction.fat == "avocado" - assert prediction.garnish == "tomato" - assert prediction.sauce == "mustard" + assert prediction.garnish == "" # This field is assigned as "" when the generation is extended + assert prediction.sauce == "tomato \n\nSauce: mustard" def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): lm = DummyLM( [ - " white\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", - " ham\n\nFat: butter\n\nGarish: lettuce\n\nSauce: mayo", + " white\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo", + " ham\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo", ] ) dspy.settings.configure(lm=lm) + # The logged conversation: + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: white + + # Fat: butter + + # Garnish: lettuce + + # Sauce: mayo + # === + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: white Fat: butter Garnish: lettuce Sauce: mayo + + # Protein: ham + + # Fat: butter + + # Garnish: lettuce + + # Sauce: mayo + # === + predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") - assert predictor.bread == "white" - assert predictor.protein == "ham" - assert predictor.fat == "butter" + assert predictor.bread == "white\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo" + assert predictor.protein == "" + assert predictor.fat == "ham\n\nFat: butter" assert predictor.garnish == "lettuce" assert predictor.sauce == "mayo" @@ -278,15 +428,86 @@ def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): def test_extend_generation_with_empty_field(SandwichIdea): lm = DummyLM( [ - " white\n\nProtein: \n\nFat: butter\n\nGarish: lettuce", - " mayo", + " white\n\nProtein: \n\nFat: butter\n\nGarnish: lettuce", + " lettuce \n\nSauce: mayo", ] ) dspy.settings.configure(lm=lm) + # The logged conversation: + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: white + + # Protein: + + # Fat: butter + + # Garnish: lettuce + # === + # === DummyLM === + # Based on the meal and dietary requirements, suggest a sandwich idea. + + # --- + + # Follow the following format. + + # Meal: ${meal} + + # Dietary Requiements: ${dietary_requiements} + + # Bread: ${bread} + + # Protein: ${protein} + + # Fat: ${fat} + + # Garnish: ${garnish} + + # Sauce: ${sauce} + + # --- + + # Meal: lunch + + # Dietary Requiements: N/A + + # Bread: white + + # Protein: Fat: butter Garnish: lettuce + + # Fat: lettuce + + # Sauce: mayo + # === predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") assert predictor.bread == "white" - assert predictor.protein == "" - assert predictor.fat == "butter" + assert predictor.protein == "Fat: butter\n\nGarnish: lettuce" + assert predictor.fat == "" assert predictor.garnish == "lettuce" assert predictor.sauce == "mayo" From bcab220171d5f02fbc2833bce649f296519cf3bb Mon Sep 17 00:00:00 2001 From: JONEMI19 Date: Thu, 20 Jun 2024 07:35:25 +0000 Subject: [PATCH 07/11] feat(dsp): clarify what is generated in each generation --- tests/predict/test_predict.py | 129 ++++------------------------------ 1 file changed, 15 insertions(+), 114 deletions(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 0e8b67790..bd07e776e 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -254,77 +254,44 @@ def test_extend_generation(SandwichIdea): dspy.settings.configure(lm=lm) prediction = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") - - # The logged conversation: + # The logged conversation (additional newlines removed, [..] indicates the generation): # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - - # Bread: whole wheat - + # Bread: [whole wheat # Protein: turkey - - # Fat: avocado + # Fat: avocado] # === # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - # Bread: whole wheat - # Protein: turkey - # Fat: avocado - - # Garnish: tomato - - # Sauce: mustard + # Garnish: [tomato + # Sauce: mustard] # === assert prediction.bread == "whole wheat" @@ -342,79 +309,45 @@ def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): ] ) dspy.settings.configure(lm=lm) - - # The logged conversation: + # The logged conversation (additional newlines removed, [..] indicates the generation): # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - - # Bread: white - + # Bread:[ white # Fat: butter - # Garnish: lettuce - - # Sauce: mayo + # Sauce: mayo] # === # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - # Bread: white Fat: butter Garnish: lettuce Sauce: mayo - - # Protein: ham - + # Protein:[ ham # Fat: butter - # Garnish: lettuce - - # Sauce: mayo + # Sauce: mayo] # === predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") @@ -433,76 +366,44 @@ def test_extend_generation_with_empty_field(SandwichIdea): ] ) dspy.settings.configure(lm=lm) - # The logged conversation: + # The logged conversation (additional newlines removed, [..] indicates the generation): # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - - # Bread: white - + # Bread:[ white # Protein: - # Fat: butter - - # Garnish: lettuce + # Garnish: lettuce] # === # === DummyLM === # Based on the meal and dietary requirements, suggest a sandwich idea. - # --- - # Follow the following format. - # Meal: ${meal} - # Dietary Requiements: ${dietary_requiements} - # Bread: ${bread} - # Protein: ${protein} - # Fat: ${fat} - # Garnish: ${garnish} - # Sauce: ${sauce} - # --- - # Meal: lunch - # Dietary Requiements: N/A - # Bread: white - # Protein: Fat: butter Garnish: lettuce - - # Fat: lettuce - - # Sauce: mayo + # Fat:[ lettuce + # Sauce: mayo] # === predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") From da053783468eac40d18fa74bccd9440657a872ef Mon Sep 17 00:00:00 2001 From: Michael Jones Date: Thu, 3 Oct 2024 14:13:54 +0000 Subject: [PATCH 08/11] add comment --- tests/predict/test_predict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index bd07e776e..3ff771c6b 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -352,7 +352,7 @@ def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): predictor = Predict(SandwichIdea)(meal="lunch", dietary_requiements="N/A") assert predictor.bread == "white\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo" - assert predictor.protein == "" + assert predictor.protein == "" # This field is assigned as "" when the generation is rolled back assert predictor.fat == "ham\n\nFat: butter" assert predictor.garnish == "lettuce" assert predictor.sauce == "mayo" From bf4e702f24fb2504f4440ce3558773df7a7b4950 Mon Sep 17 00:00:00 2001 From: Michael Jones Date: Thu, 3 Oct 2024 14:17:26 +0000 Subject: [PATCH 09/11] revert removal --- .../local_language_model_clients/LlamaCpp.md | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 docs/api/local_language_model_clients/LlamaCpp.md diff --git a/docs/api/local_language_model_clients/LlamaCpp.md b/docs/api/local_language_model_clients/LlamaCpp.md new file mode 100644 index 000000000..337c39a9d --- /dev/null +++ b/docs/api/local_language_model_clients/LlamaCpp.md @@ -0,0 +1,59 @@ + dspy.LlamaCpp + +## Prerequisites + +Install Llama Cpp Python by following the instructions provided in the [Llama Cpp Python repository](https://github.com/abetlen/llama-cpp-python). + +```shell +pip install llama-cpp-python +``` + +alternatively, to install with CUDA support: + +```shell +CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python +``` + + +### Initializing the Llama model + +Initialize the model within your program with the desired parameters. + +```python +from llama_cpp import Llama + +llm = Llama( + model_path="./sppo_finetuned_llama_3_8b.gguf", + n_gpu_layers=-1, + n_ctx=0, + verbose=False +) +``` + + +### Sending requests to the model + +After initializing the Llama model, you can interact with it using the `LlamaCpp` client. + +```python +import dspy + +llamalm = dspy.LlamaCpp(model="llama", llama_model=llm, model_type="chat", temperature=0.4) +dspy.settings.configure(lm=llamalm) + + +#Define a simple signature for basic question answering +class BasicQA(dspy.Signature): + """Answer questions with short factoid answers.""" + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + +#Pass signature to Predict module +generate_answer = dspy.Predict(BasicQA) + +# Call the predictor on a particular input. +question='What is the color of the sky?' +pred = generate_answer(question=question) + +print(f"Question: {question}") +print(f"Predicted Answer: {pred.answer}") From 06d4173e7241721eea65f3dec359ff2338232e98 Mon Sep 17 00:00:00 2001 From: Michael Jones Date: Thu, 3 Oct 2024 14:53:11 +0000 Subject: [PATCH 10/11] (feat) add DummyLiteLLM --- dspy/utils/dummies.py | 12 +++++++++++- tests/predict/test_predict.py | 12 ++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.py index d97d59b5a..88bdf43c9 100644 --- a/dspy/utils/dummies.py +++ b/dspy/utils/dummies.py @@ -6,7 +6,9 @@ from dsp.modules import LM from dsp.utils.utils import dotdict +from dspy.clients import LM as LiteLLM +RED, GREEN, RESET = "\033[91m", "\033[92m", "\033[0m" class DummyLM(LM): """Dummy language model for unit testing purposes.""" @@ -64,7 +66,6 @@ def basic_request(self, prompt, n=1, **kwargs) -> dict[str, list[dict[str, str]] }, ) - RED, GREEN, RESET = "\033[91m", "\033[92m", "\033[0m" print("=== DummyLM ===") print(prompt, end="") print(f"{RED}{answer}{RESET}") @@ -94,6 +95,15 @@ def get_convo(self, index) -> str: return self.history[index]["prompt"] + " " + self.history[index]["response"]["choices"][0]["text"] +class DummyLiteLLM(LiteLLM): + def __init__(self, answers: list[str], model_type='chat', temperature=0.0, max_tokens=1000, cache=True, **kwargs): + super().__init__("dummy", model_type, temperature, max_tokens, cache, **kwargs) + self.answers = iter([[ans] for ans in answers]) + + + def __call__(self, **kwargs): + return next(self.answers) + def dummy_rm(passages=()) -> callable: if not passages: diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 3ff771c6b..657e80a5e 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -7,7 +7,7 @@ import dspy from dspy import Predict, Signature, TypedPredictor -from dspy.utils.dummies import DummyLM +from dspy.utils.dummies import DummyLM, DummyLiteLLM def test_initialization_with_string_signature(): @@ -242,13 +242,13 @@ class SandwichIdea(dspy.Signature): def test_extend_generation(SandwichIdea): - lm = DummyLM( + lm = DummyLiteLLM( [ - " whole wheat\n\nProtein: turkey\n\nFat: avocado", + "\n[[ ## bread ## ]]\n whole wheat\n\n[[ ## protein ## ]]\n turkey\n\n[[ ## fat ## ]]\n avocado", # Incomplete generation leads to tomato field being assigned as an # empty string ("") in dsp.primitives.predict l98 the generation # therefores continues with the next field. - " tomato \n\nSauce: mustard", + "\n[[ ## garnish ## ]]\ntomato\n\n[[ ## sauce ## ]]\n mustard\n\n", ] ) dspy.settings.configure(lm=lm) @@ -302,7 +302,7 @@ def test_extend_generation(SandwichIdea): def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): - lm = DummyLM( + lm = DummyLiteLLM( [ " white\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo", " ham\n\nFat: butter\n\nGarnish: lettuce\n\nSauce: mayo", @@ -359,7 +359,7 @@ def test_extend_generation_rolled_back_when_field_is_skipped(SandwichIdea): def test_extend_generation_with_empty_field(SandwichIdea): - lm = DummyLM( + lm = DummyLiteLLM( [ " white\n\nProtein: \n\nFat: butter\n\nGarnish: lettuce", " lettuce \n\nSauce: mayo", From 778135f438167eb81ca02856588ce75269b6c227 Mon Sep 17 00:00:00 2001 From: Michael Jones Date: Thu, 3 Oct 2024 14:54:14 +0000 Subject: [PATCH 11/11] (feat) add DummyLiteLLM --- dspy/utils/dummies.py | 5 +++-- tests/predict/test_predict.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.py index 88bdf43c9..80658eb93 100644 --- a/dspy/utils/dummies.py +++ b/dspy/utils/dummies.py @@ -10,6 +10,7 @@ RED, GREEN, RESET = "\033[91m", "\033[92m", "\033[0m" + class DummyLM(LM): """Dummy language model for unit testing purposes.""" @@ -96,14 +97,14 @@ def get_convo(self, index) -> str: class DummyLiteLLM(LiteLLM): - def __init__(self, answers: list[str], model_type='chat', temperature=0.0, max_tokens=1000, cache=True, **kwargs): + def __init__(self, answers: list[str], model_type="chat", temperature=0.0, max_tokens=1000, cache=True, **kwargs): super().__init__("dummy", model_type, temperature, max_tokens, cache, **kwargs) self.answers = iter([[ans] for ans in answers]) - def __call__(self, **kwargs): return next(self.answers) + def dummy_rm(passages=()) -> callable: if not passages: diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 657e80a5e..79e7fc289 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -7,7 +7,7 @@ import dspy from dspy import Predict, Signature, TypedPredictor -from dspy.utils.dummies import DummyLM, DummyLiteLLM +from dspy.utils.dummies import DummyLiteLLM, DummyLM def test_initialization_with_string_signature():