From d26a52199aabd6a032efaa604c70d23c14d7669f Mon Sep 17 00:00:00 2001 From: Jithin James Date: Wed, 13 Nov 2024 00:22:34 +0530 Subject: [PATCH] docs: persona generator (#1664) --- .../metrics/_write_your_own_metric.md | 32 +-- .../testgenerator/_persona_generator.md | 160 ++++++++++++ .../testgenerator/persona_generator.ipynb | 241 ++++++++++++++++++ .../_langgraph_agent_evaluation.md | 2 +- .../_langgraph_agent_evaluation_23_0.jpg | Bin 0 -> 8204 bytes docs/howtos/integrations/_llamaindex.md | 4 +- mkdocs.yml | 1 + src/ragas/testset/synthesizers/__init__.py | 1 - .../synthesizers/multi_hop/abstract.py | 4 + 9 files changed, 427 insertions(+), 18 deletions(-) create mode 100644 docs/howtos/customizations/testgenerator/_persona_generator.md create mode 100644 docs/howtos/customizations/testgenerator/persona_generator.ipynb create mode 100644 docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg diff --git a/docs/howtos/customizations/metrics/_write_your_own_metric.md b/docs/howtos/customizations/metrics/_write_your_own_metric.md index b5386ceff..4913309b1 100644 --- a/docs/howtos/customizations/metrics/_write_your_own_metric.md +++ b/docs/howtos/customizations/metrics/_write_your_own_metric.md @@ -34,7 +34,7 @@ choose_evaluator_llm.md ```python from ragas.llms import llm_factory -evaluator_llm = llm_factory('gpt-4o') +evaluator_llm = llm_factory("gpt-4o") ``` ## Aspect Critic - Simple Criteria Scoring @@ -56,7 +56,7 @@ from ragas.metrics import AspectCritic hallucinations_binary = AspectCritic( name="hallucinations_binary", definition="Did the model hallucinate or add any information that was not present in the retrieved context?", - llm=evaluator_llm + llm=evaluator_llm, ) await hallucinations_binary.single_turn_ascore(eval_dataset[0]) @@ -93,9 +93,7 @@ Now lets init the metric with the rubric and evaluator llm and evaluate the data from ragas.metrics import RubricsScoreWithoutReference hallucinations_rubric = RubricsScoreWithoutReference( - name="hallucinations_rubric", - llm=evaluator_llm, - rubrics=rubric + name="hallucinations_rubric", llm=evaluator_llm, rubrics=rubric ) await hallucinations_rubric.single_turn_ascore(eval_dataset[0]) @@ -125,7 +123,6 @@ For our example, we need to to use LLMs to evaluate our metric so we will subcla As for the implementation, we will use the [Faithfulness][ragas.metrics.Faithfulness] metric to evaluate our metric to measure the hallucinations with the formula - $$ \text{Hallucinations} = 1 - \text{Faithfulness} $$ @@ -144,19 +141,28 @@ import typing as t from ragas.callbacks import Callbacks from ragas.dataset_schema import SingleTurnSample + @dataclass class HallucinationsMetric(MetricWithLLM, SingleTurnMetric): # name of the metric name: str = "hallucinations_metric" # we need to define the required columns for the metric - _required_columns: t.Dict[MetricType, t.Set[str]] = field(default_factory=lambda: {MetricType.SINGLE_TURN: {"user_input", "response", "retrieved_contexts"}}) + _required_columns: t.Dict[MetricType, t.Set[str]] = field( + default_factory=lambda: { + MetricType.SINGLE_TURN: {"user_input", "response", "retrieved_contexts"} + } + ) def __post_init__(self): # init the faithfulness metric self.faithfulness_metric = Faithfulness(llm=self.llm) - async def _single_turn_ascore(self, sample: SingleTurnSample, callbacks: Callbacks) -> float: - faithfulness_score = await self.faithfulness_metric.single_turn_ascore(sample, callbacks) + async def _single_turn_ascore( + self, sample: SingleTurnSample, callbacks: Callbacks + ) -> float: + faithfulness_score = await self.faithfulness_metric.single_turn_ascore( + sample, callbacks + ) return 1 - faithfulness_score ``` @@ -181,12 +187,8 @@ Now let's evaluate the entire dataset with the metrics we have created. from ragas import evaluate results = evaluate( - eval_dataset, - metrics=[ - hallucinations_metric, - hallucinations_rubric, - hallucinations_binary - ], + eval_dataset, + metrics=[hallucinations_metric, hallucinations_rubric, hallucinations_binary], ) ``` diff --git a/docs/howtos/customizations/testgenerator/_persona_generator.md b/docs/howtos/customizations/testgenerator/_persona_generator.md new file mode 100644 index 000000000..d4c6d0db0 --- /dev/null +++ b/docs/howtos/customizations/testgenerator/_persona_generator.md @@ -0,0 +1,160 @@ +## Persona's in Testset Generation + +You can add different persona's to the testset generation process by defining the [Persona][ragas.testset.persona.Persona] class with the name and role description of the different persona's that might be relevant to your use case and you want to generate testset for. + +For example, for the [gitlab handbook](https://about.gitlab.com/handbook/) we might want to generate testset for different persona's like a new joinee, a manager, a senior manager, etc. And hence we will define them as follows: + +1. New Joinee: Don't know much about the company and is looking for information on how to get started. +2. Manager: Wants to know about the different teams and how they collaborate with each other. +3. Senior Manager: Wants to know about the company vision and how it is executed. + +Which we can define as follows: + + +```python +from ragas.testset.persona import Persona + +persona_new_joinee = Persona(name="New Joinee", role_description="Don't know much about the company and is looking for information on how to get started.") +persona_manager = Persona(name="Manager", role_description="Wants to know about the different teams and how they collaborate with each other.") +persona_senior_manager = Persona(name="Senior Manager", role_description="Wants to know about the company vision and how it is executed.") + +personas = [persona_new_joinee, persona_manager, persona_senior_manager] +personas +``` + + + + + [Persona(name='New Joinee', role_description="Don't know much about the company and is looking for information on how to get started."), + Persona(name='Manager', role_description='Wants to know about the different teams and how they collaborate with each other.'), + Persona(name='Senior Manager', role_description='Wants to know about the company vision and how it is executed.')] + + + +And then you can use these persona's in the testset generation process by passing them to the [TestsetGenerator][ragas.testset.generator.TestsetGenerator] class. + + +```python +from ragas.testset import TestsetGenerator +from ragas.testset.graph import KnowledgeGraph +from ragas.llms import llm_factory + +# Load the knowledge graph +kg = KnowledgeGraph.load("../../../../experiments/gitlab_kg.json") +# Initialize the Generator LLM +llm = llm_factory("gpt-4o-mini") + +# Initialize the Testset Generator +testset_generator = TestsetGenerator(knowledge_graph=kg, persona_list=personas, llm=llm) +# Generate the Testset +testset = testset_generator.generate(testset_size=10) +testset + +``` + + +```python +testset.to_pandas().head() +``` + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
user_inputreference_contextsreferencesynthesizer_name
0What the Director do in GitLab and how they wo...[09db4f3e-1c10-4863-9024-f869af48d3e0\n\ntitle...The Director at GitLab, such as the Director o...single_hop_specifc_query_synthesizer
1Wht is the rol of the VP in GitLab?[56c84f1b-3558-4c80-b8a9-348e69a4801b\n\nJob F...The VP, or Vice President, at GitLab is respon...single_hop_specifc_query_synthesizer
2What GitLab do for career progression?[ead619a5-930f-4e2b-b797-41927a04d2e3\n\nGoals...The Job frameworks at GitLab help team members...single_hop_specifc_query_synthesizer
3Wht is the S-grop and how do they work with ot...[42babb12-b033-493f-b684-914e2b1b1d0f\n\nPeopl...Members of the S-group are expected to demonst...single_hop_specifc_query_synthesizer
4How does Google execute its company vision?[c3ed463d-1cdc-4ba4-a6ca-2c4ab12da883\n\nof mo...To effectively execute the company vision, man...single_hop_specifc_query_synthesizer
+
+ + + +## Automatic Persona Generation + +If you want to automatically generate persona's from a knowledge graph, you can use the [generate_personas_from_kg][ragas.testset.persona.generate_personas_from_kg] function. + + + +```python +from ragas.testset.persona import generate_personas_from_kg +from ragas.testset.graph import KnowledgeGraph +from ragas.llms import llm_factory + +kg = KnowledgeGraph.load("../../../../experiments/gitlab_kg.json") +llm = llm_factory("gpt-4o-mini") + +personas = generate_personas_from_kg(kg=kg, llm=llm, num_personas=5) +``` + + +```python +personas +``` + + + + + [Persona(name='Organizational Development Manager', role_description='Responsible for implementing job frameworks and career development strategies to enhance employee growth and clarify roles within the company.'), + Persona(name='DevSecOps Product Manager', role_description='Responsible for overseeing the development and strategy of DevSecOps solutions, ensuring alignment with company goals and user needs.'), + Persona(name='Product Pricing Analyst', role_description='Responsible for developing and analyzing pricing strategies that align with customer needs and market demands.'), + Persona(name='Site Reliability Engineer', role_description='Responsible for maintaining service reliability and performance, focusing on implementing rate limits to prevent outages and enhance system stability.'), + Persona(name='Security Operations Engineer', role_description="Works on enhancing security logging processes and ensuring compliance within GitLab's infrastructure.")] + + diff --git a/docs/howtos/customizations/testgenerator/persona_generator.ipynb b/docs/howtos/customizations/testgenerator/persona_generator.ipynb new file mode 100644 index 000000000..7ed8e7744 --- /dev/null +++ b/docs/howtos/customizations/testgenerator/persona_generator.ipynb @@ -0,0 +1,241 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Persona's in Testset Generation\n", + "\n", + "You can add different persona's to the testset generation process by defining the [Persona][ragas.testset.persona.Persona] class with the name and role description of the different persona's that might be relevant to your use case and you want to generate testset for.\n", + "\n", + "For example, for the [gitlab handbook](https://about.gitlab.com/handbook/) we might want to generate testset for different persona's like a new joinee, a manager, a senior manager, etc. And hence we will define them as follows:\n", + "\n", + "1. New Joinee: Don't know much about the company and is looking for information on how to get started.\n", + "2. Manager: Wants to know about the different teams and how they collaborate with each other.\n", + "3. Senior Manager: Wants to know about the company vision and how it is executed.\n", + "\n", + "Which we can define as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Persona(name='New Joinee', role_description=\"Don't know much about the company and is looking for information on how to get started.\"),\n", + " Persona(name='Manager', role_description='Wants to know about the different teams and how they collaborate with each other.'),\n", + " Persona(name='Senior Manager', role_description='Wants to know about the company vision and how it is executed.')]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from ragas.testset.persona import Persona\n", + "\n", + "persona_new_joinee = Persona(name=\"New Joinee\", role_description=\"Don't know much about the company and is looking for information on how to get started.\")\n", + "persona_manager = Persona(name=\"Manager\", role_description=\"Wants to know about the different teams and how they collaborate with each other.\")\n", + "persona_senior_manager = Persona(name=\"Senior Manager\", role_description=\"Wants to know about the company vision and how it is executed.\")\n", + "\n", + "personas = [persona_new_joinee, persona_manager, persona_senior_manager]\n", + "personas" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And then you can use these persona's in the testset generation process by passing them to the [TestsetGenerator][ragas.testset.generator.TestsetGenerator] class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset import TestsetGenerator\n", + "from ragas.testset.graph import KnowledgeGraph\n", + "from ragas.llms import llm_factory\n", + "\n", + "# Load the knowledge graph\n", + "kg = KnowledgeGraph.load(\"../../../../experiments/gitlab_kg.json\")\n", + "# Initialize the Generator LLM\n", + "llm = llm_factory(\"gpt-4o-mini\")\n", + "\n", + "# Initialize the Testset Generator\n", + "testset_generator = TestsetGenerator(knowledge_graph=kg, persona_list=personas, llm=llm)\n", + "# Generate the Testset\n", + "testset = testset_generator.generate(testset_size=10)\n", + "testset\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
user_inputreference_contextsreferencesynthesizer_name
0What the Director do in GitLab and how they wo...[09db4f3e-1c10-4863-9024-f869af48d3e0\\n\\ntitle...The Director at GitLab, such as the Director o...single_hop_specifc_query_synthesizer
1Wht is the rol of the VP in GitLab?[56c84f1b-3558-4c80-b8a9-348e69a4801b\\n\\nJob F...The VP, or Vice President, at GitLab is respon...single_hop_specifc_query_synthesizer
2What GitLab do for career progression?[ead619a5-930f-4e2b-b797-41927a04d2e3\\n\\nGoals...The Job frameworks at GitLab help team members...single_hop_specifc_query_synthesizer
3Wht is the S-grop and how do they work with ot...[42babb12-b033-493f-b684-914e2b1b1d0f\\n\\nPeopl...Members of the S-group are expected to demonst...single_hop_specifc_query_synthesizer
4How does Google execute its company vision?[c3ed463d-1cdc-4ba4-a6ca-2c4ab12da883\\n\\nof mo...To effectively execute the company vision, man...single_hop_specifc_query_synthesizer
\n", + "
" + ], + "text/plain": [ + " user_input ... synthesizer_name\n", + "0 What the Director do in GitLab and how they wo... ... single_hop_specifc_query_synthesizer\n", + "1 Wht is the rol of the VP in GitLab? ... single_hop_specifc_query_synthesizer\n", + "2 What GitLab do for career progression? ... single_hop_specifc_query_synthesizer\n", + "3 Wht is the S-grop and how do they work with ot... ... single_hop_specifc_query_synthesizer\n", + "4 How does Google execute its company vision? ... single_hop_specifc_query_synthesizer\n", + "\n", + "[5 rows x 4 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "testset.to_pandas().head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Automatic Persona Generation\n", + "\n", + "If you want to automatically generate persona's from a knowledge graph, you can use the [generate_personas_from_kg][ragas.testset.persona.generate_personas_from_kg] function.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset.persona import generate_personas_from_kg\n", + "from ragas.testset.graph import KnowledgeGraph\n", + "from ragas.llms import llm_factory\n", + "\n", + "kg = KnowledgeGraph.load(\"../../../../experiments/gitlab_kg.json\")\n", + "llm = llm_factory(\"gpt-4o-mini\")\n", + "\n", + "personas = generate_personas_from_kg(kg=kg, llm=llm, num_personas=5)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Persona(name='Organizational Development Manager', role_description='Responsible for implementing job frameworks and career development strategies to enhance employee growth and clarify roles within the company.'),\n", + " Persona(name='DevSecOps Product Manager', role_description='Responsible for overseeing the development and strategy of DevSecOps solutions, ensuring alignment with company goals and user needs.'),\n", + " Persona(name='Product Pricing Analyst', role_description='Responsible for developing and analyzing pricing strategies that align with customer needs and market demands.'),\n", + " Persona(name='Site Reliability Engineer', role_description='Responsible for maintaining service reliability and performance, focusing on implementing rate limits to prevent outages and enhance system stability.'),\n", + " Persona(name='Security Operations Engineer', role_description=\"Works on enhancing security logging processes and ensuring compliance within GitLab's infrastructure.\")]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "personas" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ragas", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/howtos/integrations/_langgraph_agent_evaluation.md b/docs/howtos/integrations/_langgraph_agent_evaluation.md index 8ffa47a10..800f0678c 100644 --- a/docs/howtos/integrations/_langgraph_agent_evaluation.md +++ b/docs/howtos/integrations/_langgraph_agent_evaluation.md @@ -230,7 +230,7 @@ display(Image(react_graph.get_graph(xray=True).draw_mermaid_png())) -![jpeg](../../_static/imgs/_langgraph_agent_evaluation_28_0.jpg) +![jpeg](_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg) diff --git a/docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg b/docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e580db4d425ef600f44d4f19447c8de697fb3ced GIT binary patch literal 8204 zcmb7o1z1#FxBsD0TDlPsL`ph_1_@CRkQz$5Ylf5*q@+PWT1t>~2x*k=9Gam^y1VZ9 zzV}|=d!PG#|L4EXIeY)kUTd$lcAmA*-Q?W@K=@o7y6Oe)FB8og3i zl~;Q4>~BUFfF$&%0ATOn0#R3xVbIY9Ghi+JUE?>;*c9sYJO3Ai)O$MqTRQ;2Isb*s zzbn3HW)3w)3hW~vW(d+a(peHDCb9e*KmLtP{>EazvAc_t3sUCQZwz^@E{nuwNc`CH zAK2s{*whK~TRs9QBW7>s`rFoT`t32Uxr3$#GRHu~+ z0)YE^fANsXzwnI#$)Z8}<%qm20Xx7PU;vZ=2f!5ILLy$^DZm2=-pv5A02(Ul?;AFt4+!z_iSTi72+0VEh)GCENpT6tDac4D9*~fd z{8oa3hOC2*aSsFI9tj=}9?AbV-TefJuz?F;91VpLKqW#!BSN`r189-mjf#eXr2lFy zfPsmIj)jeadJoA~BLq;;QIOta;^1OoVj*RIi=$&;5@9`fNX$b@#we}!5u1dZS5wQ_ zWq1V6Bp~?O+4Ws)d_sBUBOw`C^?*1EW-~})79YQc39NY^sgnA)t=|^^+aQ26fPs#Q zg=F#(A=d*c$~}yGNSVI{P*91`&>uYHkycZW9VX#3cFw{eCS@{-gEcR{W86Qun+9;v zkWLYy5do6G?bmEZR7M6yM$}fD)wgO#lNqAhcK}Hl9({2EsCsw3Y%r?*g|%j>C+Eua z(7M1KaQP|(b!2gFy@ssXX^3WNSGB6fFMrW9QD2{YJH58V$nm>{Gack?*H8Od)8j80 zhkjjpjt=KzLFa}c#T6T>RNsE!zB)*>X)FJC=ORe(3E(F1@Tp zzW9Nzd@xVUrsbMPc3k&3|3i(@yI*fBB;Frr+QeAkKVrYtcXWzFtdywy_`0z~o47)c4ZwPOjB@3G zkC@rj=h!lEu{dvj%E&k#@K?Y7NkYe_LN;!=6S~^azLe-5%1yYF@2mt~x~XqAwmpH& z4U2qQw2rfwa49yGUmdVa1nA=Q%4Y(Jgg>@}%W$YhsQd`%(D2mXTc*M+lUOs9?#=E= zvZv`U`7nPa4Nfm8dmYe+zaDHMC=VJL5yCnlO*VerH^!$`Nva%5qcwuy3a7hKis#Ra zi5-3&Kn2%LaTjP9T=_sIzQXUyY9G7|A+2&&H7mIv9jLOh+_{Ch`2xD#9XepO?OZQ# z&wyGsnO{m+ROBt{1b^k>3Fj{@CQG*MPw?}(CmW3{(-S3X{fpk*_$sBMh=rqW9Ahgx zr}}QF88rPN1D{0*Bnjh9Rp$Y$-I1!u^^3*9V3+9e44hSe;u)l z+553GY5c9b!G67wEiG$ey+cQAMOdF~eu2dD##|g2j_;9!s4Csg5TXvwcz-d57VdFl z`E_lyl_z-O!d^Oh%t^0vw4?cHOKVubin;=`eMIQCqKP%L-G%3%yuT>c7tVtjIrz9x zWmb<4tiYumuQPxMn(R*L#yUQ1$Z`}>rVyQmFX|K3#cEzwy1{xQ%G6n~ zYNp;)KGnydC~XJKE)a4OO?15~OUSZFs7V#qB>@f3mTh{Bv z+p_myYNHj?ZX5AO?+54XWXQl>Y9Bn&-^5YTEU0qrI?t>=nLZVi7m2p=q7meDLJ<9W zQh7^nn#H_qxAOG!9pD=E{d|coG8@z}KQw#?ylEuXq)4$%RR!Gv^CyX6+R?D;AdIZ0 zwM)H^52W^AP@}Ei4t9~ID(vqmJDpPVSVxLSzz=kx1|Xg)O5kJUPcA$(pK9^jgpjsW zda8=~?;+dqf6(mW!R2$XPziS7E%W60SjJ7Iq*R>F#xWAE;$aUw_Xr1&+M<=E^G?i*=~}9tjHmn9 z_^VD#PBij&02^51ztv~_L3;$zBg*o}Xg-k$wl_|W`26*<9Rp~@SZXyOde#xFP4%~y z0d;8d=1V;b{DxR~@u4CxL{9VvN-&L}6TYo9b%3m*gd$t<)}n@6S>H~h=Lavu=5swS@2_7N+{{;2Ywav` z7-gaPU5^XZ4_2>i%>GEml9H1{m-}}K#vtUUUnPEmSU*;~cV~RH#_d_eR@3Z!{(z){ zjQ$yC^_8@>VSZ}6cSmc|EjQ)*9dLFNeX&u{!(vMBTGmf%wIde>_1+K zf2afyh&2-F?;9vdV`8337anw z^A?RhiH!#XWW9Ay6hd4>mMA3kQn%r)b*%@u^dF3gGz;as4OqoK8B+%0+o5G) zRV}r%1D@wr;_~z zJ~1A?oI0J0!IrezFaeA(T#dfrUXoq$jO%UdNxz=K{S!CvS*I|^arwV><3sG?tnqm; z!T8!0|Nic`Ygw!PdH+k$bjjf}B#1)4mtBFO*N(*>AB>*M?;pMi`{d zPB+HZDd7ElSkF@ms(A-@xcwkMTk}{mr9U~R3`{1fWRcKZY$}DQZ94`~Y9C1_kTnT0 zY4b!C`F=c@&a+`rMLz%~$H_0yOz>YcqZ3_dwR{fOW=qHq?3nIav6qz((n;q(wG-nN z&as*ufJRFJ=y6&$N1ZmK8%N z^x#p0U)PwHSKajwryhalOQGS3ZU4Ii=vZ!+vM#1Wj(Cb71qVWVqyz(_lF5;8Ar80E z!saS9_Js;mCj#P3t+;FQ+f#~+D^c~d?_*)JF6CX5*>)=K^0k9rdVO#4&W_mYK8%U# zk(RjJAm|_;J!v#M+@Rx0J{dyjuIPl)Kmay_zcf1ci%pXwu8$>cr)I*bt(Tr&H4$xf zZeNt~AyX6Pt9kr!FR4Smr&H_CIa6#@1QKY)*A0?@lQf)ePqCE2l2fRlO6=2iP;sf?m|g#d^2*cE%B5a4C~|5+DjZDi9a2|B%M_j4 zB-?2vOw{oiuAb{Zb8W~w8JT0anTq~ zIMk}3&dGm~+s;dAW>tD&>J6!%_cZU1y-GTwMX35|=4YQ%W}W-EjWy8SODNVUwr)I{ zOc~^act&tMXl#y+hjBRcgYeTy`(rx}PHZ-eOn=}#t=XFXeru$uAUmyja2cRu;r%5h zcALzMOJGYN0vc{QV;}N(Q`07~a2)2e#guP);bagn~u#1+I z#a$$>MSUcpg8$}6e2V8n>~_Vp!i-4!ZdeEAgU$!D;lVS)LBBFcH|nO1WK5on)y6cE zBrIoEbsHtFq}E1)uzJ=B=VhfQJp*PlrL3=XBj(h-?VyN$wP@-Xms04INd-tn2hrl` zc2%6ZjUqPd{_6)7LC7aM%0+`Pcoj=tG_PA@#P>B=k0*FG#oQ|#Of~L5Bt`9l^Eg}) zRziz%Hlus%9(_So^v~6$$68<3dxo3yy7X~qt)1Ne+|@Duv7b~wE_oSv5JJEy*uP^P zxST>+>sc>i>fv>dIo|FkW6zN9v#-k%Z|3a+5ym4=tHwv=V)ZE<1`>;Wl`8kF@AvNgCk#NM^L0%A+BSjD4obWC zF@+{aPpO$9FO`YV*XKeJh+GS<_aKNKIp3f=d)lveSQU~k1br7cq90%C5pz^-d|FIM z?y6^F{75NQvq&9h&;_sW^;EEGnS;#I^*F$q?Csb-E8qN5UQ$q5 zqJO<+H^3!iP-zm7r}QRN>X7}SJ+v7hboX6{-{9-t0S|kH=%3PL<&7N=D5ULrrZ872 zB;~txmb+j2J;DDgWDZ~#%C}w9xd5LHb$>Y2DoZHrx~bAEg12@0zbNOuVeT>0Pk9zy7OI7ON3$H z=KY^z(Qe~sGoIq0&^z_*M;uRm`nUNvVyoyFvH0bUP`X{E{RG-aVhQY@ZasDzhhfc; zUe`(TLr48R6R0<-eYlXPQzZC-(ecj~X{T3Ncwa}ETuk5TUhB(&vP~dc>xz1RdCt9M4AKSmb6ujBgQ$N6A zH+^o(;^OgP4|j0oAv9&zAh0oyIK(9y-7N~sO@1eMFt*fuv(ckanubum%Q!46B1$m0 zDiZAtK>L$`7dP}j&nllChJ7(-i%ZBVzGbM|nPoxEWB2f)lQLK;QhQfzu^wBmucBpb zTE=322O(pE=px_9(4CPgV#|OwE_^ds4VFMZQx@{ed7;%)wrCy~mD^{s;;Rm)p^Odg zuX$HSZO&y~Mv~J28zxVzFOutB>Jf>eQ7sm*fz)gL2WpK z4W)zsXtJH2z`Jx9u3^fanL4Op{a9+ZtN6>1!P>+8YYu+yyoS0Mw!ds8SJYZR(>hZho~LrAtEV5y*aU-?RhTABFEuKb zomAQQEFmLbYqxi0UzxAKQydIi6vO4U`Fi^hUfA>H;m$o*T#itcC6*e+efKDlY+il& zqT-|CkxM-hb*x@`@rDMCj&+$9@l}g@XO2!d%*?c?TCCW03o9_|%YoA}tSVJ2O{^VO z8K<1KwB9%XN)j1>rmYJjGv+uYcFotCa?>^C;K5Hccz8UaB@rFkq+hksdD7b3kQI=t zzIG28Z)7a#uM5X1F@HsW7Vzf~%l^H~k?=s)yPLZmu|YmJ+$-RC?I^dNy6c;G@8*76 z&6?^IX-na?e7h_+@jgua)69+QDg_2gM2{wOOhudq!e)?;^f49hX2K+5Rk z{%Z{F%87+F_O)0)zyKGILSwD|fnsQd^zWsES*lE}$EossJm-Pj9gt))A{xRURpJC9 zTpT6D-mE^RQ}-SZZ=S2X9q&6{@-DZp2Gz>%&h*!ARrkO+#5itCUDo?dZb_rK(kS;o zU8_EutjuOyXF#6kKU9}irBSgHa)|O+tn6(Uj;*ck7|kh3SX4SF9peSB_`4Y|j)m*1 z$r+lxSUDVZN}zRQf;8SQIfG9pz`prv3;P#6I0#XYIfSPLwYkg*uXx-T=W@}6^XSRqai{vQ(7j!i|s77H#?^eoUgz&MmhL!q2sUY=`P;mR=ISEF(qBi94Q6YYf#(>16cisp{MxyXl$T%(_)A)3Cp z77~_1`=o|&qrgvPDbKD^T%5d;k8Zd+PC^vBm|ZyRDlp?ZL=ag6GoM*atBoYuJ!gU% z-i}cW++P#OXlQ?C(x^PXP$VevE2V$x`r#vwEo?7}>HhR?YJ)yc|FwsW+Bw=8iwk6> zNm82z7!nFNWGEABy$g_`WCI8_%YH4fWoRW|LK&Aerzf-h5S!3sIGRsP{D!S=N}4Pj zOA(wElq5-B1;Oi5u&7kQODOR$6-+3FVzJE7Iah@0upj&YiGQexRL%J^47b5GsMk^E zS(&jO^nP_(`r*|$f9qr8b%31U(VW(z{F(<*p>S^e)wTZ5I6jJDrt(&$f5fGsbQ< z%x!=^!vHaAO)UeSskSxqsSDi&Idv zyr~ow_oD1O7$hiATx24lC0uUdzqBBB1!>@#`h0xT$d>Iyp?AT?M-Cfh_$nb|f3e*d z8(b$_%j!<3s;+97TlDfYY%C-wb8H~(BEqPp;vehN89CC?ZKdsbMC2mKUmp56;UcVN zHDM-L!k=JOrU8N(Yjvu1V$nSy2EP@Yrc#XdG+S7jY=%33Xt}FthET@3gk)_4K^KVW z)~SB3y?+PnI&HskS+|{_OV%WRty9N0llI&=C}@wc<-+LOF^_+3=L(RZNuvi}zban8 z0mWQloSvAd$$$9ruml1P1S>J1jK;6&t0a{N61YE9+YY32O57>W)u<3aEAv!cv5Wd5 z_N`@!pot3)lo;e?N#b&I_PxwBMZ9+V)Hk!KBmL6V*&h%+i!U#8x{|Tk!6K7Xt-AC* zP0nIUCEZAU+_ZTWH%^Aq-6w>WnOfsDI}!h&ZmTd8sulUYa<_U%Cx6-Xa(|Uq-P&YU zJiMz^3^kow} zRmCTq)Ax4D^VIqiwBc^zD6^Jb`>EWHA>dY^mp`bl!_6+;y>@SVhIps^4V|YD{;D-K zAb;y8+FamaUCe%d6b(;wDmNB*m12$g63%P${gMB(hmK5&UIsnz3H#{$!R)U+eD*?c zI`WxoTKZ>$oF%jYDc1D-Bd^}=#VL@B4DeLODeW;|jJ9MipER+{FF$^>SIzba;bUd{ zV$(wp-ljh}ea|l9eO2o!hUJ+7yz{Df@2@e~$tTBWWV8vt$X?5<)KZ%C@$xV4Bn!*u z&qKSiOUOmJg;D?LDe}-Cv%3`qRbd4GJd`3UQZ`fxF|B8|)dw{!7nWP4*;a_2q3A}g z*%7Cf$KC|Xk4WDEx)FZKFS4!D?yr?-Wwt(XZDa@yKf-vnS=JoU*|&C!S~P9D4<$WP z{Nc~8Vv$-d96m-N<-M;&$l9krB6+2W_Xq-_pmGZAwy5L?Xv6W)G_DpUb`!f1|A+m{#2t9VvfO5&*f*~ z@@9!>QL7vn=p$J&gYErbx{%+QAB1iE=z_Hm2TJhKhi%pq7lhV!AvKrUrc+{8wSE|k z(W=I|0)Madj8NU(6m=L8LU%;GC6r?92)Cenq@&sH^70E-5~PaL`Z}bNcWgjn-B7Dx zw_D*n#}C@+?ygpY;6K)x$mi9m(@jB!e!1jDlJl|_EM2V^VCnlXZft&+QZ8Yt9oEv} z+HLv*Lh|uNFDSnCMdizD$-JzVSa)vcM(TYUqpJ8V)|$D|A4@H7PXi(iha5ov*!Y)qcDQ z7OF8Y=8`#jJRG2WHos>xfift`wpF2^wkqr|w5jd3*bzV_7y+O5V(pz-R#`;Fk)=npaObA84IU-As{vIe%D-0GwuQcf@$)>jzmgJ>gT$<@F2hP+r z0Kb>h=SPOD&#Z4`-{|zbIjE)lx%y!GD@`=oaQ!ElRq#pJ?Wh?G8tFimf_=#vCDyln1K!u9~ zG)HkqjsHpmaiyvApa?f;$fE==|EZRqC9QwQt8cRx7A5M!bF){c3=TpXq2ZdEWBPMn S>uiaL-uwyV{vgS_ss8{IdMRcA literal 0 HcmV?d00001 diff --git a/docs/howtos/integrations/_llamaindex.md b/docs/howtos/integrations/_llamaindex.md index abaaa79de..bdff4c78a 100644 --- a/docs/howtos/integrations/_llamaindex.md +++ b/docs/howtos/integrations/_llamaindex.md @@ -11,13 +11,15 @@ You will need an testset to evaluate your `QueryEngine` against. You can either Let's see how that works with Llamaindex # load the documents + + ```python from llama_index.core import SimpleDirectoryReader documents = SimpleDirectoryReader("./nyc_wikipedia").load_data() ``` -Now lets init the `TestsetGenerator` object with the corresponding generator and critic llms +Now lets init the `TestsetGenerator` object with the corresponding generator and critic llms ```python diff --git a/mkdocs.yml b/mkdocs.yml index 49954d63d..04a6b81ce 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,6 +84,7 @@ nav: - Write your own Metrics - (advanced): howtos/customizations/metrics/_write_your_own_metric_advanced.md - Testset Generation: - howtos/customizations/testgenerator/index.md + - Persona Generation: howtos/customizations/testgenerator/_persona_generator.md - Applications: - howtos/applications/index.md - Metrics: diff --git a/src/ragas/testset/synthesizers/__init__.py b/src/ragas/testset/synthesizers/__init__.py index 48679a179..7ccde116c 100644 --- a/src/ragas/testset/synthesizers/__init__.py +++ b/src/ragas/testset/synthesizers/__init__.py @@ -15,7 +15,6 @@ def default_query_distribution(llm: BaseRagasLLM) -> QueryDistribution: - """ """ return [ (SingleHopSpecificQuerySynthesizer(llm=llm), 0.5), (MultiHopAbstractQuerySynthesizer(llm=llm), 0.25), diff --git a/src/ragas/testset/synthesizers/multi_hop/abstract.py b/src/ragas/testset/synthesizers/multi_hop/abstract.py index 20162ff4e..65020ddf8 100644 --- a/src/ragas/testset/synthesizers/multi_hop/abstract.py +++ b/src/ragas/testset/synthesizers/multi_hop/abstract.py @@ -70,6 +70,10 @@ async def _generate_scenarios( logger.info("found %d clusters", len(node_clusters)) scenarios = [] + if len(node_clusters) == 0: + raise ValueError( + "No clusters found in the knowledge graph. Use a different Synthesizer." + ) num_sample_per_cluster = int(np.ceil(n / len(node_clusters))) for cluster in node_clusters: