-
- 
-
-
-
-
- diff --git a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py index d0e76a8f8..56a842c47 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py +++ b/agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py @@ -6,7 +6,7 @@ import click from sqlalchemy.future import select -from sqlalchemy import create_engine +from sqlalchemy import create_engine, delete from sqlalchemy.orm import sessionmaker, Session from agenta_backend.models.deprecated_models import ( @@ -34,6 +34,7 @@ def update_evaluators_with_app_name(): records = ( session.execute( select(DeprecatedEvaluatorConfigDB) + .filter(DeprecatedEvaluatorConfigDB.app_id.isnot(None)) .offset(offset) .limit(BATCH_SIZE) ) @@ -54,6 +55,13 @@ def update_evaluators_with_app_name(): session.commit() offset += BATCH_SIZE + # Delete deprecated evaluator configs with app_id as None + session.execute( + delete(DeprecatedEvaluatorConfigDB).where( + DeprecatedEvaluatorConfigDB.app_id.is_(None) + ) + ) + session.commit() except Exception as e: session.rollback() click.echo( diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/5c29a64204f4_added_modified_by_id_column_to_apps_db.py b/agenta-backend/agenta_backend/migrations/postgres/versions/5c29a64204f4_added_modified_by_id_column_to_apps_db.py index e8a444b10..cd3ea9cb2 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/versions/5c29a64204f4_added_modified_by_id_column_to_apps_db.py +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/5c29a64204f4_added_modified_by_id_column_to_apps_db.py @@ -5,6 +5,8 @@ Create Date: 2024-08-25 17:56:11.732929 """ + +import os from typing import Sequence, Union from alembic import op @@ -18,16 +20,32 @@ depends_on: Union[str, Sequence[str], None] = None +def unique_constraint_exists(engine: sa.Engine, table_name: str, constraint_name: str): + with engine.connect() as conn: + result = conn.execute( + sa.text( + f""" + SELECT conname FROM pg_constraint + WHERE conname = '{constraint_name}' AND conrelid = '{table_name}'::regclass; + """ + ) + ) + return result.fetchone() is not None + + def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("app_db", sa.Column("modified_by_id", sa.UUID(), nullable=True)) - op.create_foreign_key(None, "app_db", "users", ["modified_by_id"], ["id"]) + op.create_foreign_key( + "app_db_modified_by_id_fkey", "app_db", "users", ["modified_by_id"], ["id"] + ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint(None, "app_db", type_="foreignkey") - op.drop_constraint(None, "app_db", type_="unique") + engine = sa.create_engine(os.getenv("POSTGRES_URI")) # type: ignore + if not unique_constraint_exists(engine, "app_db", "app_db_modified_by_id_fkey"): + op.drop_constraint("app_db_modified_by_id_fkey", "app_db", type_="foreignkey") op.drop_column("app_db", "modified_by_id") # ### end Alembic commands ### diff --git a/agenta-backend/agenta_backend/migrations/postgres/versions/b80c708c21bb_initial_migration.py b/agenta-backend/agenta_backend/migrations/postgres/versions/b80c708c21bb_initial_migration.py index f69ed47b8..6bc66e751 100644 --- a/agenta-backend/agenta_backend/migrations/postgres/versions/b80c708c21bb_initial_migration.py +++ b/agenta-backend/agenta_backend/migrations/postgres/versions/b80c708c21bb_initial_migration.py @@ -5,6 +5,8 @@ Create Date: 2024-07-11 13:20:31.705894 """ + +import os from typing import Sequence, Union from alembic import op @@ -23,6 +25,19 @@ depends_on: Union[str, Sequence[str], None] = None +def unique_constraint_exists(engine: sa.Engine, table_name: str, constraint_name: str): + with engine.connect() as conn: + result = conn.execute( + sa.text( + f""" + SELECT conname FROM pg_constraint + WHERE conname = '{constraint_name}' AND conrelid = '{table_name}'::regclass; + """ + ) + ) + return result.fetchone() is not None + + def first_time_user_from_agenta_v019_upwards_upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( @@ -475,26 +490,104 @@ def first_time_user_from_agenta_v019_upwards_upgrade() -> None: def returning_user_from_agenta_v018_downwards_upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_unique_constraint(None, "app_db", ["id"]) - op.create_unique_constraint(None, "app_variant_revisions", ["id"]) - op.create_unique_constraint(None, "app_variants", ["id"]) - op.create_unique_constraint(None, "bases", ["id"]) - op.create_unique_constraint(None, "deployments", ["id"]) - op.create_unique_constraint(None, "docker_images", ["id"]) - op.create_unique_constraint(None, "environments", ["id"]) - op.create_unique_constraint(None, "environments_revisions", ["id"]) - op.create_unique_constraint(None, "evaluation_aggregated_results", ["id"]) - op.create_unique_constraint(None, "evaluation_scenario_results", ["id"]) - op.create_unique_constraint(None, "evaluation_scenarios", ["id"]) - op.create_unique_constraint(None, "evaluations", ["id"]) - op.create_unique_constraint(None, "evaluators_configs", ["id"]) - op.create_unique_constraint(None, "human_evaluation_variants", ["id"]) - op.create_unique_constraint(None, "human_evaluations", ["id"]) - op.create_unique_constraint(None, "human_evaluations_scenarios", ["id"]) - op.create_unique_constraint(None, "ids_mapping", ["id"]) - op.create_unique_constraint(None, "templates", ["id"]) - op.create_unique_constraint(None, "testsets", ["id"]) - op.create_unique_constraint(None, "users", ["id"]) + engine = sa.create_engine(os.getenv("POSTGRES_URI")) # type: ignore + if not unique_constraint_exists(engine, "app_db", "app_db_pkey"): + op.create_unique_constraint("app_db_pkey", "app_db", ["id"]) + + if not unique_constraint_exists( + engine, "app_variant_revisions", "app_variant_revisions_pkey" + ): + op.create_unique_constraint( + "app_variant_revisions_pkey", "app_variant_revisions", ["id"] + ) + + if not unique_constraint_exists(engine, "app_variants", "app_variants_pkey"): + op.create_unique_constraint("app_variants_pkey", "app_variants", ["id"]) + + if not unique_constraint_exists(engine, "bases", "bases_pkey"): + op.create_unique_constraint("bases_pkey", "bases", ["id"]) + + if not unique_constraint_exists(engine, "deployments", "deployments_pkey"): + op.create_unique_constraint("deployments_pkey", "deployments", ["id"]) + + if not unique_constraint_exists(engine, "docker_images", "docker_images_pkey"): + op.create_unique_constraint("docker_images_pkey", "docker_images", ["id"]) + + if not unique_constraint_exists(engine, "environments", "environments_pkey"): + op.create_unique_constraint("environments_pkey", "environments", ["id"]) + + if not unique_constraint_exists( + engine, "environments_revisions", "environments_revisions_pkey" + ): + op.create_unique_constraint( + "environments_revisions_pkey", "environments_revisions", ["id"] + ) + + if not unique_constraint_exists( + engine, "evaluation_aggregated_results", "evaluation_aggregated_results_pkey" + ): + op.create_unique_constraint( + "evaluation_aggregated_results_pkey", + "evaluation_aggregated_results", + ["id"], + ) + + if not unique_constraint_exists( + engine, "evaluation_scenario_results", "evaluation_scenario_results_pkey" + ): + op.create_unique_constraint( + "evaluation_scenario_results_pkey", "evaluation_scenario_results", ["id"] + ) + + if not unique_constraint_exists( + engine, "evaluation_scenarios", "evaluation_scenarios_pkey" + ): + op.create_unique_constraint( + "evaluation_scenarios_pkey", "evaluation_scenarios", ["id"] + ) + + if not unique_constraint_exists(engine, "evaluations", "evaluations_pkey"): + op.create_unique_constraint("evaluations_pkey", "evaluations", ["id"]) + + if not unique_constraint_exists( + engine, "evaluators_configs", "evaluators_configs_pkey" + ): + op.create_unique_constraint( + "evaluators_configs_pkey", "evaluators_configs", ["id"] + ) + + if not unique_constraint_exists( + engine, "human_evaluation_variants", "human_evaluation_variants_pkey" + ): + op.create_unique_constraint( + "human_evaluation_variants_pkey", "human_evaluation_variants", ["id"] + ) + + if not unique_constraint_exists( + engine, "human_evaluations", "human_evaluations_pkey" + ): + op.create_unique_constraint( + "human_evaluations_pkey", "human_evaluations", ["id"] + ) + + if not unique_constraint_exists( + engine, "human_evaluations_scenarios", "human_evaluations_scenarios_pkey" + ): + op.create_unique_constraint( + "human_evaluations_scenarios_pkey", "human_evaluations_scenarios", ["id"] + ) + + if not unique_constraint_exists(engine, "ids_mapping", "ids_mapping_pkey"): + op.create_unique_constraint("ids_mapping_pkey", "ids_mapping", ["id"]) + + if not unique_constraint_exists(engine, "templates", "templates_pkey"): + op.create_unique_constraint("templates_pkey", "templates", ["id"]) + + if not unique_constraint_exists(engine, "testsets", "testsets_pkey"): + op.create_unique_constraint("testsets_pkey", "testsets", ["id"]) + + if not unique_constraint_exists(engine, "users", "users_pkey"): + op.create_unique_constraint("users_pkey", "users", ["id"]) # ### end Alembic commands ### @@ -528,26 +621,46 @@ def first_time_user_from_agenta_v019_upwards_downgrade() -> None: def returning_user_from_agenta_v018_downwards_downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint(None, "users", type_="unique") - op.drop_constraint(None, "testsets", type_="unique") - op.drop_constraint(None, "templates", type_="unique") - op.drop_constraint(None, "ids_mapping", type_="unique") - op.drop_constraint(None, "human_evaluations_scenarios", type_="unique") - op.drop_constraint(None, "human_evaluations", type_="unique") - op.drop_constraint(None, "human_evaluation_variants", type_="unique") - op.drop_constraint(None, "evaluators_configs", type_="unique") - op.drop_constraint(None, "evaluations", type_="unique") - op.drop_constraint(None, "evaluation_scenarios", type_="unique") - op.drop_constraint(None, "evaluation_scenario_results", type_="unique") - op.drop_constraint(None, "evaluation_aggregated_results", type_="unique") - op.drop_constraint(None, "environments_revisions", type_="unique") - op.drop_constraint(None, "environments", type_="unique") - op.drop_constraint(None, "docker_images", type_="unique") - op.drop_constraint(None, "deployments", type_="unique") - op.drop_constraint(None, "bases", type_="unique") - op.drop_constraint(None, "app_variants", type_="unique") - op.drop_constraint(None, "app_variant_revisions", type_="unique") - op.drop_constraint(None, "app_db", type_="unique") + op.drop_constraint("users_pkey", "users", type_="unique") + op.drop_constraint("testsets_pkey", "testsets", type_="unique") + op.drop_constraint("templates_pkey", "templates", type_="unique") + op.drop_constraint("ids_mapping_pkey", "ids_mapping", type_="unique") + op.drop_constraint( + "human_evaluations_scenarios_pkey", + "human_evaluations_scenarios", + type_="unique", + ) + op.drop_constraint("human_evaluations_pkey", "human_evaluations", type_="unique") + op.drop_constraint( + "human_evaluation_variants_pkey", "human_evaluation_variants", type_="unique" + ) + op.drop_constraint("evaluators_configs_pkey", "evaluators_configs", type_="unique") + op.drop_constraint("evaluations_pkey", "evaluations", type_="unique") + op.drop_constraint( + "evaluation_scenarios_pkey", "evaluation_scenarios", type_="unique" + ) + op.drop_constraint( + "evaluation_scenario_results_pkey", + "evaluation_scenario_results", + type_="unique", + ) + op.drop_constraint( + "evaluation_aggregated_results_pkey", + "evaluation_aggregated_results", + type_="unique", + ) + op.drop_constraint( + "environments_revisions_pkey", "environments_revisions", type_="unique" + ) + op.drop_constraint("environments_pkey", "environments", type_="unique") + op.drop_constraint("docker_images_pkey", "docker_images", type_="unique") + op.drop_constraint("deployments_pkey", "deployments", type_="unique") + op.drop_constraint("bases_key", "bases", type_="unique") + op.drop_constraint("app_variants_pkey", "app_variants", type_="unique") + op.drop_constraint( + "app_variant_revisions_pkey", "app_variant_revisions", type_="unique" + ) + op.drop_constraint("app_db_pkey", "app_db", type_="unique") # ### end Alembic commands ### diff --git a/agenta-backend/agenta_backend/models/api/evaluation_model.py b/agenta-backend/agenta_backend/models/api/evaluation_model.py index f79ebba16..0f2b1b364 100644 --- a/agenta-backend/agenta_backend/models/api/evaluation_model.py +++ b/agenta-backend/agenta_backend/models/api/evaluation_model.py @@ -1,7 +1,9 @@ from enum import Enum from datetime import datetime -from pydantic import BaseModel from typing import Optional, List, Dict, Any, Union + +from pydantic import BaseModel, Field, model_validator + from agenta_backend.models.api.api_models import Result @@ -12,6 +14,8 @@ class Evaluator(BaseModel): settings_template: dict description: Optional[str] = None oss: Optional[bool] = False + requires_llm_api_keys: Optional[bool] = False + tags: List[str] class EvaluatorConfig(BaseModel): @@ -80,6 +84,25 @@ class Evaluation(BaseModel): updated_at: datetime +class EvaluatorInputInterface(BaseModel): + inputs: Dict[str, Any] = Field(default_factory=dict) + settings: Optional[Dict[str, Any]] = None + credentials: Optional[Dict[str, Any]] = None + + +class EvaluatorOutputInterface(BaseModel): + outputs: Dict[str, Any] + + +class EvaluatorMappingInputInterface(BaseModel): + inputs: Dict[str, Any] + mapping: Dict[str, Any] + + +class EvaluatorMappingOutputInterface(BaseModel): + outputs: Dict[str, Any] + + class SimpleEvaluationOutput(BaseModel): id: str variant_ids: List[str] diff --git a/agenta-backend/agenta_backend/models/api/testset_model.py b/agenta-backend/agenta_backend/models/api/testset_model.py index a5c82b230..02ccbe62b 100644 --- a/agenta-backend/agenta_backend/models/api/testset_model.py +++ b/agenta-backend/agenta_backend/models/api/testset_model.py @@ -48,6 +48,7 @@ class TestSetOutputResponse(BaseModel): id: str = Field(..., alias="_id") name: str created_at: str + updated_at: str class Config: allow_population_by_field_name = True diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py index 180d3bbd2..ce2ca870b 100644 --- a/agenta-backend/agenta_backend/models/converters.py +++ b/agenta-backend/agenta_backend/models/converters.py @@ -413,6 +413,7 @@ async def environment_db_and_revision_to_extended_output( environment_output_extended = EnvironmentOutputExtended( name=environment_db.name, app_id=str(environment_db.app_id), + project_id=str(environment_db.project_id), deployed_app_variant_id=deployed_app_variant_id, deployed_variant_name=deployed_variant_name, deployed_app_variant_revision_id=str( diff --git a/agenta-backend/agenta_backend/resources/evaluators/evaluators.py b/agenta-backend/agenta_backend/resources/evaluators/evaluators.py index 2ecf7b6f7..23393a2e7 100644 --- a/agenta-backend/agenta_backend/resources/evaluators/evaluators.py +++ b/agenta-backend/agenta_backend/resources/evaluators/evaluators.py @@ -41,6 +41,7 @@ }, "description": "Exact Match evaluator determines if the output exactly matches the specified correct answer, ensuring precise alignment with expected results.", "oss": True, + "tags": ["functional"], }, { "name": "Contains JSON", @@ -49,6 +50,7 @@ "settings_template": {}, "description": "'Contains JSON' evaluator checks if the output contains the a valid JSON.", "oss": True, + "tags": ["functional", "classifiers"], }, { "name": "Similarity Match", @@ -75,11 +77,13 @@ }, "description": "Similarity Match evaluator checks if the generated answer is similar to the expected answer. You need to provide the similarity threshold. It uses the Jaccard similarity to compare the answers.", "oss": True, + "tags": ["similarity", "functional"], }, { "name": "Semantic Similarity Match", "key": "auto_semantic_similarity", "direct_use": False, + "requires_llm_api_keys": True, "description": "Semantic Similarity Match evaluator measures the similarity between two pieces of text by analyzing their meaning and context. It compares the semantic content, providing a score that reflects how closely the texts match in terms of meaning, rather than just exact word matches.", "settings_template": { "correct_answer_key": { @@ -92,6 +96,7 @@ }, }, "oss": True, + "tags": ["similarity", "ai_llm"], }, { "name": "Regex Test", @@ -114,6 +119,7 @@ }, }, "oss": True, + "tags": ["classifiers", "functional"], }, { "name": "JSON Field Match", @@ -138,6 +144,7 @@ }, "description": "JSON Field Match evaluator compares specific fields within JSON (JavaScript Object Notation) data. This matching can involve finding similarities or correspondences between fields in different JSON objects.", "oss": True, + "tags": ["functional"], }, { "name": "JSON Diff Match", @@ -176,11 +183,13 @@ }, }, "oss": True, + "tags": ["similarity", "functional"], }, { "name": "LLM-as-a-judge", "key": "auto_ai_critique", "direct_use": False, + "requires_llm_api_keys": True, "settings_template": { "prompt_template": { "label": "Prompt Template", @@ -200,16 +209,25 @@ }, "description": "AI Critique evaluator sends the generated answer and the correct_answer to an LLM model and uses it to evaluate the correctness of the answer. You need to provide the evaluation prompt (or use the default prompt).", "oss": True, + "tags": ["ai_llm", "functional"], }, { "name": "Code Evaluation", "key": "auto_custom_code_run", "direct_use": False, "settings_template": { + "requires_llm_api_keys": { + "label": "Requires LLM API Key(s)", + "type": "boolean", + "required": True, + "default": False, + "advanced": True, + "description": "Indicates whether the evaluation requires LLM API key(s) to function.", + }, "code": { "label": "Evaluation Code", "type": "code", - "default": "from typing import Dict\n\ndef evaluate(\n app_params: Dict[str, str],\n inputs: Dict[str, str],\n output: Union[str, Dict[str, Any]], # output of the llm app\n datapoint: Dict[str, str] # contains the testset row \n) -> float:\n if output in datapoint.get('correct_answer', None):\n return 1.0\n else:\n return 0.0\n", + "default": "from typing import Dict, Union, Any\n\ndef evaluate(\n app_params: Dict[str, str],\n inputs: Dict[str, str],\n output: Union[str, Dict[str, Any]], # output of the llm app\n correct_answer: str # contains the testset row \n) -> float:\n if output in correct_answer:\n return 1.0\n else:\n return 0.0\n", "description": "Code for evaluating submissions", "required": True, }, @@ -224,12 +242,21 @@ }, "description": "Code Evaluation allows you to write your own evaluator in Python. You need to provide the Python code for the evaluator.", "oss": True, + "tags": ["functional"], }, { "name": "Webhook test", "key": "auto_webhook_test", "direct_use": False, "settings_template": { + "requires_llm_api_keys": { + "label": "Requires LLM API Key(s)", + "type": "boolean", + "required": True, + "default": False, + "advanced": True, + "description": "Indicates whether the evaluation requires LLM API key(s) to function.", + }, "webhook_url": { "label": "Webhook URL", "type": "string", @@ -247,6 +274,7 @@ }, "description": "Webhook test evaluator sends the generated answer and the correct_answer to a webhook and expects a response, in JSON format, indicating the correctness of the answer, along with a 200 HTTP status. You need to provide the URL of the webhook and the response of the webhook must be between 0 and 1.", "oss": True, + "tags": ["functional"], }, { "name": "Starts With", @@ -268,6 +296,7 @@ }, "description": "Starts With evaluator checks if the output starts with a specified prefix, considering case sensitivity based on the settings.", "oss": True, + "tags": ["classifiers", "functional"], }, { "name": "Ends With", @@ -289,6 +318,7 @@ }, "description": "Ends With evaluator checks if the output ends with a specified suffix, considering case sensitivity based on the settings.", "oss": True, + "tags": ["classifiers", "functional"], }, { "name": "Contains", @@ -310,6 +340,7 @@ }, "description": "Contains evaluator checks if the output contains a specified substring, considering case sensitivity based on the settings.", "oss": True, + "tags": ["classifiers", "functional"], }, { "name": "Contains Any", @@ -331,6 +362,7 @@ }, "description": "Contains Any evaluator checks if the output contains any of the specified substrings from a comma-separated list, considering case sensitivity based on the settings.", "oss": True, + "tags": ["classifiers", "functional"], }, { "name": "Contains All", @@ -352,6 +384,7 @@ }, "description": "Contains All evaluator checks if the output contains all of the specified substrings from a comma-separated list, considering case sensitivity based on the settings.", "oss": True, + "tags": ["classifiers", "functional"], }, { "name": "Levenshtein Distance", @@ -375,20 +408,25 @@ }, "description": "This evaluator calculates the Levenshtein distance between the output and the correct answer. If a threshold is provided in the settings, it returns a boolean indicating whether the distance is within the threshold. If no threshold is provided, it returns the actual Levenshtein distance as a numerical value.", "oss": True, + "tags": ["functional"], }, { "name": "RAG Faithfulness", "key": "rag_faithfulness", "direct_use": False, + "requires_llm_api_keys": True, "settings_template": rag_evaluator_settings_template, "description": "RAG Faithfulness evaluator assesses the accuracy and reliability of responses generated by Retrieval-Augmented Generation (RAG) models. It evaluates how faithfully the responses adhere to the retrieved documents or sources, ensuring that the generated text accurately reflects the information from the original sources.", + "tags": ["rag"], }, { "name": "RAG Context Relevancy", "key": "rag_context_relevancy", "direct_use": False, + "requires_llm_api_keys": True, "settings_template": rag_evaluator_settings_template, "description": "RAG Context Relevancy evaluator measures how relevant the retrieved documents or contexts are to the given question or prompt. It ensures that the selected documents provide the necessary information for generating accurate and meaningful responses, improving the overall quality of the RAG model's output.", + "tags": ["rag"], }, ] diff --git a/agenta-backend/agenta_backend/routers/app_router.py b/agenta-backend/agenta_backend/routers/app_router.py index d2f2e2fc4..ae89ecae0 100644 --- a/agenta-backend/agenta_backend/routers/app_router.py +++ b/agenta-backend/agenta_backend/routers/app_router.py @@ -550,7 +550,7 @@ async def create_app_and_variant_from_template( ) app_variant_db = await app_manager.add_variant_based_on_image( app=app, - project_id=request.state.project_id, + project_id=str(app.project_id), variant_name="app.default", docker_id_or_template_uri=( # type: ignore template_db.template_uri if isCloudProd() else template_db.digest @@ -570,7 +570,7 @@ async def create_app_and_variant_from_template( await db_manager.add_testset_to_app_variant( template_name=template_db.name, # type: ignore app_name=app.app_name, # type: ignore - project_id=request.state.project_id, + project_id=str(app.project_id), ) logger.debug( @@ -579,7 +579,7 @@ async def create_app_and_variant_from_template( else "Step 6: We create ready-to use evaluators" ) await evaluator_manager.create_ready_to_use_evaluators( - app_name=app.app_name, project_id=request.state.project_id + app_name=app.app_name, project_id=str(app.project_id) ) logger.debug( @@ -619,7 +619,7 @@ async def create_app_and_variant_from_template( envvars = {} if payload.env_vars is None else payload.env_vars await app_manager.start_variant( app_variant_db, - request.state.project_id, + str(app.project_id), envvars, user_uid=request.state.user_id, ) diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 479bf68e1..d3cd29818 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -5,6 +5,7 @@ from fastapi.responses import JSONResponse from fastapi import HTTPException, Request, status, Response, Query +from agenta_backend.services import helpers from agenta_backend.models import converters from agenta_backend.tasks.evaluations import evaluate @@ -15,9 +16,6 @@ NewEvaluation, DeleteEvaluation, ) -from agenta_backend.services.evaluator_manager import ( - check_ai_critique_inputs, -) from agenta_backend.services import evaluation_service, db_manager, app_manager if isCloudEE(): @@ -113,8 +111,9 @@ async def create_evaluation( status_code=403, ) - success, response = await check_ai_critique_inputs( - payload.evaluators_configs, payload.lm_providers_keys + llm_provider_keys = helpers.format_llm_provider_keys(payload.lm_providers_keys) + success, response = await helpers.ensure_required_llm_keys_exist( + payload.evaluators_configs, llm_provider_keys ) if not success: return response @@ -136,8 +135,8 @@ async def create_evaluation( evaluators_config_ids=payload.evaluators_configs, testset_id=payload.testset_id, evaluation_id=evaluation.id, - rate_limit_config=payload.rate_limit.dict(), - lm_providers_keys=payload.lm_providers_keys, + rate_limit_config=payload.rate_limit.model_dump(), + lm_providers_keys=llm_provider_keys, ) evaluations.append(evaluation) @@ -435,9 +434,8 @@ async def delete_evaluations( ) logger.debug("Successfully updated last_modified_by app information") - await evaluation_service.delete_evaluations( - payload.evaluations_ids, str(evaluation.project_id) - ) + logger.debug(f"Deleting evaluations {payload.evaluations_ids}...") + await evaluation_service.delete_evaluations(payload.evaluations_ids) return Response(status_code=status.HTTP_204_NO_CONTENT) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) diff --git a/agenta-backend/agenta_backend/routers/evaluators_router.py b/agenta-backend/agenta_backend/routers/evaluators_router.py index 7729c97ab..5ad1a9bf4 100644 --- a/agenta-backend/agenta_backend/routers/evaluators_router.py +++ b/agenta-backend/agenta_backend/routers/evaluators_router.py @@ -1,4 +1,5 @@ import logging +import traceback from typing import List, Optional from fastapi import HTTPException, Request @@ -6,13 +7,22 @@ from agenta_backend.utils.common import APIRouter, isCloudEE -from agenta_backend.services import evaluator_manager, db_manager, app_manager +from agenta_backend.services import ( + evaluator_manager, + db_manager, + evaluators_service, + app_manager, +) from agenta_backend.models.api.evaluation_model import ( Evaluator, EvaluatorConfig, NewEvaluatorConfig, UpdateEvaluatorConfig, + EvaluatorInputInterface, + EvaluatorOutputInterface, + EvaluatorMappingInputInterface, + EvaluatorMappingOutputInterface, ) if isCloudEE(): @@ -48,6 +58,63 @@ async def get_evaluators_endpoint(): raise HTTPException(status_code=500, detail=str(e)) +@router.post("/map/", response_model=EvaluatorMappingOutputInterface) +async def evaluator_data_map(request: Request, payload: EvaluatorMappingInputInterface): + """Endpoint to map the experiment data tree to evaluator interface. + + Args: + request (Request): The request object. + payload (EvaluatorMappingInputInterface): The payload containing the request data. + + Returns: + EvaluatorMappingOutputInterface: the evaluator mapping output object + """ + + try: + mapped_outputs = await evaluators_service.map(mapping_input=payload) + return mapped_outputs + except Exception as e: + logger.error(f"Error mapping data tree: {str(e)}") + raise HTTPException( + status_code=500, + detail={ + "message": "Error mapping data tree", + "stacktrace": traceback.format_exc(), + }, + ) + + +@router.post("/{evaluator_key}/run/", response_model=EvaluatorOutputInterface) +async def evaluator_run( + request: Request, evaluator_key: str, payload: EvaluatorInputInterface +): + """Endpoint to evaluate LLM app run + + Args: + request (Request): The request object. + evaluator_key (str): The key of the evaluator. + payload (EvaluatorInputInterface): The payload containing the request data. + + Returns: + result: EvaluatorOutputInterface object containing the outputs. + """ + + try: + result = await evaluators_service.run( + evaluator_key=evaluator_key, evaluator_input=payload + ) + return result + except Exception as e: + logger.error(f"Error while running {evaluator_key} evaluator: {str(e)}") + raise HTTPException( + status_code=500, + detail={ + "message": f"Error while running {evaluator_key} evaluator", + "stacktrace": traceback.format_exc(), + }, + ) + + @router.get("/configs/", response_model=List[EvaluatorConfig]) async def get_evaluator_configs( app_id: str, diff --git a/agenta-backend/agenta_backend/routers/human_evaluation_router.py b/agenta-backend/agenta_backend/routers/human_evaluation_router.py index 846c551ee..a5573923e 100644 --- a/agenta-backend/agenta_backend/routers/human_evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/human_evaluation_router.py @@ -481,7 +481,9 @@ async def delete_evaluations( """ try: - evaluation = await db_manager.fetch_evaluation_by_id(payload.evaluations_ids[0]) + evaluation = await db_manager.fetch_human_evaluation_by_id( + payload.evaluations_ids[0] + ) if isCloudEE(): has_permission = await check_action_access( user_uid=request.state.user_id, diff --git a/agenta-backend/agenta_backend/routers/testset_router.py b/agenta-backend/agenta_backend/routers/testset_router.py index ddd39bcc2..05268749c 100644 --- a/agenta-backend/agenta_backend/routers/testset_router.py +++ b/agenta-backend/agenta_backend/routers/testset_router.py @@ -5,7 +5,7 @@ import logging import requests from typing import Optional, List - +from datetime import datetime, timezone from pydantic import ValidationError from fastapi.responses import JSONResponse @@ -298,6 +298,7 @@ async def update_testset( testset_update = { "name": csvdata.name, "csvdata": csvdata.csvdata, + "updated_at": datetime.now(timezone.utc), } await db_manager.update_testset( testset_id=str(testset.id), values_to_update=testset_update @@ -351,6 +352,7 @@ async def get_testsets( _id=str(testset.id), # type: ignore name=testset.name, created_at=str(testset.created_at), + updated_at=str(testset.updated_at), ) for testset in testsets ] diff --git a/agenta-backend/agenta_backend/services/app_manager.py b/agenta-backend/agenta_backend/services/app_manager.py index 1ac1f2b28..84036b769 100644 --- a/agenta-backend/agenta_backend/services/app_manager.py +++ b/agenta-backend/agenta_backend/services/app_manager.py @@ -44,6 +44,7 @@ from agenta_backend.services import deployment_manager if isCloudEE(): + from agenta_backend.commons.services import db_manager_ee from agenta_backend.commons.services import ( api_key_service, ) # noqa pylint: disable-all @@ -106,9 +107,10 @@ async def start_variant( ) if isCloudEE(): user = await db_manager.get_user(user_uid=user_uid) + project = await db_manager_ee.get_project_by_id(project_id=project_id) api_key = await api_key_service.create_api_key( str(user.id), - project_id=project_id, + workspace_id=str(project.workspace_id), expiration_date=None, hidden=True, ) @@ -213,9 +215,21 @@ async def get_appdb_str_by_id(object_id: str, object_type: str) -> str: f"Deployment with id {object_id} not found" ) return str(deployment_db.app_id) + elif object_type == "evaluation": + evaluation_db = await db_manager.fetch_evaluation_by_id(object_id) + if evaluation_db is None: + raise db_manager.NoResultFound( + f"Evaluation with id {object_id} not found" + ) + return str(evaluation_db.app_id) + else: + raise ValueError( + f"Could not update last_modified_by application information. Unsupported type: {object_type}" + ) user = await db_manager.get_user(user_uid=user_uid) app_id = await get_appdb_str_by_id(object_id=object_id, object_type=object_type) + assert app_id is not None, f"app_id in {object_type} cannot be None" await db_manager.update_app( app_id=app_id, values_to_update={ diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py index 26526d155..70293e23b 100644 --- a/agenta-backend/agenta_backend/services/db_manager.py +++ b/agenta-backend/agenta_backend/services/db_manager.py @@ -2644,7 +2644,7 @@ async def fetch_evaluations_by_resource( .join(HumanEvaluationVariantDB) .filter( HumanEvaluationVariantDB.variant_id.in_(ids), - EvaluationDB.project_id == uuid.UUID(project_id), + HumanEvaluationDB.project_id == uuid.UUID(project_id), ) .options(load_only(HumanEvaluationDB.id)) # type: ignore ) @@ -2652,7 +2652,7 @@ async def fetch_evaluations_by_resource( res_human_evaluations = result_human_evaluations.scalars().all() return res_evaluations + res_human_evaluations - if resource_type == "testset": + elif resource_type == "testset": result_evaluations = await session.execute( select(EvaluationDB) .filter( @@ -2665,7 +2665,8 @@ async def fetch_evaluations_by_resource( select(HumanEvaluationDB) .filter( HumanEvaluationDB.testset_id.in_(ids), - EvaluationDB.project_id == uuid.UUID(project_id), + HumanEvaluationDB.project_id + == uuid.UUID(project_id), # Fixed to match HumanEvaluationDB ) .options(load_only(HumanEvaluationDB.id)) # type: ignore ) @@ -2673,7 +2674,7 @@ async def fetch_evaluations_by_resource( res_human_evaluations = result_human_evaluations.scalars().all() return res_evaluations + res_human_evaluations - if resource_type == "evaluator_config": + elif resource_type == "evaluator_config": query = ( select(EvaluationDB) .join(EvaluationDB.evaluator_configs) @@ -2692,19 +2693,15 @@ async def fetch_evaluations_by_resource( ) -async def delete_evaluations(evaluation_ids: List[str], project_id: str) -> None: +async def delete_evaluations(evaluation_ids: List[str]) -> None: """Delete evaluations based on the ids provided from the db. Args: evaluations_ids (list[str]): The IDs of the evaluation - project_id (str): The ID of the project """ async with db_engine.get_session() as session: - query = select(EvaluationDB).where( - EvaluationDB.id.in_(evaluation_ids), - EvaluationDB.project_id == uuid.UUID(project_id), - ) + query = select(EvaluationDB).where(EvaluationDB.id.in_(evaluation_ids)) result = await session.execute(query) evaluations = result.scalars().all() for evaluation in evaluations: @@ -2860,13 +2857,17 @@ async def fetch_evaluator_config(evaluator_config_id: str): return evaluator_config -async def check_if_ai_critique_exists_in_list_of_evaluators_configs( - evaluators_configs_ids: List[str], +async def check_if_evaluators_exist_in_list_of_evaluators_configs( + evaluators_configs_ids: List[str], evaluators_keys: List[str] ) -> bool: - """Fetch evaluator configurations from the database. + """Check if the provided evaluators exist in the database within the given evaluator configurations. + + Arguments: + evaluators_configs_ids (List[str]): List of evaluator configuration IDs to search within. + evaluators_keys (List[str]): List of evaluator keys to check for existence. Returns: - EvaluatorConfigDB: the evaluator configuration object. + bool: True if all evaluators exist, False otherwise. """ async with db_engine.get_session() as session: @@ -2875,15 +2876,18 @@ async def check_if_ai_critique_exists_in_list_of_evaluators_configs( for evaluator_config_id in evaluators_configs_ids ] - query = select(EvaluatorConfigDB).where( + query = select(EvaluatorConfigDB.id, EvaluatorConfigDB.evaluator_key).where( EvaluatorConfigDB.id.in_(evaluator_config_uuids), - EvaluatorConfigDB.evaluator_key == "auto_ai_critique", + EvaluatorConfigDB.evaluator_key.in_(evaluators_keys), ) - result = await session.execute(query) - evaluators_configs = result.scalars().all() - return bool(evaluators_configs) + # NOTE: result.all() returns the records as a list of tuples + # 0 is the evaluator_id and 1 is evaluator_key + fetched_evaluators_keys = {config[1] for config in result.all()} + + # Ensure the passed evaluators are found in the fetched evaluator keys + return any(key in fetched_evaluators_keys for key in evaluators_keys) async def fetch_evaluator_config_by_appId( diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py index 43908791b..7f8f2290f 100644 --- a/agenta-backend/agenta_backend/services/evaluation_service.py +++ b/agenta-backend/agenta_backend/services/evaluation_service.py @@ -425,6 +425,11 @@ async def create_new_evaluation( variant_db = await db_manager.get_app_variant_instance_by_id( variant_id=variant_id, project_id=project_id ) + + assert variant_db is not None, f"App variant with ID {variant_id} cannot be None." + assert ( + variant_db.revision is not None + ), f"Revision of App variant with ID {variant_id} cannot be None" variant_revision = await db_manager.fetch_app_variant_revision_by_variant( app_variant_id=variant_id, project_id=project_id, revision=variant_db.revision # type: ignore ) diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py index 3ebbc6d8c..bb5d6b19f 100644 --- a/agenta-backend/agenta_backend/services/evaluator_manager.py +++ b/agenta-backend/agenta_backend/services/evaluator_manager.py @@ -167,28 +167,3 @@ async def create_ready_to_use_evaluators(app_name: str, project_id: str): evaluator_key=evaluator.key, settings_values=settings_values, ) - - -async def check_ai_critique_inputs( - evaluators_configs: List[str], lm_providers_keys: Optional[Dict[str, Any]] -) -> Tuple[bool, Optional[JSONResponse]]: - """ - Checks if AI critique exists in evaluators configs and validates lm_providers_keys. - - Args: - evaluators_configs (List[str]): List of evaluator configurations. - lm_providers_keys (Optional[Dict[str, Any]]): Language model provider keys. - - Returns: - Tuple[bool, Optional[JSONResponse]]: Returns a tuple containing a boolean indicating success, - and a JSONResponse in case of error. - """ - if await db_manager.check_if_ai_critique_exists_in_list_of_evaluators_configs( - evaluators_configs - ): - if not lm_providers_keys: - return False, JSONResponse( - {"detail": "Missing LM provider Key"}, - status_code=400, - ) - return True, None diff --git a/agenta-backend/agenta_backend/services/evaluators_service.py b/agenta-backend/agenta_backend/services/evaluators_service.py index 42db9a2da..a9637dee5 100644 --- a/agenta-backend/agenta_backend/services/evaluators_service.py +++ b/agenta-backend/agenta_backend/services/evaluators_service.py @@ -2,7 +2,6 @@ import json import asyncio import logging -import asyncio import traceback from typing import Any, Dict, Union @@ -14,15 +13,63 @@ from agenta_backend.services.security import sandbox from agenta_backend.models.shared_models import Error, Result +from agenta_backend.models.api.evaluation_model import ( + EvaluatorInputInterface, + EvaluatorOutputInterface, + EvaluatorMappingInputInterface, + EvaluatorMappingOutputInterface, +) from agenta_backend.utils.traces import ( process_distributed_trace_into_trace_tree, get_field_value_from_trace_tree, ) + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) +def validate_string_output( + evaluator_key: str, output: Union[str, Dict[str, Any]] +) -> str: + """Checks and validate the output to be of type string. + + Args: + evaluator_key (str): the key of the evaluator + output (Union[str, Dict[str, Any]]): the llm response + + Raises: + Exception: requires output to be a string + + Returns: + str: output + """ + + output = output.get("data", "") if isinstance(output, dict) else output + if not isinstance(output, str): + raise Exception( + f"Evaluator {evaluator_key} requires the output to be a string, but received {type(output).__name__} instead. " + ) + return output + + +async def map( + mapping_input: EvaluatorMappingInputInterface, +) -> EvaluatorMappingOutputInterface: + """ + Maps the evaluator inputs based on the provided mapping and data tree. + + Returns: + EvaluatorMappingOutputInterface: A dictionary containing the mapped evaluator inputs. + """ + + mapping_outputs = {} + trace = process_distributed_trace_into_trace_tree(mapping_input.inputs["trace"]) + for to_key, from_key in mapping_input.mapping.items(): + mapping_outputs[to_key] = get_field_value_from_trace_tree(trace, from_key) + return {"outputs": mapping_outputs} + + def get_correct_answer( data_point: Dict[str, Any], settings_values: Dict[str, Any] ) -> Any: @@ -42,6 +89,10 @@ def get_correct_answer( correct_answer_key = settings_values.get("correct_answer_key") if correct_answer_key is None: raise ValueError("No correct answer keys provided.") + if isinstance(correct_answer_key, str) and correct_answer_key.startswith( + "testcase." + ): + correct_answer_key = correct_answer_key[len("testcase.") :] if correct_answer_key not in data_point: raise ValueError( f"Correct answer column '{correct_answer_key}' not found in the test set." @@ -49,7 +100,7 @@ def get_correct_answer( return data_point[correct_answer_key] -def auto_exact_match( +async def auto_exact_match( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -71,12 +122,15 @@ def auto_exact_match( Returns: Result: A Result object containing the evaluation result. """ - if not isinstance(output, str): - output = output.get("data", "") + try: + output = validate_string_output("exact_match", output) correct_answer = get_correct_answer(data_point, settings_values) - exact_match = True if output == correct_answer else False - result = Result(type="bool", value=exact_match) + inputs = {"ground_truth": correct_answer, "prediction": output} + response = await exact_match( + input=EvaluatorInputInterface(**{"inputs": inputs}) + ) + result = Result(type="bool", value=response["outputs"]["success"]) return result except ValueError as e: return Result( @@ -97,7 +151,14 @@ def auto_exact_match( ) -def auto_regex_test( +async def exact_match(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + prediction = input.inputs.get("prediction", "") + ground_truth = input.inputs.get("ground_truth", "") + success = True if prediction == ground_truth else False + return {"outputs": {"success": success}} + + +async def auto_regex_test( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -105,14 +166,15 @@ def auto_regex_test( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - re_pattern = re.compile(settings_values["regex_pattern"], re.IGNORECASE) - result = ( - bool(re_pattern.search(output)) == settings_values["regex_should_match"] + output = validate_string_output("regex_test", output) + inputs = {"ground_truth": data_point, "prediction": output} + response = await regex_test( + input=EvaluatorInputInterface( + **{"inputs": inputs, "settings": settings_values} + ) ) - return Result(type="bool", value=result) + return Result(type="bool", value=response["outputs"]["success"]) except Exception as e: # pylint: disable=broad-except return Result( type="error", @@ -124,7 +186,16 @@ def auto_regex_test( ) -def field_match_test( +async def regex_test(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + pattern = re.compile(input.settings["regex_pattern"], re.IGNORECASE) + result = ( + bool(pattern.search(input.inputs["prediction"])) + == input.settings["regex_should_match"] + ) + return {"outputs": {"success": result}} + + +async def auto_field_match_test( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], @@ -132,13 +203,14 @@ def field_match_test( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: + output = validate_string_output("field_match_test", output) correct_answer = get_correct_answer(data_point, settings_values) - output_json = json.loads(output) - result = output_json[settings_values["json_field"]] == correct_answer - return Result(type="bool", value=result) + inputs = {"ground_truth": correct_answer, "prediction": output} + response = await field_match_test( + input=EvaluatorInputInterface(**{"inputs": inputs}) + ) + return Result(type="bool", value=response["outputs"]["success"]) except ValueError as e: return Result( type="error", @@ -152,7 +224,13 @@ def field_match_test( return Result(type="bool", value=False) -def auto_webhook_test( +async def field_match_test(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + prediction_json = json.loads(input.inputs["prediction"]) + result = prediction_json == input.inputs["ground_truth"] + return {"outputs": {"success": result}} + + +async def auto_webhook_test( inputs: Dict[str, Any], output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], @@ -160,38 +238,16 @@ def auto_webhook_test( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: + output = validate_string_output("webhook_test", output) correct_answer = get_correct_answer(data_point, settings_values) - - with httpx.Client() as client: - payload = { - "correct_answer": correct_answer, - "output": output, - "inputs": inputs, - } - response = client.post(url=settings_values["webhook_url"], json=payload) - response.raise_for_status() - response_data = response.json() - score = response_data.get("score", None) - if score is None and not isinstance(score, (int, float)): - return Result( - type="error", - value=None, - error=Error( - message="Error during Auto Webhook evaluation; Webhook did not return a score", - ), - ) - if score < 0 or score > 1: - return Result( - type="error", - value=None, - error=Error( - message="Error during Auto Webhook evaluation; Webhook returned an invalid score. Score must be between 0 and 1", - ), - ) - return Result(type="number", value=score) + inputs = {"prediction": output, "ground_truth": correct_answer} + response = await webhook_test( + input=EvaluatorInputInterface( + **{"inputs": inputs, "settings": settings_values} + ) + ) + return Result(type="number", value=response["outputs"]["score"]) except httpx.HTTPError as e: return Result( type="error", @@ -221,7 +277,21 @@ def auto_webhook_test( ) -def auto_custom_code_run( +async def webhook_test(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + with httpx.Client() as client: + payload = { + "correct_answer": input.inputs["ground_truth"], + "output": input.inputs["prediction"], + "inputs": input.inputs, + } + response = client.post(url=input.settings["webhook_url"], json=payload) + response.raise_for_status() + response_data = response.json() + score = response_data.get("score", None) + return {"outputs": {"score": score}} + + +async def auto_custom_code_run( inputs: Dict[str, Any], output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], @@ -229,20 +299,20 @@ def auto_custom_code_run( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - result = sandbox.execute_code_safely( - app_params=app_params, - inputs=inputs, - output=output, - correct_answer=data_point.get( - "correct_answer", None - ), # for backward compatibility - code=settings_values["code"], - datapoint=data_point, + output = validate_string_output("custom_code_run", output) + correct_answer = get_correct_answer(data_point, settings_values) + inputs = { + "app_config": app_params, + "prediction": output, + "ground_truth": correct_answer, + } + response = await custom_code_run( + input=EvaluatorInputInterface( + **{"inputs": inputs, "settings": settings_values} + ) ) - return Result(type="number", value=result) + return Result(type="number", value=response["outputs"]["score"]) except Exception as e: # pylint: disable=broad-except return Result( type="error", @@ -254,7 +324,19 @@ def auto_custom_code_run( ) -def auto_ai_critique( +async def custom_code_run(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + result = sandbox.execute_code_safely( + app_params=input.inputs["app_config"], + inputs=input.inputs, + output=input.inputs["prediction"], + correct_answer=input.inputs["ground_truth"], + code=input.settings["code"], + datapoint=input.inputs["ground_truth"], + ) + return {"outputs": {"score": result}} + + +async def auto_ai_critique( inputs: Dict[str, Any], output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], @@ -276,34 +358,28 @@ def auto_ai_critique( Returns: Result: Evaluation result. """ - if not isinstance(output, str): - output = output.get("data", "") + try: + output = validate_string_output("ai_critique", output) correct_answer = get_correct_answer(data_point, settings_values) - openai_api_key = lm_providers_keys["OPENAI_API_KEY"] - - chain_run_args = { - "llm_app_prompt_template": app_params.get("prompt_user", ""), - "variant_output": output, - "correct_answer": correct_answer, + inputs = { + "prompt_user": app_params.get("prompt_user", "").format(**data_point), + "prediction": output, + "ground_truth": correct_answer, } - - for key, value in inputs.items(): - chain_run_args[key] = value - - prompt_template = settings_values["prompt_template"] - messages = [ - {"role": "system", "content": prompt_template}, - {"role": "user", "content": str(chain_run_args)}, - ] - - client = OpenAI(api_key=openai_api_key) - response = client.chat.completions.create( - model="gpt-3.5-turbo", messages=messages, temperature=0.01 + settings = { + "prompt_template": settings_values.get("prompt_template", ""), + } + response = await ai_critique( + input=EvaluatorInputInterface( + **{ + "inputs": inputs, + "settings": settings, + "credentials": lm_providers_keys, + } + ) ) - - evaluation_output = response.choices[0].message.content.strip() - return Result(type="text", value=evaluation_output) + return Result(type="text", value=str(response["outputs"]["score"])) except Exception as e: # pylint: disable=broad-except return Result( type="error", @@ -315,7 +391,39 @@ def auto_ai_critique( ) -def auto_starts_with( +async def ai_critique(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + openai_api_key = input.credentials.get("OPENAI_API_KEY", None) + + if not openai_api_key: + raise Exception( + "No OpenAI key was found. AI Critique evaluator requires a valid OpenAI API key to function. Please configure your OpenAI API and try again." + ) + + chain_run_args = { + "llm_app_prompt_template": input.inputs.get("prompt_user", ""), + "variant_output": input.inputs["prediction"], + "correct_answer": input.inputs["ground_truth"], + } + for key, value in input.inputs.items(): + chain_run_args[key] = value + + prompt_template = input.settings.get("prompt_template", "") + messages = [ + {"role": "system", "content": prompt_template}, + {"role": "user", "content": str(chain_run_args)}, + ] + + print(input) + + client = AsyncOpenAI(api_key=openai_api_key) + response = await client.chat.completions.create( + model="gpt-3.5-turbo", messages=messages, temperature=0.8 + ) + evaluation_output = response.choices[0].message.content.strip() + return {"outputs": {"score": evaluation_output}} + + +async def auto_starts_with( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -323,18 +431,15 @@ def auto_starts_with( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - prefix = settings_values.get("prefix", "") - case_sensitive = settings_values.get("case_sensitive", True) - - if not case_sensitive: - output = output.lower() - prefix = prefix.lower() - - result = Result(type="bool", value=output.startswith(prefix)) - return result + output = validate_string_output("starts_with", output) + inputs = {"prediction": output} + response = await starts_with( + input=EvaluatorInputInterface( + **{"inputs": inputs, "settings": settings_values} + ) + ) + return Result(type="text", value=response["outputs"]["success"]) except Exception as e: # pylint: disable=broad-except return Result( type="error", @@ -346,7 +451,20 @@ def auto_starts_with( ) -def auto_ends_with( +async def starts_with(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + prefix = input.settings.get("prefix", "") + case_sensitive = input.settings.get("case_sensitive", True) + + output = str(input.inputs["prediction"]) + if not case_sensitive: + output = output.lower() + prefix = prefix.lower() + + result = output.startswith(prefix) + return {"outputs": {"success": result}} + + +async def auto_ends_with( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -354,17 +472,15 @@ def auto_ends_with( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - suffix = settings_values.get("suffix", "") - case_sensitive = settings_values.get("case_sensitive", True) - - if not case_sensitive: - output = output.lower() - suffix = suffix.lower() - - result = Result(type="bool", value=output.endswith(suffix)) + output = validate_string_output("ends_with", output) + inputs = {"prediction": output} + response = await ends_with( + input=EvaluatorInputInterface( + **{"inputs": inputs, "settings": settings_values} + ) + ) + result = Result(type="bool", value=response["outputs"]["success"]) return result except Exception as e: # pylint: disable=broad-except return Result( @@ -377,7 +493,20 @@ def auto_ends_with( ) -def auto_contains( +async def ends_with(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + suffix = input.settings.get("suffix", "") + case_sensitive = input.settings.get("case_sensitive", True) + + output = str(input.inputs["prediction"]) + if not case_sensitive: + output = output.lower() + suffix = suffix.lower() + + result = output.endswith(suffix) + return {"outputs": {"success": result}} + + +async def auto_contains( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -385,17 +514,15 @@ def auto_contains( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - substring = settings_values.get("substring", "") - case_sensitive = settings_values.get("case_sensitive", True) - - if not case_sensitive: - output = output.lower() - substring = substring.lower() - - result = Result(type="bool", value=substring in output) + output = validate_string_output("contains", output) + inputs = {"prediction": output} + response = await contains( + input=EvaluatorInputInterface( + **{"inputs": inputs, "settings": settings_values} + ) + ) + result = Result(type="bool", value=response["outputs"]["success"]) return result except Exception as e: # pylint: disable=broad-except return Result( @@ -408,7 +535,20 @@ def auto_contains( ) -def auto_contains_any( +async def contains(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + substring = input.settings.get("substring", "") + case_sensitive = input.settings.get("case_sensitive", True) + + output = str(input.inputs["prediction"]) + if not case_sensitive: + output = output.lower() + substring = substring.lower() + + result = substring in output + return {"outputs": {"success": result}} + + +async def auto_contains_any( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -416,20 +556,15 @@ def auto_contains_any( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - substrings_str = settings_values.get("substrings", "") - substrings = [substring.strip() for substring in substrings_str.split(",")] - case_sensitive = settings_values.get("case_sensitive", True) - - if not case_sensitive: - output = output.lower() - substrings = [substring.lower() for substring in substrings] - - result = Result( - type="bool", value=any(substring in output for substring in substrings) + output = validate_string_output("contains_any", output) + inputs = {"prediction": output} + response = await contains_any( + input=EvaluatorInputInterface( + **{"inputs": inputs, "settings": settings_values} + ) ) + result = Result(type="bool", value=response["outputs"]["success"]) return result except Exception as e: # pylint: disable=broad-except return Result( @@ -442,7 +577,22 @@ def auto_contains_any( ) -def auto_contains_all( +async def contains_any(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + substrings_str = input.settings.get("substrings", "") + substrings = [substring.strip() for substring in substrings_str.split(",")] + case_sensitive = input.settings.get("case_sensitive", True) + + output = str(input.inputs["prediction"]) + if not case_sensitive: + output = output.lower() + substrings = [substring.lower() for substring in substrings] + + return { + "outputs": {"success": any(substring in output for substring in substrings)} + } + + +async def auto_contains_all( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -450,20 +600,14 @@ def auto_contains_all( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - substrings_str = settings_values.get("substrings", "") - substrings = [substring.strip() for substring in substrings_str.split(",")] - case_sensitive = settings_values.get("case_sensitive", True) - - if not case_sensitive: - output = output.lower() - substrings = [substring.lower() for substring in substrings] - - result = Result( - type="bool", value=all(substring in output for substring in substrings) + output = validate_string_output("contains_all", output) + response = await contains_all( + input=EvaluatorInputInterface( + **{"inputs": {"prediction": output}, "settings": settings_values} + ) ) + result = Result(type="bool", value=response["outputs"]["success"]) return result except Exception as e: # pylint: disable=broad-except return Result( @@ -476,7 +620,21 @@ def auto_contains_all( ) -def auto_contains_json( +async def contains_all(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + substrings_str = input.settings.get("substrings", "") + substrings = [substring.strip() for substring in substrings_str.split(",")] + case_sensitive = input.settings.get("case_sensitive", True) + + output = str(input.inputs["prediction"]) + if not case_sensitive: + output = output.lower() + substrings = [substring.lower() for substring in substrings] + + result = all(substring in output for substring in substrings) + return {"outputs": {"success": result}} + + +async def auto_contains_json( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -484,20 +642,21 @@ def auto_contains_json( settings_values: Dict[str, Any], # pylint: disable=unused-argument lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - try: - start_index = output.index("{") - end_index = output.rindex("}") + 1 - potential_json = output[start_index:end_index] - - json.loads(potential_json) - contains_json = True - except (ValueError, json.JSONDecodeError): - contains_json = False - - return Result(type="bool", value=contains_json) + # parsing llm app output format if v2 + output = output.get("data", "") if isinstance(output, dict) else output + if isinstance(output, dict): + output = json.dumps( + output + ) # contains_json expects inputs.prediction to be a string + elif not isinstance(output, (str, dict)): + raise Exception( + f"Evaluator contains_json requires the app output to be either a JSON string or object, but received {type(output).__name__} instead." + ) + response = await contains_json( + input=EvaluatorInputInterface(**{"inputs": {"prediction": output}}) + ) + return Result(type="bool", value=response["outputs"]["success"]) except Exception as e: # pylint: disable=broad-except return Result( type="error", @@ -509,6 +668,19 @@ def auto_contains_json( ) +async def contains_json(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + try: + start_index = str(input.inputs["prediction"]).index("{") + end_index = str(input.inputs["prediction"]).rindex("}") + 1 + potential_json = str(input.inputs["prediction"])[start_index:end_index] + json.loads(potential_json) + contains_json = True + except (ValueError, json.JSONDecodeError) as e: + contains_json = False + + return {"outputs": {"success": contains_json}} + + def flatten_json(json_obj: Union[list, dict]) -> Dict[str, Any]: """ This function takes a (nested) JSON object and flattens it into a single-level dictionary where each key represents the path to the value in the original JSON structure. This is done recursively, ensuring that the full hierarchical context is preserved in the keys. @@ -612,7 +784,7 @@ def diff(ground_truth: Any, app_output: Any, compare_schema_only: bool) -> float return 0.0 -def auto_json_diff( +async def auto_json_diff( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Any, data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -621,30 +793,28 @@ def auto_json_diff( lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: try: - output = output.get("data", "") if isinstance(output, dict) else output + # 2. extract ground truth from data point + correct_answer = get_correct_answer(data_point, settings_values) - if isinstance(output, dict): - output = json.dumps(output) - elif isinstance(output, str): - try: - json.loads(output) - except: - raise Exception( - f"Evaluator 'auto_json_diff' requires string outputs to be JSON strings." - ) - else: - raise Exception( - f"Evaluator 'auto_json_diff' requires the output to be either a JSON string or a JSON object, but received {type(output).__name__} instead." + response = await json_diff( + input=EvaluatorInputInterface( + **{ + "inputs": {"prediction": output, "ground_truth": correct_answer}, + "settings": settings_values, + } ) - - correct_answer = get_correct_answer(data_point, settings_values) - average_score = compare_jsons( - ground_truth=json.loads(correct_answer), - app_output=json.loads(output), - settings_values=settings_values, ) - return Result(type="number", value=average_score) - except (ValueError, json.JSONDecodeError, Exception): + return Result(type="number", value=response["outputs"]["score"]) + except json.JSONDecodeError: + return Result( + type="error", + value=None, + error=Error( + message="Expected answer is not a valid JSON", + stacktrace=traceback.format_exc(), + ), + ) + except (ValueError, Exception): return Result( type="error", value=None, @@ -655,7 +825,55 @@ def auto_json_diff( ) -def rag_faithfulness( +async def json_diff(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + ground_truth = input.inputs["ground_truth"] + if isinstance(ground_truth, str): + ground_truth = json.loads(ground_truth) # if this fails we will return an error + + # 1. extract llm app output if app output format is v2+ + app_output = input.inputs["prediction"] + assert isinstance( + app_output, (str, dict) + ), "App output is expected to be a string or a JSON object" + app_output = ( + app_output.get("data", "") if isinstance(app_output, dict) else app_output + ) + if isinstance(app_output, str): + try: + app_output = json.loads(app_output) + except json.JSONDecodeError: + app_output = ( + {} + ) # we will return 0 score for json diff in case we cannot parse the output as json + + score = compare_jsons( + ground_truth=ground_truth, + app_output=app_output, + settings_values=input.settings, + ) + return {"outputs": {"score": score}} + + +async def measure_rag_consistency( + input: EvaluatorInputInterface, +) -> EvaluatorOutputInterface: + openai_api_key = input.credentials.get("OPENAI_API_KEY", None) + if not openai_api_key: + raise Exception( + "No OpenAI key was found. RAG evaluator requires a valid OpenAI API key to function. Please configure your OpenAI API and try again." + ) + + # Initialize RAG evaluator to calculate faithfulness score + faithfulness = Faithfulness(api_key=openai_api_key) + eval_score = await faithfulness._run_eval_async( + output=input.inputs["answer_key"], + input=input.inputs["question_key"], + context=input.inputs["contexts_key"], + ) + return {"outputs": {"score": eval_score.score}} + + +async def rag_faithfulness( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -711,23 +929,20 @@ def rag_faithfulness( raise ValueError(message) - openai_api_key = lm_providers_keys.get("OPENAI_API_KEY", None) - - if not openai_api_key: - raise Exception( - "No LLM keys OpenAI key found. Please configure your OpenAI keys and try again." - ) - - # Initialize RAG evaluator to calculate faithfulness score - loop = asyncio.get_event_loop() - faithfulness = Faithfulness(api_key=openai_api_key) - eval_score = loop.run_until_complete( - faithfulness._run_eval_async( - output=answer_val, input=question_val, context=contexts_val + measurement = await measure_rag_consistency( + input=EvaluatorInputInterface( + **{ + "inputs": { + "question_key": question_val, + "contexts_key": contexts_val, + "answer_key": answer_val, + }, + "settings": settings_values, + "credentials": lm_providers_keys, + } ) ) - - return Result(type="number", value=eval_score.score) + return Result(type="number", value=measurement["outputs"]["score"]) except Exception: return Result( @@ -740,7 +955,26 @@ def rag_faithfulness( ) -def rag_context_relevancy( +async def measure_context_coherence( + input: EvaluatorInputInterface, +) -> EvaluatorOutputInterface: + openai_api_key = input.credentials.get("OPENAI_API_KEY", None) + if not openai_api_key: + raise Exception( + "No OpenAI key was found. RAG evaluator requires a valid OpenAI API key to function. Please configure your OpenAI API and try again." + ) + + # Initialize RAG evaluator to calculate context relevancy score + context_rel = ContextRelevancy(api_key=openai_api_key) + eval_score = await context_rel._run_eval_async( + output=input.inputs["answer_key"], + input=input.inputs["question_key"], + context=input.inputs["contexts_key"], + ) + return {"outputs": {"score": eval_score.score}} + + +async def rag_context_relevancy( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], # pylint: disable=unused-argument @@ -796,22 +1030,20 @@ def rag_context_relevancy( raise ValueError(message) - openai_api_key = lm_providers_keys.get("OPENAI_API_KEY", None) - - if not openai_api_key: - raise Exception( - "No LLM keys OpenAI key found. Please configure your OpenAI keys and try again." - ) - - # Initialize RAG evaluator to calculate context relevancy score - loop = asyncio.get_event_loop() - context_rel = ContextRelevancy(api_key=openai_api_key) - eval_score = loop.run_until_complete( - context_rel._run_eval_async( - output=answer_val, input=question_val, context=contexts_val + measurement = await measure_context_coherence( + input=EvaluatorInputInterface( + **{ + "inputs": { + "question_key": question_val, + "contexts_key": contexts_val, + "answer_key": answer_val, + }, + "settings": settings_values, + "credentials": lm_providers_keys, + } ) ) - return Result(type="number", value=eval_score.score) + return Result(type="number", value=measurement["outputs"]["score"]) except Exception: return Result( @@ -824,27 +1056,35 @@ def rag_context_relevancy( ) -def levenshtein_distance(s1, s2): - if len(s1) < len(s2): - return levenshtein_distance(s2, s1) # pylint: disable=arguments-out-of-order +async def levenshtein_distance( + input: EvaluatorInputInterface, +) -> EvaluatorOutputInterface: + prediction = input.inputs["prediction"] + ground_truth = input.inputs["ground_truth"] - if len(s2) == 0: - return len(s1) + if len(ground_truth) == 0: + return len(prediction) - previous_row = range(len(s2) + 1) - for i, c1 in enumerate(s1): + previous_row = range(len(ground_truth) + 1) + for i, c1 in enumerate(prediction): current_row = [i + 1] - for j, c2 in enumerate(s2): + for j, c2 in enumerate(ground_truth): insertions = previous_row[j + 1] + 1 deletions = current_row[j] + 1 substitutions = previous_row[j] + (c1 != c2) current_row.append(min(insertions, deletions, substitutions)) previous_row = current_row - return previous_row[-1] + distance = previous_row[-1] + if "threshold" in input.settings: + threshold = input.settings["threshold"] + is_within_threshold = distance <= threshold + return {"outputs": {"success": is_within_threshold}} + + return {"outputs": {"score": distance}} -def auto_levenshtein_distance( +async def auto_levenshtein_distance( inputs: Dict[str, Any], # pylint: disable=unused-argument output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], @@ -852,19 +1092,21 @@ def auto_levenshtein_distance( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], # pylint: disable=unused-argument ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: + output = validate_string_output("levenshtein_distance", output) correct_answer = get_correct_answer(data_point, settings_values) + response = await levenshtein_distance( + input=EvaluatorInputInterface( + **{ + "inputs": {"prediction": output, "ground_truth": correct_answer}, + "settings": settings_values, + } + ) + ) + if "success" in response["outputs"]: + return Result(type="number", value=response["outputs"]["success"]) + return Result(type="number", value=response["outputs"]["score"]) - distance = levenshtein_distance(output, correct_answer) - - if "threshold" in settings_values: - threshold = settings_values["threshold"] - is_within_threshold = distance <= threshold - return Result(type="bool", value=is_within_threshold) - - return Result(type="number", value=distance) except ValueError as e: return Result( type="error", @@ -884,7 +1126,7 @@ def auto_levenshtein_distance( ) -def auto_similarity_match( +async def auto_similarity_match( inputs: Dict[str, Any], output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], @@ -892,21 +1134,18 @@ def auto_similarity_match( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: + output = validate_string_output("similarity_match", output) correct_answer = get_correct_answer(data_point, settings_values) - set1 = set(output.split()) - set2 = set(correct_answer.split()) - intersect = set1.intersection(set2) - union = set1.union(set2) - - similarity = len(intersect) / len(union) - - is_similar = ( - True if similarity > settings_values["similarity_threshold"] else False + response = await similarity_match( + input=EvaluatorInputInterface( + **{ + "inputs": {"prediction": output, "ground_truth": correct_answer}, + "settings": settings_values, + } + ) ) - result = Result(type="bool", value=is_similar) + result = Result(type="bool", value=response["outputs"]["success"]) return result except ValueError as e: return Result( @@ -927,25 +1166,39 @@ def auto_similarity_match( ) +async def similarity_match(input: EvaluatorInputInterface) -> EvaluatorOutputInterface: + set1 = set(input.inputs["prediction"].split()) + set2 = set(input.inputs["ground_truth"].split()) + intersect = set1.intersection(set2) + union = set1.union(set2) + print(set1) + print(set2) + print(union) + + similarity = len(intersect) / len(union) + is_similar = True if similarity > input.settings["similarity_threshold"] else False + return {"outputs": {"success": is_similar}} + + async def semantic_similarity( - output: Union[str, Dict[str, Any]], - correct_answer: str, - api_key: str, -) -> float: - """ - Calculate the semantic similarity score of the LLM app using OpenAI's Embeddings API. + input: EvaluatorInputInterface, +) -> EvaluatorOutputInterface: + """Calculate the semantic similarity score of the LLM app using OpenAI's Embeddings API. Args: - output (str): the output text - correct_answer (str): the correct answer text + input (EvaluatorInputInterface): the evaluator input Returns: float: the semantic similarity score """ - if not isinstance(output, str): - output = output.get("data", "") - openai = AsyncOpenAI(api_key=api_key) + openai_api_key = input.credentials.get("OPENAI_API_KEY", None) + if not openai_api_key: + raise Exception( + "No OpenAI key was found. Semantic evaluator requires a valid OpenAI API key to function. Please configure your OpenAI API and try again." + ) + + openai = AsyncOpenAI(api_key=openai_api_key) async def encode(text: str): response = await openai.embeddings.create( @@ -956,13 +1209,13 @@ async def encode(text: str): def cosine_similarity(output_vector: array, correct_answer_vector: array) -> float: return np.dot(output_vector, correct_answer_vector) - output_vector = await encode(output) - correct_answer_vector = await encode(correct_answer) + output_vector = await encode(input.inputs["prediction"]) + correct_answer_vector = await encode(input.inputs["ground_truth"]) similarity_score = cosine_similarity(output_vector, correct_answer_vector) - return similarity_score + return {"outputs": {"score": similarity_score}} -def auto_semantic_similarity( +async def auto_semantic_similarity( inputs: Dict[str, Any], output: Union[str, Dict[str, Any]], data_point: Dict[str, Any], @@ -970,19 +1223,19 @@ def auto_semantic_similarity( settings_values: Dict[str, Any], lm_providers_keys: Dict[str, Any], ) -> Result: - if not isinstance(output, str): - output = output.get("data", "") try: - loop = asyncio.get_event_loop() - openai_api_key = lm_providers_keys["OPENAI_API_KEY"] + output = validate_string_output("semantic_similarity", output) correct_answer = get_correct_answer(data_point, settings_values) - - score = loop.run_until_complete( - semantic_similarity( - output=output, correct_answer=correct_answer, api_key=openai_api_key + inputs = {"prediction": output, "ground_truth": correct_answer} + response = await semantic_similarity( + input=EvaluatorInputInterface( + **{ + "inputs": inputs, + "credentials": lm_providers_keys, + } ) ) - return Result(type="number", value=score) + return Result(type="number", value=response["outputs"]["score"]) except Exception: return Result( type="error", @@ -997,7 +1250,7 @@ def auto_semantic_similarity( EVALUATOR_FUNCTIONS = { "auto_exact_match": auto_exact_match, "auto_regex_test": auto_regex_test, - "field_match_test": field_match_test, + "field_match_test": auto_field_match_test, "auto_webhook_test": auto_webhook_test, "auto_custom_code_run": auto_custom_code_run, "auto_ai_critique": auto_ai_critique, @@ -1015,8 +1268,29 @@ def auto_semantic_similarity( "rag_context_relevancy": rag_context_relevancy, } +RUN_EVALUATOR_FUNCTIONS = { + "auto_exact_match": exact_match, + "auto_regex_test": regex_test, + "field_match_test": field_match_test, + "auto_webhook_test": webhook_test, + "auto_custom_code_run": custom_code_run, + "auto_ai_critique": ai_critique, + "auto_starts_with": starts_with, + "auto_ends_with": ends_with, + "auto_contains": contains, + "auto_contains_any": contains_any, + "auto_contains_all": contains_all, + "auto_contains_json": contains_json, + "auto_json_diff": json_diff, + "auto_levenshtein_distance": levenshtein_distance, + "auto_similarity_match": similarity_match, + "auto_semantic_similarity": semantic_similarity, + "rag_faithfulness": measure_rag_consistency, + "rag_context_relevancy": measure_context_coherence, +} + -def evaluate( +async def evaluate( evaluator_key: str, inputs: Dict[str, Any], output: Union[str, Dict[str, Any]], @@ -1035,7 +1309,7 @@ def evaluate( ), ) try: - return evaluation_function( + return await evaluation_function( inputs, output, data_point, @@ -1052,3 +1326,14 @@ def evaluate( stacktrace=str(exc), ), ) + + +async def run( + evaluator_key: str, evaluator_input: EvaluatorInputInterface +) -> EvaluatorOutputInterface: + evaluator_function = RUN_EVALUATOR_FUNCTIONS.get(evaluator_key, None) + if not evaluator_function: + raise NotImplementedError(f"Evaluator {evaluator_key} not found") + + output = await evaluator_function(evaluator_input) + return output diff --git a/agenta-backend/agenta_backend/services/helpers.py b/agenta-backend/agenta_backend/services/helpers.py index 7b9510a0b..18951ad6f 100644 --- a/agenta-backend/agenta_backend/services/helpers.py +++ b/agenta-backend/agenta_backend/services/helpers.py @@ -1,6 +1,10 @@ import json -from typing import List, Dict, Any, Tuple, Union -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone +from typing import List, Dict, Any, Union, Tuple + +from agenta_backend.services import db_manager +from agenta_backend.models.api.evaluation_model import LMProvidersEnum +from agenta_backend.resources.evaluators.evaluators import get_all_evaluators def format_inputs(list_of_dictionaries: List[Dict[str, Any]]) -> Dict: @@ -76,3 +80,65 @@ def convert_to_utc_datetime(dt: Union[datetime, str, None]) -> datetime: if dt.tzinfo is None: return dt.replace(tzinfo=timezone.utc) return dt + + +def format_llm_provider_keys( + llm_provider_keys: Dict[LMProvidersEnum, str] +) -> Dict[str, str]: + """Formats a dictionary of LLM provider keys into a dictionary of strings. + + Args: + llm_provider_keys (Dict[LMProvidersEnum, str]): LLM provider keys + + Returns: + Dict[str, str]: formatted llm provided keys + + Example: + Input: {
- {isLoading ? (
-
-
- ) : error ? (
-
-
- ) : Array.isArray(apps) && apps.length ? (
-
-
-
+ )}
+
+ {isLoading ? (
+ Applications
+ {!isLoading && !error && ( +
+
+ ) : null}
+ App Management
+ {Array.isArray(apps) && apps.length ? ( } onClick={() => { @@ -308,7 +301,20 @@ const AppSelector: React.FC = () => { > Create new app -
+
+ ) : error ? (
+
+
+ ) : Array.isArray(apps) && apps.length ? (
+
{Array.isArray(apps) && (
<>
diff --git a/agenta-web/src/components/AppSelector/Welcome.tsx b/agenta-web/src/components/AppSelector/Welcome.tsx
index fa25a9377..9620bc4b5 100644
--- a/agenta-web/src/components/AppSelector/Welcome.tsx
+++ b/agenta-web/src/components/AppSelector/Welcome.tsx
@@ -1,125 +1,50 @@
import React from "react"
-import {useAppTheme} from "../Layout/ThemeContextProvider"
import {createUseStyles} from "react-jss"
-import {CheckCircleFilled} from "@ant-design/icons"
-import {StyleProps} from "@/lib/Types"
+import {JSSTheme} from "@/lib/Types"
import Image from "next/image"
+import {Button, Card, Typography} from "antd"
+import {ArrowRight} from "@phosphor-icons/react"
-const useStyles = createUseStyles({
- head: {
- marginBottom: 30,
- "& h2": {
- fontSize: 18,
- margin: "20px 0",
- textAlign: "center",
- },
- },
- heading: {
+const useStyles = createUseStyles((theme: JSSTheme) => ({
+ card: {
+ width: 392,
+ height: 268,
display: "flex",
- alignItems: "center",
- justifyContent: "center",
- gap: "1rem",
- "& > h1": {
- margin: 0,
- fontSize: 36,
- },
+ cursor: "pointer",
+ flexDirection: "column",
+ justifyContent: "space-between",
+ transition: "all 0.025s ease-in",
+ boxShadow:
+ "0px 2px 4px 0px rgba(0, 0, 0, 0.02), 0px 1px 6px -1px rgba(0, 0, 0, 0.02), 0px 1px 2px 0px rgba(0, 0, 0, 0.03)",
+ "& > .ant-card-head": {
+ minHeight: 0,
+ padding: theme.paddingSM,
- "& > img": {
- animation: "$wave 1.8s ease-in-out infinite",
- height: 44,
- },
- },
- "@keyframes wave": {
- "0%": {
- transform: "rotate(0deg)",
+ "& .ant-card-head-title": {
+ fontSize: theme.fontSizeLG,
+ fontWeight: theme.fontWeightMedium,
+ },
},
- "10%": {
- transform: "rotate(-10deg)",
+ "& > .ant-card-body": {
+ padding: theme.paddingSM,
+ flex: 1,
},
- "20%": {
- transform: "rotate(12deg)",
+ "& > .ant-card-actions": {
+ padding: "0 12px",
},
- "30%": {
- transform: "rotate(-10deg)",
- },
- "40%": {
- transform: "rotate(9deg)",
- },
- "50%": {
- transform: "rotate(0deg)",
- },
- "100%": {
- transform: "rotate(0deg)",
- },
- },
- description: {
- lineHeight: 1.7,
- },
- wrapper: {
- display: "flex",
- justifyContent: "space-between",
- gap: 20,
- maxWidth: "1250px",
- margin: "0 auto",
- width: "100%",
- },
- container: ({themeMode}: StyleProps) => ({
- display: "flex",
- justifyContent: "space-between",
- cursor: "pointer",
- flexDirection: "column",
- border: `1px solid ${themeMode === "dark" ? "rgb(13, 17, 23)" : "#91caff"}`,
- padding: "15px",
- borderRadius: 10,
- flex: 1,
- backgroundColor: themeMode === "dark" ? "#000" : "#fff",
- transition: "all 0.3s ease-out",
"&:hover": {
- backgroundColor: themeMode === "dark" ? "" : "#f3faff",
- boxShadow: themeMode === "dark" ? "0 0 10px rgba(225, 225, 225, 0.3)" : "",
+ boxShadow: theme.boxShadow,
},
- }),
- title: {
- display: "flex",
- alignItems: "center",
- justifyContent: "center",
- gap: "15px",
- "& h1": {
- fontWeight: 600,
- fontSize: 24,
- },
- },
- tag: {
- padding: "2px 6px",
- fontWeight: "bold",
},
- img: ({themeMode}: StyleProps) => ({
+ button: {
width: "100%",
- height: "auto",
- filter: themeMode === "dark" ? "invert(1)" : "none",
- }),
- steps: ({themeMode}: StyleProps) => ({
- fontSize: 16,
- margin: "20px 0 0",
display: "flex",
- flexDirection: "column",
- listStyleType: "none",
- padding: 20,
- "& li": {
- marginBottom: 10,
- },
- "& svg": {
- color: themeMode === "dark" ? "#fff" : "#0958d9",
- marginRight: 10,
- },
- "& span": {
- fontWeight: 600,
+ alignItems: "center",
+ "& > .ant-btn-icon": {
+ marginTop: 4,
},
- }),
- text: {
- marginLeft: 25,
},
-})
+}))
interface Props {
onWriteOwnApp: () => void
@@ -127,100 +52,97 @@ interface Props {
}
const Welcome: React.FC = ({onWriteOwnApp, onCreateFromTemplate}) => {
- const {appTheme} = useAppTheme()
- const classes = useStyles({themeMode: appTheme} as StyleProps)
+ const classes = useStyles()
- return (
- <>
-
-
-
-
-
-
- Welcome to Agenta
-The developer-first open source LLMOps platform.
-
-
+
-
+ const templatePoints = [
+ "Compare prompts and models",
+ "Create testsets",
+ "Evaluate outputs",
+ "Deploy in one click",
+ ]
+ const complexLLM = [
+ "Use Langchain, Llama Index, or any framework",
+ "Use OpenAI, Cohere, or self-hosted open-source models",
+ "Continue in the UI:Â Everything in the left",
+ "Streamline collaboration between devs and domain experts!",
+ ]
- Quickstart From a Template
-
+
+ Start building and testing your LLM
applications with Agenta AI.{" "} +
+
- applications with Agenta AI.{" "} +
-
-
-
-
Start from a template -
- -
-
Compare prompts and models -
- -
-
Create testsets -
- -
-
Evaluate outputs -
- -
-
Deploy in one click -
+ - {item} + ))}
-
-
Start from code -
- -
-
Use Langchain,{" "} - Llama Index, or any framework -
- -
-
Use OpenAI, Cohere, - or self-hosted open-source models -
- -
-
Continue in the UI: Everything in - the left -
- -
-
Streamline collaboration between - devs and domain experts! -
+ - {item} + ))}
+ = ({language, value}) => {
language={language}
style={appTheme === "dark" ? darcula : coy}
showLineNumbers
- wrapLongLines={true}
+ wrapLongLines={false}
>
{value}
diff --git a/agenta-web/src/components/Evaluations/AutomaticEvaluationResult.tsx b/agenta-web/src/components/Evaluations/AutomaticEvaluationResult.tsx
deleted file mode 100644
index 2b6ef8ef5..000000000
--- a/agenta-web/src/components/Evaluations/AutomaticEvaluationResult.tsx
+++ /dev/null
@@ -1,286 +0,0 @@
-import {
- deleteEvaluations,
- fetchEvaluationResults,
- fetchAllLoadEvaluations,
-} from "@/services/human-evaluations/api"
-import {Button, Spin, Statistic, Table, Typography} from "antd"
-import {useRouter} from "next/router"
-import {useEffect, useState} from "react"
-import {ColumnsType} from "antd/es/table"
-import {Evaluation, SingleModelEvaluationListTableDataType, StyleProps} from "@/lib/Types"
-import {DeleteOutlined} from "@ant-design/icons"
-import {EvaluationFlow, EvaluationType} from "@/lib/enums"
-import {createUseStyles} from "react-jss"
-import {useAppTheme} from "../Layout/ThemeContextProvider"
-import {calculateResultsDataAvg} from "@/lib/helpers/evaluate"
-import {
- fromEvaluationResponseToEvaluation,
- singleModelTestEvaluationTransformer,
-} from "@/lib/transformers"
-import {variantNameWithRev} from "@/lib/helpers/variantHelper"
-
-const useStyles = createUseStyles({
- container: {
- marginBottom: 20,
- },
- collapse: ({themeMode}: StyleProps) => ({
- margin: "10px 0",
- "& .ant-collapse-header": {
- alignItems: "center !important",
- padding: "0px 20px !important",
- borderTopLeftRadius: "10px !important",
- borderTopRightRadius: "10px !important",
- background: themeMode === "dark" ? "#1d1d1d" : "#f8f8f8",
- },
- }),
- stat: {
- "& .ant-statistic-content-value": {
- fontSize: 20,
- color: "#1677ff",
- },
- "& .ant-statistic-content-suffix": {
- fontSize: 20,
- color: "#1677ff",
- },
- },
- btnContainer: {
- display: "flex",
- alignItems: "center",
- justifyContent: "flex-end",
- margin: "20px 0",
- gap: 10,
- "& svg": {
- color: "red",
- },
- },
-})
-
-const {Title} = Typography
-interface AutomaticEvaluationResultProps {
- setIsEvalModalOpen: React.Dispatch>
-}
-export default function AutomaticEvaluationResult({
- setIsEvalModalOpen,
-}: AutomaticEvaluationResultProps) {
- const router = useRouter()
- const [evaluationsList, setEvaluationsList] = useState<
- SingleModelEvaluationListTableDataType[]
- >([])
- const [selectedRowKeys, setSelectedRowKeys] = useState([])
- const [selectionType] = useState<"checkbox" | "radio">("checkbox")
- const {appTheme} = useAppTheme()
- const classes = useStyles({themeMode: appTheme} as StyleProps)
- const app_id = router.query.app_id?.toString() || ""
- const [fetchingEvaluations, setFetchingEvaluations] = useState(false)
-
- useEffect(() => {
- if (!app_id) {
- return
- }
-
- const fetchEvaluations = async () => {
- try {
- setFetchingEvaluations(true)
- const evals: Evaluation[] = (await fetchAllLoadEvaluations(app_id)).map(
- fromEvaluationResponseToEvaluation,
- )
- const results = await Promise.all(evals.map((e) => fetchEvaluationResults(e.id)))
- const newEvals = results.map((result, ix) => {
- const item = evals[ix]
- if ([EvaluationType.single_model_test].includes(item.evaluationType)) {
- return singleModelTestEvaluationTransformer({item, result})
- }
- })
-
- setEvaluationsList(
- newEvals
- .filter((evaluation) => evaluation !== undefined)
- .filter(
- (item: any) =>
- item.resultsData !== undefined ||
- !(Object.keys(item.scoresData || {}).length === 0) ||
- item.avgScore !== undefined,
- ) as any,
- )
- } catch (error) {
- console.error(error)
- } finally {
- setFetchingEvaluations(false)
- }
- }
-
- fetchEvaluations()
- }, [app_id])
-
- const handleNavigation = (variantName: string, revisionNum: string) => {
- router.push(`/apps/${app_id}/playground?variant=${variantName}&revision=${revisionNum}`)
- }
-
- const onCompleteEvaluation = (evaluation: any) => {
- // TODO: improve type
- const evaluationType =
- EvaluationType[evaluation.evaluationType as keyof typeof EvaluationType]
-
- if (evaluationType === EvaluationType.single_model_test) {
- router.push(`/apps/${app_id}/annotations/single_model_test/${evaluation.key}`)
- }
- }
-
- const columns: ColumnsType = [
- {
- title: "Variant",
- dataIndex: "variants",
- key: "variants",
- render: (value, record: SingleModelEvaluationListTableDataType) => {
- return (
-
- }
- createdAt: string
- revisions: string[]
- variant_revision_ids: string[]
- variantNames: string[]
-}
-
-const useStyles = createUseStyles({
- container: {
- marginBottom: 20,
- },
- collapse: ({themeMode}: StyleProps) => ({
- margin: "10px 0",
- "& .ant-collapse-header": {
- alignItems: "center !important",
- padding: "0px 20px !important",
- borderTopLeftRadius: "10px !important",
- borderTopRightRadius: "10px !important",
- background: themeMode === "dark" ? "#1d1d1d" : "#f8f8f8",
- },
- }),
- statFlag: {
- "& .ant-statistic-content-value": {
- fontSize: 20,
- color: "#cf1322",
- },
- "& .ant-statistic-content-suffix": {
- fontSize: 20,
- color: "#cf1322",
- },
- },
- stat: {
- "& .ant-statistic-content-value": {
- fontSize: 20,
- color: "#1677ff",
- },
- "& .ant-statistic-content-suffix": {
- fontSize: 20,
- color: "#1677ff",
- },
- },
- statGood: {
- "& .ant-statistic-content-value": {
- fontSize: 20,
- color: "#3f8600",
- },
- "& .ant-statistic-content-suffix": {
- fontSize: 20,
- color: "#3f8600",
- },
- },
- btnContainer: {
- display: "flex",
- alignItems: "center",
- justifyContent: "flex-end",
- margin: "20px 0",
- gap: 10,
- "& svg": {
- color: "red",
- },
- },
-})
-
-const {Title} = Typography
-
-interface HumanEvaluationResultProps {
- setIsEvalModalOpen: React.Dispatch>
-}
-
-export default function HumanEvaluationResult({setIsEvalModalOpen}: HumanEvaluationResultProps) {
- const router = useRouter()
- const [evaluationsList, setEvaluationsList] = useState([])
- const [selectedRowKeys, setSelectedRowKeys] = useState([])
- const [selectionType] = useState<"checkbox" | "radio">("checkbox")
- const {appTheme} = useAppTheme()
- const classes = useStyles({themeMode: appTheme} as StyleProps)
- const app_id = router.query.app_id?.toString() || ""
- const [fetchingEvaluations, setFetchingEvaluations] = useState(false)
-
- useEffect(() => {
- if (!app_id) {
- return
- }
- const fetchEvaluations = async () => {
- try {
- setFetchingEvaluations(true)
- fetchAllLoadEvaluations(app_id)
- .then((response) => {
- const fetchPromises = response.map((item: EvaluationResponseType) => {
- return fetchEvaluationResults(item.id)
- .then((results) => {
- if (item.evaluation_type === EvaluationType.human_a_b_testing) {
- if (Object.keys(results.votes_data).length > 0) {
- return abTestingEvaluationTransformer({item, results})
- }
- }
- })
- .catch((err) => console.error(err))
- })
- Promise.all(fetchPromises)
- .then((evaluations) => {
- const validEvaluations = evaluations.filter(
- (evaluation) => evaluation !== undefined,
- )
- setEvaluationsList(validEvaluations)
- })
- .catch((err) => console.error(err))
- })
- .catch((err) => console.error(err))
- .finally(() => setFetchingEvaluations(false))
- } catch (error) {
- console.error(error)
- }
- }
-
- fetchEvaluations()
- }, [app_id])
-
- const onCompleteEvaluation = (evaluation: any) => {
- // TODO: improve type
- const evaluationType =
- EvaluationType[evaluation.evaluationType as keyof typeof EvaluationType]
-
- if (evaluationType === EvaluationType.human_a_b_testing) {
- router.push(`/apps/${app_id}/annotations/human_a_b_testing/${evaluation.key}`)
- }
- }
-
- const handleNavigation = (variantName: string, revisionNum: string) => {
- router.push(`/apps/${app_id}/playground?variant=${variantName}&revision=${revisionNum}`)
- }
-
- const columns: ColumnsType = [
- {
- title: "Test set",
- dataIndex: "testsetName",
- key: "testsetName",
- render: (_, record: HumanEvaluationListTableDataType, index: number) => {
- return {record.testset.name}
- },
- },
- {
- title: "Variant 1",
- dataIndex: "variantNames",
- key: "variant1",
- render: (value, record) => {
- const percentage = getVotesPercentage(record, 0)
- return (
-
@@ -373,7 +386,7 @@ const HumanEvaluationModal = ({
>
()
const [isDeleteEvalModalOpen, setIsDeleteEvalModalOpen] = useState(false)
+ const [isDeleteMultipleEvalModalOpen, setIsDeleteMultipleEvalModalOpen] = useState(false)
+ const [selectedRowKeys, setSelectedRowKeys] = useState([])
useEffect(() => {
if (!appId) return
@@ -105,9 +110,8 @@ const AbTestingEvalOverview = () => {
new Date(b.createdAt || 0).getTime() -
new Date(a.createdAt || 0).getTime(),
)
- .slice(0, 5)
- setEvaluationsList(results)
+ setEvaluationsList(viewType === "overview" ? results.slice(0, 5) : results)
} catch (error) {
console.error(error)
} finally {
@@ -122,6 +126,31 @@ const AbTestingEvalOverview = () => {
router.push(`/apps/${appId}/playground?variant=${variantName}&revision=${revisionNum}`)
}
+ const rowSelection = {
+ onChange: (selectedRowKeys: React.Key[]) => {
+ setSelectedRowKeys(selectedRowKeys)
+ },
+ }
+
+ const handleDeleteMultipleEvaluations = async () => {
+ const evaluationsIds = selectedRowKeys.map((key) => key.toString())
+ try {
+ setFetchingEvaluations(true)
+ await deleteEvaluations(evaluationsIds)
+ setEvaluationsList((prevEvaluationsList) =>
+ prevEvaluationsList.filter(
+ (evaluation) => !evaluationsIds.includes(evaluation.key),
+ ),
+ )
+ setSelectedRowKeys([])
+ message.success("Evaluations Deleted")
+ } catch (error) {
+ console.error(error)
+ } finally {
+ setFetchingEvaluations(false)
+ }
+ }
+
const handleDeleteEvaluation = async (record: HumanEvaluationListTableDataType) => {
try {
setFetchingEvaluations(true)
@@ -288,7 +317,7 @@ const AbTestingEvalOverview = () => {
onClick: (e) => {
e.domEvent.stopPropagation()
router.push(
- `/apps/${appId}/annotations/human_a_b_testing/${record.key}`,
+ `/apps/${appId}/evaluations/human_a_b_testing/${record.key}`,
)
},
},
@@ -357,25 +386,64 @@ const AbTestingEvalOverview = () => {
return (
+
+ Setup an app using our preset LLM config and explore Agenta AI
+
+
- -
+ {templatePoints.map((item) => (
+
-
+
)
}
-
export default Welcome
diff --git a/agenta-web/src/components/DynamicCodeBlock/CodeBlock.tsx b/agenta-web/src/components/DynamicCodeBlock/CodeBlock.tsx
index df6ebc800..6a611b3fc 100644
--- a/agenta-web/src/components/DynamicCodeBlock/CodeBlock.tsx
+++ b/agenta-web/src/components/DynamicCodeBlock/CodeBlock.tsx
@@ -29,7 +29,7 @@ const CodeBlock: FC
-
-
-
-
+
- Build Complex LLM apps
--
-
+
+ Create your own complex application using any framework.
+
+
-
-
-
+
+ -
+ {complexLLM.map((item) => (
+
handleNavigation(value[0].variantName, record.revisions[0])}
- style={{cursor: "pointer"}}
- >
-
- {variantNameWithRev({
- variant_name: value[0].variantName,
- revision: record.revisions[0],
- })}
-
-
- )
- },
- },
- {
- title: "Test set",
- dataIndex: "testsetName",
- key: "testsetName",
- render: (value: any, record: SingleModelEvaluationListTableDataType, index: number) => {
- return {record.testset.name}
- },
- },
- {
- title: "Average score",
- dataIndex: "averageScore",
- key: "averageScore",
- render: (value: any, record: SingleModelEvaluationListTableDataType, index: number) => {
- let score = 0
- if (record.scoresData) {
- score =
- ((record.scoresData.correct?.length ||
- record.scoresData.true?.length ||
- 0) /
- record.scoresData.nb_of_rows) *
- 100
- } else if (record.resultsData) {
- const multiplier = {
- [EvaluationType.auto_webhook_test]: 100,
- [EvaluationType.single_model_test]: 1,
- }
- score = calculateResultsDataAvg(
- record.resultsData,
- multiplier[record.evaluationType as keyof typeof multiplier],
- )
- score = isNaN(score) ? 0 : score
- } else if (record.avgScore) {
- score = record.avgScore * 100
- }
-
- return (
-
-
-
-
- )
- },
- },
- ]
-
- const rowSelection = {
- onChange: (
- selectedRowKeys: React.Key[],
- selectedRows: SingleModelEvaluationListTableDataType[],
- ) => {
- setSelectedRowKeys(selectedRowKeys)
- },
- }
-
- const onDelete = async () => {
- const evaluationsIds = selectedRowKeys.map((key) => key.toString())
- try {
- await deleteEvaluations(evaluationsIds)
- setEvaluationsList((prevEvaluationsList) =>
- prevEvaluationsList.filter(
- (evaluation) => !evaluationsIds.includes(evaluation.key),
- ),
- )
-
- setSelectedRowKeys([])
- } catch (error) {
- console.error(error)
- }
- }
-
- return (
-
-
-
-
-
- )
-}
diff --git a/agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx b/agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx
index 87720a438..e69de29bb 100644
--- a/agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx
+++ b/agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx
@@ -1,373 +0,0 @@
-import {
- deleteEvaluations,
- fetchAllLoadEvaluations,
- fetchEvaluationResults,
-} from "@/services/human-evaluations/api"
-import {Button, Spin, Statistic, Table, Typography} from "antd"
-import {useRouter} from "next/router"
-import {useEffect, useState} from "react"
-import {ColumnsType} from "antd/es/table"
-import {EvaluationResponseType, StyleProps} from "@/lib/Types"
-import {DeleteOutlined} from "@ant-design/icons"
-import {EvaluationFlow, EvaluationType} from "@/lib/enums"
-import {createUseStyles} from "react-jss"
-import {useAppTheme} from "../Layout/ThemeContextProvider"
-import {getVotesPercentage} from "@/lib/helpers/evaluate"
-import {isDemo} from "@/lib/helpers/utils"
-import {variantNameWithRev} from "@/lib/helpers/variantHelper"
-import {abTestingEvaluationTransformer} from "@/lib/transformers"
-
-interface VariantVotesData {
- number_of_votes: number
- percentage: number
-}
-
-export interface HumanEvaluationListTableDataType {
- key: string
- variants: string[]
- testset: {
- _id: string
- name: string
- }
- evaluationType: string
- status: EvaluationFlow
- votesData: {
- nb_of_rows: number
- variants: string[]
- flag_votes: {
- number_of_votes: number
- percentage: number
- }
- positive_votes: {
- number_of_votes: number
- percentage: number
- }
- variants_votes_data: Record
-
-
-
-
-
- Single Model Test Results
-
-
-
-
- )
- },
- },
- {
- title: "Variant 2",
- dataIndex: "variantNames",
- key: "variant2",
- render: (value, record) => {
- const percentage = getVotesPercentage(record, 1)
- return (
- handleNavigation(value[0], record.revisions[0])}
- >
- (
- {variantNameWithRev({
- variant_name: value[0],
- revision: record.revisions[0],
- })}
- )
-
-
-
- )
- },
- },
- {
- title: "Both are good",
- dataIndex: "positive",
- key: "positive",
- render: (value: any, record: HumanEvaluationListTableDataType) => {
- let percentage = record.votesData.positive_votes.percentage
- return (
-
- handleNavigation(value[1], record.revisions[1])}
- >
- (
- {variantNameWithRev({
- variant_name: value[1],
- revision: record.revisions[1],
- })}
- )
-
-
-
-
- )
- },
- },
- ],
- )
-
- const rowSelection = {
- onChange: (selectedRowKeys: React.Key[]) => {
- setSelectedRowKeys(selectedRowKeys)
- },
- }
-
- const onDelete = async () => {
- const evaluationsIds = selectedRowKeys.map((key) => key.toString())
- try {
- await deleteEvaluations(evaluationsIds)
- setEvaluationsList((prevEvaluationsList) =>
- prevEvaluationsList.filter(
- (evaluation) => !evaluationsIds.includes(evaluation.key),
- ),
- )
-
- setSelectedRowKeys([])
- } catch {}
- }
-
- return (
-
-
-
-
-
- )
-}
diff --git a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx
index 142ea39a0..084bf22fb 100644
--- a/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx
+++ b/agenta-web/src/components/HumanEvaluationModal/HumanEvaluationModal.tsx
@@ -5,7 +5,6 @@ import {createNewEvaluation} from "@/services/human-evaluations/api"
import {isDemo} from "@/lib/helpers/utils"
import {Button, Col, Dropdown, MenuProps, Modal, ModalProps, Row, Spin, message} from "antd"
import {getErrorMessage} from "@/lib/helpers/errorHandler"
-import {DownOutlined} from "@ant-design/icons"
import {EvaluationType} from "@/lib/enums"
import {PERMISSION_ERR_MSG} from "@/lib/helpers/axiosConfig"
import {getAllVariantParameters} from "@/lib/helpers/variantHelper"
@@ -15,6 +14,7 @@ import {createUseStyles} from "react-jss"
import EvaluationErrorModal from "../Evaluations/EvaluationErrorModal"
import {dynamicComponent} from "@/lib/helpers/dynamic"
import {useLoadTestsetsList} from "@/services/testsets/api"
+import {CaretDown, Play} from "@phosphor-icons/react"
const useStyles = createUseStyles((theme: JSSTheme) => ({
evaluationContainer: {
@@ -111,6 +111,11 @@ const useStyles = createUseStyles((theme: JSSTheme) => ({
alignItems: "center",
justifyContent: "space-between",
},
+ dropdownItemLabels: {
+ fontSize: theme.fontSizeSM,
+ lineHeight: theme.lineHeightSM,
+ color: theme.colorTextDescription,
+ },
}))
interface HumanEvaluationModalProps {
@@ -278,7 +283,15 @@ const HumanEvaluationModal = ({
filteredVariants.push({
label: (
<>
-
-
-
-
-
-
- A/B Test Results
-
-
- {variant.variantName}
+
+ {variant.variantName}
+
+ #{variant.variantId.split("-")[0]}
+
+
),
key: label,
@@ -340,9 +353,9 @@ const HumanEvaluationModal = ({
setVariants(selectedVariants)
if (evaluationType === EvaluationType.human_a_b_testing) {
- router.push(`/apps/${appId}/annotations/human_a_b_testing/${evaluationTableId}`)
+ router.push(`/apps/${appId}/evaluations/human_a_b_testing/${evaluationTableId}`)
} else if (evaluationType === EvaluationType.single_model_test) {
- router.push(`/apps/${appId}/annotations/single_model_test/${evaluationTableId}`)
+ router.push(`/apps/${appId}/evaluations/single_model_test/${evaluationTableId}`)
}
}
@@ -356,7 +369,7 @@ const HumanEvaluationModal = ({
setSelectedTestset({name: "Select a Test set"})
setSelectedVariants(new Array(1).fill({variantName: "Select a variant"}))
}}
- title="Start a New Evaluation"
+ title="New Evaluation"
footer={null}
>
{selectedTestset.name}
-
@@ -393,7 +406,7 @@ const HumanEvaluationModal = ({
{selectedVariants[index]?.variantName ||
"Select a variant"}
-
@@ -430,6 +443,8 @@ const HumanEvaluationModal = ({
onClick={onStartEvaluation}
type="primary"
data-cy="start-new-evaluation-button"
+ icon={
-
{
style: {cursor: "pointer"},
onClick: () =>
router.push(
- `/apps/${appId}/annotations/human_a_b_testing/${record.key}`,
+ `/apps/${appId}/evaluations/human_a_b_testing/${record.key}`,
),
})}
/>
@@ -409,8 +477,20 @@ const AbTestingEvalOverview = () => {
evaluationType={"a/b testing evaluation"}
/>
)}
+
+ {isDeleteMultipleEvalModalOpen && (
+
-
- A/B Testing Evaluations
-
+
+
+ )}