merge main

README.md

@@ -1,18 +1,4 @@
-  <div align="center" style="margin: 30px">
-  <a href="https://docs.agenta.ai/self-host/migration/migration-to-postgres">
-    Important: We are migrating from MongoDB to PostgreSQL in v0.19! Follow this guide to migrate your data.
-  </a>
-  </div>
-  </br>
-  </br>
-
 <div align="center" style="margin: 30px">
-  <div>
-    <a href="https://agentaai.notion.site/Careers-at-agenta-fb8a76d4d9044e27bd3a32678818dbc8/">
-      <img src="https://github.com/Agenta-AI/agenta/raw/main/.github/images/we-are-hiring.svg" alt="We are hiring! Join our team!" width="350" height="60" alt="We are hiring! Join our team!" width="350" height="60">
-    </a> 
-  </div>
-
   <a href="https://agenta.ai/">
     <div  align="center" >
       <picture >
@@ -76,12 +62,9 @@
   <br />
       <div  align="center" >
       <picture >
-        <source width="800" media="(prefers-color-scheme: dark)" srcset="https://github.com/Agenta-AI/agenta/assets/4510758/cf6d4713-4558-4c6c-9e1b-ee4eab261f4c"  >
-        <source width="800" media="(prefers-color-scheme: light)" srcset="https://github.com/Agenta-AI/agenta/assets/4510758/ae9cf11f-8ef9-4b67-98c7-4a40341fa87a"  >
-        <img alt="Mockup agenta" src="https://github.com/Agenta-AI/agenta/assets/4510758/ae9cf11f-8ef9-4b67-98c7-4a40341fa87a" >
+        <img alt="Glamour Shot" src="https://github.com/user-attachments/assets/32e95ddb-e001-4462-b92e-72bf4cc78597" >
       </picture>
     </div>
-
 </div>
 <br />
 <br />

agenta-backend/agenta_backend/migrations/postgres/data_migrations/applications.py

@@ -6,7 +6,7 @@
 
 import click
 from sqlalchemy.future import select
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, delete
 from sqlalchemy.orm import sessionmaker, Session
 
 from agenta_backend.models.deprecated_models import (
@@ -34,6 +34,7 @@ def update_evaluators_with_app_name():
                 records = (
                     session.execute(
                         select(DeprecatedEvaluatorConfigDB)
+                        .filter(DeprecatedEvaluatorConfigDB.app_id.isnot(None))
                         .offset(offset)
                         .limit(BATCH_SIZE)
                     )
@@ -54,6 +55,13 @@ def update_evaluators_with_app_name():
                 session.commit()
                 offset += BATCH_SIZE
 
+            # Delete deprecated evaluator configs with app_id as None
+            session.execute(
+                delete(DeprecatedEvaluatorConfigDB).where(
+                    DeprecatedEvaluatorConfigDB.app_id.is_(None)
+                )
+            )
+            session.commit()
         except Exception as e:
             session.rollback()
             click.echo(

agenta-backend/agenta_backend/migrations/postgres/versions/5c29a64204f4_added_modified_by_id_column_to_apps_db.py

@@ -5,6 +5,8 @@
 Create Date: 2024-08-25 17:56:11.732929
 
 """
+
+import os
 from typing import Sequence, Union
 
 from alembic import op
@@ -18,16 +20,32 @@
 depends_on: Union[str, Sequence[str], None] = None
 
 
+def unique_constraint_exists(engine: sa.Engine, table_name: str, constraint_name: str):
+    with engine.connect() as conn:
+        result = conn.execute(
+            sa.text(
+                f"""
+        SELECT conname FROM pg_constraint
+        WHERE conname = '{constraint_name}' AND conrelid = '{table_name}'::regclass;
+        """
+            )
+        )
+        return result.fetchone() is not None
+
+
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column("app_db", sa.Column("modified_by_id", sa.UUID(), nullable=True))
-    op.create_foreign_key(None, "app_db", "users", ["modified_by_id"], ["id"])
+    op.create_foreign_key(
+        "app_db_modified_by_id_fkey", "app_db", "users", ["modified_by_id"], ["id"]
+    )
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_constraint(None, "app_db", type_="foreignkey")
-    op.drop_constraint(None, "app_db", type_="unique")
+    engine = sa.create_engine(os.getenv("POSTGRES_URI"))  # type: ignore
+    if not unique_constraint_exists(engine, "app_db", "app_db_modified_by_id_fkey"):
+        op.drop_constraint("app_db_modified_by_id_fkey", "app_db", type_="foreignkey")
     op.drop_column("app_db", "modified_by_id")
     # ### end Alembic commands ###

agenta-backend/agenta_backend/migrations/postgres/versions/b80c708c21bb_initial_migration.py

@@ -5,6 +5,8 @@
 Create Date: 2024-07-11 13:20:31.705894
 
 """
+
+import os
 from typing import Sequence, Union
 
 from alembic import op
@@ -23,6 +25,19 @@
 depends_on: Union[str, Sequence[str], None] = None
 
 
+def unique_constraint_exists(engine: sa.Engine, table_name: str, constraint_name: str):
+    with engine.connect() as conn:
+        result = conn.execute(
+            sa.text(
+                f"""
+        SELECT conname FROM pg_constraint
+        WHERE conname = '{constraint_name}' AND conrelid = '{table_name}'::regclass;
+        """
+            )
+        )
+        return result.fetchone() is not None
+
+
 def first_time_user_from_agenta_v019_upwards_upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table(
@@ -475,26 +490,104 @@ def first_time_user_from_agenta_v019_upwards_upgrade() -> None:
 
 def returning_user_from_agenta_v018_downwards_upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_unique_constraint(None, "app_db", ["id"])
-    op.create_unique_constraint(None, "app_variant_revisions", ["id"])
-    op.create_unique_constraint(None, "app_variants", ["id"])
-    op.create_unique_constraint(None, "bases", ["id"])
-    op.create_unique_constraint(None, "deployments", ["id"])
-    op.create_unique_constraint(None, "docker_images", ["id"])
-    op.create_unique_constraint(None, "environments", ["id"])
-    op.create_unique_constraint(None, "environments_revisions", ["id"])
-    op.create_unique_constraint(None, "evaluation_aggregated_results", ["id"])
-    op.create_unique_constraint(None, "evaluation_scenario_results", ["id"])
-    op.create_unique_constraint(None, "evaluation_scenarios", ["id"])
-    op.create_unique_constraint(None, "evaluations", ["id"])
-    op.create_unique_constraint(None, "evaluators_configs", ["id"])
-    op.create_unique_constraint(None, "human_evaluation_variants", ["id"])
-    op.create_unique_constraint(None, "human_evaluations", ["id"])
-    op.create_unique_constraint(None, "human_evaluations_scenarios", ["id"])
-    op.create_unique_constraint(None, "ids_mapping", ["id"])
-    op.create_unique_constraint(None, "templates", ["id"])
-    op.create_unique_constraint(None, "testsets", ["id"])
-    op.create_unique_constraint(None, "users", ["id"])
+    engine = sa.create_engine(os.getenv("POSTGRES_URI"))  # type: ignore
+    if not unique_constraint_exists(engine, "app_db", "app_db_pkey"):
+        op.create_unique_constraint("app_db_pkey", "app_db", ["id"])
+
+    if not unique_constraint_exists(
+        engine, "app_variant_revisions", "app_variant_revisions_pkey"
+    ):
+        op.create_unique_constraint(
+            "app_variant_revisions_pkey", "app_variant_revisions", ["id"]
+        )
+
+    if not unique_constraint_exists(engine, "app_variants", "app_variants_pkey"):
+        op.create_unique_constraint("app_variants_pkey", "app_variants", ["id"])
+
+    if not unique_constraint_exists(engine, "bases", "bases_pkey"):
+        op.create_unique_constraint("bases_pkey", "bases", ["id"])
+
+    if not unique_constraint_exists(engine, "deployments", "deployments_pkey"):
+        op.create_unique_constraint("deployments_pkey", "deployments", ["id"])
+
+    if not unique_constraint_exists(engine, "docker_images", "docker_images_pkey"):
+        op.create_unique_constraint("docker_images_pkey", "docker_images", ["id"])
+
+    if not unique_constraint_exists(engine, "environments", "environments_pkey"):
+        op.create_unique_constraint("environments_pkey", "environments", ["id"])
+
+    if not unique_constraint_exists(
+        engine, "environments_revisions", "environments_revisions_pkey"
+    ):
+        op.create_unique_constraint(
+            "environments_revisions_pkey", "environments_revisions", ["id"]
+        )
+
+    if not unique_constraint_exists(
+        engine, "evaluation_aggregated_results", "evaluation_aggregated_results_pkey"
+    ):
+        op.create_unique_constraint(
+            "evaluation_aggregated_results_pkey",
+            "evaluation_aggregated_results",
+            ["id"],
+        )
+
+    if not unique_constraint_exists(
+        engine, "evaluation_scenario_results", "evaluation_scenario_results_pkey"
+    ):
+        op.create_unique_constraint(
+            "evaluation_scenario_results_pkey", "evaluation_scenario_results", ["id"]
+        )
+
+    if not unique_constraint_exists(
+        engine, "evaluation_scenarios", "evaluation_scenarios_pkey"
+    ):
+        op.create_unique_constraint(
+            "evaluation_scenarios_pkey", "evaluation_scenarios", ["id"]
+        )
+
+    if not unique_constraint_exists(engine, "evaluations", "evaluations_pkey"):
+        op.create_unique_constraint("evaluations_pkey", "evaluations", ["id"])
+
+    if not unique_constraint_exists(
+        engine, "evaluators_configs", "evaluators_configs_pkey"
+    ):
+        op.create_unique_constraint(
+            "evaluators_configs_pkey", "evaluators_configs", ["id"]
+        )
+
+    if not unique_constraint_exists(
+        engine, "human_evaluation_variants", "human_evaluation_variants_pkey"
+    ):
+        op.create_unique_constraint(
+            "human_evaluation_variants_pkey", "human_evaluation_variants", ["id"]
+        )
+
+    if not unique_constraint_exists(
+        engine, "human_evaluations", "human_evaluations_pkey"
+    ):
+        op.create_unique_constraint(
+            "human_evaluations_pkey", "human_evaluations", ["id"]
+        )
+
+    if not unique_constraint_exists(
+        engine, "human_evaluations_scenarios", "human_evaluations_scenarios_pkey"
+    ):
+        op.create_unique_constraint(
+            "human_evaluations_scenarios_pkey", "human_evaluations_scenarios", ["id"]
+        )
+
+    if not unique_constraint_exists(engine, "ids_mapping", "ids_mapping_pkey"):
+        op.create_unique_constraint("ids_mapping_pkey", "ids_mapping", ["id"])
+
+    if not unique_constraint_exists(engine, "templates", "templates_pkey"):
+        op.create_unique_constraint("templates_pkey", "templates", ["id"])
+
+    if not unique_constraint_exists(engine, "testsets", "testsets_pkey"):
+        op.create_unique_constraint("testsets_pkey", "testsets", ["id"])
+
+    if not unique_constraint_exists(engine, "users", "users_pkey"):
+        op.create_unique_constraint("users_pkey", "users", ["id"])
     # ### end Alembic commands ###
 
 
@@ -528,26 +621,46 @@ def first_time_user_from_agenta_v019_upwards_downgrade() -> None:
 
 def returning_user_from_agenta_v018_downwards_downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_constraint(None, "users", type_="unique")
-    op.drop_constraint(None, "testsets", type_="unique")
-    op.drop_constraint(None, "templates", type_="unique")
-    op.drop_constraint(None, "ids_mapping", type_="unique")
-    op.drop_constraint(None, "human_evaluations_scenarios", type_="unique")
-    op.drop_constraint(None, "human_evaluations", type_="unique")
-    op.drop_constraint(None, "human_evaluation_variants", type_="unique")
-    op.drop_constraint(None, "evaluators_configs", type_="unique")
-    op.drop_constraint(None, "evaluations", type_="unique")
-    op.drop_constraint(None, "evaluation_scenarios", type_="unique")
-    op.drop_constraint(None, "evaluation_scenario_results", type_="unique")
-    op.drop_constraint(None, "evaluation_aggregated_results", type_="unique")
-    op.drop_constraint(None, "environments_revisions", type_="unique")
-    op.drop_constraint(None, "environments", type_="unique")
-    op.drop_constraint(None, "docker_images", type_="unique")
-    op.drop_constraint(None, "deployments", type_="unique")
-    op.drop_constraint(None, "bases", type_="unique")
-    op.drop_constraint(None, "app_variants", type_="unique")
-    op.drop_constraint(None, "app_variant_revisions", type_="unique")
-    op.drop_constraint(None, "app_db", type_="unique")
+    op.drop_constraint("users_pkey", "users", type_="unique")
+    op.drop_constraint("testsets_pkey", "testsets", type_="unique")
+    op.drop_constraint("templates_pkey", "templates", type_="unique")
+    op.drop_constraint("ids_mapping_pkey", "ids_mapping", type_="unique")
+    op.drop_constraint(
+        "human_evaluations_scenarios_pkey",
+        "human_evaluations_scenarios",
+        type_="unique",
+    )
+    op.drop_constraint("human_evaluations_pkey", "human_evaluations", type_="unique")
+    op.drop_constraint(
+        "human_evaluation_variants_pkey", "human_evaluation_variants", type_="unique"
+    )
+    op.drop_constraint("evaluators_configs_pkey", "evaluators_configs", type_="unique")
+    op.drop_constraint("evaluations_pkey", "evaluations", type_="unique")
+    op.drop_constraint(
+        "evaluation_scenarios_pkey", "evaluation_scenarios", type_="unique"
+    )
+    op.drop_constraint(
+        "evaluation_scenario_results_pkey",
+        "evaluation_scenario_results",
+        type_="unique",
+    )
+    op.drop_constraint(
+        "evaluation_aggregated_results_pkey",
+        "evaluation_aggregated_results",
+        type_="unique",
+    )
+    op.drop_constraint(
+        "environments_revisions_pkey", "environments_revisions", type_="unique"
+    )
+    op.drop_constraint("environments_pkey", "environments", type_="unique")
+    op.drop_constraint("docker_images_pkey", "docker_images", type_="unique")
+    op.drop_constraint("deployments_pkey", "deployments", type_="unique")
+    op.drop_constraint("bases_key", "bases", type_="unique")
+    op.drop_constraint("app_variants_pkey", "app_variants", type_="unique")
+    op.drop_constraint(
+        "app_variant_revisions_pkey", "app_variant_revisions", type_="unique"
+    )
+    op.drop_constraint("app_db_pkey", "app_db", type_="unique")
     # ### end Alembic commands ###
 
 

agenta-backend/agenta_backend/models/api/evaluation_model.py

@@ -1,7 +1,9 @@
 from enum import Enum
 from datetime import datetime
-from pydantic import BaseModel
 from typing import Optional, List, Dict, Any, Union
+
+from pydantic import BaseModel, Field, model_validator
+
 from agenta_backend.models.api.api_models import Result
 
 
@@ -12,6 +14,8 @@ class Evaluator(BaseModel):
     settings_template: dict
     description: Optional[str] = None
     oss: Optional[bool] = False
+    requires_llm_api_keys: Optional[bool] = False
+    tags: List[str]
 
 
 class EvaluatorConfig(BaseModel):
@@ -80,6 +84,25 @@ class Evaluation(BaseModel):
     updated_at: datetime
 
 
+class EvaluatorInputInterface(BaseModel):
+    inputs: Dict[str, Any] = Field(default_factory=dict)
+    settings: Optional[Dict[str, Any]] = None
+    credentials: Optional[Dict[str, Any]] = None
+
+
+class EvaluatorOutputInterface(BaseModel):
+    outputs: Dict[str, Any]
+
+
+class EvaluatorMappingInputInterface(BaseModel):
+    inputs: Dict[str, Any]
+    mapping: Dict[str, Any]
+
+
+class EvaluatorMappingOutputInterface(BaseModel):
+    outputs: Dict[str, Any]
+
+
 class SimpleEvaluationOutput(BaseModel):
     id: str
     variant_ids: List[str]

agenta-backend/agenta_backend/models/api/testset_model.py

@@ -48,6 +48,7 @@ class TestSetOutputResponse(BaseModel):
     id: str = Field(..., alias="_id")
     name: str
     created_at: str
+    updated_at: str
 
     class Config:
         allow_population_by_field_name = True

agenta-backend/agenta_backend/models/converters.py

@@ -413,6 +413,7 @@ async def environment_db_and_revision_to_extended_output(
     environment_output_extended = EnvironmentOutputExtended(
         name=environment_db.name,
         app_id=str(environment_db.app_id),
+        project_id=str(environment_db.project_id),
         deployed_app_variant_id=deployed_app_variant_id,
         deployed_variant_name=deployed_variant_name,
         deployed_app_variant_revision_id=str(