Add LLM Observability to ListStreamSource

.apigentools-info

@@ -4,13 +4,13 @@
     "spec_versions": {
         "v1": {
             "apigentools_version": "1.6.6",
-            "regenerated": "2024-10-16 20:07:16.049805",
-            "spec_repo_commit": "86072741"
+            "regenerated": "2024-10-17 14:10:51.942099",
+            "spec_repo_commit": "fb024a45"
         },
         "v2": {
             "apigentools_version": "1.6.6",
-            "regenerated": "2024-10-16 20:07:16.068600",
-            "spec_repo_commit": "86072741"
+            "regenerated": "2024-10-17 14:10:51.959206",
+            "spec_repo_commit": "fb024a45"
         }
     }
 }

.generator/schemas/v1/openapi.yaml

@@ -4796,6 +4796,7 @@ components:
       - logs_transaction_stream
       - event_stream
       - rum_stream
+      - llm_observability_stream
       example: apm_issue_stream
       type: string
       x-enum-varnames:
@@ -4811,6 +4812,7 @@ components:
       - LOGS_TRANSACTION_STREAM
       - EVENT_STREAM
       - RUM_STREAM
+      - LLM_OBSERVABILITY_STREAM
     ListStreamWidgetDefinition:
       description: 'The list stream visualization displays a table of recent events
         in your application that

examples/v1/dashboards/CreateDashboard_3298564989.py

@@ -0,0 +1,84 @@
+"""
+Create a new dashboard with llm_observability_stream list_stream widget
+"""
+
+from datadog_api_client import ApiClient, Configuration
+from datadog_api_client.v1.api.dashboards_api import DashboardsApi
+from datadog_api_client.v1.model.dashboard import Dashboard
+from datadog_api_client.v1.model.dashboard_layout_type import DashboardLayoutType
+from datadog_api_client.v1.model.list_stream_column import ListStreamColumn
+from datadog_api_client.v1.model.list_stream_column_width import ListStreamColumnWidth
+from datadog_api_client.v1.model.list_stream_query import ListStreamQuery
+from datadog_api_client.v1.model.list_stream_response_format import ListStreamResponseFormat
+from datadog_api_client.v1.model.list_stream_source import ListStreamSource
+from datadog_api_client.v1.model.list_stream_widget_definition import ListStreamWidgetDefinition
+from datadog_api_client.v1.model.list_stream_widget_definition_type import ListStreamWidgetDefinitionType
+from datadog_api_client.v1.model.list_stream_widget_request import ListStreamWidgetRequest
+from datadog_api_client.v1.model.widget import Widget
+
+body = Dashboard(
+    layout_type=DashboardLayoutType.ORDERED,
+    title="Example-Dashboard with list_stream widget",
+    widgets=[
+        Widget(
+            definition=ListStreamWidgetDefinition(
+                type=ListStreamWidgetDefinitionType.LIST_STREAM,
+                requests=[
+                    ListStreamWidgetRequest(
+                        response_format=ListStreamResponseFormat.EVENT_LIST,
+                        query=ListStreamQuery(
+                            data_source=ListStreamSource.LLM_OBSERVABILITY_STREAM,
+                            query_string="@event_type:span @parent_id:undefined",
+                            indexes=[],
+                        ),
+                        columns=[
+                            ListStreamColumn(
+                                field="@status",
+                                width=ListStreamColumnWidth.COMPACT,
+                            ),
+                            ListStreamColumn(
+                                field="@content.prompt",
+                                width=ListStreamColumnWidth.AUTO,
+                            ),
+                            ListStreamColumn(
+                                field="@content.response.content",
+                                width=ListStreamColumnWidth.AUTO,
+                            ),
+                            ListStreamColumn(
+                                field="timestamp",
+                                width=ListStreamColumnWidth.AUTO,
+                            ),
+                            ListStreamColumn(
+                                field="@ml_app",
+                                width=ListStreamColumnWidth.AUTO,
+                            ),
+                            ListStreamColumn(
+                                field="service",
+                                width=ListStreamColumnWidth.AUTO,
+                            ),
+                            ListStreamColumn(
+                                field="@meta.evaluations.quality",
+                                width=ListStreamColumnWidth.AUTO,
+                            ),
+                            ListStreamColumn(
+                                field="@meta.evaluations.security",
+                                width=ListStreamColumnWidth.AUTO,
+                            ),
+                            ListStreamColumn(
+                                field="@duration",
+                                width=ListStreamColumnWidth.AUTO,
+                            ),
+                        ],
+                    ),
+                ],
+            ),
+        ),
+    ],
+)
+
+configuration = Configuration()
+with ApiClient(configuration) as api_client:
+    api_instance = DashboardsApi(api_client)
+    response = api_instance.create_dashboard(body=body)
+
+    print(response)

src/datadog_api_client/v1/model/list_stream_source.py

@@ -16,7 +16,7 @@ class ListStreamSource(ModelSimple):
     """
     Source from which to query items to display in the stream.
 
-    :param value: If omitted defaults to "apm_issue_stream". Must be one of ["logs_stream", "audit_stream", "ci_pipeline_stream", "ci_test_stream", "rum_issue_stream", "apm_issue_stream", "trace_stream", "logs_issue_stream", "logs_pattern_stream", "logs_transaction_stream", "event_stream", "rum_stream"].
+    :param value: If omitted defaults to "apm_issue_stream". Must be one of ["logs_stream", "audit_stream", "ci_pipeline_stream", "ci_test_stream", "rum_issue_stream", "apm_issue_stream", "trace_stream", "logs_issue_stream", "logs_pattern_stream", "logs_transaction_stream", "event_stream", "rum_stream", "llm_observability_stream"].
     :type value: str
     """
 
@@ -33,6 +33,7 @@ class ListStreamSource(ModelSimple):
         "logs_transaction_stream",
         "event_stream",
         "rum_stream",
+        "llm_observability_stream",
     }
     LOGS_STREAM: ClassVar["ListStreamSource"]
     AUDIT_STREAM: ClassVar["ListStreamSource"]
@@ -46,6 +47,7 @@ class ListStreamSource(ModelSimple):
     LOGS_TRANSACTION_STREAM: ClassVar["ListStreamSource"]
     EVENT_STREAM: ClassVar["ListStreamSource"]
     RUM_STREAM: ClassVar["ListStreamSource"]
+    LLM_OBSERVABILITY_STREAM: ClassVar["ListStreamSource"]
 
     @cached_property
     def openapi_types(_):
@@ -66,3 +68,4 @@ def openapi_types(_):
 ListStreamSource.LOGS_TRANSACTION_STREAM = ListStreamSource("logs_transaction_stream")
 ListStreamSource.EVENT_STREAM = ListStreamSource("event_stream")
 ListStreamSource.RUM_STREAM = ListStreamSource("rum_stream")
+ListStreamSource.LLM_OBSERVABILITY_STREAM = ListStreamSource("llm_observability_stream")

tests/v1/cassettes/test_scenarios/test_create_a_new_dashboard_with_llm_observability_stream_list_stream_widget.frozen

@@ -0,0 +1 @@
+2024-10-15T21:46:06.749Z

tests/v1/cassettes/test_scenarios/test_create_a_new_dashboard_with_llm_observability_stream_list_stream_widget.yaml

@@ -0,0 +1,45 @@
+interactions:
+- request:
+    body: '{"layout_type":"ordered","title":"Test-Create_a_new_dashboard_with_llm_observability_stream_list_stream_widget-1729028766
+      with list_stream widget","widgets":[{"definition":{"requests":[{"columns":[{"field":"@status","width":"compact"},{"field":"@content.prompt","width":"auto"},{"field":"@content.response.content","width":"auto"},{"field":"timestamp","width":"auto"},{"field":"@ml_app","width":"auto"},{"field":"service","width":"auto"},{"field":"@meta.evaluations.quality","width":"auto"},{"field":"@meta.evaluations.security","width":"auto"},{"field":"@duration","width":"auto"}],"query":{"data_source":"llm_observability_stream","indexes":[],"query_string":"@event_type:span
+      @parent_id:undefined"},"response_format":"event_list"}],"type":"list_stream"}}]}'
+    headers:
+      accept:
+      - application/json
+      content-type:
+      - application/json
+    method: POST
+    uri: https://api.datadoghq.com/api/v1/dashboard
+  response:
+    body:
+      string: '{"id":"k3w-qcg-ug8","title":"Test-Create_a_new_dashboard_with_llm_observability_stream_list_stream_widget-1729028766
+        with list_stream widget","description":null,"author_handle":"9919ec9b-ebc7-49ee-8dc8-03626e717cca","author_name":"CI
+        Account","layout_type":"ordered","url":"/dashboard/k3w-qcg-ug8/test-createanewdashboardwithllmobservabilitystreamliststreamwidget-1729028766-wi","is_read_only":false,"template_variables":null,"widgets":[{"definition":{"requests":[{"columns":[{"field":"@status","width":"compact"},{"field":"@content.prompt","width":"auto"},{"field":"@content.response.content","width":"auto"},{"field":"timestamp","width":"auto"},{"field":"@ml_app","width":"auto"},{"field":"service","width":"auto"},{"field":"@meta.evaluations.quality","width":"auto"},{"field":"@meta.evaluations.security","width":"auto"},{"field":"@duration","width":"auto"}],"query":{"data_source":"llm_observability_stream","indexes":[],"query_string":"@event_type:span
+        @parent_id:undefined"},"response_format":"event_list"}],"type":"list_stream"},"id":8221646523831060}],"notify_list":null,"created_at":"2024-10-15T21:46:06.954265+00:00","modified_at":"2024-10-15T21:46:06.954265+00:00","restricted_roles":[]}
+
+        '
+    headers:
+      content-type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      accept:
+      - application/json
+    method: DELETE
+    uri: https://api.datadoghq.com/api/v1/dashboard/k3w-qcg-ug8
+  response:
+    body:
+      string: '{"deleted_dashboard_id":"k3w-qcg-ug8"}
+
+        '
+    headers:
+      content-type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+version: 1

tests/v1/features/dashboards.feature

@@ -500,6 +500,14 @@ Feature: Dashboards
     And the response "widgets[0].definition.requests[0].query.sort.column" is equal to "timestamp"
     And the response "widgets[0].definition.requests[0].query.sort.order" is equal to "desc"
 
+  @team:DataDog/dashboards-backend
+  Scenario: Create a new dashboard with llm_observability_stream list_stream widget
+    Given new "CreateDashboard" request
+    And body with value {"layout_type":"ordered","title":"{{ unique }} with list_stream widget","widgets":[{"definition":{"type":"list_stream","requests":[{"response_format":"event_list","query":{"data_source":"llm_observability_stream","query_string":"@event_type:span @parent_id:undefined","indexes":[]},"columns":[{"field":"@status","width":"compact"},{"field":"@content.prompt","width":"auto"},{"field":"@content.response.content","width":"auto"},{"field":"timestamp","width":"auto"},{"field":"@ml_app","width":"auto"},{"field":"service","width":"auto"},{"field":"@meta.evaluations.quality","width":"auto"},{"field":"@meta.evaluations.security","width":"auto"},{"field":"@duration","width":"auto"}]}]}}]}
+    When the request is sent
+    Then the response status is 200 OK
+    And the response "widgets[0].definition.requests[0].query.data_source" is equal to "llm_observability_stream"
+
   @team:DataDog/dashboards-backend
   Scenario: Create a new dashboard with log_stream widget
     Given new "CreateDashboard" request