Add list custom LLM Obs evaluator configs endpoint

.generator/schemas/v2/openapi.yaml

@@ -52338,6 +52338,10 @@ components:
     LLMObsCustomEvalConfigBedrockOptions:
       description: AWS Bedrock-specific options for LLM provider configuration.
       properties:
+        inference_profile:
+          description: Bedrock inference profile identifier, such as an application inference profile ARN.
+          example: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123"
+          type: string
         region:
           description: AWS region for Bedrock.
           example: "us-east-1"
@@ -52428,6 +52432,11 @@ components:
       properties:
         assessment_criteria:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigAssessmentCriteria"
+        context_query:
+          description: Query used to extract additional context for the evaluation.
+          example: "@input.context"
+          nullable: true
+          type: string
         inference_params:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigInferenceParams"
         last_used_library_prompt_template_name:
@@ -52452,6 +52461,15 @@ components:
           items:
             $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage"
           type: array
+        target_query:
+          description: Query used to extract the target value to evaluate.
+          example: "@output.value"
+          nullable: true
+          type: string
+        user_specified_json_post_processing_function:
+          description: User-provided function applied to post-process the JSON output of the LLM judge.
+          nullable: true
+          type: string
       required:
         - inference_params
       type: object
@@ -52473,16 +52491,29 @@ components:
         vertex_ai:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigVertexAIOptions"
       type: object
+    LLMObsCustomEvalConfigListResponse:
+      description: Response containing a list of custom LLM Observability evaluator configurations.
+      properties:
+        data:
+          description: List of custom evaluator configuration data objects.
+          items:
+            $ref: "#/components/schemas/LLMObsCustomEvalConfigData"
+          type: array
+      required:
+        - data
+      type: object
     LLMObsCustomEvalConfigParsingType:
       description: Output parsing type for a custom LLM judge evaluator.
       enum:
         - structured_output
         - json
+        - keyword_search
       example: "structured_output"
       type: string
       x-enum-varnames:
         - STRUCTURED_OUTPUT
         - JSON
+        - KEYWORD_SEARCH
     LLMObsCustomEvalConfigPromptContent:
       description: A content block within a prompt message.
       properties:
@@ -52589,6 +52620,13 @@ components:
         eval_scope:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigEvalScope"
           nullable: true
+        experiment_project_ids:
+          description: Experiment project IDs this evaluator is scoped to.
+          items:
+            description: An experiment project ID.
+            format: uuid
+            type: string
+          type: array
         filter:
           description: Filter expression to select which spans to evaluate.
           example: "@service:my-service"
@@ -116425,6 +116463,67 @@ paths:
       x-unstable: |-
         This endpoint is in Preview and may introduce breaking changes.
         If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/).
+  /api/unstable/llm-obs/config/evaluators/custom:
+    get:
+      description: List all custom LLM Observability evaluator configurations for the organization.
+      operationId: ListLLMObsCustomEvalConfigs
+      responses:
+        "200":
+          content:
+            application/json:
+              examples:
+                default:
+                  value:
+                    data:
+                      - attributes:
+                          category: "Custom"
+                          created_at: "2024-01-15T10:30:00Z"
+                          created_by:
+                            email: "user@example.com"
+                          eval_name: "my-custom-evaluator"
+                          last_updated_by:
+                            email: "user@example.com"
+                          llm_judge_config:
+                            inference_params:
+                              max_tokens: 1024
+                              temperature: 0.7
+                            parsing_type: "structured_output"
+                          llm_provider:
+                            integration_provider: "openai"
+                            model_name: "gpt-4o"
+                          target:
+                            application_name: "my-llm-app"
+                            enabled: true
+                            sampling_percentage: 50.0
+                          updated_at: "2024-01-15T10:30:00Z"
+                        id: "my-custom-evaluator"
+                        type: "evaluator_config"
+              schema:
+                $ref: "#/components/schemas/LLMObsCustomEvalConfigListResponse"
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JSONAPIErrorResponse"
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JSONAPIErrorResponse"
+          description: Forbidden
+        "429":
+          $ref: "#/components/responses/TooManyRequestsResponse"
+      security:
+        - apiKeyAuth: []
+          appKeyAuth: []
+      summary: List custom evaluator configurations
+      tags:
+        - LLM Observability
+      x-unstable: |-
+        **Note**: This endpoint is in preview and is subject to change.
+        If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/).
   /api/unstable/llm-obs/config/evaluators/custom/{eval_name}:
     delete:
       description: Delete a custom LLM Observability evaluator configuration by its name.

docs/datadog_api_client.v2.model.rst

@@ -22180,6 +22180,13 @@ datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_integration\_provi
    :members:
    :show-inheritance:
 
+datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_list\_response module
+-----------------------------------------------------------------------------------
+
+.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_list_response
+   :members:
+   :show-inheritance:
+
 datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_llm\_judge\_config module
 ---------------------------------------------------------------------------------------
 

examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.py

@@ -0,0 +1,14 @@
+"""
+List custom evaluator configurations returns "OK" response
+"""
+
+from datadog_api_client import ApiClient, Configuration
+from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi
+
+configuration = Configuration()
+configuration.unstable_operations["list_llm_obs_custom_eval_configs"] = True
+with ApiClient(configuration) as api_client:
+    api_instance = LLMObservabilityApi(api_client)
+    response = api_instance.list_llm_obs_custom_eval_configs()
+
+    print(response)

examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py

@@ -53,6 +53,7 @@
                     ],
                     pass_when=True,
                 ),
+                context_query="@input.context",
                 inference_params=LLMObsCustomEvalConfigInferenceParams(
                     frequency_penalty=0.0,
                     max_tokens=1024,
@@ -91,9 +92,12 @@
                         role="user",
                     ),
                 ],
+                target_query="@output.value",
+                user_specified_json_post_processing_function=None,
             ),
             llm_provider=LLMObsCustomEvalConfigLLMProvider(
                 bedrock=LLMObsCustomEvalConfigBedrockOptions(
+                    inference_profile="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
                     region="us-east-1",
                 ),
                 integration_account_id="my-account-id",
@@ -108,6 +112,7 @@
                 application_name="my-llm-app",
                 enabled=True,
                 eval_scope=LLMObsCustomEvalConfigEvalScope.SPAN,
+                experiment_project_ids=[],
                 filter="@service:my-service",
                 root_spans_only=True,
                 sampling_percentage=50.0,

src/datadog_api_client/configuration.py

@@ -298,6 +298,7 @@ def __init__(
                 "v2.get_llm_obs_prompt": False,
                 "v2.get_llm_obs_prompt_version": False,
                 "v2.list_llm_obs_annotation_queues": False,
+                "v2.list_llm_obs_custom_eval_configs": False,
                 "v2.list_llm_obs_dataset_records": False,
                 "v2.list_llm_obs_datasets": False,
                 "v2.list_llm_obs_dataset_versions": False,

src/datadog_api_client/v2/api/llm_observability_api.py

@@ -13,6 +13,7 @@
     UnsetType,
     unset,
 )
+from datadog_api_client.v2.model.llm_obs_custom_eval_config_list_response import LLMObsCustomEvalConfigListResponse
 from datadog_api_client.v2.model.llm_obs_custom_eval_config_response import LLMObsCustomEvalConfigResponse
 from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_request import LLMObsCustomEvalConfigUpdateRequest
 from datadog_api_client.v2.model.llm_obs_data_deletion_response import LLMObsDataDeletionResponse
@@ -1029,6 +1030,22 @@ def __init__(self, api_client=None):
             api_client=api_client,
         )
 
+        self._list_llm_obs_custom_eval_configs_endpoint = _Endpoint(
+            settings={
+                "response_type": (LLMObsCustomEvalConfigListResponse,),
+                "auth": ["apiKeyAuth", "appKeyAuth"],
+                "endpoint_path": "/api/unstable/llm-obs/config/evaluators/custom",
+                "operation_id": "list_llm_obs_custom_eval_configs",
+                "http_method": "GET",
+                "version": "v2",
+            },
+            params_map={},
+            headers_map={
+                "accept": ["application/json"],
+            },
+            api_client=api_client,
+        )
+
         self._list_llm_obs_dataset_records_endpoint = _Endpoint(
             settings={
                 "response_type": (LLMObsDatasetRecordsListResponse,),
@@ -2921,6 +2938,18 @@ def list_llm_obs_annotation_queues(
 
         return self._list_llm_obs_annotation_queues_endpoint.call_with_http_info(**kwargs)
 
+    def list_llm_obs_custom_eval_configs(
+        self,
+    ) -> LLMObsCustomEvalConfigListResponse:
+        """List custom evaluator configurations.
+
+        List all custom LLM Observability evaluator configurations for the organization.
+
+        :rtype: LLMObsCustomEvalConfigListResponse
+        """
+        kwargs: Dict[str, Any] = {}
+        return self._list_llm_obs_custom_eval_configs_endpoint.call_with_http_info(**kwargs)
+
     def list_llm_obs_dataset_records(
         self,
         project_id: str,

src/datadog_api_client/v2/model/llm_obs_custom_eval_config_bedrock_options.py

@@ -17,20 +17,29 @@ class LLMObsCustomEvalConfigBedrockOptions(ModelNormal):
     @cached_property
     def openapi_types(_):
         return {
+            "inference_profile": (str,),
             "region": (str,),
         }
 
     attribute_map = {
+        "inference_profile": "inference_profile",
         "region": "region",
     }
 
-    def __init__(self_, region: Union[str, UnsetType] = unset, **kwargs):
+    def __init__(
+        self_, inference_profile: Union[str, UnsetType] = unset, region: Union[str, UnsetType] = unset, **kwargs
+    ):
         """
         AWS Bedrock-specific options for LLM provider configuration.
 
+        :param inference_profile: Bedrock inference profile identifier, such as an application inference profile ARN.
+        :type inference_profile: str, optional
+
         :param region: AWS region for Bedrock.
         :type region: str, optional
         """
+        if inference_profile is not unset:
+            kwargs["inference_profile"] = inference_profile
         if region is not unset:
             kwargs["region"] = region
         super().__init__(kwargs)

src/datadog_api_client/v2/model/llm_obs_custom_eval_config_list_response.py

@@ -0,0 +1,40 @@
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019-Present Datadog, Inc.
+from __future__ import annotations
+
+from typing import List, TYPE_CHECKING
+
+from datadog_api_client.model_utils import (
+    ModelNormal,
+    cached_property,
+)
+
+
+if TYPE_CHECKING:
+    from datadog_api_client.v2.model.llm_obs_custom_eval_config_data import LLMObsCustomEvalConfigData
+
+
+class LLMObsCustomEvalConfigListResponse(ModelNormal):
+    @cached_property
+    def openapi_types(_):
+        from datadog_api_client.v2.model.llm_obs_custom_eval_config_data import LLMObsCustomEvalConfigData
+
+        return {
+            "data": ([LLMObsCustomEvalConfigData],),
+        }
+
+    attribute_map = {
+        "data": "data",
+    }
+
+    def __init__(self_, data: List[LLMObsCustomEvalConfigData], **kwargs):
+        """
+        Response containing a list of custom LLM Observability evaluator configurations.
+
+        :param data: List of custom evaluator configuration data objects.
+        :type data: [LLMObsCustomEvalConfigData]
+        """
+        super().__init__(kwargs)
+
+        self_.data = data