diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 9d6b430aee..b556f5b826 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -52338,6 +52338,10 @@ components: LLMObsCustomEvalConfigBedrockOptions: description: AWS Bedrock-specific options for LLM provider configuration. properties: + inference_profile: + description: Bedrock inference profile identifier, such as an application inference profile ARN. + example: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123" + type: string region: description: AWS region for Bedrock. example: "us-east-1" @@ -52428,6 +52432,11 @@ components: properties: assessment_criteria: $ref: "#/components/schemas/LLMObsCustomEvalConfigAssessmentCriteria" + context_query: + description: Query used to extract additional context for the evaluation. + example: "@input.context" + nullable: true + type: string inference_params: $ref: "#/components/schemas/LLMObsCustomEvalConfigInferenceParams" last_used_library_prompt_template_name: @@ -52452,6 +52461,15 @@ components: items: $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage" type: array + target_query: + description: Query used to extract the target value to evaluate. + example: "@output.value" + nullable: true + type: string + user_specified_json_post_processing_function: + description: User-provided function applied to post-process the JSON output of the LLM judge. + nullable: true + type: string required: - inference_params type: object @@ -52473,16 +52491,29 @@ components: vertex_ai: $ref: "#/components/schemas/LLMObsCustomEvalConfigVertexAIOptions" type: object + LLMObsCustomEvalConfigListResponse: + description: Response containing a list of custom LLM Observability evaluator configurations. + properties: + data: + description: List of custom evaluator configuration data objects. + items: + $ref: "#/components/schemas/LLMObsCustomEvalConfigData" + type: array + required: + - data + type: object LLMObsCustomEvalConfigParsingType: description: Output parsing type for a custom LLM judge evaluator. enum: - structured_output - json + - keyword_search example: "structured_output" type: string x-enum-varnames: - STRUCTURED_OUTPUT - JSON + - KEYWORD_SEARCH LLMObsCustomEvalConfigPromptContent: description: A content block within a prompt message. properties: @@ -52589,6 +52620,13 @@ components: eval_scope: $ref: "#/components/schemas/LLMObsCustomEvalConfigEvalScope" nullable: true + experiment_project_ids: + description: Experiment project IDs this evaluator is scoped to. + items: + description: An experiment project ID. + format: uuid + type: string + type: array filter: description: Filter expression to select which spans to evaluate. example: "@service:my-service" @@ -116425,6 +116463,67 @@ paths: x-unstable: |- This endpoint is in Preview and may introduce breaking changes. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/unstable/llm-obs/config/evaluators/custom: + get: + description: List all custom LLM Observability evaluator configurations for the organization. + operationId: ListLLMObsCustomEvalConfigs + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + - attributes: + category: "Custom" + created_at: "2024-01-15T10:30:00Z" + created_by: + email: "user@example.com" + eval_name: "my-custom-evaluator" + last_updated_by: + email: "user@example.com" + llm_judge_config: + inference_params: + max_tokens: 1024 + temperature: 0.7 + parsing_type: "structured_output" + llm_provider: + integration_provider: "openai" + model_name: "gpt-4o" + target: + application_name: "my-llm-app" + enabled: true + sampling_percentage: 50.0 + updated_at: "2024-01-15T10:30:00Z" + id: "my-custom-evaluator" + type: "evaluator_config" + schema: + $ref: "#/components/schemas/LLMObsCustomEvalConfigListResponse" + description: OK + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List custom evaluator configurations + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/unstable/llm-obs/config/evaluators/custom/{eval_name}: delete: description: Delete a custom LLM Observability evaluator configuration by its name. diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index ed534e9046..e1d7c65ef8 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -22180,6 +22180,13 @@ datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_integration\_provi :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_list\_response module +----------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_list_response + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_llm\_judge\_config module --------------------------------------------------------------------------------------- diff --git a/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.py b/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.py new file mode 100644 index 0000000000..d8b1b1a908 --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.py @@ -0,0 +1,14 @@ +""" +List custom evaluator configurations returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +configuration = Configuration() +configuration.unstable_operations["list_llm_obs_custom_eval_configs"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.list_llm_obs_custom_eval_configs() + + print(response) diff --git a/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py index 6ee8f657b1..8e53ae12ff 100644 --- a/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py +++ b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py @@ -53,6 +53,7 @@ ], pass_when=True, ), + context_query="@input.context", inference_params=LLMObsCustomEvalConfigInferenceParams( frequency_penalty=0.0, max_tokens=1024, @@ -91,9 +92,12 @@ role="user", ), ], + target_query="@output.value", + user_specified_json_post_processing_function=None, ), llm_provider=LLMObsCustomEvalConfigLLMProvider( bedrock=LLMObsCustomEvalConfigBedrockOptions( + inference_profile="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", region="us-east-1", ), integration_account_id="my-account-id", @@ -108,6 +112,7 @@ application_name="my-llm-app", enabled=True, eval_scope=LLMObsCustomEvalConfigEvalScope.SPAN, + experiment_project_ids=[], filter="@service:my-service", root_spans_only=True, sampling_percentage=50.0, diff --git a/src/datadog_api_client/configuration.py b/src/datadog_api_client/configuration.py index fbb2249e6b..19b8d3d76f 100644 --- a/src/datadog_api_client/configuration.py +++ b/src/datadog_api_client/configuration.py @@ -298,6 +298,7 @@ def __init__( "v2.get_llm_obs_prompt": False, "v2.get_llm_obs_prompt_version": False, "v2.list_llm_obs_annotation_queues": False, + "v2.list_llm_obs_custom_eval_configs": False, "v2.list_llm_obs_dataset_records": False, "v2.list_llm_obs_datasets": False, "v2.list_llm_obs_dataset_versions": False, diff --git a/src/datadog_api_client/v2/api/llm_observability_api.py b/src/datadog_api_client/v2/api/llm_observability_api.py index a01126b496..360532f054 100644 --- a/src/datadog_api_client/v2/api/llm_observability_api.py +++ b/src/datadog_api_client/v2/api/llm_observability_api.py @@ -13,6 +13,7 @@ UnsetType, unset, ) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_list_response import LLMObsCustomEvalConfigListResponse from datadog_api_client.v2.model.llm_obs_custom_eval_config_response import LLMObsCustomEvalConfigResponse from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_request import LLMObsCustomEvalConfigUpdateRequest from datadog_api_client.v2.model.llm_obs_data_deletion_response import LLMObsDataDeletionResponse @@ -1029,6 +1030,22 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._list_llm_obs_custom_eval_configs_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsCustomEvalConfigListResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/unstable/llm-obs/config/evaluators/custom", + "operation_id": "list_llm_obs_custom_eval_configs", + "http_method": "GET", + "version": "v2", + }, + params_map={}, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + self._list_llm_obs_dataset_records_endpoint = _Endpoint( settings={ "response_type": (LLMObsDatasetRecordsListResponse,), @@ -2921,6 +2938,18 @@ def list_llm_obs_annotation_queues( return self._list_llm_obs_annotation_queues_endpoint.call_with_http_info(**kwargs) + def list_llm_obs_custom_eval_configs( + self, + ) -> LLMObsCustomEvalConfigListResponse: + """List custom evaluator configurations. + + List all custom LLM Observability evaluator configurations for the organization. + + :rtype: LLMObsCustomEvalConfigListResponse + """ + kwargs: Dict[str, Any] = {} + return self._list_llm_obs_custom_eval_configs_endpoint.call_with_http_info(**kwargs) + def list_llm_obs_dataset_records( self, project_id: str, diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_bedrock_options.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_bedrock_options.py index 2c44cd8000..0f3e8fb3ee 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_bedrock_options.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_bedrock_options.py @@ -17,20 +17,29 @@ class LLMObsCustomEvalConfigBedrockOptions(ModelNormal): @cached_property def openapi_types(_): return { + "inference_profile": (str,), "region": (str,), } attribute_map = { + "inference_profile": "inference_profile", "region": "region", } - def __init__(self_, region: Union[str, UnsetType] = unset, **kwargs): + def __init__( + self_, inference_profile: Union[str, UnsetType] = unset, region: Union[str, UnsetType] = unset, **kwargs + ): """ AWS Bedrock-specific options for LLM provider configuration. + :param inference_profile: Bedrock inference profile identifier, such as an application inference profile ARN. + :type inference_profile: str, optional + :param region: AWS region for Bedrock. :type region: str, optional """ + if inference_profile is not unset: + kwargs["inference_profile"] = inference_profile if region is not unset: kwargs["region"] = region super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_list_response.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_list_response.py new file mode 100644 index 0000000000..e80dcb4775 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_list_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_data import LLMObsCustomEvalConfigData + + +class LLMObsCustomEvalConfigListResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_data import LLMObsCustomEvalConfigData + + return { + "data": ([LLMObsCustomEvalConfigData],), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: List[LLMObsCustomEvalConfigData], **kwargs): + """ + Response containing a list of custom LLM Observability evaluator configurations. + + :param data: List of custom evaluator configuration data objects. + :type data: [LLMObsCustomEvalConfigData] + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py index 6d6b6fc885..cfda678d1f 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py @@ -48,6 +48,7 @@ def openapi_types(_): return { "assessment_criteria": (LLMObsCustomEvalConfigAssessmentCriteria,), + "context_query": (str, none_type), "inference_params": (LLMObsCustomEvalConfigInferenceParams,), "last_used_library_prompt_template_name": (str, none_type), "modified_library_prompt_template": (bool, none_type), @@ -70,27 +71,35 @@ def openapi_types(_): ), "parsing_type": (LLMObsCustomEvalConfigParsingType,), "prompt_template": ([LLMObsCustomEvalConfigPromptMessage],), + "target_query": (str, none_type), + "user_specified_json_post_processing_function": (str, none_type), } attribute_map = { "assessment_criteria": "assessment_criteria", + "context_query": "context_query", "inference_params": "inference_params", "last_used_library_prompt_template_name": "last_used_library_prompt_template_name", "modified_library_prompt_template": "modified_library_prompt_template", "output_schema": "output_schema", "parsing_type": "parsing_type", "prompt_template": "prompt_template", + "target_query": "target_query", + "user_specified_json_post_processing_function": "user_specified_json_post_processing_function", } def __init__( self_, inference_params: LLMObsCustomEvalConfigInferenceParams, assessment_criteria: Union[LLMObsCustomEvalConfigAssessmentCriteria, UnsetType] = unset, + context_query: Union[str, none_type, UnsetType] = unset, last_used_library_prompt_template_name: Union[str, none_type, UnsetType] = unset, modified_library_prompt_template: Union[bool, none_type, UnsetType] = unset, output_schema: Union[Dict[str, Any], none_type, UnsetType] = unset, parsing_type: Union[LLMObsCustomEvalConfigParsingType, UnsetType] = unset, prompt_template: Union[List[LLMObsCustomEvalConfigPromptMessage], UnsetType] = unset, + target_query: Union[str, none_type, UnsetType] = unset, + user_specified_json_post_processing_function: Union[str, none_type, UnsetType] = unset, **kwargs, ): """ @@ -99,6 +108,9 @@ def __init__( :param assessment_criteria: Criteria used to assess the pass/fail result of a custom evaluator. :type assessment_criteria: LLMObsCustomEvalConfigAssessmentCriteria, optional + :param context_query: Query used to extract additional context for the evaluation. + :type context_query: str, none_type, optional + :param inference_params: LLM inference parameters for a custom evaluator. :type inference_params: LLMObsCustomEvalConfigInferenceParams @@ -116,9 +128,17 @@ def __init__( :param prompt_template: List of messages forming the LLM judge prompt template. :type prompt_template: [LLMObsCustomEvalConfigPromptMessage], optional + + :param target_query: Query used to extract the target value to evaluate. + :type target_query: str, none_type, optional + + :param user_specified_json_post_processing_function: User-provided function applied to post-process the JSON output of the LLM judge. + :type user_specified_json_post_processing_function: str, none_type, optional """ if assessment_criteria is not unset: kwargs["assessment_criteria"] = assessment_criteria + if context_query is not unset: + kwargs["context_query"] = context_query if last_used_library_prompt_template_name is not unset: kwargs["last_used_library_prompt_template_name"] = last_used_library_prompt_template_name if modified_library_prompt_template is not unset: @@ -129,6 +149,10 @@ def __init__( kwargs["parsing_type"] = parsing_type if prompt_template is not unset: kwargs["prompt_template"] = prompt_template + if target_query is not unset: + kwargs["target_query"] = target_query + if user_specified_json_post_processing_function is not unset: + kwargs["user_specified_json_post_processing_function"] = user_specified_json_post_processing_function super().__init__(kwargs) self_.inference_params = inference_params diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py index b913e071ba..0a30fbef8c 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py @@ -16,16 +16,18 @@ class LLMObsCustomEvalConfigParsingType(ModelSimple): """ Output parsing type for a custom LLM judge evaluator. - :param value: Must be one of ["structured_output", "json"]. + :param value: Must be one of ["structured_output", "json", "keyword_search"]. :type value: str """ allowed_values = { "structured_output", "json", + "keyword_search", } STRUCTURED_OUTPUT: ClassVar["LLMObsCustomEvalConfigParsingType"] JSON: ClassVar["LLMObsCustomEvalConfigParsingType"] + KEYWORD_SEARCH: ClassVar["LLMObsCustomEvalConfigParsingType"] @cached_property def openapi_types(_): @@ -36,3 +38,4 @@ def openapi_types(_): LLMObsCustomEvalConfigParsingType.STRUCTURED_OUTPUT = LLMObsCustomEvalConfigParsingType("structured_output") LLMObsCustomEvalConfigParsingType.JSON = LLMObsCustomEvalConfigParsingType("json") +LLMObsCustomEvalConfigParsingType.KEYWORD_SEARCH = LLMObsCustomEvalConfigParsingType("keyword_search") diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py index 42b2d5b6ef..c289909867 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py @@ -3,7 +3,7 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import Union, TYPE_CHECKING +from typing import List, Union, TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, @@ -11,6 +11,7 @@ none_type, unset, UnsetType, + UUID, ) @@ -27,6 +28,7 @@ def openapi_types(_): "application_name": (str,), "enabled": (bool,), "eval_scope": (LLMObsCustomEvalConfigEvalScope,), + "experiment_project_ids": ([UUID],), "filter": (str, none_type), "root_spans_only": (bool, none_type), "sampling_percentage": (float, none_type), @@ -36,6 +38,7 @@ def openapi_types(_): "application_name": "application_name", "enabled": "enabled", "eval_scope": "eval_scope", + "experiment_project_ids": "experiment_project_ids", "filter": "filter", "root_spans_only": "root_spans_only", "sampling_percentage": "sampling_percentage", @@ -46,6 +49,7 @@ def __init__( application_name: str, enabled: bool, eval_scope: Union[LLMObsCustomEvalConfigEvalScope, UnsetType] = unset, + experiment_project_ids: Union[List[UUID], UnsetType] = unset, filter: Union[str, none_type, UnsetType] = unset, root_spans_only: Union[bool, none_type, UnsetType] = unset, sampling_percentage: Union[float, none_type, UnsetType] = unset, @@ -63,6 +67,9 @@ def __init__( :param eval_scope: Scope at which to evaluate spans. :type eval_scope: LLMObsCustomEvalConfigEvalScope, optional + :param experiment_project_ids: Experiment project IDs this evaluator is scoped to. + :type experiment_project_ids: [UUID], optional + :param filter: Filter expression to select which spans to evaluate. :type filter: str, none_type, optional @@ -74,6 +81,8 @@ def __init__( """ if eval_scope is not unset: kwargs["eval_scope"] = eval_scope + if experiment_project_ids is not unset: + kwargs["experiment_project_ids"] = experiment_project_ids if filter is not unset: kwargs["filter"] = filter if root_spans_only is not unset: diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index 2c7fd2b69e..49b92b4ae7 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -3997,6 +3997,7 @@ ) from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config import LLMObsCustomEvalConfigLLMJudgeConfig from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider import LLMObsCustomEvalConfigLLMProvider +from datadog_api_client.v2.model.llm_obs_custom_eval_config_list_response import LLMObsCustomEvalConfigListResponse from datadog_api_client.v2.model.llm_obs_custom_eval_config_parsing_type import LLMObsCustomEvalConfigParsingType from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content import LLMObsCustomEvalConfigPromptContent from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content_value import ( @@ -13065,6 +13066,7 @@ "LLMObsCustomEvalConfigIntegrationProvider", "LLMObsCustomEvalConfigLLMJudgeConfig", "LLMObsCustomEvalConfigLLMProvider", + "LLMObsCustomEvalConfigListResponse", "LLMObsCustomEvalConfigParsingType", "LLMObsCustomEvalConfigPromptContent", "LLMObsCustomEvalConfigPromptContentValue", diff --git a/tests/v2/features/llm_observability.feature b/tests/v2/features/llm_observability.feature index 2c23f9f6d7..b1897f532b 100644 --- a/tests/v2/features/llm_observability.feature +++ b/tests/v2/features/llm_observability.feature @@ -338,7 +338,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 400 Bad Request @@ -347,7 +347,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 404 Not Found @@ -356,7 +356,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 200 OK @@ -365,7 +365,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 422 Unprocessable Entity @@ -1110,6 +1110,13 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: List custom evaluator configurations returns "OK" response + Given operation "ListLLMObsCustomEvalConfigs" enabled + And new "ListLLMObsCustomEvalConfigs" request + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List events for an LLM Observability experiment returns "Bad Request" response Given operation "ListLLMObsExperimentEvents" enabled diff --git a/tests/v2/features/undo.json b/tests/v2/features/undo.json index d93991ac7d..2a7fb64a0a 100644 --- a/tests/v2/features/undo.json +++ b/tests/v2/features/undo.json @@ -123,6 +123,12 @@ "type": "safe" } }, + "ListLLMObsCustomEvalConfigs": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "DeleteLLMObsCustomEvalConfig": { "tag": "LLM Observability", "undo": {