feat(sdk): add create_decision_evaluator and DecisionEvaluator (SAO-17973)

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `create_decision_evaluator()` (also `Evaluators.create_decision_evaluator`) and `DecisionEvaluator` create and
+  read decision-model evaluators: one typed question per evaluator, answered with probabilities. Supports
+  `boolean_probability`, `categorical` and `discrete` output types; when `output_type` is omitted it is inferred
+  from `answer_options`. If the version request is clearly rejected (400, 403, 404, 422) the new scorer is
+  deleted; on a timeout, network error or 5xx it is kept and the raised error names its id.
+- Scorer requests now send `X-Galileo-Scorer-Capabilities: decision`, and `ScorerTypes` and `ModelType` include
+  `decision`, so decision scorers appear in list and get responses and decode.
+
 ### Fixed
 
 - `SplunkAOTracingProcessor` (OpenAI Agents) now records Responses API LLM spans, the Agents SDK

codegen_templates/endpoint_module.py.jinja

@@ -12,7 +12,11 @@ from splunk_ao.exceptions import (
     RateLimitError,
     ServerError,
 )
-from splunk_ao.utils.headers_data import get_sdk_header
+{# Endpoints whose responses carry scorer_type; the API leaves decision scorers out unless the header is sent.
+   The blank line after the import is needed: trim_blocks drops the newline that follows {% endif %}. #}
+{% set scorer_discovery = endpoint.path.startswith("/scorers") or endpoint.path.endswith("/scorer-settings") or endpoint.path.endswith("/metric_settings") %}
+from splunk_ao.utils.headers_data import get_sdk_header{% if scorer_discovery %}, get_scorer_capabilities_header{% endif %}
+
 from galileo_core.constants.request_method import RequestMethod
 from galileo_core.helpers.api_client import ApiClient
 from ...types import Response, UNSET
@@ -73,6 +77,9 @@ def _get_kwargs(
 {% endif %}
 
     headers["Splunk-AO-SDK"] = get_sdk_header()
+{% if scorer_discovery %}
+    headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header()
+{% endif %}
 
     _kwargs["content_headers"] = headers
     return _kwargs

openapi.yaml

@@ -6877,6 +6877,46 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/HTTPValidationError'
+  /scorers/{scorer_id}/version/decision:
+    post:
+      tags:
+        - data
+        - prompts
+        - rows
+      summary: Create Decision Scorer Version
+      description: Create a decision evaluator version that holds its question, answer options and model.
+      operationId: create_decision_scorer_version_scorers__scorer_id__version_decision_post
+      security:
+        - APIKeyHeader: []
+        - OAuth2PasswordBearer: []
+        - HTTPBasic: []
+      parameters:
+        - name: scorer_id
+          in: path
+          required: true
+          schema:
+            type: string
+            format: uuid4
+            title: Scorer Id
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateDecisionScorerVersionRequest'
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/BaseScorerVersionResponse'
+        '422':
+          description: Validation Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
   /scorers/code/validate:
     post:
       tags:
@@ -13824,6 +13864,10 @@ components:
           anyOf:
             - $ref: '#/components/schemas/BaseFinetunedScorerDB'
             - type: 'null'
+        decision_scorer:
+          anyOf:
+            - $ref: '#/components/schemas/DecisionScorerDB'
+            - type: 'null'
         model_name:
           anyOf:
             - type: string
@@ -13894,6 +13938,10 @@ components:
           anyOf:
             - $ref: '#/components/schemas/FineTunedScorerResponse'
             - type: 'null'
+        decision_scorer:
+          anyOf:
+            - $ref: '#/components/schemas/DecisionScorerDB'
+            - type: 'null'
         model_name:
           anyOf:
             - type: string
@@ -15530,6 +15578,66 @@ components:
         - lora_task_id
         - prompt
       title: CreateCustomLunaScorerVersionRequest
+    CreateDecisionScorerVersionRequest:
+      properties:
+        model_name:
+          type: string
+          maxLength: 255
+          minLength: 1
+          title: Model Name
+        num_judges:
+          anyOf:
+            - type: integer
+            - type: 'null'
+          title: Num Judges
+        scoreable_node_types:
+          anyOf:
+            - items:
+                type: string
+              type: array
+            - type: 'null'
+          title: Scoreable Node Types
+        cot_enabled:
+          anyOf:
+            - type: boolean
+            - type: 'null'
+          title: Cot Enabled
+        output_type:
+          anyOf:
+            - $ref: '#/components/schemas/OutputTypeEnum'
+            - type: 'null'
+        input_type:
+          anyOf:
+            - $ref: '#/components/schemas/InputTypeEnum'
+            - type: 'null'
+        question:
+          type: string
+          maxLength: 2000
+          minLength: 1
+          title: Question
+        answer_options:
+          anyOf:
+            - items:
+                type: string
+              type: array
+            - items:
+                $ref: '#/components/schemas/DecisionLevel'
+              type: array
+            - type: 'null'
+          title: Answer Options
+      type: object
+      required:
+        - model_name
+        - question
+      title: CreateDecisionScorerVersionRequest
+      description: 'The question and model for a new decision scorer version.
+
+
+        The inherited scoreable_node_types, output_type and input_type are optional, but each must
+
+        match the scorer when sent. The service checks these fields, and it checks answer_options
+
+        against the stored output_type.'
     CreateJobRequest:
       properties:
         resource_limits:
@@ -22724,6 +22832,77 @@ components:
           default: column
       type: object
       title: DatasetVersionIndexSort
+    DecisionLevel:
+      properties:
+        level:
+          type: integer
+          maximum: 9.0
+          minimum: 0.0
+          title: Level
+        label:
+          type: string
+          title: Label
+        description:
+          anyOf:
+            - type: string
+            - type: 'null'
+          title: Description
+      type: object
+      required:
+        - level
+        - label
+      title: DecisionLevel
+      description: One level of a score question. A question's levels run 0, 1, 2 and so on, with no gaps.
+    DecisionScorerDB:
+      properties:
+        id:
+          type: string
+          format: uuid4
+          title: Id
+        organization_id:
+          type: string
+          format: uuid4
+          title: Organization Id
+        created_by:
+          anyOf:
+            - type: string
+              format: uuid4
+            - type: 'null'
+          title: Created By
+        question:
+          type: string
+          title: Question
+        answer_options:
+          anyOf:
+            - items:
+                type: string
+              type: array
+            - items:
+                $ref: '#/components/schemas/DecisionLevel'
+              type: array
+            - type: 'null'
+          title: Answer Options
+        created_at:
+          type: string
+          format: date-time
+          title: Created At
+      type: object
+      required:
+        - id
+        - organization_id
+        - question
+        - created_at
+      title: DecisionScorerDB
+      description: 'A row of the decision_scorers table.
+
+
+        answer_options depends on the output type of the scorer:
+
+        - categorical: a list of labels (list[str]).
+
+        - discrete: a list of DecisionLevel.
+
+        - boolean_probability: None.'
     DeletePromptResponse:
       properties:
         message:
@@ -35838,6 +36017,7 @@ components:
         - slm
         - llm
         - code
+        - decision
       title: ModelType
     MultiModalModelIntegrationConfig:
       properties:
@@ -36305,6 +36485,7 @@ components:
         - multilabel
         - retrieved_chunk_list_boolean
         - boolean_multilabel
+        - boolean_probability
       title: OutputTypeEnum
       description: Enumeration of output types.
     OverrideAction:
@@ -42460,6 +42641,7 @@ components:
         - code
         - luna
         - preset
+        - decision
       title: ScorerTypes
     ScorerUpdatedAtFilter:
       properties:

src/splunk_ao/__future__/__init__.py

@@ -6,9 +6,18 @@
 from splunk_ao.configuration import Configuration
 from splunk_ao.dataset import Dataset
 from splunk_ao.experiment import Experiment
+from splunk_ao.evaluators import DecisionVersionOutcomeUnknownError
 from splunk_ao.integration import Integration
 from splunk_ao.agent_stream import AgentStream
-from splunk_ao.evaluator import BuiltInEvaluators, CodeEvaluator, Evaluator, LlmEvaluator, LocalEvaluator, SplunkAOEvaluator
+from splunk_ao.evaluator import (
+    BuiltInEvaluators,
+    CodeEvaluator,
+    DecisionEvaluator,
+    Evaluator,
+    LlmEvaluator,
+    LocalEvaluator,
+    SplunkAOEvaluator,
+)
 from splunk_ao.model import Model
 from splunk_ao.project import Project
 from splunk_ao.prompt import Prompt
@@ -34,6 +43,8 @@
     "Configuration",
     "ConfigurationError",
     "Dataset",
+    "DecisionEvaluator",
+    "DecisionVersionOutcomeUnknownError",
     "Evaluator",
     "Experiment",
     "Integration",

src/splunk_ao/__init__.py

@@ -44,7 +44,15 @@
 from splunk_ao.configuration import Configuration
 from splunk_ao.dataset import Dataset
 from splunk_ao.decorator import SplunkAODecorator, log, splunk_ao_context, start_session
-from splunk_ao.evaluator import CodeEvaluator, Evaluator, LlmEvaluator, LocalEvaluator, SplunkAOEvaluator
+from splunk_ao.evaluator import (
+    CodeEvaluator,
+    DecisionEvaluator,
+    Evaluator,
+    LlmEvaluator,
+    LocalEvaluator,
+    SplunkAOEvaluator,
+)
+from splunk_ao.evaluators import DecisionVersionOutcomeUnknownError
 from splunk_ao.exceptions import (
     AuthenticationError,
     BadRequestError,
@@ -117,6 +125,8 @@
     "ControlResult",
     "ControlSpan",
     "Dataset",
+    "DecisionEvaluator",
+    "DecisionVersionOutcomeUnknownError",
     "Document",
     "Evaluator",
     "Experiment",

src/splunk_ao/configuration.py

@@ -107,6 +107,12 @@ def parse_log_level(value: str) -> str:
         value_type=int,
         parser=int,
     ),
+    ConfigKey(
+        name="default_decision_model",
+        env_var="SPLUNK_AO_DEFAULT_DECISION_MODEL",
+        description="Default model for decision evaluators",
+        default="jev-1.13.0",
+    ),
     ConfigKey(
         name="logging_disabled",
         env_var="SPLUNK_AO_LOGGING_DISABLED",