Create a test metric

POST

tests

test_framework

metrics

JavaScript

import Telnyx from 'telnyx';

const client = new Telnyx({
  apiKey: process.env['TELNYX_API_KEY'], // This is the default and can be omitted
});

const AiTestMetric = await client.ai.tests.testFramework.v1.metrics.create({
  name: 'name',
});

console.log(AiTestMetric.id);

import os
from telnyx import Telnyx

client = Telnyx(
    api_key=os.environ.get("TELNYX_API_KEY"),  # This is the default and can be omitted
)
ai_test_metric = client.ai.tests.test_framework.v1.metrics.create(
    name="name",
)
print(ai_test_metric.id)

package main

import (
	"context"
	"fmt"

	"github.com/team-telnyx/telnyx-go"
	"github.com/team-telnyx/telnyx-go/option"
)

func main() {
	client := telnyx.NewClient(
		option.WithAPIKey("My API Key"),
	)
	metric, err := client.AI.Tests.TestFramework.V1.Metrics.New(
		context.TODO(),
		telnyx.AITestTestFrameworkV1MetricNewParams{
			Name: "name",
		},
	)
	if err != nil {
		panic(err.Error())
	}
	fmt.Printf("%+v\n", metric.Data)
}

package com.telnyx.sdk.example;

import com.telnyx.sdk.client.TelnyxClient;
import com.telnyx.sdk.client.okhttp.TelnyxOkHttpClient;
import com.telnyx.sdk.models.ai.tests.testFramework.v1.metrics.MetricCreateParams;

public final class Main {
    private Main() {}

    public static void main(String[] args) {
        TelnyxClient client = TelnyxOkHttpClient.fromEnv();

        MetricCreateParams params = MetricCreateParams.builder()
            .name("name")
            .build();
        var response = client.ai().tests().testFramework().v1().metrics().create(params);
    }
}

<?php

require_once dirname(__DIR__) . '/vendor/autoload.php';

use Telnyx\Client;
use Telnyx\Core\Exceptions\APIException;

$client = new Client(apiKey: getenv('TELNYX_API_KEY') ?: 'My API Key');

try {
  $ai_test_metric = $client->ai->tests->testFramework->v1->metrics->create(
    name: 'name',
  );

  var_dump($ai_test_metric);
} catch (APIException $e) {
  echo $e->getMessage();
}

curl --request POST \
  --url https://api.telnyx.com/v2/ai/tests/test_framework/v1/metrics \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data @- <<EOF
{
  "name": "Escalates on refund denial",
  "description": "Returns a score for whether the assistant escalates when the caller's refund is denied.",
  "type": "llm_judge",
  "eval_type": "continuous_qualitative",
  "prompt": "Score the assistant's escalation behaviour between 0.0 and 1.0.",
  "agents": [
    1259
  ],
  "add_to_new_agents": false,
  "simulation_enabled": true,
  "observability_enabled": true
}
EOF

{
  "id": 266881,
  "slug": "ai_interrupting_user",
  "project": 16003,
  "agent": null,
  "agents": [],
  "agent_names": [],
  "name": "AI interrupting user",
  "description": "Tells if the AI (main agent) interrupted user (testing agent) during the interaction.",
  "function_name": "get_main_agent_interruption",
  "type": "custom_code",
  "eval_type": "numeric",
  "enum_values": [],
  "audio_enabled": false,
  "observability_enabled": true,
  "simulation_enabled": true,
  "sampling_enabled": false,
  "sample_rate": null,
  "prompt": "",
  "display_order": 1,
  "evaluation_trigger": "always",
  "trigger_type": "llm_judge",
  "evaluation_trigger_prompt": "",
  "evaluation_trigger_custom_code": "",
  "priority_assignment_prompt": "",
  "configuration": {},
  "overall_score": 0,
  "total_score": 0,
  "kb_file_ids": [],
  "knowledge_base_files": [],
  "metric_cost": 0,
  "alert_type": "disabled",
  "significant_change_alert_status": "disabled",
  "significant_change_alert_direction": "",
  "window_size": 50,
  "std_multiplier": 2,
  "ewma_alpha": 0.1,
  "add_to_new_agents": true,
  "alert_filters": {},
  "uses_llm_judge": false,
  "user_defined": false
}

{
  "errors": [
    {
      "code": "10015",
      "title": "Bad Request",
      "detail": "The AI Tests service rejected the request."
    }
  ]
}

{
  "errors": [
    {
      "code": "10003",
      "title": "Unauthorized",
      "detail": "The API key is missing or invalid."
    }
  ]
}

{
  "errors": [
    {
      "code": "10005",
      "title": "Not Found",
      "detail": "The requested resource or URL could not be found."
    }
  ]
}

{
  "errors": [
    {
      "code": "10018",
      "title": "Unprocessable Entity",
      "detail": "Validation failed for the request payload."
    }
  ]
}

{
  "errors": [
    {
      "code": "10029",
      "title": "Too Many Requests",
      "detail": "Too many requests; retry later."
    }
  ]
}

{
  "errors": [
    {
      "code": "10063",
      "title": "Service Unavailable",
      "detail": "The test framework service is temporarily unavailable."
    }
  ]
}

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Body

application/json

name

string

required

description

string

type

string

How the metric is evaluated: custom_code or llm_judge.

eval_type

string

What the metric scores: numeric, continuous_qualitative, or binary_workflow_adherence.

prompt

string

Judge prompt used by llm_judge metrics.

enum_values

string[]

agents

integer[]

assistant_id

string

add_to_new_agents

boolean | null

simulation_enabled

boolean

observability_enabled

boolean

sampling_enabled

boolean

audio_enabled

boolean

display_order

integer

Response

Test metric created.

integer

required

name

string

required

type

string

required

How the metric is evaluated: custom_code or llm_judge.

eval_type

string

required

What the metric scores: numeric, continuous_qualitative, or binary_workflow_adherence.

slug

string | null

Stable identifier of a built-in metric.

project

integer | null

agent

integer | null

agents

integer[]

Test agent IDs the metric is attached to.

agent_names

string[]

description

string

function_name

string | null

Built-in evaluation function used by custom_code metrics.

enum_values

string[]

Allowed values for qualitative enum metrics.

audio_enabled

boolean

observability_enabled

boolean

simulation_enabled

boolean

sampling_enabled

boolean

sample_rate

number | null

prompt

string

Judge prompt used by llm_judge metrics.

display_order

integer

evaluation_trigger

string

trigger_type

string

evaluation_trigger_prompt

string

evaluation_trigger_custom_code

string

priority_assignment_prompt

string

configuration

object | null

overall_score

total_score

kb_file_ids

integer[]

knowledge_base_files

string[]

metric_cost

number

alert_type

string

significant_change_alert_status

string

significant_change_alert_direction

string

window_size

integer

Sliding window size the alert status is computed over.

std_multiplier

number

Standard-deviation multiplier for significant-change alerts.

ewma_alpha

number

Smoothing factor for the alert trend.

add_to_new_agents

boolean | null

Whether the metric is attached to newly created test agents by default.

alert_filters

object | null

uses_llm_judge

boolean

user_defined

boolean

JavaScript

import Telnyx from 'telnyx';

const client = new Telnyx({
  apiKey: process.env['TELNYX_API_KEY'], // This is the default and can be omitted
});

const AiTestMetric = await client.ai.tests.testFramework.v1.metrics.create({
  name: 'name',
});

console.log(AiTestMetric.id);

import os
from telnyx import Telnyx

client = Telnyx(
    api_key=os.environ.get("TELNYX_API_KEY"),  # This is the default and can be omitted
)
ai_test_metric = client.ai.tests.test_framework.v1.metrics.create(
    name="name",
)
print(ai_test_metric.id)

package main

import (
	"context"
	"fmt"

	"github.com/team-telnyx/telnyx-go"
	"github.com/team-telnyx/telnyx-go/option"
)

func main() {
	client := telnyx.NewClient(
		option.WithAPIKey("My API Key"),
	)
	metric, err := client.AI.Tests.TestFramework.V1.Metrics.New(
		context.TODO(),
		telnyx.AITestTestFrameworkV1MetricNewParams{
			Name: "name",
		},
	)
	if err != nil {
		panic(err.Error())
	}
	fmt.Printf("%+v\n", metric.Data)
}

package com.telnyx.sdk.example;

import com.telnyx.sdk.client.TelnyxClient;
import com.telnyx.sdk.client.okhttp.TelnyxOkHttpClient;
import com.telnyx.sdk.models.ai.tests.testFramework.v1.metrics.MetricCreateParams;

public final class Main {
    private Main() {}

    public static void main(String[] args) {
        TelnyxClient client = TelnyxOkHttpClient.fromEnv();

        MetricCreateParams params = MetricCreateParams.builder()
            .name("name")
            .build();
        var response = client.ai().tests().testFramework().v1().metrics().create(params);
    }
}

<?php

require_once dirname(__DIR__) . '/vendor/autoload.php';

use Telnyx\Client;
use Telnyx\Core\Exceptions\APIException;

$client = new Client(apiKey: getenv('TELNYX_API_KEY') ?: 'My API Key');

try {
  $ai_test_metric = $client->ai->tests->testFramework->v1->metrics->create(
    name: 'name',
  );

  var_dump($ai_test_metric);
} catch (APIException $e) {
  echo $e->getMessage();
}

curl --request POST \
  --url https://api.telnyx.com/v2/ai/tests/test_framework/v1/metrics \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data @- <<EOF
{
  "name": "Escalates on refund denial",
  "description": "Returns a score for whether the assistant escalates when the caller's refund is denied.",
  "type": "llm_judge",
  "eval_type": "continuous_qualitative",
  "prompt": "Score the assistant's escalation behaviour between 0.0 and 1.0.",
  "agents": [
    1259
  ],
  "add_to_new_agents": false,
  "simulation_enabled": true,
  "observability_enabled": true
}
EOF

{
  "id": 266881,
  "slug": "ai_interrupting_user",
  "project": 16003,
  "agent": null,
  "agents": [],
  "agent_names": [],
  "name": "AI interrupting user",
  "description": "Tells if the AI (main agent) interrupted user (testing agent) during the interaction.",
  "function_name": "get_main_agent_interruption",
  "type": "custom_code",
  "eval_type": "numeric",
  "enum_values": [],
  "audio_enabled": false,
  "observability_enabled": true,
  "simulation_enabled": true,
  "sampling_enabled": false,
  "sample_rate": null,
  "prompt": "",
  "display_order": 1,
  "evaluation_trigger": "always",
  "trigger_type": "llm_judge",
  "evaluation_trigger_prompt": "",
  "evaluation_trigger_custom_code": "",
  "priority_assignment_prompt": "",
  "configuration": {},
  "overall_score": 0,
  "total_score": 0,
  "kb_file_ids": [],
  "knowledge_base_files": [],
  "metric_cost": 0,
  "alert_type": "disabled",
  "significant_change_alert_status": "disabled",
  "significant_change_alert_direction": "",
  "window_size": 50,
  "std_multiplier": 2,
  "ewma_alpha": 0.1,
  "add_to_new_agents": true,
  "alert_filters": {},
  "uses_llm_judge": false,
  "user_defined": false
}

{
  "errors": [
    {
      "code": "10015",
      "title": "Bad Request",
      "detail": "The AI Tests service rejected the request."
    }
  ]
}

{
  "errors": [
    {
      "code": "10003",
      "title": "Unauthorized",
      "detail": "The API key is missing or invalid."
    }
  ]
}

{
  "errors": [
    {
      "code": "10005",
      "title": "Not Found",
      "detail": "The requested resource or URL could not be found."
    }
  ]
}

{
  "errors": [
    {
      "code": "10018",
      "title": "Unprocessable Entity",
      "detail": "Validation failed for the request payload."
    }
  ]
}

{
  "errors": [
    {
      "code": "10029",
      "title": "Too Many Requests",
      "detail": "Too many requests; retry later."
    }
  ]
}

{
  "errors": [
    {
      "code": "10063",
      "title": "Service Unavailable",
      "detail": "The test framework service is temporarily unavailable."
    }
  ]
}

Guides

Missions

AI Insights

Assistants API

Missions API

AI Tests API

Conversations API

For AI Agents

Create a test metric

Authorizations

Body

Response