Suite YAML Reference

1
name: string (required)
2
description: string (optional)
3
dataset: path (required)
4
max_samples: integer (optional)
5
sample_tags: array (optional)
6
num_runs: integer (optional)
7
setup_script: string (optional)
8
9
target: object (required)
10
  kind: "agent"
11
  base_url: string
12
  api_key: string
13
  timeout: float
14
  project_id: string
15
  agent_id: string (one of: agent_id, agent_file, agent_script)
16
  agent_file: path
17
  agent_script: string
18
  model_configs: array
19
  model_handles: array
20
21
graders: object (required)
22
  <metric_key>: object
23
    kind: "tool" | "rubric"
24
    display_name: string
25
    extractor: string
26
    extractor_config: object
27
    # Tool grader fields
28
    function: string
29
    # Rubric grader fields (LLM API)
30
    prompt: string
31
    prompt_path: path
32
    model: string
33
    temperature: float
34
    provider: string
35
    max_retries: integer
36
    timeout: float
37
    rubric_vars: array
38
    # Rubric grader fields (agent-as-judge)
39
    agent_file: path
40
    judge_tool_name: string
41
42
gate: object (required)
43
  metric_key: string
44
  metric: "avg_score" | "accuracy"
45
  op: "gte" | "gt" | "lte" | "lt" | "eq"
46
  value: float
47
  pass_op: "gte" | "gt" | "lte" | "lt" | "eq"
48
  pass_value: float

1
name: question-answering-eval

1
description: Tests agent's ability to answer factual questions accurately

1
dataset: ./datasets/qa.jsonl
2
dataset: /absolute/path/to/dataset.jsonl

1
max_samples: 10  # Only evaluate first 10 samples

1
sample_tags: [math, easy]  # Only samples tagged with both

1
num_runs: 5  # Run the evaluation 5 times

1
setup_script: setup.py:prepare_environment

1
target:
2
  kind: agent

1
target:
2
  base_url: http://localhost:8283
3
  # or
4
  base_url: https://api.letta.com

1
target:
2
  api_key: your-api-key-here

1
target:
2
  timeout: 600.0  # 10 minutes

1
target:
2
  agent_id: agent-123-abc

1
target:
2
  agent_file: ./agents/my_agent.af

1
target:
2
  agent_script: factory.py:MyAgentFactory

1
target:
2
  model_configs: [gpt-4o-mini, claude-3-5-sonnet]

1
target:
2
  model_handles: ["openai/gpt-4o-mini", "anthropic/claude-3-5-sonnet"]

1
graders:
2
  my_metric:
3
    kind: tool

1
graders:
2
  my_metric:
3
    extractor: last_assistant

1
graders:
2
  accuracy:
3
    kind: tool
4
    function: exact_match

1
graders:
2
  quality:
3
    kind: rubric
4
    prompt: |
5
      Evaluate response quality from 0.0 to 1.0.

1
graders:
2
  quality:
3
    kind: rubric
4
    model: gpt-4o

1
graders:
2
  quality:
3
    kind: rubric
4
    temperature: 0.0

1
graders:
2
  agent_judge:
3
    kind: rubric
4
    agent_file: judge.af
5
    prompt_path: rubric.txt

1
gate:
2
  metric_key: accuracy

1
gate:
2
  metric: avg_score

1
gate:
2
  op: gte  # Greater than or equal

1
gate:
2
  value: 0.8  # Require >= 0.8

1
name: basic-eval
2
dataset: dataset.jsonl
3
4
target:
5
  kind: agent
6
  agent_file: agent.af
7
8
graders:
9
  accuracy:
10
    kind: tool
11
    function: exact_match
12
    extractor: last_assistant
13
14
gate:
15
  op: gte
16
  value: 0.8

1
name: comprehensive-eval
2
description: Tests accuracy and quality
3
dataset: test_data.jsonl
4
5
target:
6
  kind: agent
7
  agent_file: agent.af
8
9
graders:
10
  accuracy:
11
    kind: tool
12
    function: contains
13
    extractor: last_assistant
14
15
  quality:
16
    kind: rubric
17
    prompt_path: rubrics/quality.txt
18
    model: gpt-4o-mini
19
    extractor: last_assistant
20
21
gate:
22
  metric_key: accuracy
23
  op: gte
24
  value: 0.85