Custom Graders

1
from letta_evals.decorators import grader
2
from letta_evals.models import GradeResult, Sample
3
4
@grader
5
def my_custom_grader(sample: Sample, submission: str) -> GradeResult:
6
    """Custom grading logic."""
7
8
    # Your evaluation logic
9
    score = calculate_score(submission, sample.ground_truth)
10
11
    # Ensure score is between 0.0 and 1.0
12
    score = max(0.0, min(1.0, score))
13
14
    return GradeResult(
15
        score=score,
16
        rationale=f"Score based on custom logic: {score}"
17
    )

1
import json
2
from letta_evals.decorators import grader
3
from letta_evals.models import GradeResult, Sample
4
5
@grader
6
def valid_json(sample: Sample, submission: str) -> GradeResult:
7
    """Check if submission is valid JSON."""
8
    try:
9
        json.loads(submission)
10
        return GradeResult(score=1.0, rationale="Valid JSON")
11
    except json.JSONDecodeError as e:
12
        return GradeResult(score=0.0, rationale=f"Invalid JSON: {e}")