AI 原生开放教材

外观

Anthropic 平台文档（英文全量）官方文档·英文原文 ↗原件 ↓仓库 ↗

Grade your evaluations ​

When deciding which method to use to grade evals, choose the fastest, most reliable, most scalable method:

  1. Code-based grading: Fastest and most reliable, extremely scalable, but also lacks nuance for more complex judgments that require less rule-based rigidity.

    • Exact match: output == golden_answer
    • String match: key_phrase in output

  2. Human grading: Most flexible and high quality, but slow and expensive. Avoid if possible.

  3. LLM-based grading: Fast and flexible, scalable and suitable for complex judgment. Test to ensure reliability first then scale.

Tips for LLM-based grading ​

  • Have detailed, clear rubrics: "The answer should always mention 'Acme Inc.' in the first sentence. If it does not, the answer is automatically graded as 'incorrect.'" A given use case, or even a specific success criteria for that use case, might require several rubrics for holistic evaluation.

  • Empirical or specific: For example, instruct the LLM to output only 'correct' or 'incorrect', or to judge from a scale of 1–5. Purely qualitative evaluations are hard to assess quickly and at scale.

  • Encourage reasoning: Ask the LLM to reason first before producing an evaluation score, and then discard the reasoning. This increases evaluation performance, particularly for tasks requiring complex judgment.

    python
    client = anthropic.Anthropic()

def build_grader_prompt(answer, rubric):
    return f"""Grade this answer based on the rubric:
    <rubric>\{rubric\}</rubric>
    <answer>\{answer\}</answer>
    Think through your reasoning in <thinking> tags, then output 'correct' or 'incorrect' in <result> tags."""

def grade_completion(output, golden_answer):
    grader_message = client.messages.create(
        model="claude-opus-5",
        max_tokens=2048,
        messages=[
            \{"role": "user", "content": build_grader_prompt(output, golden_answer)\}
        ],
    )
    grader_response = next(
        block.text for block in grader_message.content if block.type == "text"
    )

    return (
        "correct"
        if "<result>correct</result>" in grader_response.lower()
        else "incorrect"
    )

# Example usage
eval_data = [
    \{
        "question": "Is 42 the answer to life, the universe, and everything?",
        "golden_answer": "Yes, according to 'The Hitchhiker's Guide to the Galaxy'.",
    \},
    \{
        "question": "What is the capital of France?",
        "golden_answer": "The capital of France is Paris.",
    \},
]

def get_completion(prompt: str):
    message = client.messages.create(
        model="claude-opus-5",
        max_tokens=1024,
        messages=[\{"role": "user", "content": prompt\}],
    )
    return next(block.text for block in message.content if block.type == "text")

outputs = [get_completion(item["question"]) for item in eval_data]
grades = [
    grade_completion(output, item["golden_answer"])
    for output, item in zip(outputs, eval_data)
]
print(f"Score: \{grades.count('correct') / len(grades) * 100\}%")
    typescript
    const client = new Anthropic();

function buildGraderPrompt(answer: string, rubric: string): string \{
  return `Grade this answer based on the rubric:
<rubric>${rubric}</rubric>
<answer>${answer\}</answer>
Think through your reasoning in <thinking> tags, then output 'correct' or 'incorrect' in <result> tags.`;
\}

async function gradeCompletion(output: string, goldenAnswer: string): Promise<string> \{
  const graderResponse = await client.messages.create(\{
    model: "claude-opus-5",
    max_tokens: 2048,
    messages: [\{ role: "user", content: buildGraderPrompt(output, goldenAnswer) \}]
  \});
  const textBlock = graderResponse.content.find((block) => block.type === "text");
  const graderText = textBlock ? textBlock.text : "";
  return graderText.toLowerCase().includes("<result>correct</result>")
    ? "correct"
    : "incorrect";
\}

// Example usage
const evalData = [
  \{
    question: "Is 42 the answer to life, the universe, and everything?",
    goldenAnswer: "Yes, according to 'The Hitchhiker's Guide to the Galaxy'."
  \},
  \{
    question: "What is the capital of France?",
    goldenAnswer: "The capital of France is Paris."
  \}
];

async function getCompletion(prompt: string): Promise<string> \{
  const message = await client.messages.create(\{
    model: "claude-opus-5",
    max_tokens: 1024,
    messages: [\{ role: "user", content: prompt \}]
  \});
  const textBlock = message.content.find((block) => block.type === "text");
  return textBlock ? textBlock.text : "";
\}

const grades: string[] = [];
for (const item of evalData) \{
  const output = await getCompletion(item.question);
  grades.push(await gradeCompletion(output, item.goldenAnswer));
\}
const score = (grades.filter((grade) => grade === "correct").length / grades.length) * 100;
console.log(`Score: ${score}%`);
    csharp
    var client = new AnthropicClient();

string BuildGraderPrompt(string answer, string rubric)
{
    return $"""
        Grade this answer based on the rubric:
        <rubric>\{rubric\}</rubric>
        <answer>\{answer\}</answer>
        Think through your reasoning in <thinking> tags, then output 'correct' or 'incorrect' in <result> tags.
        """;
\}

async Task<string> GradeCompletion(string output, string goldenAnswer)
\{
    var graderResponse = await client.Messages.Create(new MessageCreateParams
    \{
        Model = Model.ClaudeOpus5,
        MaxTokens = 2048,
        Messages = [new() \{ Role = Role.User, Content = BuildGraderPrompt(output, goldenAnswer) \}],
    \});
    return ContentText(graderResponse).ToLowerInvariant().Contains("<result>correct</result>")
        ? "correct"
        : "incorrect";
\}

// Example usage
EvalItem[] evalData =
[
    new("Is 42 the answer to life, the universe, and everything?",
        "Yes, according to 'The Hitchhiker's Guide to the Galaxy'."),
    new("What is the capital of France?",
        "The capital of France is Paris."),
];

async Task<string> GetCompletion(string prompt)
\{
    var message = await client.Messages.Create(new MessageCreateParams
    \{
        Model = Model.ClaudeOpus5,
        MaxTokens = 1024,
        Messages = [new() \{ Role = Role.User, Content = prompt \}],
    \});
    return ContentText(message);
\}

string ContentText(Message message)
\{
    var text = "";
    foreach (var block in message.Content)
    \{
        if (block.TryPickText(out var textBlock))
        \{
            text += textBlock.Text;
        \}
    \}
    return text;
\}

var correct = 0;
foreach (var item in evalData)
\{
    var output = await GetCompletion(item.Question);
    if (await GradeCompletion(output, item.GoldenAnswer) == "correct")
    \{
        correct++;
    \}
\}
Console.WriteLine($"Score: {100.0 * correct / evalData.Length}%");

record EvalItem(string Question, string GoldenAnswer);
    go
    var client = anthropic.NewClient()

func contentText(message *anthropic.Message) string {
	var text strings.Builder
	for _, block := range message.Content {
		if textBlock, ok := block.AsAny().(anthropic.TextBlock); ok {
			text.WriteString(textBlock.Text)
		}
	}
	return text.String()
}

func buildGraderPrompt(answer, rubric string) string {
	return fmt.Sprintf(`Grade this answer based on the rubric:
<rubric>%s</rubric>
<answer>%s</answer>
Think through your reasoning in <thinking> tags, then output 'correct' or 'incorrect' in <result> tags.`, rubric, answer)
}

func gradeCompletion(output, goldenAnswer string) string {
	graderResponse, err := client.Messages.New(context.Background(), anthropic.MessageNewParams{
		Model:     anthropic.ModelClaudeOpus5,
		MaxTokens: 2048,
		Messages: []anthropic.MessageParam{
			anthropic.NewUserMessage(anthropic.NewTextBlock(buildGraderPrompt(output, goldenAnswer))),
		},
	})
	if err != nil {
		log.Fatal(err)
	}
	if strings.Contains(strings.ToLower(contentText(graderResponse)), "<result>correct</result>") {
		return "correct"
	}
	return "incorrect"
}

func getCompletion(prompt string) string {
	message, err := client.Messages.New(context.Background(), anthropic.MessageNewParams{
		Model:     anthropic.ModelClaudeOpus5,
		MaxTokens: 1024,
		Messages: []anthropic.MessageParam{
			anthropic.NewUserMessage(anthropic.NewTextBlock(prompt)),
		},
	})
	if err != nil {
		log.Fatal(err)
	}
	return contentText(message)
}

func main() {
	evalData := []struct {
		Question     string
		GoldenAnswer string
	}{
		{"Is 42 the answer to life, the universe, and everything?", "Yes, according to 'The Hitchhiker's Guide to the Galaxy'."},
		{"What is the capital of France?", "The capital of France is Paris."},
	}

	correct := 0
	for _, item := range evalData {
		output := getCompletion(item.Question)
		if gradeCompletion(output, item.GoldenAnswer) == "correct" {
			correct++
		}
	}
	fmt.Printf("Score: %.1f%%\n", float64(correct)/float64(len(evalData))*100)
}
    java
    record EvalItem(String question, String goldenAnswer) {}

// Example usage
List<EvalItem> evalData = List.of(
    new EvalItem(
        "Is 42 the answer to life, the universe, and everything?",
        "Yes, according to 'The Hitchhiker's Guide to the Galaxy'."),
    new EvalItem(
        "What is the capital of France?",
        "The capital of France is Paris."));

AnthropicClient client = AnthropicOkHttpClient.fromEnv();

String contentText(Message message) {
    var text = new StringBuilder();
    for (var block : message.content()) {
        block.text().ifPresent(textBlock -> text.append(textBlock.text()));
    }
    return text.toString();
}

String buildGraderPrompt(String answer, String rubric) {
    return """
        Grade this answer based on the rubric:
        <rubric>%s</rubric>
        <answer>%s</answer>
        Think through your reasoning in <thinking> tags, then output 'correct' or 'incorrect' in <result> tags.""".formatted(rubric, answer);
}

String gradeCompletion(String output, String goldenAnswer) {
    var params = MessageCreateParams.builder()
        .model(Model.CLAUDE_OPUS_5)
        .maxTokens(2048L)
        .addUserMessage(buildGraderPrompt(output, goldenAnswer))
        .build();
    var graderResponse = contentText(client.messages().create(params));
    return graderResponse.toLowerCase().contains("<result>correct</result>") ? "correct" : "incorrect";
}

String getCompletion(String prompt) {
    var params = MessageCreateParams.builder()
        .model(Model.CLAUDE_OPUS_5)
        .maxTokens(1024L)
        .addUserMessage(prompt)
        .build();
    return contentText(client.messages().create(params));
}

void main() {
    int correct = 0;
    for (var item : evalData) {
        var output = getCompletion(item.question());
        if (gradeCompletion(output, item.goldenAnswer()).equals("correct")) {
            correct++;
        }
    }
    IO.println("Score: " + (100.0 * correct / evalData.size()) + "%");
}
    php
    $client = new Client();

function buildGraderPrompt(string $answer, string $rubric): string
\{
    return <<&lt;PROMPT
    Grade this answer based on the rubric:
    &lt;rubric>\{$rubric}&lt;/rubric>
    &lt;answer>{$answer\}&lt;/answer>
    Think through your reasoning in &lt;thinking> tags, then output 'correct' or 'incorrect' in &lt;result> tags.
    PROMPT;
\}

function gradeCompletion(Client $client, string $output, string $goldenAnswer): string
{
    $graderResponse = $client->messages->create(
        model: Model::CLAUDE_OPUS_5,
        maxTokens: 2048,
        messages: [
            [
                'role' => 'user',
                'content' => buildGraderPrompt($output, $goldenAnswer),
            ],
        ],
    );
    return str_contains(strtolower(contentText($graderResponse)), '&lt;result>correct&lt;/result>')
        ? 'correct'
        : 'incorrect';
\}

// Example usage
$evalData = [
    [
        'question' => 'Is 42 the answer to life, the universe, and everything?',
        'goldenAnswer' => "Yes, according to 'The Hitchhiker's Guide to the Galaxy'.",
    ],
    [
        'question' => 'What is the capital of France?',
        'goldenAnswer' => 'The capital of France is Paris.',
    ],
];

function getCompletion(Client $client, string $prompt): string
{
    $message = $client->messages->create(
        model: Model::CLAUDE_OPUS_5,
        maxTokens: 1024,
        messages: [
            [
                'role' => 'user',
                'content' => $prompt,
            ],
        ],
    );
    return contentText($message);
}

function contentText($message): string
\{
    $text = '';
    foreach ($message->content as $block) {
        if ($block instanceof TextBlock) \{
            $text .= $block->text;
        \}
    \}
    return $text;
}

$correct = 0;
foreach ($evalData as $item) \{
    $output = getCompletion($client, $item['question']);
    if (gradeCompletion($client, $output, $item['goldenAnswer']) === 'correct') \{
        $correct++;
    }
}
echo 'Score: ' . (100 * $correct / count($evalData)) . '%' . PHP_EOL;
    ruby
    client = Anthropic::Client.new

def content_text(message)
  message.content.filter_map \{ |block| block.text if block.type == :text \}.join
end

def build_grader_prompt(answer, rubric)
  <<~PROMPT
    Grade this answer based on the rubric:
    &lt;rubric>#\{rubric\}&lt;/rubric>
    &lt;answer>#\{answer\}&lt;/answer>
    Think through your reasoning in &lt;thinking> tags, then output 'correct' or 'incorrect' in &lt;result> tags.
  PROMPT
end

def grade_completion(client, output, golden_answer)
  grader_response = client.messages.create(
    model: Anthropic::Model::CLAUDE_OPUS_5,
    max_tokens: 2048,
    messages: [
      \{
        role: "user",
        content: build_grader_prompt(output, golden_answer)
      \}
    ]
  )
  content_text(grader_response).downcase.include?("&lt;result>correct&lt;/result>") ? "correct" : "incorrect"
end

# Example usage
eval_data = [
  \{
    question: "Is 42 the answer to life, the universe, and everything?",
    golden_answer: "Yes, according to 'The Hitchhiker's Guide to the Galaxy'."
  \},
  \{
    question: "What is the capital of France?",
    golden_answer: "The capital of France is Paris."
  \}
]

def get_completion(client, prompt)
  message = client.messages.create(
    model: Anthropic::Model::CLAUDE_OPUS_5,
    max_tokens: 1024,
    messages: [
      \{
        role: "user",
        content: prompt
      \}
    ]
  )
  content_text(message)
end

grades = eval_data.map do |item|
  output = get_completion(client, item[:question])
  grade_completion(client, output, item[:golden_answer])
end
puts "Score: #\{100.0 * grades.count("correct") / grades.length\}%"