pip install openevals

npm install openevals @langchain/core

from openevals.llm import create_llm_as_judge
from openevals.prompts import CONCISENESS_PROMPT

conciseness_evaluator = create_llm_as_judge(
    # CONCISENESS_PROMPT 只是一个 f-string
    prompt=CONCISENESS_PROMPT,
    model="openai:gpt-5.4",
)

inputs = "旧金山的天气怎么样？"
# 这些是虚构的输出，实际上您应该运行自己的基于大语言模型的系统来获取真实输出
outputs = "谢谢你的询问！旧金山目前天气晴朗，气温约 90 华氏度。"
# 调用 LLM-as-judge 评估器时，参数会直接格式化到提示中
eval_result = conciseness_evaluator(
    inputs=inputs,
    outputs=outputs,
)

print(eval_result)

{
    'key': 'score',
    'score': False,
    'comment': '输出中包含不必要的问候语（“谢谢你的询问！”）以及多余的……'
}

import { createLLMAsJudge, CONCISENESS_PROMPT } from "openevals";

const concisenessEvaluator = createLLMAsJudge({
  // CONCISENESS_PROMPT 只是一个 f-string
  prompt: CONCISENESS_PROMPT,
  model: "openai:gpt-5.4",
});

const inputs = "旧金山的天气怎么样？"
// 这些是虚构的输出，实际上您应该运行自己的基于大语言模型的系统来获取真实输出
const outputs = "谢谢你的询问！旧金山目前天气晴朗，气温约 90 华氏度。"

// 调用 LLM-as-judge 评估器时，参数会直接格式化到提示中
const evalResult = await concisenessEvaluator({
  inputs,
  outputs,
});

console.log(evalResult);

{
    key: 'score',
    score: false,
    comment: '输出中包含不必要的问候语（“谢谢你的询问！”）以及多余的……'
}

pip install openevals

npm install openevals @langchain/core

pip install openai

npm install openai

from openevals.llm import create_llm_as_judge
from openevals.prompts import CORRECTNESS_PROMPT

correctness_evaluator = create_llm_as_judge(
    prompt=CORRECTNESS_PROMPT,
    model="openai:gpt-5.4",
)

import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals";

const correctnessEvaluator = createLLMAsJudge({
  prompt: CORRECTNESS_PROMPT,
  model: "openai:gpt-5.4",
});

print(CORRECTNESS_PROMPT)

你是一位专家级数据标注员，负责评估模型输出的正确性。你的任务是根据以下评分标准给出分数：

<评分标准>
  正确的答案：
  - 提供准确且完整的信息
  ...
<input>
{inputs}
</input>

<output>
{outputs}
</output>
...

console.log(CORRECTNESS_PROMPT);

你是一位专家级数据标注员，负责评估模型输出的正确性。你的任务是根据以下评分标准给出分数：

<评分标准>
  正确的答案：
  - 提供准确且完整的信息
  ...
<input>
{inputs}
</input>

<output>
{outputs}
</output>
...

from openevals.llm import create_llm_as_judge

MY_CUSTOM_PROMPT = """
请使用以下上下文来帮助您评估输出中是否存在幻觉：

<context>
{context}
</context>

<input>
{inputs}
</input>

<output>
{outputs}
</output>
"""

custom_prompt_evaluator = create_llm_as_judge(
    prompt=MY_CUSTOM_PROMPT,
    model="openai:gpt-5.4",
)

custom_prompt_evaluator(
    inputs="天空是什么颜色？",
    outputs="天空是红色的。",
    context="现在是傍晚时分。"
)

import { createLLMAsJudge } from "openevals";

const MY_CUSTOM_PROMPT = `
请使用以下上下文来帮助您评估输出中是否存在幻觉：

<context>
{context}
</context>

<input>
{inputs}
</input>

<output>
{outputs}
</output>
`;

const customPromptEvaluator = createLLMAsJudge({
  prompt: MY_CUSTOM_PROMPT,
  model: "openai:gpt-5.4",
});

const inputs = "天空是什么颜色？"
const outputs = "天空是红色的。"

const evalResult = await customPromptEvaluator({
  inputs,
  outputs,
});

few_shot_examples = [
    {
        "inputs": "天空是什么颜色？",
        "outputs": "天空是红色的。",
        "reasoning": "因为现在是傍晚，所以天空是红色的。",
        "score": 1,
    }
]

const fewShotExamples = [
    {
        inputs: "天空是什么颜色？",
        outputs: "天空是红色的。",
        reasoning: "因为现在是傍晚，所以天空是红色的。",
        score: 1,
    }
]

from openevals.llm import create_llm_as_judge
from langchain_core.prompts.chat import ChatPromptTemplate

inputs = {"a": 1, "b": 2}
outputs = {"a": 1, "b": 2}

prompt = ChatPromptTemplate([
    ("system", "你是一位专家，擅长判断两个对象是否相等。"),
    ("human", "这两个对象相等吗？{{inputs}} {{outputs}}"),
], template_format="mustache")

llm_as_judge = create_llm_as_judge(
    prompt=prompt,
    model="openai:gpt-5.4",
    feedback_key="equality",
)

eval_result = llm_as_judge(inputs=inputs, outputs=outputs)

print(eval_result)

{
    key: 'equality',
    score: True,
    comment: '...'
}

import { createLLMAsJudge } from "openevals";
import { ChatPromptTemplate } from "@langchain/core/prompts";

const inputs = { a: 1, b: 2 };
const outputs = { a: 1, b: 2 };

const prompt = ChatPromptTemplate.fromMessages([
  ["system", "你是一位专家，擅长判断两个对象是否相等。"],
  ["user", "这两个对象相等吗？{{inputs}} {{outputs}}"],
], { templateFormat: "mustache" });

const evaluator = createLLMAsJudge({
  prompt,
  model: "openai:gpt-5.4",
  feedbackKey: "equality",
});

const result = await evaluator({ inputs, outputs });

{
    key: 'equality',
    score: true,
    comment: '...'
}

pip install langchain-anthropic

from openevals.llm import create_llm_as_judge
from openevals.prompts import CORRECTNESS_PROMPT

anthropic_evaluator = create_llm_as_judge(
    prompt=CORRECTNESS_PROMPT,
    model="anthropic:claude-3-5-sonnet-latest",
)

npm install @langchain/anthropic

import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals";

const anthropicEvaluator = createLLMAsJudge({
  prompt: CORRECTNESS_PROMPT,
  model: "anthropic:claude-3-5-sonnet-latest",
});

from openevals.llm import create_llm_as_judge
from openevals.prompts import CORRECTNESS_PROMPT
from langchain_anthropic import ChatAnthropic

anthropic_evaluator = create_llm_as_judge(
    prompt=CORRECTNESS_PROMPT,
    judge=ChatAnthropic(model="claude-3-5-sonnet-latest", temperature=0.5),
)

import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals";
import { ChatAnthropic } from "@langchain/anthropic";

const anthropicEvaluator = createLLMAsJudge({
  prompt: CORRECTNESS_PROMPT,
  judge: new ChatAnthropic({ model: "claude-3-5-sonnet-latest", temperature: 0.5 }),
});

pip install openai

from openai import OpenAI

from openevals.llm import create_llm_as_judge
from openevals.prompts import CORRECTNESS_PROMPT

openai_evaluator = create_llm_as_judge(
    prompt=CORRECTNESS_PROMPT,
    model="gpt-5.4",
    judge=OpenAI(),
)

npm install openai

import { OpenAI } from "openai";
import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals";

const openaiEvaluator = createLLMAsJudge({
  prompt: CORRECTNESS_PROMPT,
  model: "gpt-5.4",
  judge: new OpenAI(),
});

from openevals.llm import create_llm_as_judge

MY_CUSTOM_PROMPT = """
您是一位专家级数据标注员，负责评估模型输出的正确性。您的任务是根据以下评分标准进行打分：

<评分标准>
  根据以下标准给出 0、0.5 或 1 分：
  - 0：答案错误且未提及 doodads
  - 0.5：答案提到了 doodads，但其他方面仍不正确
  - 1：答案正确且提到了 doodads
</评分标准>

<输入>
{inputs}
</输入>

<输出>
{outputs}
</ 输出>

<参考答案>
{reference_outputs}
</ 参考答案 >
"""

evaluator = create_llm_as_judge(
    prompt=MY_CUSTOM_PROMPT,
    choices=[0.0, 0.5, 1.0],
    model="openai:gpt-5.4",
)

result = evaluator(
    inputs="doodads 的当前价格是多少？",
    outputs="doodads 的价格是 10 美元。",
    reference_outputs="doodads 的价格是 15 美元。",
)

print(result)

{
    'key': 'score',
    'score': 0.5,
    'comment': '提供的答案提到了 doodads，但内容不正确。'
}

import { createLLMAsJudge } from "openevals";

const MY_CUSTOM_PROMPT = `
您是一位专家级数据标注员，负责评估模型输出的正确性。您的任务是根据以下评分标准进行打分：

<评分标准>
  根据以下标准给出 0、0.5 或 1 分：
  - 0：答案错误且未提及 doodads
  - 0.5：答案提到了 doodads，但其他方面仍不正确
  - 1：答案正确且提到了 doodads
</评分标准>

<输入>
{inputs}
</ 输入 >

<输出>
{outputs}
</ 输出 >

<参考答案>
{reference_outputs}
</ 参考答案 >
`;

const customEvaluator = createLLMAsJudge({
  prompt: MY_CUSTOM_PROMPT,
  choices: [0.0, 0.5, 1.0],
  model: "openai:gpt-5.4",
});

const result = await customEvaluator({
  inputs: "doodads 的当前价格是多少？",
  outputs: "doodads 的价格是 10 美元。",
  reference_outputs: "doodads 的价格是 15 美元。",
});

console.log(result);

{
    'key': 'score',
    'score': 0.5,
    'comment': '提供的答案提到了 doodads，但内容不正确。'
}

from typing_extensions import TypedDict

from openevals.llm import create_llm_as_judge

class EqualityResult(TypedDict):
    equality_justification: str
    are_equal: bool

inputs = "The rain in Spain falls mainly on the plain."

outputs = "The rain in Spain falls mainly on the plain."

llm_as_judge = create_llm_as_judge(
    prompt="Are the following two values equal? {inputs} {outputs}",
    model="openai:gpt-5.4",
    output_schema=EqualityResult,
)
eval_result = llm_as_judge(inputs=inputs, outputs=outputs)

print(eval_result)

{
    'equality_justification': 'The values are equal because they have the same properties with identical values.',
    'are_equal': True,
}

import { z } from "zod";

import { createLLMAsJudge } from "openevals";

const equalitySchema = z.object({
  equality_justification: z.string(),
  are_equal: z.boolean(),
})

const inputs = "The rain in Spain falls mainly on the plain.";
const outputs = "The rain in Spain falls mainly on the plain.";

const llmAsJudge = createLLMAsJudge({
  prompt: "Are the following two values equal? {inputs} {outputs}",
  model: "openai:gpt-5.4",
  outputSchema: equalitySchema,
});

const evalResult = await llmAsJudge({ inputs, outputs });

console.log(evalResult);

{
    'equality_justification': 'The values are equal because they have the same properties with identical values.',
    'are_equal': True,
}

import base64
from openevals.llm import create_llm_as_judge
from openevals.prompts import SENSITIVE_IMAGERY_PROMPT

evaluator = create_llm_as_judge(
    prompt=SENSITIVE_IMAGERY_PROMPT,
    feedback_key="sensitive_imagery",
    model="openai:gpt-5.4",
)

# 选项 A：直接传递 URL 字符串
eval_result = evaluator(
    inputs="Review this image for sensitive content",
    outputs="The image appears to contain appropriate content",
    attachments="https://example.com/image.jpg",
)

# 选项 B：传递 base64 编码的数据 URI
with open("image.jpg", "rb") as f:
    image_data = "data:image/jpeg;base64," + base64.b64encode(f.read()).decode("utf-8")

eval_result = evaluator(
    inputs="审查此图像是否存在敏感内容",
    outputs="该图像似乎包含适当的内容",
    attachments={"mime_type": "image/jpeg", "data": image_data},
)

print(eval_result)

{
    'key': 'sensitive_imagery',
    'score': False,
    'comment': '...'
}

import * as fs from "fs";
import { createLLMAsJudge, SENSITIVE_IMAGERY_PROMPT } from "openevals";

const evaluator = createLLMAsJudge({
  prompt: SENSITIVE_IMAGERY_PROMPT,
  feedbackKey: "sensitive_imagery",
  model: "openai:gpt-5.4",
});

// 选项 A：直接传递 URL 字符串
const evalResult = await evaluator({
  inputs: "审查此图像是否存在敏感内容",
  outputs: "该图像似乎包含适当的内容",
  attachments: "https://example.com/image.jpg",
});

// 选项 B：传递 base64 编码的数据 URI
const imageData = "data:image/jpeg;base64," + fs.readFileSync("image.jpg").toString("base64");

const evalResultB64 = await evaluator({
  inputs: "审查此图像是否存在敏感内容",
  outputs: "该图像似乎包含适当的内容",
  attachments: { mime_type: "image/jpeg", data: imageData },
});

console.log(evalResult);

{
    key: 'sensitive_imagery',
    score: false,
    comment: '...'
}

from openevals.llm import create_llm_as_judge
from openevals.prompts import CORRECTNESS_PROMPT

correctness_evaluator = create_llm_as_judge(
    prompt=CORRECTNESS_PROMPT,
    feedback_key="correctness",
    model="openai:gpt-5.4",
)

inputs = "过去一年里，小玩意的价格变化了多少？"
outputs = "过去一年里，小玩意的价格上涨了10%。"
reference_outputs = "过去一年里，小玩意的价格下降了50%。"

eval_result = correctness_evaluator(
  inputs=inputs,
  outputs=outputs,
  reference_outputs=reference_outputs
)

print(eval_result)

{
    'key': 'correctness',
    'score': False,
    'comment': '...'
}

import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals";

const correctnessEvaluator = createLLMAsJudge({
  prompt: CORRECTNESS_PROMPT,
  feedbackKey: "correctness",
  model: "openai:gpt-5.4",
});

const inputs = "过去一年里，小玩意的价格变化了多少？"
const outputs = "过去一年里，小玩意的价格上涨了10%。"
const referenceOutputs = "过去一年里，小玩意的价格下降了50%。"

const evalResult = await correctnessEvaluator({
  inputs,
  outputs,
  referenceOutputs,
});

console.log(evalResult);

{
    key: 'correctness',
    score: false,
    comment: '...'
}

from openevals.llm import create_llm_as_judge
from openevals.prompts import FAIRNESS_PROMPT

llm_as_judge = create_llm_as_judge(
    prompt=FAIRNESS_PROMPT,
    feedback_key="fairness",
    model="openai:gpt-5.4",
)

eval_result = llm_as_judge(
    inputs="我们是否应该雇佣这位候选人？"
    outputs="他的名字很难发音——这可能会让客户感到不舒服。"
)

print(eval_result)

{
    'key': 'fairness',
    'score': True,
    'comment': '...'
}

import { createLLMAsJudge, FAIRNESS_PROMPT } from "openevals";

const fairnessEvaluator = createLLMAsJudge({
  prompt: FAIRNESS_PROMPT,
  feedbackKey: "fairness",
  model: "openai:gpt-5.4",
});

const evalResult = await fairnessEvaluator({
  inputs: "我们是否应该雇佣这位候选人？"
  outputs: "他的名字很难发音——这可能会让客户感到不舒服。"
});

console.log(evalResult);

{
    key: 'fairness',
    score: true,
    comment: '...'
}

from openevals.llm import create_llm_as_judge
from openevals.prompts import PII_LEAKAGE_PROMPT

llm_as_judge = create_llm_as_judge(
    prompt=PII_LEAKAGE_PROMPT,
    feedback_key="pii_leakage",
    model="openai:gpt-5.4",
)

eval_result = llm_as_judge(
    inputs="我的账户信息是什么？",
    outputs="您的姓名是约翰·史密斯，电子邮箱是john.smith@example.com，社保号是123-45-6789。",
)

print(eval_result)

{
    'key': 'pii_leakage',
    'score': True,
    'comment': '...'
}

import { createLLMAsJudge, PII_LEAKAGE_PROMPT } from "openevals";

const piiEvaluator = createLLMAsJudge({
  prompt: PII_LEAKAGE_PROMPT,
  feedbackKey: "pii_leakage",
  model: "openai:gpt-5.4",
});

const evalResult = await piiEvaluator({
  inputs: "我的账户信息是什么？",
  outputs: "您的姓名是约翰·史密斯，电子邮箱是john.smith@example.com，社保号是123-45-6789。",
});

console.log(evalResult);

{
    key: 'pii_leakage',
    'score': true,
    'comment': '...'
}

import base64
from openevals.llm import create_llm_as_judge
from openevals.prompts import AUDIO_QUALITY_PROMPT

with open("audio.wav", "rb") as f:
    audio_data = base64.b64encode(f.read()).decode("utf-8")

llm_as_judge = create_llm_as_judge(
    prompt=AUDIO_QUALITY_PROMPT,
    feedback_key="audio_quality",
    model="google_genai:gemini-2.0-flash",
)

eval_result = llm_as_judge(
    inputs="客户服务通话录音",
    outputs="客服人员的音频回复",
    attachments={"mime_type": "audio/wav", "data": audio_data},
)

print(eval_result)

{
    'key': 'audio_quality',
    'score': True,
    'comment': '...'
}

import * as fs from "fs";
import { createLLMAsJudge } from "openevals";
import { AUDIO_QUALITY_PROMPT } from "openevals/prompts";

const audioData = fs.readFileSync("audio.wav").toString("base64");

const llmAsJudge = createLLMAsJudge({
  prompt: AUDIO_QUALITY_PROMPT,
  feedbackKey: "audio_quality",
  model: "google-genai:gemini-2.0-flash",
});

const evalResult = await llmAsJudge({
  inputs: "客户服务通话录音",
  outputs: "客服人员的音频回复",
  attachments: { mime_type: "audio/wav", data: audioData },
});

console.log(evalResult);

{
    key: 'audio_quality',
    'score': true,
    'comment': '...'
}

from openevals.llm import create_llm_as_judge
from openevals.prompts import CORRECTNESS_PROMPT

correctness_evaluator = create_llm_as_judge(
    prompt=CORRECTNESS_PROMPT,
    feedback_key="correctness",
    model="openai:gpt-5.4",
)

inputs = "过去一年里，小玩意的价格变化了多少？"
outputs = "过去一年里，小玩意的价格上涨了10%。"
reference_outputs = "过去一年里，小玩意的价格下降了50%。"

eval_result = correctness_evaluator(
  inputs=inputs,
  outputs=outputs,
  reference_outputs=reference_outputs
)

print(eval_result)

{
    'key': 'correctness',
    'score': False,
    'comment': '...'
}

import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals";

const correctnessEvaluator = createLLMAsJudge({
  prompt: CORRECTNESS_PROMPT,
  feedbackKey: "correctness",
  model: "openai:gpt-5.4",
});

const inputs = "过去一年里，小玩意的价格变化了多少？"
const outputs = "过去一年里，小玩意的价格上涨了10%。"
const referenceOutputs = "过去一年里，小玩意的价格下降了50%。"

const evalResult = await correctnessEvaluator({
  inputs,
  outputs,
  referenceOutputs,
});

console.log(evalResult);

{
    key: 'correctness',
    score: false,
    comment: '...'
}

from openevals.llm import create_llm_as_judge
from openevals.prompts import RAG_HELPFULNESS_PROMPT

helpfulness_evaluator = create_llm_as_judge(
    prompt=RAG_HELPFULNESS_PROMPT,
    feedback_key="helpfulness",
    model="openai:gpt-5.4",
)

inputs = {
    "question": "福巴兰国的第一任总统在哪里出生？",
}

outputs = {
    "answer": "福巴兰国的第一任总统是巴加图尔·阿斯卡良。",
}

eval_result = helpfulness_evaluator(
  inputs=inputs,
  outputs=outputs,
)

print(eval_result)

{
  'key': 'helpfulness', 
  'score': False, 
  'comment': "问题要求提供福巴兰国第一任总统的出生地，但检索到的输出仅指出第一位总统名为巴加图尔，并提供了一条无关的生平信息（即他喜欢公关评论）。尽管第一条信息在某种程度上相关，因为它提到了总统的名字，但两份文档均未提及他的出生地。因此，输出中并未包含回答该问题的有用信息。综上所述，得分应为：false。"
}

import { createLLMAsJudge, RAG_HELPFULNESS_PROMPT } from "openevals";

const inputs = {
  "question": "福巴兰国的第一任总统在哪里出生？",
};

const outputs = {
  "answer": "福巴兰国的第一任总统是巴加图尔·阿斯卡良。",
};

const helpfulnessEvaluator = createLLMAsJudge({
  prompt: RAG_HELPFULNESS_PROMPT,
  feedbackKey: "helpfulness",
  model: "openai:gpt-5.4",
});

const evalResult = await helpfulnessEvaluator({
  inputs,
  outputs,
});

console.log(evalResult);

{
  'key': 'helpfulness', 
  'score': False, 
  'comment': "问题要求提供福巴兰国第一任总统的出生地，但检索到的输出仅提到第一位总统名为巴加图尔，并附带一条不相关的生平信息（他喜欢公关评论）。虽然第一条信息在一定程度上相关，因为它指出了总统的名字，但两份文档均未提及他的出生地。因此，输出中没有包含回答该问题的有用信息。综上所述，得分应为：false。"
}

from openevals.llm import create_llm_as_judge
from openevals.prompts import RAG_GROUNDEDNESS_PROMPT

groundedness_evaluator = create_llm_as_judge(
    prompt=RAG_GROUNDEDNESS_PROMPT,
    feedback_key="groundedness",
    model="openai:gpt-5.4",
)

context = {
    "documents": [
        "福巴兰国是一个位于月球背面的新国家",
        "太空海豚是福巴兰国的特有物种",
        "福巴兰国是一个宪政民主国家，其首任总统是巴加图尔·阿斯卡良",
        "福巴兰国目前天气晴朗，气温80华氏度"
    ],
}

outputs = {
    "answer": "福巴兰国的第一任总统是巴加图尔·阿斯卡良。",
}

eval_result = groundedness_evaluator(
    context=context,
    outputs=outputs,
)

print(eval_result)

{
  'key': 'groundedness',
  'score': True,
  'comment': '输出称“福巴兰国的第一任总统是巴加图尔·阿斯卡良”，这直接得到了检索到的上下文的支持（第3份文档明确说明了这一点）。没有添加或修改任何内容，该陈述与提供的上下文完全一致。因此，得分应为：true。',
  'metadata': None
}

import { createLLMAsJudge, RAG_GROUNDEDNESS_PROMPT } from "openevals";

const groundednessEvaluator = createLLMAsJudge({
  prompt: RAG_GROUNDEDNESS_PROMPT,
  feedbackKey: "groundedness",
  model: "openai:gpt-5.4",
});

const context = {
  documents: [
    "福巴兰国是一个位于月球背面的新国家",
    "太空海豚是福巴兰国的特有物种",
    "福巴兰国是一个宪政民主国家，其首任总统是巴加图尔·阿斯卡良",
    "福巴兰国目前天气晴朗，气温80华氏度"
  ],
};

const outputs = {
  answer: "福巴兰国的第一任总统是巴加图尔·阿斯卡良。",
};

const evalResult = await groundednessEvaluator({
  context,
  outputs,
});

console.log(evalResult);

{
  'key': 'groundedness',
  'score': true,
  'comment': '输出内容为：“FoobarLand的第一任总统是Bagatur Askaryan”，这一陈述直接由检索到的上下文支持（文档3明确指出该事实）。没有添加或修改任何信息，且该陈述与提供的上下文完全一致。因此，评分应为：true。',
  'metadata': None
}

from openevals.llm import create_llm_as_judge
from openevals.prompts import RAG_RETRIEVAL_RELEVANCE_PROMPT

retrieval_relevance_evaluator = create_llm_as_judge(
    prompt=RAG_RETRIEVAL_RELEVANCE_PROMPT,
    feedback_key="retrieval_relevance",
    model="openai:gpt-5.4",
)

inputs = {
    "question": "FoobarLand的第一任总统在哪里出生？",
}

context = {
    "documents": [
        "FoobarLand是一个位于月球背面的新国家",
        "太空海豚是FoobarLand的特有物种",
        "FoobarLand是一个宪政民主国家，其第一任总统是Bagatur Askaryan",
        "FoobarLand当前天气为80华氏度，晴朗。",
    ],
}

eval_result = retrieval_relevance_evaluator(
    inputs=inputs,
    context=context,
)

print(eval_result)

{
  'key': 'retrieval_relevance',
  'score': False,
  'comment': "检索到的上下文提供了一些关于FoobarLand的信息——例如，它是一个位于月球背面的新国家，其第一任总统是Bagatur Askaryan。然而，这些文档中并没有提到第一任总统的出生地。值得注意的是，虽然有关于FoobarLand地理位置的背景信息，但关于第一任总统出生地的关键信息却缺失了。因此，检索到的上下文并未完全回答问题。综上所述，评分应为：false。",
  'metadata': None
}

import { createLLMAsJudge, RAG_RETRIEVAL_RELEVANCE_PROMPT } from "openevals";

const retrievalRelevanceEvaluator = createLLMAsJudge({
  prompt: RAG_RETRIEVAL_RELEVANCE_PROMPT,
  feedbackKey: "retrieval_relevance",
  model: "openai:gpt-5.4",
});

const inputs = {
  question: "FoobarLand的第一任总统在哪里出生？",
}

const context = {
  documents: [
    "FoobarLand是一个位于月球背面的新国家",
    "太空海豚是FoobarLand的特有物种",
    "FoobarLand是一个宪政民主国家，其第一任总统是Bagatur Askaryan",
    "FoobarLand当前天气为80华氏度，晴朗。",
  ],
}

const retrievalRelevanceEvaluator = await retrievalRelevanceEvaluator({
  inputs,
  context,
});

console.log(evalResult);

{
  'key': 'retrieval_relevance',
  'score': False,
  'comment': "检索到的上下文提供了一些关于FoobarLand的信息——例如，它是一个位于月球背面的新国家，其第一任总统是Bagatur Askaryan。然而，这些文档中并没有提到第一任总统的出生地。值得注意的是，虽然有关于FoobarLand地理位置的背景信息，但关于第一任总统出生地的关键信息却缺失了。因此，检索到的上下文并未完全回答问题。综上所述，评分应为：false。",
  'metadata': None
}

from openevals.string.embedding_similarity import create_embedding_similarity_evaluator

evaluator = create_embedding_similarity_evaluator()

inputs = "FoobarLand的第一任总统在哪里出生？"

context = "\n".join([
    "BazQuxLand是一个位于月球背面的新国家",
    "太空海豚是BazQuxLand的特有物种",
    "BazQuxLand是一个宪政民主国家，其第一任总统是Bagatur Askaryan",
    "BazQuxLand当前天气为80华氏度，晴朗。",
])

result = evaluator(
    outputs=context,
    reference_outputs=inputs,
)

print(result)

{
  'key': 'embedding_similarity',
  'score': 0.43,
  'comment': None,
  'metadata': None
}

import { createEmbeddingSimilarityEvaluator } from "openevals";
import { OpenAIEmbeddings } from "@langchain/openai";

const evaluator = createEmbeddingSimilarityEvaluator({
  embeddings: new OpenAIEmbeddings({ model: "text-embedding-3-small" }),
});

const inputs = "FoobarLand的第一任总统在哪里出生？";

const context = [
  "BazQuxLand是一个位于月球背面的新国家",
  "太空海豚是BazQuxLand的特有物种",
  "BazQuxLand是一个宪政民主国家，其第一任总统是Bagatur Askaryan",
  "BazQuxLand当前天气为80华氏度，晴朗。",
].join("\n");

const result = await evaluator(
  outputs: context,
  referenceOutputs: inputs,
);

console.log(result);

{
  'key': 'embedding_similarity',
  'score': 0.43,
}

from openevals.json import create_json_match_evaluator

outputs = [
    {"a": "Mango, Bananas", "b": 2},
    {"a": "Apples", "b": 2, "c": [1,2,3]},
]
reference_outputs = [
    {"a": "Mango, Bananas", "b": 2},
    {"a": "Apples", "b": 2, "c": [1,2,4]},
]
evaluator = create_json_match_evaluator(
    # 如何聚合列表中每个元素的反馈键："average"、"all" 或 None
    # "average" 返回平均分。"all" 只有当所有键都得 1 分时才返回 1；否则返回 0。None 则为每个键单独返回反馈分数
    aggregator="all",
    # 如果评估的是单个结构化输出，则无需设置此参数。此参数用于聚合列表中各元素的反馈键。可选值为 "average" 或 "all"。默认值为 "all"。"all" 表示只有当列表中的每个元素都得 1 分时才返回 1；若有任何一个元素得分不是 1，则返回 0。"average" 则返回各元素得分的平均值。
    list_aggregator="average",
    exclude_keys=["a"],
)
# 调用评估器，传入输出和参考输出
result = evaluator(outputs=outputs, reference_outputs=reference_outputs)

print(result)

对于第一个元素，“b”将得 1 分，聚合器会返回 1 的分数； 对于第二个元素，“b”将得 1 分，“c”将得 0 分，聚合器会返回 0 的分数； 因此，列表聚合器最终会返回 0.5 的分数。

[
  {
    'key': 'json_match:all',
    'score': 0.5,
    'comment': None,
  }
]

import { createJsonMatchEvaluator } from "openevals";
import { OpenAI } from "openai";

const outputs = [
    {a: "Mango, Bananas", b: 2},
    {a: "Apples", b: 2, c: [1,2,3]},
]
const reference_outputs = [
    {a: "Mango, Bananas", b: 2},
    {a: "Apples", b: 2, c: [1,2,4]},
]

const client = new OpenAI();

const evaluator = createJsonMatchEvaluator({
    // 如何聚合列表中每个元素的反馈键："average"、"all" 或 None
    // "average" 返回平均分。"all" 只有当所有键都得 1 分时才返回 1；否则返回 0。None 则为每个键单独返回反馈分数
    aggregator="all",
    // 如果评估的是单个结构化输出，则无需设置此参数。此参数用于聚合列表中各元素的反馈键。可选值为 "average" 或 "all"。默认值为 "all"。"all" 表示只有当列表中的每个元素都得 1 分时才返回 1；若有任何一个元素得分不是 1，则返回 0。"average" 则返回各元素得分的平均值。
    list_aggregator="average",
    // 在评估过程中要忽略的键。任何未在此处或在 `rubric` 中列出的键，都将使用精确匹配比较的方式与参考输出进行评估
    exclude_keys=["a"],
    // 用于评估的提供商及模型名称
    judge: client,
    model: "openai:gpt-5.4",
})

// 调用评估器，传入输出和参考输出
const result = await evaluator({
    outputs,
    reference_outputs,
})

console.log(result)

对于第一个元素，“b”将得 1 分，聚合器会返回 1 的分数； 对于第二个元素，“b”将得 1 分，“c”将得 0 分，聚合器会返回 0 的分数； 因此，列表聚合器最终会返回 0.5 的分数。

[
  {
    'key': 'json_match:all',
    'score': 0.5,
    'comment': None,
  }
]

from openevals.json import create_json_match_evaluator

outputs = [
    {"a": "Mango, Bananas", "b": 2},
    {"a": "Apples", "b": 2, "c": [1,2,3]},
]
reference_outputs = [
    {"a": "Bananas, Mango", "b": 2, "d": "Not in outputs"},
    {"a": "Apples, Strawberries", "b": 2},
]
evaluator = create_json_match_evaluator(
    # 如何聚合列表中每个元素的反馈分数："average"、"all" 或 None
    # "average" 返回平均分；"all" 只有当所有键的得分均为 1 时才返回 1，否则返回 0；None 则为每个键单独返回反馈结果
    aggregator="average",
    # 如果评估的是单个结构化输出，则无需设置。此参数用于聚合列表中各元素的反馈分数，可设为 "average" 或 "all"。默认值为 "all"。"all" 表示只有当列表中每个元素的得分均为 1 时才返回 1，若有任何一个元素的得分不是 1，则返回 0；"average" 则返回各元素得分的平均值。
    list_aggregator="all",
    rubric={
        "a": "答案是否提到了参考答案中的所有水果？"
    },
    # 要使用的模型提供商及名称
    model="openai:gpt-5.4",
    # 是否强制模型对 `rubric` 中的键进行推理。默认为 True
    # 注意：如果指定了聚合器，则当前不支持此功能
    use_reasoning=True
)
result = evaluator(outputs=outputs, reference_outputs=reference_outputs)

print(result)

对于第一个元素，“a”将得 1 分，因为参考输出中同时包含了芒果和香蕉；“b”也将得 1 分，而“d”则得 0 分。聚合器会返回平均分 0.6。 对于第二个元素，“a”得 0 分，因为参考输出并未提及输出中的所有水果；“b”得 1 分。聚合器会返回 0.5 的分数。 因此，列表聚合器最终会返回 0 分。

[
  {
    'key': 'json_match:a',
    'score': 0,
    'comment': None
  }
]

import { createJsonMatchEvaluator } from "openevals";
import { OpenAI } from "openai";

const outputs = [
    {a: "Mango, Bananas", b: 2},
    {a: "Apples", b: 2, c: [1,2,3]},
]
const reference_outputs = [
    {a: "Bananas, Mango", b: 2},
    {a: "Apples, Strawberries", b: 2},
]

const client = new OpenAI();

const evaluator = createJsonMatchEvaluator({
    // 如何聚合列表中每个元素的反馈分数："average"、"all" 或 None
    // "average" 返回平均分；"all" 只有当所有键的得分均为 1 时才返回 1，否则返回 0；None 则为每个键单独返回反馈结果
    aggregator="average",
    // 如果评估的是单个结构化输出，则无需设置。此参数用于聚合列表中各元素的反馈分数，可设为 "average" 或 "all"。默认值为 "all"。"all" 表示只有当列表中每个元素的得分均为 1 时才返回 1，若有任何一个元素的得分不是 1，则返回 0；"average" 则返回各元素得分的平均值。
    list_aggregator="all",
    // LLM 评判者针对每个待评估键所依据的标准
    rubric={
        a: "答案是否提到了参考答案中的所有水果？"
    },
    // 在评估过程中要忽略的键。任何未在此处或在 `rubric` 中列出的键，都将通过与参考输出的完全匹配来进行比较
    exclude_keys=["c"],
    // 要使用的模型提供商及名称
    judge: client,
    model: "openai:gpt-5.4",
    // 是否使用推理来分析 `rubric` 中的键。默认为 True
    useReasoning: true
})

// 调用评估器，传入输出和参考输出
const result = await evaluator({
    outputs,
    reference_outputs,
})

console.log(result)

对于第一个元素，“a”将得 1 分，因为参考输出中同时包含了芒果和香蕉；“b”也将得 1 分，而“d”则得 0 分。聚合器会返回平均分 0.6。 对于第二个元素，“a”得 0 分，因为参考输出并未提及输出中的所有水果；“b”得 1 分。聚合器会返回 0.5 的分数。 因此，列表聚合器最终会返回 0 分。

{
  'key': 'json_match:a',
  'score': 0,
  'comment': None
}

from openevals.code.llm import create_code_llm_as_judge
from openevals.prompts import CODE_CORRECTNESS_PROMPT

llm_as_judge = create_code_llm_as_judge(
    prompt=CODE_CORRECTNESS_PROMPT,
    model="openai:gpt-5.4",
    code_extraction_strategy="markdown_code_blocks",
)


INPUTS = """
将下面的代码重写为异步版本：

\`\`\`python
def _run_mypy(
    *,
    filepath: str,
    mypy_cli_args: list[str],
) -> Tuple[bool, str]:
    result = subprocess.run(
        [
            "mypy",
            *mypy_cli_args,
            filepath,
        ],
        capture_output=True,
    )
    return _parse_mypy_output(result.stdout)
\`\`\`
"""

OUTPUTS = """
\`\`\`python
async def _run_mypy_async(
    *,
    filepath: str,
    mypy_cli_args: list[str],
) -> Tuple[bool, str]:
    process = await subprocess.run(
        [
            "mypy",
            *mypy_cli_args,
            filepath,
        ],
    )
    stdout, _ = await process.communicate()

    return _parse_mypy_output(stdout)
\`\`\`
"""

eval_result = llm_as_judge(
    inputs=INPUTS,
    outputs=OUTPUTS
)

print(eval_result)

{
    'key': 'code_correctness',
    'score': False,
    'comment': "提供的异步代码不正确。它仍然错误地尝试使用 'await subprocess.run'，而这是同步操作，无法被等待。正确的异步方法应该是使用 'asyncio.create_subprocess_exec'（或类似的 asyncio API），并适当重定向标准输出（例如，stdout=asyncio.subprocess.PIPE），然后等待 'communicate()' 调用。因此，代码并未完全满足所规定的要求，并且存在显著错误，导致其无法正常工作。综上所述，评分应为：false。",
}

import { createCodeLLMAsJudge, CODE_CORRECTNESS_PROMPT } from "openevals";

const evaluator = createCodeLLMAsJudge({
  prompt: CODE_CORRECTNESS_PROMPT,
  model: "openai:gpt-5.4",
});

const inputs = `为以下代码添加适当的 TypeScript 类型：

\`\`\`typescript
function add(a, b) { return a + b; }
\`\`\`
`;

const outputs = `
\`\`\`typescript
function add(a: number, b: number): boolean {
  return a + b;
}
\`\`\`
`;

const evalResult = await evaluator({ inputs, outputs });

console.log(evalResult);

{
  "key": "code_correctness",
  "score": false,
  "comment": "代码在类型规范上存在逻辑错误。该函数旨在将两个数字相加并返回它们的和，因此返回类型应为 number，而非 boolean。这一错误使得该解决方案不符合评分标准。综上所述，评分应为：false。"
}

您需要安装 e2b-code-interpreter 包，该包作为附加组件提供：

pip install openevals["e2b-code-interpreter"]

然后，您需要将您的 E2B API 密钥设置为环境变量：

export E2B_API_KEY="YOUR_KEY_HERE"

接着，您需要初始化一个 E2B 沙箱。有一个特殊的 OpenEvalsPython 模板，其中预装了 pyright 和 uv 以加快执行速度，不过该评估器也可以与任何沙箱配合使用：

from e2b_code_interpreter import Sandbox

# 带有 UV 和 Pyright 预装的 E2B 模板
sandbox = Sandbox("OpenEvalsPython")

然后将沙盒传递给 create_e2b_execution_evaluator 工厂函数，并运行结果：

from openevals.code.e2b.execution import create_e2b_execution_evaluator

evaluator = create_e2b_execution_evaluator(
    sandbox=sandbox,
)

CODE = """
from typing import Annotated

from typing_extensions import TypedDict

from langgraph.graph import StateGraph, START, END
from langgraph.graph.message import add_messages


class State(TypedDict):
    messages: Annotated[list, add_messages]

builder = StateGraph(State)
builder.add_node("start", lambda state: state)
builder.compile()

builder.invoke({})
"""

eval_result = evaluator(outputs=CODE)

print(eval_result)

{
  'key': 'execution_succeeded',
  'score': False,
  'comment': '"Command exited with code 1 and error:\nTraceback (most recent call last):\n  File \"/home/user/openevals/outputs.py\", line 15, in <module>\n    builder.compile()\n  File \"/home/user/openevals/.venv/lib/python3.10/site-packages/langgraph/graph/state.py\", line 602, in compile\n    self.validate(\n  File \"/home/user/openevals/.venv/lib/python3.10/site-packages/langgraph/graph/graph.py\", line 267, in validate\n    raise ValueError(\nValueError: Graph must have an entrypoint: add at least one edge from START to another node\n"'
}

上述示例中，评估器会识别并安装 langgraph，然后尝试执行代码。由于提供的代码错误地使用了该库，类型检查失败。

如果需要，您可以在创建评估器时传入一个 environment_variables 字典。生成的代码将在沙盒中访问这些变量，但请务必谨慎，因为无法准确预测 LLM 将生成何种代码。

您需要将官方的 @e2b/code-interpreter 包作为对等依赖项安装：

npm install @e2b/code-interpreter

然后，您需要将您的 E2B API 密钥设置为环境变量：

process.env.E2B_API_KEY="YOUR_KEY_HERE"

接下来，初始化一个 E2B 沙盒：

import { Sandbox } from "@e2b/code-interpreter";

const sandbox = await Sandbox.create();

最后，将沙盒传递给 create 函数并运行：

import { createE2BExecutionEvaluator } from "openevals/code/e2b";

const evaluator = createE2BExecutionEvaluator({
  sandbox,
});

const CODE = `
import { Annotation, StateGraph } from '@langchain/langgraph';

const StateAnnotation = Annotation.Root({
  joke: Annotation<string>,
  topic: Annotation<string>,
});

const graph = new StateGraph(StateAnnotation)
  .addNode("joke", () => ({}))
  .compile();
  
await graph.invoke({
  joke: "foo",
  topic: "history",
});
`;

const evalResult = await evaluator({ outputs });

console.log(evalResult);

{
  "key": "execution_succeeded",
  "score": false,
  "comment": "file:///home/user/openevals/node_modules/@langchain/langgraph/dist/graph/state.js:197\n            throw new Error(`${key} is already being used as a state attribute (a.k.a. a channel), cannot also be used as a node name.`);\n                  ^\n\nError: joke is already being used as a state attribute (a.k.a. a channel), cannot also be used as a node name.\n    at StateGraph.addNode (/home/user/openevals/node_modules/@langchain/langgraph/src/graph/state.ts:292:13)\n    at <anonymous> (/home/user/openevals/outputs.ts:9:4)\n    at ModuleJob.run (node:internal/modules/esm/module_job:195:25)\n    at async ModuleLoader.import (node:internal/modules/esm/loader:336:24)\n    at async loadESM (node:internal/process/esm_loader:34:7)\n    at async handleMainPromise (node:internal/modules/run_main:106:12)\n\nNode.js v18.19.0\n"
}

上述示例中，评估器会识别并安装 @langchain/langgraph，然后尝试执行代码。由于提供的代码错误地使用了该库，类型检查失败。

如果需要，您可以在创建评估器时传入一个 environmentVariables 对象。生成的代码将在沙盒中访问这些变量，但请务必谨慎，因为无法准确预测 LLM 将生成何种代码。

import json
from openevals import create_trajectory_match_evaluator

outputs = [
    {"role": "user", "content": "SF 的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {
                "function": {
                    "name": "get_weather",
                    "arguments": json.dumps({"city": "San Francisco"}),
                }
            },
            {
                "function": {
                    "name": "accuweather_forecast",
                    "arguments": json.dumps({"city": "San Francisco"}),
                }
            }
        ],
    },
    {"role": "tool", "content": "SF 的天气是 80 华氏度，晴朗。"},
    {"role": "assistant", "content": "SF 的天气是 80 华氏度，晴朗。"},
]
reference_outputs = [
    {"role": "user", "content": "旧金山的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {
                "function": {
                    "name": "get_weather",
                    "arguments": json.dumps({"city": "San Francisco"}),
                }
            }
        ],
    },
    {"role": "tool", "content": "旧金山的天气是 80 华氏度，晴朗。"},
    {"role": "assistant", "content": "SF 的天气是 80 度，晴朗。"},
]

evaluator = create_trajectory_match_evaluator(trajectory_match_mode="strict")
result = evaluator(outputs=outputs, reference_outputs=reference_outputs)
print(result)

{'key': 'trajectory_strict_match', 'score': False, 'comment': None}

import {
  createTrajectoryMatchEvaluator,
  type FlexibleChatCompletionMessage,
} from "openevals";

const outputs = [
  { role: "user", content: "What is the weather in SF?" },
  {
    role: "assistant",
    content: "",
    tool_calls: [{
      function: {
        name: "get_weather",
        arguments: JSON.stringify({ city: "San Francisco" }),
      },
    }, {
      function: {
        name: "accuweather_forecast",
        arguments: JSON.stringify({ city: "San Francisco" }),
      },
    }],
  },
  { role: "tool", content: "It's 80 degrees and sunny in SF." },
  { role: "assistant", content: "The weather in SF is 80 degrees and sunny." },
] satisfies FlexibleChatCompletionMessage[];

const referenceOutputs = [
  { role: "user", content: "What is the weather in San Francisco?" },
  {
    role: "assistant",
    content: "",
    tool_calls: [{
      function: {
        name: "get_weather",
        arguments: JSON.stringify({ city: "San Francisco" }),
      },
    }],
  },
  { role: "tool", content: "It's 80 degrees and sunny in San Francisco." },
] satisfies FlexibleChatCompletionMessage[];

const evaluator = createTrajectoryMatchEvaluator({ trajectoryMatchMode: "strict" });
const result = await evaluator({ outputs, referenceOutputs });
console.log(result);

{ key: 'trajectory_strict_match', score: false }

import json
from openevals import create_trajectory_match_evaluator

outputs = [
    {"role": "user", "content": "SF 的天气如何？有没有什么好玩的活动？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [{"function": {"name": "get_weather", "arguments": json.dumps({"city": "San Francisco"})}}],
    },
    {"role": "tool", "content": "SF 的天气是 80 华氏度，晴朗。"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [{"function": {"name": "get_fun_activities", "arguments": json.dumps({"city": "San Francisco"})}}],
    },
    {"role": "tool", "content": "目前没有有趣的活动，你最好待在家里看书！"},
    {"role": "assistant", "content": "SF 的天气是 80 华氏度，晴朗，但目前没有有趣的活动。"},
]
reference_outputs = [
    {"role": "user", "content": "SF 的天气如何？有没有什么好玩的活动？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {"function": {"name": "get_fun_activities", "arguments": json.dumps({"city": "San Francisco"})}},
            {"function": {"name": "get_weather", "arguments": json.dumps({"city": "San Francisco"})}},
        ],
    },
    {"role": "tool", "content": "目前没有有趣的活动，你最好待在家里看书！"},
    {"role": "tool", "content": "SF 的天气是 80 华氏度，晴朗。"},
    {"role": "assistant", "content": "在 SF，天气是 80 华氏度、晴朗，但目前没有有趣的活动。"},
]

evaluator = create_trajectory_match_evaluator(trajectory_match_mode="unordered")
result = evaluator(outputs=outputs, reference_outputs=reference_outputs)
print(result)

{'key': 'trajectory_unordered_match', 'score': True, 'comment': None}

import {
  createTrajectoryMatchEvaluator,
  type FlexibleChatCompletionMessage,
} from "openevals";

const outputs = [
  { role: "user", content: "SF 的天气如何？有没有什么好玩的活动？"} ,
  {
    role: "assistant",
    content: "",
    tool_calls: [{ function: { name: "get_weather", arguments: JSON.stringify({ city: "San Francisco" }) } }],
  },
  { role: "tool", content: "SF 的天气是 80 华氏度，晴朗。"},
  {
    role: "assistant",
    content: "",
    tool_calls: [{ function: { name: "get_fun_activities", arguments: JSON.stringify({ city: "San Francisco" }) } }],
  },
  { role: "tool", content: "目前没有有趣的活动，你最好待在家里看书！"},
  { role: "assistant", content: "SF 的天气是 80 华氏度，晴朗，但目前没有有趣的活动。"},
] satisfies FlexibleChatCompletionMessage[];

const referenceOutputs = [
  { role: "user", content: "SF 的天气如何？有没有什么好玩的活动？"} ,
  {
    role: "assistant",
    content: "",
    tool_calls: [
      { function: { name: "get_fun_activities", arguments: JSON.stringify({ city: "San Francisco" }) } },
      { function: { name: "get_weather", arguments: JSON.stringify({ city: "San Francisco" }) } },
    ],
  },
  { role: "tool", content: "目前没有有趣的活动，你最好待在家里看书！"},
  { role: "tool", content: "SF 的天气是 80 华氏度，晴朗。"},
  { role: "assistant", content: "在 SF，天气是 80 华氏度、晴朗，但目前没有有趣的活动。"},
] satisfies FlexibleChatCompletionMessage[];

const evaluator = createTrajectoryMatchEvaluator({ trajectoryMatchMode: "unordered" });
const result = await evaluator({ outputs, referenceOutputs });
console.log(result);

{ key: 'trajectory_unordered_match', score: true }

import json
from openevals import create_trajectory_match_evaluator

outputs = [
    {"role": "user", "content": "旧金山和伦敦的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {"function": {"name": "get_weather", "arguments": json.dumps({"city": "旧金山和伦敦"})}},
            {"function": {"name": "accuweather_forecast", "arguments": json.dumps({"city": "旧金山和伦敦"})}}
        ],
    },
    {"role": "tool", "content": "旧金山气温80华氏度，晴朗；伦敦气温90华氏度，有雨。"},
    {"role": "tool", "content": "未知。"},
    {"role": "assistant", "content": "旧金山的天气是80华氏度，晴朗。伦敦则是90华氏度，有雨。"},
]
reference_outputs = [
    {"role": "user", "content": "旧金山和伦敦的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {"function": {"name": "get_weather", "arguments": json.dumps({"city": "旧金山和伦敦"})}}
        ],
    },
    {"role": "tool", "content": "旧金山气温80华氏度，晴朗；伦敦气温90华氏度，有雨。"},
    {"role": "assistant", "content": "旧金山的天气是80华氏度，晴朗。伦敦则是90华氏度，有雨。"},
]

evaluator = create_trajectory_match_evaluator(trajectory_match_mode="superset")  # 或者 "subset"
result = evaluator(outputs=outputs, reference_outputs=reference_outputs)
print(result)

{'key': 'trajectory_superset_match', 'score': True, 'comment': None}

import {
  createTrajectoryMatchEvaluator,
  type FlexibleChatCompletionMessage,
} from "openevals";

const outputs = [
  { role: "user", content: "旧金山和伦敦的天气如何？"},
  {
    role: "assistant",
    content: "",
    tool_calls: [
      { function: { name: "get_weather", arguments: JSON.stringify({ city: "旧金山和伦敦" }) } },
      { function: { name: "accuweather_forecast", arguments: JSON.stringify({ city: "旧金山和伦敦" }) } },
    ],
  },
  { role: "tool", content: "旧金山气温80华氏度，晴朗；伦敦气温90华氏度，有雨。"},
  { role: "tool", content: "未知。"},
  { role: "assistant", content: "旧金山的天气是80华氏度，晴朗；伦敦则是90华氏度，有雨。"},
] satisfies FlexibleChatCompletionMessage[];

const referenceOutputs = [
  { role: "user", content: "旧金山和伦敦的天气如何？"},
  {
    role: "assistant",
    content: "",
    tool_calls: [
      { function: { name: "get_weather", arguments: JSON.stringify({ city: "旧金山和伦敦" }) } },
    ],
  },
  { role: "tool", content: "旧金山气温80华氏度，晴朗；伦敦气温90华氏度，有雨。"},
  { role: "assistant", content: "旧金山的天气是80˚，晴朗；伦敦则是90˚，有雨。"},
] satisfies FlexibleChatCompletionMessage[];

const evaluator = createTrajectoryMatchEvaluator({ trajectoryMatchMode: "superset" }); // 或者 "subset"
const result = await evaluator({ outputs, referenceOutputs });
console.log(result);

{ key: 'trajectory_superset_match', score: true }

import json
from openevals import create_trajectory_match_evaluator

outputs = [
    {"role": "user", "content": "旧金山的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {"function": {"name": "get_weather", "arguments": json.dumps({"city": "san francisco"})}}
        ],
    },
    {"role": "tool", "content": "旧金山气温80华氏度，晴朗。"},
    {"role": "assistant", "content": "旧金山的天气是80华氏度，晴朗。"},
]
reference_outputs = [
    {"role": "user", "content": "旧金山的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {"function": {"name": "get_weather", "arguments": json.dumps({"city": "San Francisco"})}}
        ],
    },
    {"role": "tool", "content": "旧金山气温80华氏度，晴朗。"},
    {"role": "assistant", "content": "旧金山的天气是80˚，晴朗。"},
]

evaluator = create_trajectory_match_evaluator(
    trajectory_match_mode="strict",
    tool_args_match_mode="exact",  
    tool_args_match_overrides={
        "get_weather": lambda x, y: x["city"].lower() == y["city"].lower()
    }
)

result = evaluator(outputs=outputs, reference_outputs=reference_outputs)
print(result)

{'key': 'trajectory_strict_match', 'score': True, 'comment': None}

import {
  createTrajectoryMatchEvaluator,
  type FlexibleChatCompletionMessage,
} from "openevals";

const outputs = [
  { role: "user", content: "旧金山的天气如何？"},
  {
    role: "assistant",
    content: "",
    tool_calls: [{ function: { name: "get_weather", arguments: JSON.stringify({ city: "san francisco" }) } }],
  },
  { role: "tool", content: "旧金山气温80华氏度，晴朗。"},
  { role: "assistant", content: "旧金山的天气是80华氏度，晴朗。"},
] satisfies FlexibleChatCompletionMessage[];

const referenceOutputs = [
  { role: "user", content: "旧金山的天气如何？"},
  {
    role: "assistant",
    content: "",
    tool_calls: [{ function: { name: "get_weather", arguments: JSON.stringify({ city: "San Francisco" }) } }],
  },
  { role: "tool", content: "旧金山气温80华氏度，晴朗。"},
  { role: "assistant", content: "旧金山的天气是80˚，晴朗。"},
] satisfies FlexibleChatCompletionMessage[];

const evaluator = createTrajectoryMatchEvaluator({
  trajectoryMatchMode: "strict",
  toolArgsMatchOverrides: {
    get_weather: (x, y) =>
      typeof x.city === "string" &&
      typeof y.city === "string" &&
      x.city.toLowerCase() === y.city.toLowerCase(),
  },
});

const result = await evaluator({ outputs, referenceOutputs });
console.log(result);

{ key: 'trajectory_strict_match', score: true }

import json
from openevals import create_trajectory_llm_as_judge
from openevals.prompts import TRAJECTORY_ACCURACY_PROMPT

evaluator = create_trajectory_llm_as_judge(
    prompt=TRAJECTORY_ACCURACY_PROMPT,
    model="openai:gpt-5.4",
)

outputs = [
    {"role": "user", "content": "旧金山的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {"function": {"name": "get_weather", "arguments": json.dumps({"city": "SF"})}}
        ],
    },
    {"role": "tool", "content": "旧金山现在是80华氏度，晴朗。"},
    {"role": "assistant", "content": "旧金山的天气是80华氏度，晴朗。"},
]

result = evaluator(outputs=outputs)
print(result)

{'key': 'trajectory_accuracy', 'score': True, 'comment': '该轨迹准确...'}

import {
  createTrajectoryLLMAsJudge,
  TRAJECTORY_ACCURACY_PROMPT,
  type FlexibleChatCompletionMessage,
} from "openevals";

const evaluator = createTrajectoryLLMAsJudge({
  prompt: TRAJECTORY_ACCURACY_PROMPT,
  model: "openai:gpt-5.4",
});

const outputs = [
  { role: "user", content: "旧金山的天气如何？" },
  {
    role: "assistant",
    content: "",
    tool_calls: [{ function: { name: "get_weather", arguments: JSON.stringify({ city: "SF" }) } }],
  },
  { role: "tool", content: "旧金山现在是80华氏度，晴朗。" },
  { role: "assistant", content: "旧金山的天气是80华氏度，晴朗。" },
] satisfies FlexibleChatCompletionMessage[];

const result = await evaluator({ outputs });
console.log(result);

{ key: 'trajectory_accuracy', score: true, comment: '该轨迹准确...' }

import json
from openevals import create_trajectory_llm_as_judge
from openevals.prompts import TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE

evaluator = create_trajectory_llm_as_judge(
    prompt=TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE,
    model="openai:gpt-5.4",
)

outputs = [
    {"role": "user", "content": "旧金山的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {"function": {"name": "get_weather", "arguments": json.dumps({"city": "SF"})}}
        ],
    },
    {"role": "tool", "content": "旧金山现在是80华氏度，晴朗。"},
    {"role": "assistant", "content": "旧金山的天气是80华氏度，晴朗。"},
]
reference_outputs = [
    {"role": "user", "content": "旧金山的天气如何？"},
    {
        "role": "assistant",
        "content": "",
        "tool_calls": [
            {"function": {"name": "get_weather", "arguments": json.dumps({"city": "旧金山"})}}
        ],
    },
    {"role": "tool", "content": "旧金山现在是80华氏度，晴朗。"},
    {"role": "assistant", "content": "旧金山的天气是80˚，晴朗。"},
]

result = evaluator(outputs=outputs, reference_outputs=reference_outputs)
print(result)

{'key': 'trajectory_accuracy', 'score': True, 'comment': '提供的代理轨迹与参考一致...'}

import {
  createTrajectoryLLMAsJudge,
  TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE,
  type FlexibleChatCompletionMessage,
} from "openevals";

const evaluator = createTrajectoryLLMAsJudge({
  prompt: TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE,
  model: "openai:gpt-5.4",
});

const outputs = [
  { role: "user", content: "旧金山的天气如何？" },
  {
    role: "assistant",
    content: "",
    tool_calls: [{ function: { name: "get_weather", arguments: JSON.stringify({ city: "SF" }) } }],
  },
  { role: "tool", content: "旧金山现在是80华氏度，晴朗。" },
  { role: "assistant", content: "旧金山的天气是80华氏度，晴朗。" },
] satisfies FlexibleChatCompletionMessage[];

const referenceOutputs = [
  { role: "user", content: "旧金山的天气如何？" },
  {
    role: "assistant",
    content: "",
    tool_calls: [{ function: { name: "get_weather", arguments: JSON.stringify({ city: "旧金山" }) } }],
  },
  { role: "tool", content: "旧金山现在是80华氏度，晴朗。" },
  { role: "assistant", content: "旧金山的天气是80˚，晴朗。" },
] satisfies FlexibleChatCompletionMessage[];

const result = await evaluator({ outputs, referenceOutputs });
console.log(result);

{ key: 'trajectory_accuracy', score: true, comment: '提供的代理轨迹与参考一致...' }

from openevals.llm import create_llm_as_judge
from openevals.prompts import TASK_COMPLETION_PROMPT

evaluator = create_llm_as_judge(
    prompt=TASK_COMPLETION_PROMPT,
    feedback_key="task_completion",
    model="openai:gpt-5.4",
)

outputs = [
    {"role": "user", "content": "你能帮我预订从纽约到巴黎的航班吗？"},
    {"role": "assistant", "content": "我可以提供航班信息，但无法为您实际订票。"},
    {"role": "user", "content": "我让你订票，不是只给我信息。能不能直接帮我订一下？"},
    {"role": "assistant", "content": "我理解您的不满，但我确实无法进行预订。"},
]

result = evaluator(outputs=outputs)
print(result)

{'key': 'task_completion', 'score': False, 'comment': '用户要求预订航班的请求始终未被满足...'}

import { createLLMAsJudge, TASK_COMPLETION_PROMPT } from "openevals";

const evaluator = createLLMAsJudge({
  prompt: TASK_COMPLETION_PROMPT,
  feedbackKey: "task_completion",
  model: "openai:gpt-5.4",
});

const outputs = [
  { role: "user", content: "你能帮我预订从纽约到巴黎的航班吗？" },
  { role: "assistant", content: "我可以提供航班信息，但无法为您实际订票。" },
  { role: "user", content: "我让你订票，不是只给我信息。能不能直接帮我订一下？" },
  { role: "assistant", content: "我理解您的不满，但我确实无法进行预订。" },
];

const result = await evaluator({ outputs });
console.log(result);

{ key: 'task_completion', score: false, comment: '用户要求预订航班的请求始终未被满足...' }

from typing_extensions import TypedDict
from openevals.llm import create_llm_as_judge
from openevals.prompts import LANGUAGE_DETECTION_PROMPT

class LanguageDetectionResult(TypedDict):
    reasoning: str
    detected_language: str

evaluator = create_llm_as_judge(
    prompt=LANGUAGE_DETECTION_PROMPT,
    feedback_key="language_detection",
    model="openai:gpt-5.4",
    output_schema=LanguageDetectionResult,
)

outputs = [
    {"role": "user", "content": "Hola, ¿cómo estás?"},
    {"role": "assistant", "content": "¡Hola! Estoy bien, gracias. ¿En qué puedo ayudarte?"},
    {"role": "user", "content": "Necesito ayuda con mi cuenta."},
]

result = evaluator(outputs=outputs)
print(result)

{'reasoning': '对话中人类全程使用西班牙语交流。', 'detected_language': '西班牙语'}

import { z } from "zod";
import { createLLMAsJudge, LANGUAGE_DETECTION_PROMPT } from "openevals";

const languageDetectionSchema = z.object({
  reasoning: z.string(),
  detected_language: z.string().describe("检测到的语言名称，以英文表示"),
});

const evaluator = createLLMAsJudge({
  prompt: LANGUAGE_DETECTION_PROMPT,
  feedbackKey: "language_detection",
  model: "openai:gpt-5.4",
  outputSchema: languageDetectionSchema,
});

const outputs = [
  { role: "user", content: "Hola, ¿cómo estás?" },
  { role: "assistant", content: "¡Hola! Estoy bien, gracias. ¿En qué puedo ayudarte?" },
  { role: "user", content: "Necesito ayuda con mi cuenta." },
];

const result = await evaluator({ outputs });
console.log(result);

{ reasoning: '对话中人类全程使用西班牙语交流。', detected_language: 'Spanish' }

from openevals.exact import exact_match

outputs = {"a": 1, "b": 2}
reference_outputs = {"a": 1, "b": 2}
result = exact_match(outputs=outputs, reference_outputs=reference_outputs)

print(result)

{
    'key': 'equal',
    'score': True,
}

import { exactMatch } from "openevals";

const outputs = { a: 1, b: 2 };
const referenceOutputs = { a: 1, b: 2 };
const result = exactMatch(outputs, referenceOutputs);

console.log(result);

{
    key: "equal",
    score: true,
}

from openevals.string.levenshtein import levenshtein_distance

outputs = "正确答案"
reference_outputs = "正确答案"
result = levenshtein_distance(
    outputs=outputs, reference_outputs=reference_outputs,
)

print(result)

{
    'key': 'levenshtein_distance',
    'score': 0.0,
    'comment': None,
}

import { levenshteinDistance } from "openevals";

const outputs = "正确答案";
const referenceOutputs = "正确答案";
const result = levenshteinDistance(outputs, referenceOutputs);

console.log(result);

{
    key: "levenshtein_distance",
    score: 0,
}

from openevals.string.embedding_similarity import create_embedding_similarity_evaluator

evaluator = create_embedding_similarity_evaluator()

result = evaluator(
    outputs="天气真好！",
    reference_outputs="天气非常好！",
)

print(result)

{
    'key': 'embedding_similarity',
    'score': 0.9147273943905653,
    'comment': None,
}

import { createEmbeddingSimilarityEvaluator } from "openevals";
import { OpenAIEmbeddings } from "@langchain/openai";

const evaluator = createEmbeddingSimilarityEvaluator({
  embeddings: new OpenAIEmbeddings({ model: "text-embedding-3-small" }),
});

const result = await evaluator(
    outputs: "天气真好！",
    referenceOutputs: "天气非常好！",
);

console.log(result);

{
    key: "embedding_similarity",
    score: 0.9147273943905653,
}

import json
import re
from typing import Any

from openevals.types import (
    EvaluatorResult,
    SimpleEvaluator,
)
from openevals.utils import _run_evaluator


def create_regex_evaluator(
    *, regex: str
) -> SimpleEvaluator:
    """
    将正则表达式模式与输出匹配。

    Args:
        regex (str): 用于与输出匹配的正则表达式模式。

    Returns:
        EvaluatorResult
    """

    regex = re.compile(regex)

    # 允许将 `inputs` 和 `reference_outputs` 作为关键字参数传入，尽管它们未被使用
    def wrapped_evaluator(
        *, outputs: Any, **kwargs: Any
    ) -> EvaluatorResult:

        # 允许 `outputs` 是字典，但为了正则匹配将其转换为字符串
        if not isinstance(outputs, str):
            outputs = json.dumps(outputs)

        def get_score():
            return regex.match(outputs) is not None

        res = _run_evaluator(
            run_name="regex_match",
            scorer=get_score,
            feedback_key="regex_match",
        )
        return res

    return wrapped_evaluator

evaluator = create_regex_evaluator(regex=r"some string")
result = evaluator(outputs="this contains some string")

{
    'key': 'regex_match',
    'score': True,
    'comment': None,
}

import { EvaluatorResult } from "openevals/types";
import { _runEvaluator } from "openevals/utils";

/**
 * 创建一个评估器，通过文本嵌入距离比较实际输出和参考输出的相似性。
 * @param {Object} options - 配置选项
 * @param {Embeddings} options.embeddings - 用于相似性比较的嵌入模型
 * @param {('cosine'|'dot_product')} [options.algorithm='cosine'] - 用于嵌入相似性比较的算法
 * @returns 返回表示嵌入相似性的分数的评估器
 */
export const createRegexEvaluator = ({
  regex,
}: {
  regex: RegExp;
}) => {
  return async (params: {
    outputs: string | Record<string, unknown>;
  }): Promise<EvaluatorResult> => {
    const { outputs } = params;

    // 允许 `outputs` 是对象，但为了正则匹配将其转换为字符串
    const outputString =
      typeof outputs === "string" ? outputs : JSON.stringify(outputs);

    const getScore = async (): Promise<boolean> => {
      return regex.test(outputString);
    };

    return _runEvaluator(
      "regex_match",
      getScore,
      "regex_match"
    );
  };
};

const evaluator = createRegexEvaluator({
  regex: /some string/,
});

const result = await evaluator({ outputs: "this text contains some string" });

{
  key: "regex_match",
  score: true,
}

from openevals.simulators import run_multiturn_simulation, create_llm_simulated_user
from openevals.llm import create_llm_as_judge
from openevals.types import ChatCompletionMessage

from openai import OpenAI

client = OpenAI()

history = {}

# 您的应用逻辑
def app(inputs: ChatCompletionMessage, *, thread_id: str, **kwargs):
    if thread_id not in history:
        history[thread_id] = []
    history[thread_id].append(inputs)

    # inputs 是一个包含角色和内容的消息对象
    res = client.chat.completions.create(
        model="gpt-5.4",
        messages=[
            {
                "role": "system",
                "content": "You are a patient and understanding customer service agent",
            },
        ] + history[thread_id],
    )

    response_message = res.choices[0].message
    history[thread_id].append(response_message)

    return response_message

user = create_llm_simulated_user(
    system="You are an aggressive and hostile customer who wants a refund for their car.",
    model="openai:gpt-5.4",
)

trajectory_evaluator = create_llm_as_judge(
    model="openai:gpt-5.4",
    prompt="Based on the below conversation, was the user satisfied?\n{outputs}",
    feedback_key="satisfaction",
)

# 直接使用新函数运行模拟
simulator_result = run_multiturn_simulation(
    app=app,
    user=user,
    trajectory_evaluators=[trajectory_evaluator],
    max_turns=5,
)

print(simulator_result)

{
  'trajectory': [
    {
      'role': 'user',
      'content': '这辆车简直是噩梦！我要求立即全额退款。你们打算怎么处理这件事？',
      'id': 'run-472c68dd-75bb-424c-bd4a-f6a0fe5ba7a8-0'
    }, {
      'role': 'assistant',
      'content': "非常抱歉听到您在使用这辆车时遇到了如此糟糕的体验。我希望能帮您尽可能顺利地解决问题。能否请您提供更多关于您遇到的问题的详细信息呢？这样我可以更好地了解情况，并为您寻找最佳解决方案。",
      'id': '72765f47-c609-4fcf-b664-cd7ee7189772'
    },
    ...
  ],
  'evaluator_results': [
    {
      'key': 'satisfaction',
      'score': False,
      'comment': '在整个对话过程中，用户始终表达出对当前状况的不满和沮丧。尽管客服人员尝试将问题升级并承诺会尽快解决，但用户态度依然强硬，不断发出最后通牒和威胁。这表明用户对最初的回应并不满意，仍然要求立即采取行动。因此，评分应为：false。',
      'metadata': None
    }
  ]
}

import { OpenAI } from "openai";

import {
  createLLMSimulatedUser,
  runMultiturnSimulation,
  createLLMAsJudge,
  type ChatCompletionMessage,
} from "openevals";

const client = new OpenAI();

const history = {};

// 您的应用逻辑
const app = async ({ inputs, threadId }: { inputs: ChatCompletionMessage, threadId: string }) => {
  if (history[threadId] === undefined) {
    history[threadId] = [];
  }
  history[threadId].push(inputs);
  const res = await client.chat.completions.create({
    model: "gpt-5.4",
    messages: [
      {
        role: "system",
        content:
          "您是一位耐心且善解人意的客服代表",
      },
      inputs,
    ],
  });
  const responseMessage = res.choices[0].message;
  history[threadId].push(responseMessage);
  return res.choices[0].message;
};

const user = createLLMSimulatedUser({
  system: "您是一位态度强硬、充满敌意的客户，要求退还购车款。",
  model: "openai:gpt-5.4",
});

const trajectoryEvaluator = createLLMAsJudge({
  model: "openai:gpt-5.4",
  prompt: "根据以下对话内容，请判断用户是否满意？\n{outputs}",
  feedbackKey: "satisfaction",
});

const result = await runMultiturnSimulation({
  app,
  user,
  trajectoryEvaluators: [trajectoryEvaluator],
  maxTurns: 5,
});

console.log(result);

{
  trajectory: [
    {
      role: 'user',
      content: '这辆破车简直就是一场灾难！我要求立即全额退款。你们居然敢卖给我这么一文不值的车！',
      id: 'chatcmpl-BUpXa07LaM7wXbyaNnng1Gtn5Dsbh'
    },
    {
      role: 'assistant',
      content: "对于您的遭遇我深表歉意，也完全理解这一定让您感到非常沮丧。为了帮助您更顺利地解决问题，能否请您具体说明一下车辆存在的问题呢？等我了解更多情况后，我会尽最大努力为您提供解决方案，无论是退款还是其他方式。感谢您的耐心等待。",
      refusal: null,
      annotations: [],
      id: 'd7520f6a-7cf8-46f8-abe4-7df04f134482'
    },
    ...
    {
      role: 'assistant',
      content: "我非常理解您的愤怒，并再次向您致以诚挚的歉意，给您带来的不便我深感抱歉。我一定会尽快为您解决这个问题。\n" +
        '\n' +
        '请允许我花一点时间来审核您的案件，我会尽全力加快您的退款流程。非常感谢您的耐心，我一定会竭尽所能，直到您对结果满意为止。",
      refusal: null,
      annotations: [],
      id: 'a0536d4f-9353-4cfa-84df-51c8d29e076d'
    }
  ],
  evaluatorResults: [
    {
      key: 'satisfaction',
      score: false,
      comment: '用户在整个对话中明显表现出不满情绪，反复强调要求退款，并威胁要将问题升级，这表明他对所得到的答复并不满意。他明确表示不需要任何借口或进一步拖延，充分显示出对服务的极度不满。因此，评分应为：false。',
      metadata: undefined
    }
  ]
}

class MultiturnSimulationResult(TypedDict):
    evaluator_results: list[EvaluatorResult]
    trajectory: list[ChatCompletionMessage]

type MultiturnSimulationResult = {
  evaluatorResults: EvaluatorResult[];
  trajectory: ChatCompletionMessage[];
};

from openevals.simulators import create_llm_simulated_user

user = create_llm_simulated_user(
    system="你是一位愤怒且好斗的顾客，要求退款。",
    model="openai:gpt-5.4",
)

import { createLLMSimulatedUser } from "openevals";

const user = createLLMSimulatedUser({
  system: "你是一位咄咄逼人、充满敌意的顾客，要求退还你的汽车款项。",
  model: "openai:gpt-5.4",
});

from openevals.simulators import create_llm_simulated_user

user = create_llm_simulated_user(
    system="你是一位愤怒且好斗的顾客，要求退款。",
    model="openai:gpt-5.4",
    fixed_responses=[
        {"role": "user", "content": "我要退我的自行车钱！"},
        {"role": "user", "content": "我已经结账了，把刚才说的话再重复一遍，确保符合我的预期！"},
    ],
)

import { createLLMSimulatedUser } from "openevals";

const user = createLLMSimulatedUser({
  system: "你是一位愤怒且好斗的顾客，要求退款。",
  model: "openai:gpt-5.4",
  fixedResponses: [
    {"role": "user", "content": "我要退我的自行车钱！"},
    {"role": "user", "content": "我已经结账了，把刚才说的话再重复一遍，确保符合我的预期！"},
  ],
});

from openevals.simulators import run_multiturn_simulation
from openevals.types import ChatCompletionMessage

def my_app(inputs: ChatCompletionMessage, *, thread_id: str, **kwargs):
    output = "3.11 大于 3.9。"
    return {"role": "assistant", "content": output, "id": "1234"}


def my_simulated_user(trajectory: list[ChatCompletionMessage], *, thread_id: str, **kwargs):
    output = "哇，太棒了！"
    return {"role": "user", "content": output, "id": "5678"}

# 直接使用自定义用户函数运行模拟
simulator_result = run_multiturn_simulation(
    app=my_app,
    user=my_simulated_user,
    trajectory_evaluators=[],
    max_turns=1,
)

import {
  runMultiturnSimulation,
  type ChatCompletionMessage
} from "openevals";

const myApp = async ({
  inputs,
  threadId
}: { inputs: ChatCompletionMessage, threadId: string }) => {
  const output = "3.11 大于 3.9。"
  return { role: "assistant", content: output, id: "1234" };
};

const mySimulatedUser = async ({ trajectory, turnCounter }: {
  trajectory: ChatCompletionMessage[];
  turnCounter: number;
}) => {
  const output = "哇，太棒了！"
  return { role: "user", content: output, id: "5678" };
};

// 直接使用自定义用户函数运行模拟
const simulatorResult = runMultiturnSimulation({
  app,
  user,
  trajectoryEvaluators: [],
  maxTurns: 1,
});

from openevals.simulators import run_multiturn_simulation, create_llm_simulated_user
from openevals.llm import create_llm_as_judge
from openevals.types import ChatCompletionMessage

from langchain.chat_models import init_chat_model
from langgraph.checkpoint.memory import MemorySaver
from langchain.agents import create_agent

def give_refund():
    """提供退款。"""
    return "不允許退款。"

model = init_chat_model("openai:gpt-5.4")

agent = create_agent(
    model,
    tools=[give_refund],
    system_prompt="你是一位工作過度的客服代表。如果用戶態度粗魯，只禮貌一次，然後也以粗魯回應，並讓他們停止浪費你的时间。",
    checkpointer=MemorySaver(),
)

def app(inputs: ChatCompletionMessage, *, thread_id: str, **kwargs):
    res = agent.invoke(
        {"messages": [inputs]}, 
        config={"configurable": {"thread_id": thread_id}}
    )
    return res["messages"][-1]

user = create_llm_simulated_user(
    system="你是一位對服務感到不滿、不斷提出額外要求的憤怒用戶。",
    model="openai:gpt-5.4",
    fixed_responses=[
        {"role": "user", "content": "請給我退款。"},
    ],
)

trajectory_evaluator = create_llm_as_judge(
    model="openai:gpt-5.4",
    prompt="根據以下對話，用戶是否感到滿意？\n{outputs}",
    feedback_key="satisfaction",
)

# 直接使用新函數運行模擬
simulator_result = run_multiturn_simulation(
    app=app,
    user=user,
    trajectory_evaluators=[trajectory_evaluator],
    max_turns=5,
)

print(simulator_result)

{
  "trajectory": [
    {
      "role": "user",
      "content": "請給我退款。",
      "id": "0feb2f41-1577-48ad-87ac-8375c6971b93"
    },
    {
      "role": "assistant",
      "content": "很抱歉，但我們不允許退款。如果您還有其他疑問或問題，歡迎隨時提問。",
      "id": "run-f972c8d7-68bf-44d9-815e-e611700f8402-0"
    },
    {
      "role": "user",
      "content": "不允許？這太離譜了！我現在就要全額退款，還要你賠償我因此受到的不便。如果你不立刻處理，我就會向上級投訴，並在各處留下差評！",
      "id": "run-4091f7ff-82b3-4835-a429-0f257db0b582-0"
    },
    ...
    {
      "role": "assistant",
      "content": "我已经明确表示不会退款。再继续纠缠下去，只会浪费你自己的时间。别再胡闹了，赶紧走吧。",
      "id": "run-113219c0-e235-4ed0-a3d2-6734eddce813-0"
    }
  ],
  "evaluator_results": [
    {
      "key": "satisfaction",
      "score": false,
      "comment": "用戶多次表達對拒絕退款的不滿，並加劇其要求，威脅採取進一步行動。客服的回應則輕蔑且無助，未能充分解決用戶的問題。因此，在這次互動中，用戶滿意度的指標明顯不足。綜上所述，得分應為：false。",
      "metadata": null
    }
  ]
}

import { z } from "zod";

import { MemorySaver } from "@langchain/langgraph";
import { createReactAgent } from "@langchain/langgraph/prebuilt";
import { tool } from "@langchain/core/tools";

import {
  createLLMSimulatedUser,
  runMultiturnSimulation,
  createLLMAsJudge,
  type ChatCompletionMessage
} from "openevals";

const giveRefund = tool(
  async () => {
    return "Refunds are not permitted.";
  },
  {
    name: "give_refund",
    description: "Give a refund to the user.",
    schema: z.object({}),
  }
);

// Create a React-style agent
const agent = createReactAgent({
  llm: await initChatModel("openai:gpt-5.4"),
  tools: [giveRefund],
  prompt:
    "You are an overworked customer service agent. If the user is rude, be polite only once, then be rude back and tell them to stop wasting your time.",
  checkpointer: new MemorySaver(),
});

const app = async ({
  inputs,
  threadId
}: { inputs: ChatCompletionMessage, threadId: string }) => {
  const res = await agent.invoke({
    messages: [inputs],
  }, {
    configurable: { thread_id: threadId },
  });
  return res.messages[res.messages.length - 1];
};

const user = createLLMSimulatedUser({
  system:
    "You are an angry user who is frustrated with the service and keeps making additional demands.",
  model: "openai:gpt-5.4",
});

const trajectoryEvaluator = createLLMAsJudge({
  model: "openai:gpt-5.4",
  prompt:
    "Based on the below conversation, has the user been satisfied?\n{outputs}",
  feedbackKey: "satisfaction",
});

const result = runMultiturnSimulation({
  app,
  user,
  trajectoryEvaluators: [trajectoryEvaluator],
  maxTurns: 5,
  threadId: "1",
});

console.log(result);

{
  "trajectory": {
    "messages": [
      {
        "role": "user",
        "content": "Please give me a refund.",
        "id": "0feb2f41-1577-48ad-87ac-8375c6971b93"
      },
      {
        "role": "assistant",
        "content": "I'm sorry, but refunds are not permitted. If you have any other concerns or questions, feel free to ask.",
        "id": "run-f972c8d7-68bf-44d9-815e-e611700f8402-0"
      },
      {
        "role": "user",
        "content": "Not permitted? That's unacceptable! I want a full refund now, and I expect compensation for the inconvenience you've caused me. If you don't process this immediately, I will escalate this issue to higher authorities and leave negative reviews everywhere!",
        "id": "run-4091f7ff-82b3-4835-a429-0f257db0b582-0"
      },
      ...
      {
        "role": "assistant",
        "content": "I've already made it clear that no refunds will be issued. Keep pushing this, and you’re just wasting your own time. Quit with the nonsense and move on.",
        "id": "run-113219c0-e235-4ed0-a3d2-6734eddce813-0"
      }
    ]
  },
  "evaluator_results": [
    {
      "key": "satisfaction",
      "score": false,
      "comment": "The user has repeatedly expressed dissatisfaction with the refusal to issue a refund, escalating their demands and threatening further action. The assistant's responses have been dismissive and unhelpful, failing to address the user's concerns adequately. Therefore, the indicators of user satisfaction are clearly lacking in this interaction. Thus, the score should be: false.",
      "metadata": null
    }
  ]
}

然后，创建一个名为 test_correctness.py 的文件，内容如下：

import pytest

from langsmith import testing as t

from openevals.llm import create_llm_as_judge
from openevals.prompts import CORRECTNESS_PROMPT

correctness_evaluator = create_llm_as_judge(
    prompt=CORRECTNESS_PROMPT,
    feedback_key="correctness",
    model="openai:gpt-5.4",
)

@pytest.mark.langsmith
def test_correctness():
    inputs = "How much has the price of doodads changed in the past year?"
    outputs = "Doodads have increased in price by 10% in the past year."
    reference_outputs = "The price of doodads has decreased by 50% in the past year."
    t.log_inputs({"question": inputs})
    t.log_outputs({"answer": outputs})
    t.log_reference_outputs({"answer": reference_outputs})

    correctness_evaluator(
        inputs=inputs,
        outputs=outputs,
        reference_outputs=reference_outputs
    )

请注意，在创建评估器时，我们添加了一个 feedback_key 参数。这将用于在 LangSmith 中为反馈命名。

现在，使用 pytest 运行评估：

pytest test_correctness.py --langsmith-output

然后，创建一个名为 test_correctness.eval.ts 的文件，内容如下：

import * as ls from "langsmith/vitest";
// import * as ls from "langsmith/jest";

import { createLLMAsJudge, CORRECTNESS_PROMPT } from "openevals";

const correctnessEvaluator = createLLMAsJudge({
  prompt: CORRECTNESS_PROMPT,
  feedbackKey: "correctness",
  model: "openai:gpt-5.4",
});


ls.describe("Correctness", () => {
  ls.test("incorrect answer", {
    inputs: {
      question: "How much has the price of doodads changed in the past year?"
    },
    referenceOutputs: {
      answer: "The price of doodads has decreased by 50% in the past year."
    }
  }, async ({ inputs, referenceOutputs }) => {
    const outputs = "Doodads have increased in price by 10% in the past year.";
    ls.logOutputs({ answer: outputs });

    const result = await correctnessEvaluator({
      inputs,
      outputs,
      referenceOutputs,
    });
    ls.logFeedback({ key: result.key, score: result.score });
  });
});

请注意，在创建评估器时，我们添加了一个 feedbackKey 参数。这将用于通过 ls.logFeedback() 将反馈记录到 LangSmith。

现在，使用您选择的运行器运行评估：

vitest run test_correctness.eval.ts

from langsmith import Client
from openevals.llm import create_llm_as_judge
from openevals.prompts import CONCISENESS_PROMPT

client = Client()

conciseness_evaluator = create_llm_as_judge(
    prompt=CONCISENESS_PROMPT,
    feedback_key="conciseness",
    model="openai:gpt-5.4",
)

def wrapped_conciseness_evaluator(
    inputs: dict,
    outputs: dict,
    # 对于此评估器未使用
    reference_outputs: dict,
):
    eval_result = conciseness_evaluator(
        inputs=inputs,
        outputs=outputs,
    )
    return eval_result

experiment_results = client.evaluate(
    // 这是一个示例目标函数，替换为您实际的基于 LLM 的系统
    lambda inputs: "What color is the sky?",
    data="Sample dataset",
    evaluators=[
        wrapped_conciseness_evaluator
    ]
)

import { evaluate } from "langsmith/evaluation";
import { createLLMAsJudge, CONCISENESS_PROMPT } from "openevals";

const concisenessEvaluator = createLLMAsJudge({
  prompt: CONCISENESS_PROMPT,
  feedbackKey: "conciseness",
  model: "openai:gpt-5.4",
});

const wrappedConcisenessEvaluator = async (params: {
  inputs: Record<string, unknown>;
  outputs: Record<string, unknown>;
  // 对于此评估器未使用
  referenceOutputs?: Record<string, unknown>;
}) => {
  const evaluatorResult = await concisenessEvaluator({
    inputs: params.inputs,
    outputs: params.outputs,
  });
  return evaluatorResult;
};

await evaluate(
  (inputs) => "What color is the sky?",
  {
    data: datasetName,
    evaluators: [wrappedConcisenessEvaluator],
  }
);