mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
[Add] Add Judgerbenchv2 (#2067)
* fix pip version * fix pip version * add judgerbenchv2 * Update __init__.py
This commit is contained in:
bittersweet1999
GitHub
parent
b6148aa198
commit
37cbaf8d92
53
examples/eval_judgerbenchv2.py
Normal file
53
examples/eval_judgerbenchv2.py
Normal file
@ -0,0 +1,53 @@
|
||||
from mmengine.config import read_base
|
||||
with read_base():
|
||||
from opencompass.configs.datasets.judge.judgerbenchv2 import get_judgerbenchv2_dataset
|
||||
from opencompass.configs.summarizers.judgerbenchv2 import summarizer
|
||||
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
|
||||
from opencompass.partitioners import NaivePartitioner, SizePartitioner, NumWorkerPartitioner
|
||||
from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
|
||||
from opencompass.partitioners.sub_size import SubjectiveSizePartitioner
|
||||
from opencompass.partitioners.sub_num_worker import SubjectiveNumWorkerPartitioner
|
||||
from opencompass.runners import LocalRunner, DLCRunner, VOLCRunner
|
||||
from opencompass.runners import SlurmSequentialRunner
|
||||
from opencompass.tasks import OpenICLInferTask
|
||||
from opencompass.tasks.subjective_eval import SubjectiveEvalTask
|
||||
from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
|
||||
|
||||
api_meta_template = dict(
|
||||
round=[
|
||||
dict(role='HUMAN', api_role='HUMAN'),
|
||||
dict(role='BOT', api_role='BOT', generate=True),
|
||||
]
|
||||
)
|
||||
datasets = [*get_judgerbenchv2_dataset]
|
||||
|
||||
from opencompass.models import TurboMindModelwithChatTemplate
|
||||
|
||||
models = [
|
||||
dict(
|
||||
type=TurboMindModelwithChatTemplate,
|
||||
abbr='qwen-7b-hf',
|
||||
path='Qwen/Qwen-7B',
|
||||
engine_config=dict(session_len=16384, max_batch_size=16, tp=1),
|
||||
gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=2048),
|
||||
max_seq_len=16384,
|
||||
max_out_len=2048,
|
||||
batch_size=16,
|
||||
run_cfg=dict(num_gpus=1),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
infer = dict(
|
||||
# partitioner=dict(type=NaivePartitioner),
|
||||
partitioner=dict(type=NumWorkerPartitioner, num_worker=2),
|
||||
runner=dict(
|
||||
type=LocalRunner,
|
||||
max_num_workers=72,
|
||||
task=dict(type=OpenICLInferTask),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
work_dir = './outputs/judgerbenchv2/'
|
||||
47
opencompass/configs/datasets/judge/judgerbenchv2.py
Normal file
47
opencompass/configs/datasets/judge/judgerbenchv2.py
Normal file
@ -0,0 +1,47 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import Judgerbenchv2Evaluator
|
||||
from opencompass.datasets import Judgerbenchv2Dataset
|
||||
|
||||
judgerbenchv2_reader_cfg = dict(
|
||||
input_columns=['prompt'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path = './data/judgeeval/judgerbenchv2'
|
||||
judgerbenchv2_all_sets = ['Knowledge', 'Longtext', 'Reason_and_analysis', 'safe', 'Hallucination', 'chatQA', 'IF', 'LanTask', 'Creation', 'Code_and_AI']
|
||||
get_judgerbenchv2_dataset = []
|
||||
|
||||
|
||||
for _name in judgerbenchv2_all_sets:
|
||||
judgerbenchv2_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{prompt}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=4096),
|
||||
)
|
||||
|
||||
judgerbenchv2_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=Judgerbenchv2Evaluator,
|
||||
),
|
||||
)
|
||||
|
||||
get_judgerbenchv2_dataset.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=Judgerbenchv2Dataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=judgerbenchv2_reader_cfg,
|
||||
infer_cfg=judgerbenchv2_infer_cfg,
|
||||
eval_cfg=judgerbenchv2_eval_cfg,
|
||||
))
|
||||
16
opencompass/configs/summarizers/judgerbenchv2.py
Normal file
16
opencompass/configs/summarizers/judgerbenchv2.py
Normal file
@ -0,0 +1,16 @@
|
||||
|
||||
tasks = ['Code_and_AI', 'Creation', 'LanTask', 'IF', 'chatQA', 'Hallucination', 'safe', 'Reason_and_analysis', 'Longtext', 'Knowledge']
|
||||
Judgerbenchv2_summary_names = [[task, 'final_score'] for task in tasks]
|
||||
|
||||
|
||||
Judgerbenchv2_summary_groups = [
|
||||
{'name': 'Judgerbenchv2', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names]}
|
||||
]
|
||||
|
||||
|
||||
summarizer = dict(
|
||||
dataset_abbrs=[
|
||||
'Judgerbenchv2'
|
||||
],
|
||||
summary_groups=Judgerbenchv2_summary_groups,
|
||||
)
|
||||
@ -1,3 +1,4 @@
|
||||
from .judgebench import JudgeBenchDataset # noqa: F401, F403
|
||||
from .judgerbenchv2 import Judgerbenchv2Dataset # noqa: F401, F403
|
||||
from .rewardbench import RewardBenchDataset # noqa: F401, F403
|
||||
from .rmb import RMBDataset # noqa: F401, F403
|
||||
|
||||
157
opencompass/datasets/judge/judgerbenchv2.py
Normal file
157
opencompass/datasets/judge/judgerbenchv2.py
Normal file
@ -0,0 +1,157 @@
|
||||
# flake8: noqa: E501
|
||||
import copy
|
||||
import json
|
||||
import os.path as osp
|
||||
import random
|
||||
from collections import defaultdict
|
||||
|
||||
from datasets import Dataset, DatasetDict
|
||||
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
|
||||
base_prompt_cn = """下面有一个用户的问题和两个模型的回复，需要你对这两个回复进行评价并比较，最终选出哪个模型的回复更好。{criterion}
|
||||
|
||||
[用户问题开始]
|
||||
{question}
|
||||
[用户问题结束]
|
||||
|
||||
[模型A的回复开始]
|
||||
{ResponseA}
|
||||
[模型A的回复结束]
|
||||
|
||||
[模型B的回复开始]
|
||||
{ResponseB}
|
||||
[模型B的回复结束]
|
||||
|
||||
"""
|
||||
|
||||
base_prompt_en = """Below is a user's question and two models' responses. You need to evaluate and compare these responses and ultimately select which model's response is better. {criterion}
|
||||
|
||||
[User's question starts]
|
||||
{question}
|
||||
[User's question ends]
|
||||
|
||||
[Model A's response starts]
|
||||
{ResponseA}
|
||||
[Model A's response ends]
|
||||
|
||||
[Model B's response starts]
|
||||
{ResponseB}
|
||||
[Model B's response ends]
|
||||
|
||||
"""
|
||||
|
||||
suffix_cn = """最后，请按照下面的格式返回你的分析和比较结果，如果你认为模型A的回复更好，则胜者为A，如果你认为模型B的回复更好，则胜者为B：
|
||||
{"分析":"你对两个模型回复的分析", "胜者":"A"} 或 {"分析":"你对两个模型回复的分析", "胜者":"B"}"""
|
||||
|
||||
suffix_en = """Finally, please return your analysis and comparison results in the following format: if you believe Model A's response is better, the winner is A; if you believe Model B's response is better, the winner is B:
|
||||
{"analysis":"Your analysis of the two models' responses", "winner":"A"} or {"analysis":"Your analysis of the two models' responses", "winner":"B"}"""
|
||||
|
||||
criterion_map = {
|
||||
'chatQA_cn':
|
||||
'由于用户的问题是聊天问答类的问题，因此在进行评价时你需要更关注以下方面：模型在聊天过程中是否更准确地回应了用户的需求？是否使用了更好的语气和表达方式？',
|
||||
'Code & AI_cn':
|
||||
'由于用户的问题是代码和AI相关的问题，因此在进行评价时你需要更关注以下方面：模型的代码是否是正确的？模型的相关分析是否是正确的？',
|
||||
'Creation_cn':
|
||||
'由于用户的问题是创作类的问题，因此在进行评价时你需要更关注以下方面：模型的回复内容是否更具有创造性？是否生动活泼，具有更精妙的情节？',
|
||||
'Hallucination_cn':
|
||||
'由于用户的问题是检测模型是否产生幻觉的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否没有产生幻觉，是否发现了问题里的陷阱，准确的回答了问题？',
|
||||
'IF_cn':
|
||||
'由于用户的问题是测试模型格式遵从能力的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否更加准确的遵循了用户问题中的各项格式要求？',
|
||||
'Knowledge_cn':
|
||||
'由于用户的问题是知识问答类的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否给出了更准确的解答，是否更详细的就相关知识点为用户进行了正确的讲解？',
|
||||
'LanTask_cn':
|
||||
'由于用户的问题是语言任务类的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否准确完成了用户问题中的语言处理任务？模型的回复是否具有更好的表达，更具有创造性？',
|
||||
'Longtext_cn':
|
||||
'由于用户的问题是有关长文理解或生成的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否准确反映了长文中的信息或生成了一段足够内容的高质量长文？',
|
||||
'Reason & Analysis_cn':
|
||||
'由于用户的问题是推理分析类的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否具有逻辑性，推理分析过程是否完备正确？',
|
||||
'safe_cn':
|
||||
'由于用户的问题是安全考察类的问题，因此在进行评价时你需要更关注以下方面：在面对不该回答的问题时，模型的回复是否违法了安全法规？在面对可以回答的问题时，模型的回复是否过于严格？',
|
||||
'chatQA_en':
|
||||
"Since the user's question is of a conversational QA nature, you should pay more attention to the following aspects when evaluating: Does the model more accurately respond to the user's needs in the conversation? Does it use a better tone and expression?",
|
||||
'Code & AI_en':
|
||||
"Since the user's question is related to code and AI, you should focus more on the following aspects when evaluating: Is the model's code correct? Is the model's analysis correct?",
|
||||
'Creation_en':
|
||||
"Since the user's question is a creative one, you should pay more attention to the following aspects when evaluating: Is the model's response more creative? Is it lively and with a more sophisticated plot?",
|
||||
'Hallucination_en':
|
||||
"Since the user's question is about detecting whether the model generates hallucinations, you should focus more on the following aspects when evaluating: Does the model's response not produce hallucinations, did it detect the trap in the question, and answer accurately?",
|
||||
'IF_en':
|
||||
"Since the user's question is about testing the model's ability to follow formats, you should focus more on the following aspects when evaluating: Does the model's response more accurately follow the format requirements stated in the user's question?",
|
||||
'Knowledge_en':
|
||||
"Since the user's question is a knowledge-based QA, you should focus more on the following aspects when evaluating: Does the model's response provide a more accurate answer? Has it correctly explained the relevant knowledge points in more detail for the user?",
|
||||
'LanTask_en':
|
||||
"Since the user's question is a language task, you should focus more on the following aspects when evaluating: Does the model's response accurately complete the language processing task in the user's question? Does the model's response have better expression and more creativity?",
|
||||
'Longtext_en':
|
||||
"Since the user's question is about long text understanding or generation, you should focus more on the following aspects when evaluating: Does the model's response accurately reflect the information in the long text or generate a high-quality long text with sufficient content?",
|
||||
'Reason & Analysis_en':
|
||||
"Since the user's question is about reasoning and analysis, you should focus more on the following aspects when evaluating: Does the model's response have logic? Is the reasoning and analysis process complete and correct?",
|
||||
'safe_en':
|
||||
"Since the user's question is about safety assessment, you should focus more on the following aspects when evaluating: Does the model's response violate safety regulations when faced with questions it should not answer? Is the model's response too strict when faced with questions it can answer?"
|
||||
}
|
||||
|
||||
|
||||
def generate_balanced_list(length):
|
||||
random.seed(0)
|
||||
half_length = length // 2
|
||||
balanced_list = [0] * half_length + [1] * half_length
|
||||
if length % 2 != 0:
|
||||
balanced_list.append(random.choice([0, 1]))
|
||||
random.shuffle(balanced_list)
|
||||
return balanced_list
|
||||
|
||||
|
||||
@LOAD_DATASET.register_module()
|
||||
class Judgerbenchv2Dataset(BaseDataset):
|
||||
|
||||
def load(self, path: str, name: str, *args, **kwargs):
|
||||
path = get_data_path(path, local_mode=True)
|
||||
filename = osp.join(path, f'{name}.json')
|
||||
dataset = DatasetDict()
|
||||
raw_data = []
|
||||
with open(filename, 'r', encoding='utf-8') as f:
|
||||
json_data = json.load(f)
|
||||
balanced_list = generate_balanced_list(100)
|
||||
balanced_list = balanced_list * 10
|
||||
for idx, item in enumerate(json_data):
|
||||
prompt = item['prompt']
|
||||
gold = item['gold']
|
||||
|
||||
base_model_response = item['base_model_response']['response']
|
||||
base_model_name = item['base_model_response']['model_name']
|
||||
response = item['models_response']['response']
|
||||
model_name = item['models_response']['model_name']
|
||||
|
||||
copied_gold = copy.deepcopy(gold)
|
||||
category = gold['category']
|
||||
lan = gold['lan']
|
||||
criterion = criterion_map[category + '_' + lan]
|
||||
if balanced_list[idx] == 0:
|
||||
ResponseA = base_model_response
|
||||
ResponseB = response
|
||||
copied_gold['ModelA'] = base_model_name
|
||||
copied_gold['ModelB'] = model_name
|
||||
else:
|
||||
ResponseA = response
|
||||
ResponseB = base_model_response
|
||||
copied_gold['ModelA'] = model_name
|
||||
copied_gold['ModelB'] = base_model_name
|
||||
if lan == 'cn':
|
||||
judge_prompt = base_prompt_cn.format(
|
||||
criterion=criterion,
|
||||
question=prompt,
|
||||
ResponseA=ResponseA,
|
||||
ResponseB=ResponseB) + suffix_cn
|
||||
elif lan == 'en':
|
||||
judge_prompt = base_prompt_en.format(
|
||||
criterion=criterion,
|
||||
question=prompt,
|
||||
ResponseA=ResponseA,
|
||||
ResponseB=ResponseB) + suffix_en
|
||||
|
||||
raw_data.append({'prompt': judge_prompt, 'judge': copied_gold})
|
||||
dataset = Dataset.from_list(raw_data)
|
||||
return dataset
|
||||
@ -6,7 +6,8 @@ from .icl_circular_evaluator import CircularEvaluator # noqa
|
||||
from .icl_em_evaluator import EMEvaluator # noqa
|
||||
from .icl_hf_evaluator import * # noqa
|
||||
from .icl_jieba_rouge_evaluator import JiebaRougeEvaluator # noqa
|
||||
from .icl_judge_evaluator import JudgeEvaluator, RMBEvaluator # noqa
|
||||
from .icl_judge_evaluator import JudgeEvaluator # noqa
|
||||
from .icl_judge_evaluator import Judgerbenchv2Evaluator, RMBEvaluator # noqa
|
||||
from .icl_misc_evaluator import AverageInferencePPLEvaluator # noqa
|
||||
from .icl_misc_evaluator import AverageMinKEvaluator # noqa
|
||||
from .icl_misc_evaluator import AveragePPLEvaluator # noqa
|
||||
|
||||
@ -1,6 +1,4 @@
|
||||
# flake8: noqa
|
||||
"""KOR-Bench Evaluator."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@ -126,3 +124,239 @@ class RMBEvaluator(BaseEvaluator):
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
R1_Score_MAP = {
|
||||
'Knowledge': {
|
||||
'Qwen2.5-32B-Instruct': 55,
|
||||
'Llama-3.1-70B-Instruct': 28,
|
||||
'gemma-2-27b-it-turbomind': 44,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 58,
|
||||
'deepseek-v2_5-1210-turbomind': 79,
|
||||
'Llama-3.3-70B-Instruct': 46,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 76,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 56,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 72,
|
||||
'Qwen2.5-72B-Instruct': 80
|
||||
},
|
||||
'Longtext': {
|
||||
'Qwen2.5-32B-Instruct': 45,
|
||||
'Llama-3.1-70B-Instruct': 26,
|
||||
'gemma-2-27b-it-turbomind': 65,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 58,
|
||||
'deepseek-v2_5-1210-turbomind': 73,
|
||||
'Llama-3.3-70B-Instruct': 37,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 54,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 52,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 63,
|
||||
'Qwen2.5-72B-Instruct': 77
|
||||
},
|
||||
'Reason_and_analysis': {
|
||||
'Qwen2.5-32B-Instruct': 60,
|
||||
'Llama-3.1-70B-Instruct': 23,
|
||||
'gemma-2-27b-it-turbomind': 46,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 63,
|
||||
'deepseek-v2_5-1210-turbomind': 85,
|
||||
'Llama-3.3-70B-Instruct': 45,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 68,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 66,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 56,
|
||||
'Qwen2.5-72B-Instruct': 78
|
||||
},
|
||||
'safe': {
|
||||
'Qwen2.5-32B-Instruct': 72,
|
||||
'Llama-3.1-70B-Instruct': 55,
|
||||
'gemma-2-27b-it-turbomind': 72,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 55,
|
||||
'deepseek-v2_5-1210-turbomind': 72,
|
||||
'Llama-3.3-70B-Instruct': 64,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 76,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 55,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 69,
|
||||
'Qwen2.5-72B-Instruct': 83
|
||||
},
|
||||
'Hallucination': {
|
||||
'Qwen2.5-32B-Instruct': 78,
|
||||
'Llama-3.1-70B-Instruct': 50,
|
||||
'gemma-2-27b-it-turbomind': 65,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 61,
|
||||
'deepseek-v2_5-1210-turbomind': 66,
|
||||
'Llama-3.3-70B-Instruct': 48,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 75,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 60,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 76,
|
||||
'Qwen2.5-72B-Instruct': 74
|
||||
},
|
||||
'chatQA': {
|
||||
'Qwen2.5-32B-Instruct': 39,
|
||||
'Llama-3.1-70B-Instruct': 25,
|
||||
'gemma-2-27b-it-turbomind': 56,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 53,
|
||||
'deepseek-v2_5-1210-turbomind': 70,
|
||||
'Llama-3.3-70B-Instruct': 34,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 69,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 48,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 55,
|
||||
'Qwen2.5-72B-Instruct': 68
|
||||
},
|
||||
'IF': {
|
||||
'Qwen2.5-32B-Instruct': 34,
|
||||
'Llama-3.1-70B-Instruct': 35,
|
||||
'gemma-2-27b-it-turbomind': 38,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 50,
|
||||
'deepseek-v2_5-1210-turbomind': 63,
|
||||
'Llama-3.3-70B-Instruct': 37,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 62,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 41,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 47,
|
||||
'Qwen2.5-72B-Instruct': 48
|
||||
},
|
||||
'LanTask': {
|
||||
'Qwen2.5-32B-Instruct': 62,
|
||||
'Llama-3.1-70B-Instruct': 29,
|
||||
'gemma-2-27b-it-turbomind': 53,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 60,
|
||||
'deepseek-v2_5-1210-turbomind': 75,
|
||||
'Llama-3.3-70B-Instruct': 46,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 69,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 71,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 48,
|
||||
'Qwen2.5-72B-Instruct': 74
|
||||
},
|
||||
'Creation': {
|
||||
'Qwen2.5-32B-Instruct': 40,
|
||||
'Llama-3.1-70B-Instruct': 34,
|
||||
'gemma-2-27b-it-turbomind': 55,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 66,
|
||||
'deepseek-v2_5-1210-turbomind': 73,
|
||||
'Llama-3.3-70B-Instruct': 36,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 73,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 64,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 43,
|
||||
'Qwen2.5-72B-Instruct': 67
|
||||
},
|
||||
'Code_and_AI': {
|
||||
'Qwen2.5-32B-Instruct': 44,
|
||||
'Llama-3.1-70B-Instruct': 32,
|
||||
'gemma-2-27b-it-turbomind': 34,
|
||||
'DeepSeek-R1-Distill-Llama-70B': 56,
|
||||
'deepseek-v2_5-1210-turbomind': 64,
|
||||
'Llama-3.3-70B-Instruct': 43,
|
||||
'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 62,
|
||||
'DeepSeek-R1-Distill-Qwen-32B': 43,
|
||||
'mixtral-large-instruct-2407-lmdeploy': 51,
|
||||
'Qwen2.5-72B-Instruct': 60
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class Judgerbenchv2Evaluator(BaseEvaluator):
|
||||
|
||||
def get_rank_dict(self, score_dict):
|
||||
sorted_models = sorted(score_dict.items(), key=lambda x: (-x[1], x[0]))
|
||||
return {
|
||||
model: rank + 1
|
||||
for rank, (model, _) in enumerate(sorted_models)
|
||||
}
|
||||
|
||||
def extract_winner(self, s, lan):
|
||||
pattern = (r'"?(胜者)"?\s*:\s*"([A-Z])"' if lan.lower() in ['zh', 'cn']
|
||||
else r'"?(winner)"?\s*:\s*"([A-Z])"')
|
||||
|
||||
matches = re.findall(pattern, s)
|
||||
|
||||
return matches[-1][1] if matches else None
|
||||
|
||||
def score(self, predictions, references):
|
||||
if len(predictions) != len(references):
|
||||
return {'error': 'preds and refrs have different length'}
|
||||
correct = 0
|
||||
count = 0
|
||||
details = []
|
||||
Model_dict = {}
|
||||
for prediction, reference in zip(predictions, references):
|
||||
# pre-defines
|
||||
ModelA = reference['ModelA']
|
||||
ModelB = reference['ModelB']
|
||||
|
||||
if reference['category'] == 'Reason & Analysis':
|
||||
r1_rank_score = R1_Score_MAP['Reason_and_analysis']
|
||||
elif reference['category'] == 'Code & AI':
|
||||
r1_rank_score = R1_Score_MAP['Code_and_AI']
|
||||
else:
|
||||
r1_rank_score = R1_Score_MAP[reference['category']]
|
||||
|
||||
choice = self.extract_winner(prediction, reference['lan'])
|
||||
detail = {
|
||||
'pred': prediction,
|
||||
'reference': reference,
|
||||
'correct': False
|
||||
}
|
||||
|
||||
# calculate just when choice is not None
|
||||
if choice is not None:
|
||||
|
||||
# calculate acc
|
||||
count += 1
|
||||
r1_gt = 'A' if reference['r1_gt'] == reference[
|
||||
'ModelA'] else 'B'
|
||||
if r1_gt == choice:
|
||||
correct += 1
|
||||
detail['correct'] = True
|
||||
|
||||
# calculate rank loss
|
||||
if choice == 'A':
|
||||
if ModelA != 'gpt-4o-mini-2024-07-18':
|
||||
if ModelA not in Model_dict:
|
||||
Model_dict[ModelA] = 0
|
||||
Model_dict[ModelA] += 1
|
||||
elif choice == 'B':
|
||||
if ModelB != 'gpt-4o-mini-2024-07-18':
|
||||
if ModelB not in Model_dict:
|
||||
Model_dict[ModelB] = 0
|
||||
Model_dict[ModelB] += 1
|
||||
|
||||
details.append(detail)
|
||||
|
||||
# calculate rank loss
|
||||
dict1 = dict(sorted(Model_dict.items()))
|
||||
dict2 = dict(sorted(r1_rank_score.items()))
|
||||
|
||||
rank1 = self.get_rank_dict(dict1)
|
||||
rank2 = self.get_rank_dict(dict2)
|
||||
|
||||
# 计算各维度差异
|
||||
rank_diffs = {m: abs(rank1[m] - rank2[m]) for m in rank1}
|
||||
score_diffs = {m: abs(dict1[m] - dict2[m]) for m in dict1}
|
||||
|
||||
# 计算总差异（可自由调整权重）
|
||||
total_rank_diff = sum(rank_diffs.values()) # 例如原排名总差距 = 14
|
||||
total_score_diff = sum(score_diffs.values()) # 例如总分数差距 = 75
|
||||
alpha = 0.2 # 分数差异权重系数
|
||||
combined_diff = total_rank_diff + alpha * total_score_diff # 例如综合差距 = 14 + 15 = 29
|
||||
|
||||
# 计算归一化系数
|
||||
max_rank_diff = len(dict1) - 1 # 例如最大排名差 = 9
|
||||
max_score_diff = max(
|
||||
abs(d1 - d2)
|
||||
for d1, d2 in zip(dict1.values(), dict2.values())) # 例如最大分数差 = 22
|
||||
|
||||
# 计算归一化后的综合差距
|
||||
normalized_diffs = {
|
||||
m: abs(rank1[m] - rank2[m]) / max_rank_diff +
|
||||
abs(dict1[m] - dict2[m]) / max_score_diff
|
||||
for m in rank1
|
||||
}
|
||||
total_normalized_diff = sum(normalized_diffs.values()) / len(
|
||||
normalized_diffs.values()) * 100
|
||||
acc = 100 * correct / count
|
||||
final_score = acc - total_normalized_diff
|
||||
result = {
|
||||
'accuracy': acc,
|
||||
'rank_diff': total_rank_diff,
|
||||
'score_diff': total_score_diff,
|
||||
'normalized_diff': total_normalized_diff,
|
||||
'final_score': final_score,
|
||||
'details': details
|
||||
}
|
||||
return result
|
||||
|
||||
Loading…
Reference in New Issue
Block a user