mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
[Feature] Add Judgerbench and reorg subeval (#1593)
* fix pip version * fix pip version * update (#1522) Co-authored-by: zhulin1 <zhulin1@pjlab.org.cn> * [Feature] Update Models (#1518) * Update Models * Update * Update humanevalx * Update * Update * [Feature] Dataset prompts update for ARC, BoolQ, Race (#1527) add judgerbench and reorg sub add judgerbench and reorg subeval add judgerbench and reorg subeval * add judgerbench and reorg subeval * add judgerbench and reorg subeval * add judgerbench and reorg subeval * add judgerbench and reorg subeval --------- Co-authored-by: zhulinJulia24 <145004780+zhulinJulia24@users.noreply.github.com> Co-authored-by: zhulin1 <zhulin1@pjlab.org.cn> Co-authored-by: Songyang Zhang <tonysy@users.noreply.github.com> Co-authored-by: Linchen Xiao <xxllcc1993@gmail.com>
This commit is contained in:
bittersweet1999
GitHub
parent
2b1afa7d1e
commit
fa54aa62f6
@ -1,74 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import AlignmentBenchDataset
|
||||
from opencompass.summarizers import AlignmentBenchSummarizer
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'alignment_bench',
|
||||
]
|
||||
data_path ='data/subjective/alignment_bench'
|
||||
|
||||
alignbench_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = """为上传的针对给定用户问题的回应撰写评论, 并为该回复打分:
|
||||
|
||||
[BEGIN DATA]
|
||||
***
|
||||
[用户问询]: {question}
|
||||
***
|
||||
[回应]: {prediction}
|
||||
***
|
||||
[参考答案]: {ref}
|
||||
***
|
||||
[END DATA]
|
||||
|
||||
请根据参考答案为这个回应撰写评论. 在这之后, 你应该按照如下格式给这个回应一个最终的1-10范围的评分: "[[评分]]", 例如: "评分: [[5]]"."""
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
alignbench_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=AlignmentBenchDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=AlignmentBenchSummarizer, judge_type='autoj')
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import AlignmentBenchDataset
|
||||
from opencompass.summarizers import AlignmentBenchSummarizer
|
||||
from opencompass.datasets import AlignmentBenchDataset, alignbench_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'critiquellm_prefix'],
|
||||
@ -47,6 +46,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=alignbench_postprocess, judge_type='general'),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -63,5 +63,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=AlignmentBenchSummarizer, judge_type='general')
|
||||
))
|
||||
|
||||
@ -1,62 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import AlignmentBenchDataset
|
||||
from opencompass.summarizers import AlignmentBenchSummarizer
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'alignment_bench',
|
||||
]
|
||||
data_path ='data/subjective/alignment_bench'
|
||||
|
||||
alignbench_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = """You are a helpful and precise assistant for checking the quality of the answer.\n[Question]\n{question}\n\n[The Start of Assistant 1's Answer]\n{ref}\n\n[The End of Assistant 1's Answer]\n\n[The Start of Assistant 2's Answer]\n{prediction}\n\n[The End of Assistant 2's Answer]\n\n[System]\nWe would like to request your feedback on the performance of two AI assistants in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space. In the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment.\n\n### Response:10"""
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
alignbench_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=AlignmentBenchDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=AlignmentBenchSummarizer, judge_type='judgelm')
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import AlignmentBenchDataset
|
||||
from opencompass.summarizers import AlignmentBenchSummarizer
|
||||
from opencompass.datasets import AlignmentBenchDataset, alignbench_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'critiquellm_prefix'],
|
||||
@ -47,6 +46,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=alignbench_postprocess, judge_type='general'),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -63,5 +63,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=AlignmentBenchSummarizer, judge_type='general')
|
||||
))
|
||||
|
||||
@ -1,97 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import SubjectiveCmpDataset
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'alpaca_eval',
|
||||
]
|
||||
|
||||
|
||||
alpacav1_datasets = []
|
||||
|
||||
gpt4_prompt = """
|
||||
I want you to create a leaderboard of different of large-language models. To do so, I will give you the instructions (prompts) given to the models, and the responses of two models. Please rank the models based on which responses would be preferred by humans. All inputs and outputs should be python dictionaries.
|
||||
|
||||
Here is the prompt:
|
||||
{
|
||||
"instruction": "{question}"
|
||||
}
|
||||
|
||||
Here are the outputs of the models:
|
||||
[
|
||||
{
|
||||
"model": "model_1",
|
||||
"answer": "{prediction}"
|
||||
},
|
||||
{
|
||||
"model": "model_2",
|
||||
"answer": "{prediction2}"
|
||||
}
|
||||
]
|
||||
|
||||
Now please rank the models by the quality of their answers, so that the model with rank 1 has the best output. Then return a list of the model names and ranks, i.e., produce the following output:
|
||||
[
|
||||
{"model": <model-name>, "rank": <model-rank>},
|
||||
{"model": <model-name>, "rank": <model-rank>}
|
||||
]
|
||||
|
||||
Your response must be a valid Python dictionary and should contain nothing else because we will directly execute it in Python. Please provide the ranking that the majority of humans would give.
|
||||
"""
|
||||
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=4096),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(
|
||||
begin=[
|
||||
dict(
|
||||
role='SYSTEM',
|
||||
fallback_role='HUMAN',
|
||||
prompt='You are a helpful assistant, that ranks models by the quality of their answers.')
|
||||
],
|
||||
round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = gpt4_prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
alpacav1_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=SubjectiveCmpDataset,
|
||||
path='./data/subjective/alpaca_eval',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import SubjectiveCmpDataset
|
||||
from opencompass.summarizers import AlpacaSummarizer
|
||||
from opencompass.datasets import AlpacaEvalDataset, alpacaeval_postprocess
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -95,6 +94,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=alpacaeval_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -102,7 +102,7 @@ for _name in subjective_all_sets:
|
||||
alpacav2_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=SubjectiveCmpDataset,
|
||||
type=AlpacaEvalDataset,
|
||||
path='./data/subjective/alpaca_eval',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
@ -111,6 +111,5 @@ for _name in subjective_all_sets:
|
||||
mode='m2n',
|
||||
infer_order='random',
|
||||
base_models=gpt4,
|
||||
summarizer=dict(type=AlpacaSummarizer, judge_type='v2'),
|
||||
given_pred = [{'abbr':'gpt4-turbo', 'path':'./data/subjective/alpaca_eval/gpt4-turbo'}]
|
||||
))
|
||||
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import ArenaHardDataset
|
||||
from opencompass.summarizers import ArenaHardSummarizer
|
||||
from opencompass.datasets import ArenaHardDataset, arenahard_postprocess
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -60,6 +59,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=arenahard_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -76,6 +76,5 @@ for _name in subjective_all_sets:
|
||||
mode='m2n',
|
||||
infer_order='double',
|
||||
base_models=gpt4,
|
||||
summarizer = dict(type=ArenaHardSummarizer),
|
||||
given_pred = [{'abbr':'gpt4-0314', 'path':'./data/subjective/arena_hard'}]
|
||||
))
|
||||
|
||||
@ -1,154 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CompassArenaDataset
|
||||
from opencompass.summarizers import CompassArenaSummarizer
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path ='data/subjective/compass_arena'
|
||||
|
||||
compassarena_datasets = []
|
||||
|
||||
base_prompt = """
|
||||
|
||||
[回答1开始]
|
||||
{prediction}
|
||||
[回答1结束]
|
||||
|
||||
[回答2开始]
|
||||
{prediction2}
|
||||
[回答2结束]
|
||||
|
||||
根据评分要求，在以下 3 个选项中做出选择:
|
||||
A. 回答1更好
|
||||
B. 回答2更好
|
||||
C. 回答1、2平局
|
||||
并提供你的解释原因。
|
||||
|
||||
如果你认为回答1更好，你的输出应形如：
|
||||
选择：A
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答2更好，你的输出应形如：
|
||||
选择：B
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答1、2打成平手，你的输出应形如：
|
||||
选择：C
|
||||
原因：blahblah blahblah\n
|
||||
"""
|
||||
|
||||
knowledge_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答能与参考答案吻合或表明参考答案的意思。
|
||||
2. 在都准确答对问题的前提下，更好的回答能对知识点进行额外补充，且补充的知识准确无误。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
language_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 在有明确的参考答案的情况下，越贴近参考答案或表明了参考答案的意思的回答越好。
|
||||
2. 更好的回答在语言表达上更流畅，更加符合与人类对话的习惯，包括语气、情调等
|
||||
3. 在都准确答对问题的前提下，更好的回答能进行额外补充，且补充的内容准确无误。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
math_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答的答案能和参考答案一致。
|
||||
2. 若两个回答的答案都与参考答案不一致，则更好的回答的推理过程应更加合理。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
reason_prompt = math_prompt
|
||||
|
||||
creation_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 好的回答必须首先符合用户问题里的各种需求，不能跑题
|
||||
2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答
|
||||
3. 好的回答必须具有创造性的词语和表达丰富度
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
""" + base_prompt
|
||||
|
||||
sub_map = {'language': language_prompt, 'knowledge': knowledge_prompt, 'reason_v2': reason_prompt, 'math_v2': math_prompt, 'creationv2_zh': creation_prompt}
|
||||
|
||||
gpt4 = [dict(
|
||||
abbr='gpt4-turbo',
|
||||
)]
|
||||
|
||||
for _name, _prompt in sub_map.items():
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = _prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
compassarena_datasets.append(
|
||||
dict(
|
||||
abbr=f'compassarena_{_name}',
|
||||
type=CompassArenaDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='m2n',
|
||||
infer_order='double',
|
||||
base_models=gpt4,
|
||||
summarizer = dict(type=CompassArenaSummarizer, summary_type='half_add'),
|
||||
given_pred = [{'abbr':'gpt4-turbo', 'path':'./data/subjective/compass_arena/gpt4-turbo'}]
|
||||
))
|
||||
@ -1,145 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CompassArenaDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path ='data/subjective/compass_arena'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
base_prompt = """
|
||||
|
||||
[回答1开始]
|
||||
{prediction}
|
||||
[回答1结束]
|
||||
|
||||
[回答2开始]
|
||||
{prediction2}
|
||||
[回答2结束]
|
||||
|
||||
根据评分要求，在以下 3 个选项中做出选择:
|
||||
A. 回答1更好
|
||||
B. 回答2更好
|
||||
C. 回答1、2平局
|
||||
并提供你的解释原因。
|
||||
|
||||
如果你认为回答1更好，你的输出应形如：
|
||||
选择：A
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答2更好，你的输出应形如：
|
||||
选择：B
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答1、2打成平手，你的输出应形如：
|
||||
选择：C
|
||||
原因：blahblah blahblah\n
|
||||
"""
|
||||
|
||||
knowledge_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答能与参考答案吻合或表明参考答案的意思。
|
||||
2. 在都准确答对问题的前提下，更好的回答能对知识点进行额外补充，且补充的知识准确无误。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
language_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 在有明确的参考答案的情况下，越贴近参考答案或表明了参考答案的意思的回答越好。
|
||||
2. 更好的回答在语言表达上更流畅，更加符合与人类对话的习惯，包括语气、情调等
|
||||
3. 在都准确答对问题的前提下，更好的回答能进行额外补充，且补充的内容准确无误。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
math_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答的答案能和参考答案一致。
|
||||
2. 若两个回答的答案都与参考答案不一致，则更好的回答的推理过程应更加合理。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
reason_prompt = math_prompt
|
||||
|
||||
creation_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 好的回答必须首先符合用户问题里的各种需求，不能跑题
|
||||
2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答
|
||||
3. 好的回答必须具有创造性的词语和表达丰富度
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
""" + base_prompt
|
||||
|
||||
sub_map = {'creationv3': creation_prompt}
|
||||
|
||||
for _name, _prompt in sub_map.items():
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
infer_order='double',
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = _prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=CompassArenaDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,156 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CompassArenaDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path ='data/subjective/compass_arena'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
base_prompt = """
|
||||
|
||||
[回答1开始]
|
||||
{prediction}
|
||||
[回答1结束]
|
||||
|
||||
[回答2开始]
|
||||
{prediction2}
|
||||
[回答2结束]
|
||||
|
||||
根据评分要求，在以下 3 个选项中做出选择:
|
||||
A. 回答1更好
|
||||
B. 回答2更好
|
||||
C. 回答1、2平局
|
||||
并提供你的解释原因。
|
||||
|
||||
如果你认为回答1更好，你的输出应形如：
|
||||
选择：A
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答2更好，你的输出应形如：
|
||||
选择：B
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答1、2打成平手，你的输出应形如：
|
||||
选择：C
|
||||
原因：blahblah blahblah\n
|
||||
"""
|
||||
|
||||
knowledge_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答能与参考答案吻合或表明参考答案的意思。
|
||||
2. 在都准确答对问题的前提下，更好的回答能对知识点进行额外补充，且补充的知识准确无误。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
language_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 在有明确的参考答案的情况下，越贴近参考答案或表明了参考答案的意思的回答越好。
|
||||
2. 更好的回答在语言表达上更流畅，更加符合与人类对话的习惯，包括语气、情调等
|
||||
3. 在都准确答对问题的前提下，更好的回答能进行额外补充，且补充的内容准确无误。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
math_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答的答案能和参考答案一致。
|
||||
2. 若两个回答的答案都与参考答案不一致，则更好的回答的推理过程应更加合理。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
reason_prompt = math_prompt
|
||||
|
||||
creation_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 好的回答必须首先符合用户问题里的各种需求，不能跑题
|
||||
2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答
|
||||
3. 好的回答必须具有创造性的词语和表达丰富度
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
""" + base_prompt
|
||||
|
||||
sub_map = {'knowledge': knowledge_prompt, 'language': language_prompt, 'math_v2': math_prompt, 'reason_v2': reason_prompt, 'creationv2_zh': creation_prompt}
|
||||
|
||||
meta_prompt = """
|
||||
\n你是一个评判专家，请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。\n评分要求（重要性依次递减）:\n1. 好的回答必须首先符合用户问题里的各种需求，不能跑题 \n2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答\n3. 好的回答必须具有创造性的词语和表达丰富度\n\n[用户问题]\n{question}\n[回答1开始]\n{prediction}\n[回答1结束]\n[回答2开始]\n{prediction2}\n[回答2结束]\n此外，还有两个其他评判专家的评判意见供你参考。\n[评判意见1]\n{judgement}\n[评判意见2]\n{judgement2}\n\n最终请你综合其他评判专家的评判意见与你自己的意见，在以下 3 个选项中做出选择:\nA. 回答1更好\nB. 回答2更好\nC. 回答1、2平局\n并提供你的解释原因。\n\n如果你认为回答1更好，你的输出应形如：\n选择：A\n原因：blahblah blahblah\n\n\n如果你认为回答2更好，你的输出应形如：\n选择：B\n原因：blahblah blahblah\n\n\n如果你认为回答1、2打成平手，你的输出应形如：\n选择：C\n原因：blahblah blahblah\n\n
|
||||
"""
|
||||
for _name, _prompt in sub_map.items():
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = _prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
meta_review_prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = meta_prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=CompassArenaDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,60 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CreationBenchDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'gpt4_prefix', 'gpt4_suffix'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'creationbench',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{gpt4_prefix}{prediction}{gpt4_suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=CreationBenchDataset,
|
||||
multi_dimension=True,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,60 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CreationBenchDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'score_with_ref_prefix', 'score_with_ref_suffix'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'creationv2_zh',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{score_with_ref_prefix}{prediction}{score_with_ref_suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=CreationBenchDataset,
|
||||
multi_dimension=True,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import FofoDataset
|
||||
from opencompass.summarizers import FofoSummarizer
|
||||
from opencompass.datasets import FofoDataset, fofo_postprocess
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -113,6 +112,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=fofo_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -127,5 +127,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=FofoSummarizer, judge_type='general')
|
||||
))
|
||||
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import FofoDataset
|
||||
from opencompass.summarizers import FofoSummarizer
|
||||
from opencompass.datasets import FofoDataset, fofo_postprocess
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -81,6 +80,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=fofo_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -95,5 +95,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=FofoSummarizer, judge_type='general')
|
||||
))
|
||||
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import FollowBenchDataset
|
||||
from opencompass.summarizers import FollowBenchSummarizer
|
||||
from opencompass.datasets import FollowBenchDataset, followbench_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['instruction', 'judge_prompt',],
|
||||
@ -44,6 +43,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=followbench_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -59,5 +59,4 @@ for _name in subjective_all_sets:
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
summarizer = dict(type=FollowBenchSummarizer,)
|
||||
))
|
||||
|
||||
@ -1,20 +1,19 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import Creationv01Dataset
|
||||
from opencompass.datasets.subjective import JudgerBenchDataset, JudgerBenchEvaluator
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'prefix', 'suffix'],
|
||||
input_columns=['judge_prompt'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'creation_v0.1',
|
||||
'judgerbench_A_cn', 'judgerbench_A_en', 'judgerbench_B'
|
||||
]
|
||||
|
||||
subjective_datasets = []
|
||||
judgerbench_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
@ -23,37 +22,33 @@ for _name in subjective_all_sets:
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
prompt='{judge_prompt}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=4096),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{prefix}问题: <问题开始> {question} <问题结束>\n\n回答: <回答开始> {prediction} <回答结束>\n\n{suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
type=JudgerBenchEvaluator,
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
judgerbench_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=Creationv01Dataset,
|
||||
path='./data/subjective/',
|
||||
type=JudgerBenchDataset,
|
||||
path='./data/subjective/judgerbench',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
))
|
||||
# ds1000_eval_cfg = dict(
|
||||
# evaluator=dict(type=DS1000Evaluator),
|
||||
# pred_role='BOT',
|
||||
# pred_postprocessor=dict(type=ds1000_postprocess),
|
||||
# )
|
||||
@ -1,56 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MultiroundDataset
|
||||
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'capability', 'gpt4_prefix', 'gpt4_suffix'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'FunctionalMT',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{dialogue}""",
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, infer_mode='every'),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
pack_all_predictions=True,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{gpt4_prefix}{prediction}{gpt4_suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=MultiroundDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MTBench101Dataset
|
||||
from opencompass.summarizers import MTBench101Summarizer
|
||||
from opencompass.datasets import MTBench101Dataset, mtbench101_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue','task','multi_id','turn_id','system_prompt','prompt_template'],
|
||||
@ -46,6 +45,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=mtbench101_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -60,5 +60,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=MTBench101Summarizer, judge_type='single')
|
||||
))
|
||||
|
||||
@ -1,64 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MTBenchDataset
|
||||
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'capability', 'system_prompt', 'prompt_template'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'mtbench',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{dialogue}""",
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, infer_mode='every'),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
infer_order='double',
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(
|
||||
begin=[
|
||||
dict(
|
||||
role='SYSTEM',
|
||||
fallback_role='HUMAN',
|
||||
prompt='{system_prompt}')
|
||||
],
|
||||
round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{prompt_template}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=MTBenchDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
judge_type='pair',
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,62 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MTBenchDataset
|
||||
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'capability', 'system_prompt', 'prompt_template'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'mtbench',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{dialogue}""",
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, infer_mode='every'),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(
|
||||
begin=[
|
||||
dict(
|
||||
role='SYSTEM',
|
||||
fallback_role='HUMAN',
|
||||
prompt='{system_prompt}')
|
||||
],
|
||||
round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{prompt_template}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=MTBenchDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MTBenchDataset
|
||||
from opencompass.summarizers import MTBenchSummarizer
|
||||
from opencompass.datasets import MTBenchDataset, mtbench_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'capability', 'system_prompt', 'prompt_template'],
|
||||
@ -48,6 +47,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=mtbench_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -62,5 +62,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=MTBenchSummarizer, judge_type='single')
|
||||
))
|
||||
|
||||
@ -1,61 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets.subjective_cmp import SubjectiveCmpDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'index', 'reference_answer', 'evaluating_guidance', 'capability', 'prompt'],
|
||||
output_column='judge',
|
||||
train_split='test')
|
||||
|
||||
subjective_all_sets = [
|
||||
'creation_v0.1',
|
||||
]
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
cmp_order='both',
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(
|
||||
begin=[
|
||||
dict(
|
||||
role='SYSTEM',
|
||||
fallback_role='HUMAN',
|
||||
prompt='{prompt}'
|
||||
),
|
||||
],
|
||||
round=[dict(role='HUMAN',
|
||||
prompt='回答 1: <回答 1 开始> {prediction} <回答 1 结束>\n回答 2: <回答 2 开始> {prediction2} <回答 2 结束>\n')]))),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=SubjectiveCmpDataset,
|
||||
path='./data/subjective/',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,62 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import Corev2Dataset
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'prefix', 'suffix'],
|
||||
output_column='judge',
|
||||
#train_split='test'
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'COREV2_6A_all',
|
||||
]
|
||||
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
infer_order='random',
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{prefix}问题: <问题开始> {question} <问题结束>\n\n回答 1: <回答 1 开始> {prediction} <回答 1 结束>\n\n回答 2: <回答 2 开始> {prediction2} <回答 2 结束>\n\n{suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=Corev2Dataset,
|
||||
path='./data/subjective/',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import WildBenchDataset
|
||||
from opencompass.datasets import WildBenchDataset, wildbench_postprocess
|
||||
from opencompass.summarizers import WildBenchPairSummarizer
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -30,6 +30,7 @@ subjective_eval_cfg = dict(
|
||||
type=PromptTemplate,
|
||||
template="""{prompt}"""
|
||||
),
|
||||
dict_postprocessor=dict(type=wildbench_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -62,5 +63,4 @@ wildbench_datasets.append(
|
||||
mode='m2n', # m个模型 与 n个模型进行对战
|
||||
infer_order='random',
|
||||
base_models = [llama_2_70b, gpt4, claude],
|
||||
summarizer = dict(type=WildBenchPairSummarizer),
|
||||
))
|
||||
|
||||
@ -1,47 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import WildBenchDataset
|
||||
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'prompt'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path ='./data/WildBench/wildbench.jsonl'
|
||||
|
||||
wildbench_single_datasets = []
|
||||
|
||||
# the question is a list, how to process it
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{dialogue}"""
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, infer_mode='last'),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{prompt}"""
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
wildbench_single_datasets.append(
|
||||
dict(
|
||||
abbr='wildbench',
|
||||
type=WildBenchDataset,
|
||||
path=data_path,
|
||||
eval_mode='single',
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
52
configs/eval_judgerbench.py
Normal file
52
configs/eval_judgerbench.py
Normal file
@ -0,0 +1,52 @@
|
||||
from mmengine.config import read_base
|
||||
|
||||
with read_base():
|
||||
from opencompass.configs.datasets.subjective.judgerbench.judgerbench import judgerbench_datasets
|
||||
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI, TurboMindModelwithChatTemplate
|
||||
from opencompass.partitioners import NaivePartitioner, SizePartitioner
|
||||
from opencompass.runners import LocalRunner
|
||||
from opencompass.runners import SlurmSequentialRunner
|
||||
from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
|
||||
|
||||
|
||||
api_meta_template = dict(
|
||||
round=[
|
||||
dict(role='HUMAN', api_role='HUMAN'),
|
||||
dict(role='BOT', api_role='BOT', generate=True),
|
||||
]
|
||||
)
|
||||
|
||||
# -------------Inference Stage ----------------------------------------
|
||||
# For subjective evaluation, we often set do sample for models
|
||||
models = [
|
||||
dict(
|
||||
type=TurboMindModelwithChatTemplate,
|
||||
abbr='Qwen2-7B',
|
||||
path='Qwen/Qwen2-7B-Instruct',
|
||||
engine_config=dict(session_len=16384, max_batch_size=16, tp=1),
|
||||
gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=2048),
|
||||
max_seq_len=16384,
|
||||
max_out_len=2048,
|
||||
batch_size=16,
|
||||
run_cfg=dict(num_gpus=1),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
datasets = judgerbench_datasets
|
||||
|
||||
|
||||
infer = dict(
|
||||
partitioner=dict(type=NaivePartitioner),
|
||||
runner=dict(type=LocalRunner, max_num_workers=16, task=dict(type=OpenICLInferTask)),
|
||||
)
|
||||
# -------------Evalation Stage ----------------------------------------
|
||||
|
||||
|
||||
## ------------- Evaluation Configuration
|
||||
eval = dict(
|
||||
partitioner=dict(type=NaivePartitioner, n=10,),
|
||||
runner=dict(type=LocalRunner, max_num_workers=16, task=dict(type=OpenICLEvalTask)),
|
||||
)
|
||||
|
||||
work_dir = 'outputs/judgerbench/'
|
||||
@ -1,74 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import AlignmentBenchDataset
|
||||
from opencompass.summarizers import AlignmentBenchSummarizer
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'alignment_bench',
|
||||
]
|
||||
data_path ='data/subjective/alignment_bench'
|
||||
|
||||
alignbench_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = """为上传的针对给定用户问题的回应撰写评论, 并为该回复打分:
|
||||
|
||||
[BEGIN DATA]
|
||||
***
|
||||
[用户问询]: {question}
|
||||
***
|
||||
[回应]: {prediction}
|
||||
***
|
||||
[参考答案]: {ref}
|
||||
***
|
||||
[END DATA]
|
||||
|
||||
请根据参考答案为这个回应撰写评论. 在这之后, 你应该按照如下格式给这个回应一个最终的1-10范围的评分: "[[评分]]", 例如: "评分: [[5]]"."""
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
alignbench_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=AlignmentBenchDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=AlignmentBenchSummarizer, judge_type='autoj')
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import AlignmentBenchDataset
|
||||
from opencompass.summarizers import AlignmentBenchSummarizer
|
||||
from opencompass.datasets import AlignmentBenchDataset, alignbench_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'critiquellm_prefix'],
|
||||
@ -47,6 +46,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=alignbench_postprocess, judge_type='general'),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -63,5 +63,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=AlignmentBenchSummarizer, judge_type='general')
|
||||
))
|
||||
|
||||
@ -1,62 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import AlignmentBenchDataset
|
||||
from opencompass.summarizers import AlignmentBenchSummarizer
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'alignment_bench',
|
||||
]
|
||||
data_path ='data/subjective/alignment_bench'
|
||||
|
||||
alignbench_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = """You are a helpful and precise assistant for checking the quality of the answer.\n[Question]\n{question}\n\n[The Start of Assistant 1's Answer]\n{ref}\n\n[The End of Assistant 1's Answer]\n\n[The Start of Assistant 2's Answer]\n{prediction}\n\n[The End of Assistant 2's Answer]\n\n[System]\nWe would like to request your feedback on the performance of two AI assistants in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space. In the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment.\n\n### Response:10"""
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
alignbench_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=AlignmentBenchDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=AlignmentBenchSummarizer, judge_type='judgelm')
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import AlignmentBenchDataset
|
||||
from opencompass.summarizers import AlignmentBenchSummarizer
|
||||
from opencompass.datasets import AlignmentBenchDataset, alignbench_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'critiquellm_prefix'],
|
||||
@ -47,6 +46,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=alignbench_postprocess, judge_type='general'),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -63,5 +63,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=AlignmentBenchSummarizer, judge_type='general')
|
||||
))
|
||||
|
||||
@ -1,97 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import SubjectiveCmpDataset
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'alpaca_eval',
|
||||
]
|
||||
|
||||
|
||||
alpacav1_datasets = []
|
||||
|
||||
gpt4_prompt = """
|
||||
I want you to create a leaderboard of different of large-language models. To do so, I will give you the instructions (prompts) given to the models, and the responses of two models. Please rank the models based on which responses would be preferred by humans. All inputs and outputs should be python dictionaries.
|
||||
|
||||
Here is the prompt:
|
||||
{
|
||||
"instruction": "{question}"
|
||||
}
|
||||
|
||||
Here are the outputs of the models:
|
||||
[
|
||||
{
|
||||
"model": "model_1",
|
||||
"answer": "{prediction}"
|
||||
},
|
||||
{
|
||||
"model": "model_2",
|
||||
"answer": "{prediction2}"
|
||||
}
|
||||
]
|
||||
|
||||
Now please rank the models by the quality of their answers, so that the model with rank 1 has the best output. Then return a list of the model names and ranks, i.e., produce the following output:
|
||||
[
|
||||
{"model": <model-name>, "rank": <model-rank>},
|
||||
{"model": <model-name>, "rank": <model-rank>}
|
||||
]
|
||||
|
||||
Your response must be a valid Python dictionary and should contain nothing else because we will directly execute it in Python. Please provide the ranking that the majority of humans would give.
|
||||
"""
|
||||
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=4096),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(
|
||||
begin=[
|
||||
dict(
|
||||
role='SYSTEM',
|
||||
fallback_role='HUMAN',
|
||||
prompt='You are a helpful assistant, that ranks models by the quality of their answers.')
|
||||
],
|
||||
round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = gpt4_prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
alpacav1_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=SubjectiveCmpDataset,
|
||||
path='./data/subjective/alpaca_eval',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import SubjectiveCmpDataset
|
||||
from opencompass.summarizers import AlpacaSummarizer
|
||||
from opencompass.datasets import AlpacaEvalDataset, alpacaeval_postprocess
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -95,6 +94,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=alpacaeval_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -102,7 +102,7 @@ for _name in subjective_all_sets:
|
||||
alpacav2_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=SubjectiveCmpDataset,
|
||||
type=AlpacaEvalDataset,
|
||||
path='./data/subjective/alpaca_eval',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
@ -111,6 +111,5 @@ for _name in subjective_all_sets:
|
||||
mode='m2n',
|
||||
infer_order='random',
|
||||
base_models=gpt4,
|
||||
summarizer=dict(type=AlpacaSummarizer, judge_type='v2'),
|
||||
given_pred = [{'abbr':'gpt4-turbo', 'path':'./data/subjective/alpaca_eval/gpt4-turbo'}]
|
||||
))
|
||||
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import ArenaHardDataset
|
||||
from opencompass.summarizers import ArenaHardSummarizer
|
||||
from opencompass.datasets import ArenaHardDataset, arenahard_postprocess
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -60,6 +59,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=arenahard_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -76,6 +76,5 @@ for _name in subjective_all_sets:
|
||||
mode='m2n',
|
||||
infer_order='double',
|
||||
base_models=gpt4,
|
||||
summarizer = dict(type=ArenaHardSummarizer),
|
||||
given_pred = [{'abbr':'gpt4-0314', 'path':'./data/subjective/arena_hard'}]
|
||||
))
|
||||
|
||||
@ -1,154 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CompassArenaDataset
|
||||
from opencompass.summarizers import CompassArenaSummarizer
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path ='data/subjective/compass_arena'
|
||||
|
||||
compassarena_datasets = []
|
||||
|
||||
base_prompt = """
|
||||
|
||||
[回答1开始]
|
||||
{prediction}
|
||||
[回答1结束]
|
||||
|
||||
[回答2开始]
|
||||
{prediction2}
|
||||
[回答2结束]
|
||||
|
||||
根据评分要求，在以下 3 个选项中做出选择:
|
||||
A. 回答1更好
|
||||
B. 回答2更好
|
||||
C. 回答1、2平局
|
||||
并提供你的解释原因。
|
||||
|
||||
如果你认为回答1更好，你的输出应形如：
|
||||
选择：A
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答2更好，你的输出应形如：
|
||||
选择：B
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答1、2打成平手，你的输出应形如：
|
||||
选择：C
|
||||
原因：blahblah blahblah\n
|
||||
"""
|
||||
|
||||
knowledge_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答能与参考答案吻合或表明参考答案的意思。
|
||||
2. 在都准确答对问题的前提下，更好的回答能对知识点进行额外补充，且补充的知识准确无误。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
language_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 在有明确的参考答案的情况下，越贴近参考答案或表明了参考答案的意思的回答越好。
|
||||
2. 更好的回答在语言表达上更流畅，更加符合与人类对话的习惯，包括语气、情调等
|
||||
3. 在都准确答对问题的前提下，更好的回答能进行额外补充，且补充的内容准确无误。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
math_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答的答案能和参考答案一致。
|
||||
2. 若两个回答的答案都与参考答案不一致，则更好的回答的推理过程应更加合理。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
reason_prompt = math_prompt
|
||||
|
||||
creation_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 好的回答必须首先符合用户问题里的各种需求，不能跑题
|
||||
2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答
|
||||
3. 好的回答必须具有创造性的词语和表达丰富度
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
""" + base_prompt
|
||||
|
||||
sub_map = {'language': language_prompt, 'knowledge': knowledge_prompt, 'reason_v2': reason_prompt, 'math_v2': math_prompt, 'creationv2_zh': creation_prompt}
|
||||
|
||||
gpt4 = [dict(
|
||||
abbr='gpt4-turbo',
|
||||
)]
|
||||
|
||||
for _name, _prompt in sub_map.items():
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = _prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
compassarena_datasets.append(
|
||||
dict(
|
||||
abbr=f'compassarena_{_name}',
|
||||
type=CompassArenaDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='m2n',
|
||||
infer_order='double',
|
||||
base_models=gpt4,
|
||||
summarizer = dict(type=CompassArenaSummarizer, summary_type='half_add'),
|
||||
given_pred = [{'abbr':'gpt4-turbo', 'path':'./data/subjective/compass_arena/gpt4-turbo'}]
|
||||
))
|
||||
@ -1,145 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CompassArenaDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path ='data/subjective/compass_arena'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
base_prompt = """
|
||||
|
||||
[回答1开始]
|
||||
{prediction}
|
||||
[回答1结束]
|
||||
|
||||
[回答2开始]
|
||||
{prediction2}
|
||||
[回答2结束]
|
||||
|
||||
根据评分要求，在以下 3 个选项中做出选择:
|
||||
A. 回答1更好
|
||||
B. 回答2更好
|
||||
C. 回答1、2平局
|
||||
并提供你的解释原因。
|
||||
|
||||
如果你认为回答1更好，你的输出应形如：
|
||||
选择：A
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答2更好，你的输出应形如：
|
||||
选择：B
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答1、2打成平手，你的输出应形如：
|
||||
选择：C
|
||||
原因：blahblah blahblah\n
|
||||
"""
|
||||
|
||||
knowledge_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答能与参考答案吻合或表明参考答案的意思。
|
||||
2. 在都准确答对问题的前提下，更好的回答能对知识点进行额外补充，且补充的知识准确无误。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
language_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 在有明确的参考答案的情况下，越贴近参考答案或表明了参考答案的意思的回答越好。
|
||||
2. 更好的回答在语言表达上更流畅，更加符合与人类对话的习惯，包括语气、情调等
|
||||
3. 在都准确答对问题的前提下，更好的回答能进行额外补充，且补充的内容准确无误。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
math_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答的答案能和参考答案一致。
|
||||
2. 若两个回答的答案都与参考答案不一致，则更好的回答的推理过程应更加合理。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
reason_prompt = math_prompt
|
||||
|
||||
creation_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 好的回答必须首先符合用户问题里的各种需求，不能跑题
|
||||
2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答
|
||||
3. 好的回答必须具有创造性的词语和表达丰富度
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
""" + base_prompt
|
||||
|
||||
sub_map = {'creationv3': creation_prompt}
|
||||
|
||||
for _name, _prompt in sub_map.items():
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
infer_order='double',
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = _prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=CompassArenaDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,156 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CompassArenaDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'ref'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path ='data/subjective/compass_arena'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
base_prompt = """
|
||||
|
||||
[回答1开始]
|
||||
{prediction}
|
||||
[回答1结束]
|
||||
|
||||
[回答2开始]
|
||||
{prediction2}
|
||||
[回答2结束]
|
||||
|
||||
根据评分要求，在以下 3 个选项中做出选择:
|
||||
A. 回答1更好
|
||||
B. 回答2更好
|
||||
C. 回答1、2平局
|
||||
并提供你的解释原因。
|
||||
|
||||
如果你认为回答1更好，你的输出应形如：
|
||||
选择：A
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答2更好，你的输出应形如：
|
||||
选择：B
|
||||
原因：blahblah blahblah\n
|
||||
|
||||
如果你认为回答1、2打成平手，你的输出应形如：
|
||||
选择：C
|
||||
原因：blahblah blahblah\n
|
||||
"""
|
||||
|
||||
knowledge_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答能与参考答案吻合或表明参考答案的意思。
|
||||
2. 在都准确答对问题的前提下，更好的回答能对知识点进行额外补充，且补充的知识准确无误。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
language_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 在有明确的参考答案的情况下，越贴近参考答案或表明了参考答案的意思的回答越好。
|
||||
2. 更好的回答在语言表达上更流畅，更加符合与人类对话的习惯，包括语气、情调等
|
||||
3. 在都准确答对问题的前提下，更好的回答能进行额外补充，且补充的内容准确无误。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
|
||||
math_prompt = """
|
||||
请根据提供的 评分要求，用户问题，参考答案 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 更好的回答的答案能和参考答案一致。
|
||||
2. 若两个回答的答案都与参考答案不一致，则更好的回答的推理过程应更加合理。
|
||||
3. 更好的回答更加符合与人类对话的习惯，包括语气、情调等。
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
|
||||
[参考答案]
|
||||
{ref}
|
||||
""" + base_prompt
|
||||
|
||||
reason_prompt = math_prompt
|
||||
|
||||
creation_prompt = """
|
||||
请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。
|
||||
评分要求（重要性依次递减）:
|
||||
1. 好的回答必须首先符合用户问题里的各种需求，不能跑题
|
||||
2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答
|
||||
3. 好的回答必须具有创造性的词语和表达丰富度
|
||||
|
||||
[用户问题]
|
||||
{question}
|
||||
""" + base_prompt
|
||||
|
||||
sub_map = {'knowledge': knowledge_prompt, 'language': language_prompt, 'math_v2': math_prompt, 'reason_v2': reason_prompt, 'creationv2_zh': creation_prompt}
|
||||
|
||||
meta_prompt = """
|
||||
\n你是一个评判专家，请根据提供的 评分要求，用户问题 以及 相应的两个回答（回答1，回答2），判断两个回答中哪一个更好。\n评分要求（重要性依次递减）:\n1. 好的回答必须首先符合用户问题里的各种需求，不能跑题 \n2. 好的回答必须具有逻辑连贯性，围绕一个中心进行回答\n3. 好的回答必须具有创造性的词语和表达丰富度\n\n[用户问题]\n{question}\n[回答1开始]\n{prediction}\n[回答1结束]\n[回答2开始]\n{prediction2}\n[回答2结束]\n此外，还有两个其他评判专家的评判意见供你参考。\n[评判意见1]\n{judgement}\n[评判意见2]\n{judgement2}\n\n最终请你综合其他评判专家的评判意见与你自己的意见，在以下 3 个选项中做出选择:\nA. 回答1更好\nB. 回答2更好\nC. 回答1、2平局\n并提供你的解释原因。\n\n如果你认为回答1更好，你的输出应形如：\n选择：A\n原因：blahblah blahblah\n\n\n如果你认为回答2更好，你的输出应形如：\n选择：B\n原因：blahblah blahblah\n\n\n如果你认为回答1、2打成平手，你的输出应形如：\n选择：C\n原因：blahblah blahblah\n\n
|
||||
"""
|
||||
for _name, _prompt in sub_map.items():
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = _prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
meta_review_prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = meta_prompt
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=CompassArenaDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,60 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CreationBenchDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'gpt4_prefix', 'gpt4_suffix'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'creationbench',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{gpt4_prefix}{prediction}{gpt4_suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=CreationBenchDataset,
|
||||
multi_dimension=True,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,60 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import CreationBenchDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'capability', 'score_with_ref_prefix', 'score_with_ref_suffix'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'creationv2_zh',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_seq_len=4096, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{score_with_ref_prefix}{prediction}{score_with_ref_suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=CreationBenchDataset,
|
||||
multi_dimension=True,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import FofoDataset
|
||||
from opencompass.summarizers import FofoSummarizer
|
||||
from opencompass.datasets import FofoDataset, fofo_postprocess
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -113,6 +112,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=fofo_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -127,5 +127,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=FofoSummarizer, judge_type='general')
|
||||
))
|
||||
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import FofoDataset
|
||||
from opencompass.summarizers import FofoSummarizer
|
||||
from opencompass.datasets import FofoDataset, fofo_postprocess
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -81,6 +80,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=fofo_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -95,5 +95,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=FofoSummarizer, judge_type='general')
|
||||
))
|
||||
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import FollowBenchDataset
|
||||
from opencompass.summarizers import FollowBenchSummarizer
|
||||
from opencompass.datasets import FollowBenchDataset, followbench_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['instruction', 'judge_prompt',],
|
||||
@ -44,6 +43,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=followbench_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -59,5 +59,4 @@ for _name in subjective_all_sets:
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
summarizer = dict(type=FollowBenchSummarizer,)
|
||||
))
|
||||
|
||||
@ -1,20 +1,19 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import Creationv01Dataset
|
||||
from opencompass.datasets.subjective import JudgerBenchDataset, JudgerBenchEvaluator
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'prefix', 'suffix'],
|
||||
input_columns=['judge_prompt'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'creation_v0.1',
|
||||
'judgerbench_A_cn', 'judgerbench_A_en', 'judgerbench_B'
|
||||
]
|
||||
|
||||
subjective_datasets = []
|
||||
judgerbench_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
@ -23,37 +22,33 @@ for _name in subjective_all_sets:
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
prompt='{judge_prompt}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=4096),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{prefix}问题: <问题开始> {question} <问题结束>\n\n回答: <回答开始> {prediction} <回答结束>\n\n{suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
type=JudgerBenchEvaluator,
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
judgerbench_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=Creationv01Dataset,
|
||||
path='./data/subjective/',
|
||||
type=JudgerBenchDataset,
|
||||
path='./data/subjective/judgerbench',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
))
|
||||
# ds1000_eval_cfg = dict(
|
||||
# evaluator=dict(type=DS1000Evaluator),
|
||||
# pred_role='BOT',
|
||||
# pred_postprocessor=dict(type=ds1000_postprocess),
|
||||
# )
|
||||
@ -1,56 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MultiroundDataset
|
||||
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'capability', 'gpt4_prefix', 'gpt4_suffix'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'FunctionalMT',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{dialogue}""",
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, infer_mode='every'),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
pack_all_predictions=True,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{gpt4_prefix}{prediction}{gpt4_suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=MultiroundDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MTBench101Dataset
|
||||
from opencompass.summarizers import MTBench101Summarizer
|
||||
from opencompass.datasets import MTBench101Dataset, mtbench101_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue','task','multi_id','turn_id','system_prompt','prompt_template'],
|
||||
@ -46,6 +45,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=mtbench101_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -60,5 +60,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=MTBench101Summarizer, judge_type='single')
|
||||
))
|
||||
|
||||
@ -1,64 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MTBenchDataset
|
||||
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'capability', 'system_prompt', 'prompt_template'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'mtbench',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{dialogue}""",
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, infer_mode='every'),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
infer_order='double',
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(
|
||||
begin=[
|
||||
dict(
|
||||
role='SYSTEM',
|
||||
fallback_role='HUMAN',
|
||||
prompt='{system_prompt}')
|
||||
],
|
||||
round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{prompt_template}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=MTBenchDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
judge_type='pair',
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,62 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MTBenchDataset
|
||||
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'capability', 'system_prompt', 'prompt_template'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'mtbench',
|
||||
]
|
||||
data_path ='data/subjective/'
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{dialogue}""",
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, infer_mode='every'),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(
|
||||
begin=[
|
||||
dict(
|
||||
role='SYSTEM',
|
||||
fallback_role='HUMAN',
|
||||
prompt='{system_prompt}')
|
||||
],
|
||||
round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{prompt_template}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=MTBenchDataset,
|
||||
path=data_path,
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,8 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import MTBenchDataset
|
||||
from opencompass.summarizers import MTBenchSummarizer
|
||||
from opencompass.datasets import MTBenchDataset, mtbench_postprocess
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'capability', 'system_prompt', 'prompt_template'],
|
||||
@ -48,6 +47,7 @@ for _name in subjective_all_sets:
|
||||
),
|
||||
]),
|
||||
),
|
||||
dict_postprocessor=dict(type=mtbench_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -62,5 +62,4 @@ for _name in subjective_all_sets:
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg,
|
||||
mode='singlescore',
|
||||
summarizer = dict(type=MTBenchSummarizer, judge_type='single')
|
||||
))
|
||||
|
||||
@ -1,61 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets.subjective_cmp import SubjectiveCmpDataset
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'index', 'reference_answer', 'evaluating_guidance', 'capability', 'prompt'],
|
||||
output_column='judge',
|
||||
train_split='test')
|
||||
|
||||
subjective_all_sets = [
|
||||
'creation_v0.1',
|
||||
]
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
cmp_order='both',
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(
|
||||
begin=[
|
||||
dict(
|
||||
role='SYSTEM',
|
||||
fallback_role='HUMAN',
|
||||
prompt='{prompt}'
|
||||
),
|
||||
],
|
||||
round=[dict(role='HUMAN',
|
||||
prompt='回答 1: <回答 1 开始> {prediction} <回答 1 结束>\n回答 2: <回答 2 开始> {prediction2} <回答 2 结束>\n')]))),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=SubjectiveCmpDataset,
|
||||
path='./data/subjective/',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,62 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import Corev2Dataset
|
||||
from mmengine.config import read_base
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['question', 'prefix', 'suffix'],
|
||||
output_column='judge',
|
||||
#train_split='test'
|
||||
)
|
||||
|
||||
subjective_all_sets = [
|
||||
'COREV2_6A_all',
|
||||
]
|
||||
|
||||
|
||||
subjective_datasets = []
|
||||
|
||||
for _name in subjective_all_sets:
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt='{question}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=GenInferencer, max_out_len=2048),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
infer_order='random',
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template=dict(round=[
|
||||
dict(
|
||||
role='HUMAN',
|
||||
prompt = '{prefix}问题: <问题开始> {question} <问题结束>\n\n回答 1: <回答 1 开始> {prediction} <回答 1 结束>\n\n回答 2: <回答 2 开始> {prediction2} <回答 2 结束>\n\n{suffix}'
|
||||
),
|
||||
]),
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
subjective_datasets.append(
|
||||
dict(
|
||||
abbr=f'{_name}',
|
||||
type=Corev2Dataset,
|
||||
path='./data/subjective/',
|
||||
name=_name,
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import WildBenchDataset
|
||||
from opencompass.datasets import WildBenchDataset, wildbench_postprocess
|
||||
from opencompass.summarizers import WildBenchPairSummarizer
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
@ -30,6 +30,7 @@ subjective_eval_cfg = dict(
|
||||
type=PromptTemplate,
|
||||
template="""{prompt}"""
|
||||
),
|
||||
dict_postprocessor=dict(type=wildbench_postprocess),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
@ -62,5 +63,4 @@ wildbench_datasets.append(
|
||||
mode='m2n', # m个模型 与 n个模型进行对战
|
||||
infer_order='random',
|
||||
base_models = [llama_2_70b, gpt4, claude],
|
||||
summarizer = dict(type=WildBenchPairSummarizer),
|
||||
))
|
||||
|
||||
@ -1,47 +0,0 @@
|
||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
|
||||
from opencompass.openicl.icl_evaluator import LMEvaluator
|
||||
from opencompass.datasets import WildBenchDataset
|
||||
|
||||
|
||||
subjective_reader_cfg = dict(
|
||||
input_columns=['dialogue', 'prompt'],
|
||||
output_column='judge',
|
||||
)
|
||||
|
||||
data_path ='./data/WildBench/wildbench.jsonl'
|
||||
|
||||
wildbench_single_datasets = []
|
||||
|
||||
# the question is a list, how to process it
|
||||
subjective_infer_cfg = dict(
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{dialogue}"""
|
||||
),
|
||||
retriever=dict(type=ZeroRetriever),
|
||||
inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, infer_mode='last'),
|
||||
)
|
||||
|
||||
subjective_eval_cfg = dict(
|
||||
evaluator=dict(
|
||||
type=LMEvaluator,
|
||||
prompt_template=dict(
|
||||
type=PromptTemplate,
|
||||
template="""{prompt}"""
|
||||
),
|
||||
),
|
||||
pred_role='BOT',
|
||||
)
|
||||
|
||||
wildbench_single_datasets.append(
|
||||
dict(
|
||||
abbr='wildbench',
|
||||
type=WildBenchDataset,
|
||||
path=data_path,
|
||||
eval_mode='single',
|
||||
reader_cfg=subjective_reader_cfg,
|
||||
infer_cfg=subjective_infer_cfg,
|
||||
eval_cfg=subjective_eval_cfg
|
||||
))
|
||||
@ -1,6 +1,10 @@
|
||||
# flake8: noqa: F401, F403
|
||||
from .alignbench import AlignmentBenchDataset # noqa: F401, F403
|
||||
from .alignbench import alignbench_postprocess # noqa: F401, F403
|
||||
from .alpacaeval import AlpacaEvalDataset # noqa: F401, F403
|
||||
from .alpacaeval import alpacaeval_postprocess # noqa: F401, F403
|
||||
from .arena_hard import ArenaHardDataset # noqa: F401, F403
|
||||
from .compass_arena import CompassArenaDataset # noqa: F401, F403
|
||||
from .arena_hard import arenahard_postprocess # noqa: F401, F403
|
||||
from .compassbench import CompassBenchDataset # noqa: F401, F403
|
||||
from .compassbench_checklist import \
|
||||
CompassBenchCheklistDataset # noqa: F401, F403
|
||||
@ -9,11 +13,15 @@ from .compassbench_control_length_bias import \
|
||||
from .corev2 import Corev2Dataset # noqa: F401, F403
|
||||
from .creationbench import CreationBenchDataset # noqa: F401, F403
|
||||
from .flames import FlamesDataset # noqa: F401, F403
|
||||
from .fofo import FofoDataset # noqa: F401, F403
|
||||
from .fofo import FofoDataset, fofo_postprocess # noqa: F401, F403
|
||||
from .followbench import FollowBenchDataset # noqa: F401, F403
|
||||
from .information_retrival import IRDataset # noqa: F401, F403
|
||||
from .mtbench import MTBenchDataset # noqa: F401, F403
|
||||
from .followbench import followbench_postprocess
|
||||
from .judgerbench import JudgerBenchDataset # noqa: F401, F403
|
||||
from .judgerbench import JudgerBenchEvaluator # noqa: F401, F403
|
||||
from .mtbench import MTBenchDataset, mtbench_postprocess # noqa: F401, F403
|
||||
from .mtbench101 import MTBench101Dataset # noqa: F401, F403
|
||||
from .mtbench101 import mtbench101_postprocess
|
||||
from .multiround import MultiroundDataset # noqa: F401, F403
|
||||
from .subjective_cmp import SubjectiveCmpDataset # noqa: F401, F403
|
||||
from .wildbench import WildBenchDataset # noqa: F401, F403
|
||||
from .wildbench import wildbench_postprocess # noqa: F401, F403
|
||||
|
||||
@ -1,15 +1,22 @@
|
||||
# flake8: noqa: E501
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import os.path as osp
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
from datasets import Dataset, DatasetDict
|
||||
from mmengine import ConfigDict
|
||||
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from .subjective_cmp import SubjectiveCmpDataset
|
||||
from .utils import get_judgeanswer_and_reference
|
||||
|
||||
|
||||
class Config:
|
||||
@ -111,3 +118,201 @@ class AlignmentBenchDataset(SubjectiveCmpDataset):
|
||||
alignbench_dataset.append(data)
|
||||
dataset = Dataset.from_list(alignbench_dataset)
|
||||
return dataset
|
||||
|
||||
|
||||
CATEGORIES = {
|
||||
'中文推理': ['数学计算', '逻辑推理'],
|
||||
'中文语言': ['基本任务', '中文理解', '综合问答', '文本写作', '角色扮演', '专业能力'],
|
||||
}
|
||||
|
||||
All_Dimensions = [
|
||||
'事实正确性', '满足用户需求', '安全无害', '清晰度', '逻辑性', '完备性', '创造性', '可负责程度', '逻辑连贯性',
|
||||
'公平与可负责程度', '丰富度', '综合得分'
|
||||
]
|
||||
|
||||
MAPPING = {
|
||||
'事实与解释型回答': ['事实正确性', '满足用户需求', '清晰度', '完备性'],
|
||||
'逻辑推理型回答': ['事实正确性', '满足用户需求', '逻辑连贯性', '完备性'],
|
||||
'生成型回答': ['事实正确性', '满足用户需求', '逻辑连贯性', '创造性', '丰富度'],
|
||||
'建议型回答': ['事实正确性', '满足用户需求', '公平与可负责程度', '创造性']
|
||||
}
|
||||
|
||||
|
||||
def detect_mapping(text):
|
||||
if '清晰度' in text and '完备性' in text:
|
||||
return '事实与解释型回答'
|
||||
elif '完备性' in text and '逻辑连贯性' in text:
|
||||
return '逻辑推理型回答'
|
||||
elif '创造性' in text and '丰富度' in text:
|
||||
return '生成型回答'
|
||||
elif '创造性' in text and '公平与可负责程度' in text:
|
||||
return '建议型回答'
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def extract_missing_rating(text, search_type):
|
||||
searching_keys = MAPPING[search_type]
|
||||
result_dict = {}
|
||||
for k in searching_keys:
|
||||
matches = re.findall(rf'{k}.*?\n', text)
|
||||
result_dict[k] = None
|
||||
for match in reversed(matches):
|
||||
if re.findall(r'\d{1,2}', match):
|
||||
result_dict[k] = int(re.findall(r'\d{1,2}', match)[-1])
|
||||
break
|
||||
overall_number = re.findall('\d{1,2}', text)
|
||||
try:
|
||||
result_dict['综合得分'] = int(overall_number[-1])
|
||||
except:
|
||||
return {}
|
||||
return result_dict
|
||||
|
||||
|
||||
def extract_rating(text):
|
||||
pattern = r'{(.*?)}(?![^{]*{)' # match last brackets
|
||||
match = re.search(pattern, text)
|
||||
|
||||
if match:
|
||||
dictionary_str = match.group(1)
|
||||
kv_pattern = r"'(.*?)': (\d+)"
|
||||
matches = re.findall(kv_pattern, dictionary_str)
|
||||
result_dict = {key: int(value) for key, value in matches}
|
||||
return result_dict
|
||||
else:
|
||||
match_type = detect_mapping(text=text)
|
||||
if match_type is not None:
|
||||
return extract_missing_rating(text=text, search_type=match_type)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def check_rating(rating, all_dimensions):
|
||||
for k, v in rating.items():
|
||||
if isinstance(v, (int, float)) and k in all_dimensions: # 确保值是数字
|
||||
if v >= 0 and v <= 10:
|
||||
pass
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
return rating
|
||||
|
||||
|
||||
def post_process_alignbench(judgement: dict,
|
||||
all_dimensions=All_Dimensions,
|
||||
possible_keys=['综合得分']):
|
||||
"""Input a dict item must contain string like below:
|
||||
|
||||
xxx{'事实正确性': 1, '满足用户需求': 1, '清晰度': 2, '完备性': 1, '综合得分': 1}xxx,
|
||||
and extract each score
|
||||
"""
|
||||
judgement = judgement['prediction']
|
||||
|
||||
def extract_score(text):
|
||||
keys_pattern = '|'.join(map(re.escape, possible_keys))
|
||||
pattern = rf"({'|'.join(possible_keys)}): (\d+(\.\d{{1,2}})?)"
|
||||
match = re.search(pattern, text)
|
||||
if match:
|
||||
try:
|
||||
return float(match.group(1))
|
||||
except ValueError:
|
||||
return -1
|
||||
return -1
|
||||
|
||||
# judgement = judgement.replace('\n', '')
|
||||
rating = extract_rating(judgement)
|
||||
|
||||
if rating is not None:
|
||||
score = -1
|
||||
for key in possible_keys:
|
||||
score = rating.get(key, -1)
|
||||
if score != -1:
|
||||
break
|
||||
if score == -1:
|
||||
score = extract_score(judgement)
|
||||
if score >= 0 and score <= 10:
|
||||
pass
|
||||
else:
|
||||
score = -1
|
||||
rating = check_rating(rating, all_dimensions)
|
||||
else:
|
||||
score = -1
|
||||
if rating == None or score == -1:
|
||||
return None
|
||||
else:
|
||||
return {'rating': rating, 'score': score}
|
||||
|
||||
|
||||
def get_dimension_results(judged_answers, references):
|
||||
dimension_ratings = defaultdict(int)
|
||||
dimension_counts = defaultdict(int)
|
||||
for ans, ref in zip(judged_answers, references):
|
||||
for k, v in ans['rating'].items():
|
||||
if k != '综合得分' or k != 'Overall Score':
|
||||
dimension_ratings[k] += v
|
||||
dimension_counts[k] += 1
|
||||
else:
|
||||
if k == '综合得分':
|
||||
dimension_ratings['综合得分'] += ans['score']
|
||||
dimension_counts['综合得分'] += 1
|
||||
else:
|
||||
dimension_ratings['Overall Score'] += ans['score']
|
||||
dimension_counts['Overall Score'] += 1
|
||||
|
||||
dimension_avg_ratings = defaultdict(float)
|
||||
for dimension, total_score in dimension_ratings.items():
|
||||
s = total_score / dimension_counts[dimension]
|
||||
s = round(s, 2)
|
||||
dimension_avg_ratings[dimension] = s
|
||||
|
||||
scores = {'dimensional_scores': dimension_avg_ratings}
|
||||
return scores
|
||||
|
||||
|
||||
def get_capability_results(judged_answers, references, categories=CATEGORIES):
|
||||
capability_ratings = defaultdict(int)
|
||||
capability_counts = defaultdict(int)
|
||||
for ans, ref in zip(judged_answers, references):
|
||||
capability_ratings[ref['capability']] += ans['score']
|
||||
capability_counts[ref['capability']] += 1
|
||||
|
||||
capability_avg_ratings = defaultdict(float)
|
||||
|
||||
for capability, total_score in capability_ratings.items():
|
||||
s = total_score / capability_counts[capability]
|
||||
s = round(s, 2)
|
||||
capability_avg_ratings[capability] = s
|
||||
|
||||
temp_list = []
|
||||
total_column_num = 2
|
||||
for category, sub_categories in categories.items():
|
||||
total_column_num += 1 + len(sub_categories)
|
||||
capability_avg_ratings[category + '总分'] = np.mean([
|
||||
np.mean(capability_avg_ratings[cat])
|
||||
for cat in categories[category]
|
||||
])
|
||||
capability_avg_ratings[category + '总分'] = round(
|
||||
capability_avg_ratings[category + '总分'], 2)
|
||||
temp_list.append(category + '总分')
|
||||
capability_avg_ratings['总分'] = 0
|
||||
for temp in temp_list:
|
||||
capability_avg_ratings['总分'] += capability_avg_ratings[temp]
|
||||
capability_avg_ratings['总分'] /= len(temp_list)
|
||||
capability_avg_ratings['总分'] = round(capability_avg_ratings['总分'], 2)
|
||||
return capability_avg_ratings
|
||||
|
||||
|
||||
@DICT_POSTPROCESSORS.register_module('alignbench')
|
||||
def alignbench_postprocess(output: dict,
|
||||
output_path: str,
|
||||
judge_type: Optional[str] = 'general') -> dict:
|
||||
judged_answers, references = get_judgeanswer_and_reference(
|
||||
output, output_path, post_process_alignbench)
|
||||
|
||||
if len(judged_answers) == 0:
|
||||
scores = None
|
||||
|
||||
results = get_capability_results(judged_answers, references)
|
||||
results['details'] = output
|
||||
return results
|
||||
|
||||
108
opencompass/datasets/subjective/alpacaeval.py
Normal file
108
opencompass/datasets/subjective/alpacaeval.py
Normal file
@ -0,0 +1,108 @@
|
||||
# flake8: noqa: E501
|
||||
import json
|
||||
import os.path as osp
|
||||
from collections import defaultdict
|
||||
|
||||
from datasets import Dataset, DatasetDict
|
||||
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
from .utils import get_judgeanswer_and_reference
|
||||
|
||||
|
||||
@LOAD_DATASET.register_module()
|
||||
class AlpacaEvalDataset(BaseDataset):
|
||||
|
||||
def load(self, path: str, name: str, *args, **kwargs):
|
||||
path = get_data_path(path, local_mode=True)
|
||||
filename = osp.join(path, f'{name}.json')
|
||||
dataset = DatasetDict()
|
||||
raw_data = []
|
||||
with open(filename, 'r', encoding='utf-8') as f:
|
||||
json_data = json.load(f)
|
||||
for problem in json_data:
|
||||
question = problem['question']
|
||||
capability = problem['capability']
|
||||
others = problem['others']
|
||||
raw_data.append({
|
||||
'question': question,
|
||||
'capability': capability,
|
||||
'others': others,
|
||||
'judge': {
|
||||
'capability': capability,
|
||||
'question': question
|
||||
}
|
||||
})
|
||||
dataset = Dataset.from_list(raw_data)
|
||||
return dataset
|
||||
|
||||
|
||||
def post_process_alpacav2(completion: str):
|
||||
r"""Parse a completion that contains 'm' or 'M' and returns the rank of the model1.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> ranking_parser("m")
|
||||
1
|
||||
>>> ranking_parser("M")
|
||||
2
|
||||
>>> ranking_parser("s")
|
||||
None
|
||||
"""
|
||||
completion = completion['prediction']
|
||||
try:
|
||||
if completion[0] == 'm':
|
||||
return {'rank': 1}
|
||||
elif completion[0] == 'M':
|
||||
return {'rank': 2}
|
||||
else:
|
||||
return None
|
||||
except Exception as e:
|
||||
return None
|
||||
|
||||
|
||||
@DICT_POSTPROCESSORS.register_module('alpacaeval')
|
||||
def alpacaeval_postprocess(output: dict, output_path: str) -> dict:
|
||||
judged_answers, references = get_judgeanswer_and_reference(
|
||||
output, output_path, post_process_alpacav2)
|
||||
|
||||
if len(judged_answers) == 0:
|
||||
scores = None
|
||||
|
||||
win_model1, win_model2, categories = defaultdict(float), defaultdict(
|
||||
float), defaultdict(float)
|
||||
model1, model2 = references[0]['answer1'], references[0]['answer2']
|
||||
for prediction, reference in zip(judged_answers, references):
|
||||
categories['total'] += 1
|
||||
categories[reference['capability']] += 1
|
||||
if prediction['rank'] == 1:
|
||||
if reference['answer1'] == model1:
|
||||
win_model1[reference['capability']] += 1
|
||||
win_model1['total'] += 1
|
||||
else:
|
||||
win_model2[reference['capability']] += 1
|
||||
win_model2['total'] += 1
|
||||
else:
|
||||
if reference['answer1'] == model1:
|
||||
win_model2[reference['capability']] += 1
|
||||
win_model2['total'] += 1
|
||||
else:
|
||||
win_model1[reference['capability']] += 1
|
||||
win_model1['total'] += 1
|
||||
for capability in categories:
|
||||
if capability not in win_model1:
|
||||
win_model1[capability] = 0.0
|
||||
else:
|
||||
win_model1[capability] = round(
|
||||
(win_model1[capability] / categories[capability]) * 100, 2)
|
||||
if capability not in win_model2:
|
||||
win_model2[capability] = 0.0
|
||||
else:
|
||||
win_model2[capability] = round(
|
||||
(win_model2[capability] / categories[capability]) * 100, 2)
|
||||
|
||||
results = win_model2
|
||||
results['details'] = output
|
||||
return results
|
||||
@ -1,12 +1,20 @@
|
||||
# flake8: noqa: W605
|
||||
import json
|
||||
import math
|
||||
import os.path as osp
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datasets import Dataset, DatasetDict
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
from .utils import get_judgeanswer_and_reference
|
||||
|
||||
|
||||
@LOAD_DATASET.register_module()
|
||||
@ -35,3 +43,171 @@ class ArenaHardDataset(BaseDataset):
|
||||
})
|
||||
dataset = Dataset.from_list(raw_data)
|
||||
return dataset
|
||||
|
||||
|
||||
def post_process_arenahard(completion):
|
||||
s = completion['prediction']
|
||||
if result := re.findall('\[\[([AB<>=]+)\]\]', s):
|
||||
return result[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def get_battles_from_judgment(judged_answers, references, WEIGHT=3):
|
||||
arena_hard_battles = pd.DataFrame()
|
||||
for judged_answer, reference in zip(judged_answers, references):
|
||||
output = {
|
||||
'model_a': reference['answer1'],
|
||||
'model_b': reference['answer2']
|
||||
}
|
||||
|
||||
if judged_answer is not None:
|
||||
weight = 1
|
||||
if judged_answer == 'A=B':
|
||||
output['winner'] = 'tie'
|
||||
elif judged_answer == 'A>B':
|
||||
output['winner'] = 'model_a'
|
||||
elif judged_answer == 'A>>B':
|
||||
output['winner'] = 'model_a'
|
||||
weight = WEIGHT
|
||||
elif judged_answer == 'B>A':
|
||||
output['winner'] = 'model_b'
|
||||
elif judged_answer == 'B>>A':
|
||||
output['winner'] = 'model_b'
|
||||
weight = WEIGHT
|
||||
else:
|
||||
weight = 0
|
||||
else:
|
||||
weight = 0
|
||||
|
||||
if weight:
|
||||
arena_hard_battles = pd.concat(
|
||||
[arena_hard_battles,
|
||||
pd.DataFrame([output] * weight)])
|
||||
|
||||
return arena_hard_battles
|
||||
|
||||
|
||||
def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
|
||||
models = pd.concat([df['model_a'], df['model_b']]).unique()
|
||||
models = pd.Series(np.arange(len(models)), index=models)
|
||||
|
||||
# duplicate battles
|
||||
df = pd.concat([df, df], ignore_index=True)
|
||||
p = len(models.index)
|
||||
n = df.shape[0]
|
||||
|
||||
X = np.zeros([n, p])
|
||||
X[np.arange(n), models[df['model_a']]] = +math.log(BASE)
|
||||
X[np.arange(n), models[df['model_b']]] = -math.log(BASE)
|
||||
|
||||
# one A win => two A win
|
||||
Y = np.zeros(n)
|
||||
Y[df['winner'] == 'model_a'] = 1.0
|
||||
|
||||
# one tie => one A win + one B win
|
||||
# find tie + tie (both bad) index
|
||||
tie_idx = (df['winner'] == 'tie') | (df['winner'] == 'tie (bothbad)')
|
||||
tie_idx[len(tie_idx) // 2:] = False
|
||||
Y[tie_idx] = 1.0
|
||||
lr = LogisticRegression(
|
||||
fit_intercept=False, penalty=None, tol=1e-8
|
||||
) # May need to set a small value when not use GPT4 as judge model
|
||||
lr.fit(X, Y)
|
||||
|
||||
elo_scores = SCALE * lr.coef_[0] + INIT_RATING
|
||||
|
||||
# set anchor as gpt4-0314 = 1000
|
||||
if 'gpt4-0314' in models.index:
|
||||
elo_scores += 1000 - elo_scores[models['gpt4-0314']]
|
||||
return pd.Series(elo_scores,
|
||||
index=models.index).sort_values(ascending=False)
|
||||
|
||||
|
||||
def get_bootstrap_result(battles, func_compute_elo, num_round):
|
||||
rows = []
|
||||
for i in range(num_round):
|
||||
rows.append(func_compute_elo(battles.sample(frac=1.0, replace=True)))
|
||||
df = pd.DataFrame(rows)
|
||||
return df[df.median().sort_values(ascending=False).index]
|
||||
|
||||
|
||||
def preety_print_two_ratings(ratings_1, ratings_2, column_names):
|
||||
df = pd.DataFrame(
|
||||
[[n, ratings_1[n], ratings_2[n]] for n in ratings_1.keys()],
|
||||
columns=['Model', column_names[0], column_names[1]
|
||||
]).sort_values(column_names[0],
|
||||
ascending=False).reset_index(drop=True)
|
||||
df[column_names[0]] = (df[column_names[0]] + 0.5).astype(int)
|
||||
df[column_names[1]] = (df[column_names[1]] + 0.5).astype(int)
|
||||
df.index = df.index + 1
|
||||
return df
|
||||
|
||||
|
||||
def predict_win_rate(elo_ratings, SCALE=400, BASE=10, INIT_RATING=1000):
|
||||
names = sorted(list(elo_ratings.keys()))
|
||||
wins = defaultdict(lambda: defaultdict(lambda: 0))
|
||||
for a in names:
|
||||
for b in names:
|
||||
ea = 1 / (1 + BASE**((elo_ratings[b] - elo_ratings[a]) / SCALE))
|
||||
wins[a][b] = ea
|
||||
wins[b][a] = 1 - ea
|
||||
|
||||
data = {
|
||||
a: [wins[a][b] if a != b else np.NAN for b in names]
|
||||
for a in names
|
||||
}
|
||||
|
||||
df = pd.DataFrame(data, index=names)
|
||||
df.index.name = 'model_a'
|
||||
df.columns.name = 'model_b'
|
||||
return df.T
|
||||
|
||||
|
||||
def get_win_rate_column(df, column, baseline='gpt4-0314'):
|
||||
to_dict = df[['model', column]].set_index('model').to_dict()[column]
|
||||
win_rate_table = predict_win_rate(to_dict)
|
||||
return win_rate_table[baseline].fillna(0.5).apply(
|
||||
lambda x: round(x * 100, 2))
|
||||
|
||||
|
||||
@DICT_POSTPROCESSORS.register_module('arenahard')
|
||||
def arenahard_postprocess(output: dict, output_path: str) -> dict:
|
||||
judged_answers, references = get_judgeanswer_and_reference(
|
||||
output, output_path, post_process_arenahard)
|
||||
|
||||
if len(judged_answers) == 0:
|
||||
scores = None
|
||||
|
||||
battles = get_battles_from_judgment(
|
||||
judged_answers,
|
||||
references,
|
||||
)
|
||||
|
||||
bootstrap_online_elo = compute_mle_elo(battles)
|
||||
|
||||
np.random.seed(42)
|
||||
bootstrap_elo_lu = get_bootstrap_result(battles, compute_mle_elo, 100)
|
||||
stats = pd.DataFrame()
|
||||
stats['results'] = None
|
||||
stats['results'] = stats['results'].astype('object')
|
||||
|
||||
for i, model in enumerate(bootstrap_online_elo.index):
|
||||
assert model in bootstrap_elo_lu.columns
|
||||
stats.at[i, 'model'] = model
|
||||
stats.at[i, 'score'] = bootstrap_online_elo[model]
|
||||
# stats.at[i, 'lower'] = np.percentile(bootstrap_elo_lu[model], 2.5)
|
||||
# stats.at[i, 'upper'] = np.percentile(bootstrap_elo_lu[model], 97.5)
|
||||
# stats.at[i, 'results'] = bootstrap_elo_lu[model].tolist()
|
||||
|
||||
stats['score'] = get_win_rate_column(stats, 'score', 'gpt4-0314').tolist()
|
||||
models = stats['model']
|
||||
scores = stats['score']
|
||||
if models[0] == 'gpt4-0314':
|
||||
score = scores[1]
|
||||
else:
|
||||
score = scores[0]
|
||||
|
||||
results = {'score': score}
|
||||
results['details'] = output
|
||||
return results
|
||||
|
||||
@ -1,24 +0,0 @@
|
||||
from datasets import Dataset
|
||||
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
|
||||
from .subjective_cmp import SubjectiveCmpDataset
|
||||
|
||||
|
||||
@LOAD_DATASET.register_module()
|
||||
class CompassArenaDataset(SubjectiveCmpDataset):
|
||||
|
||||
def load(self, path: str, name: str, *args, **kwargs):
|
||||
dataset = list(super().load(path, name))
|
||||
creation_dataset = []
|
||||
for data in dataset:
|
||||
if 'reference' in data['others']:
|
||||
if data['others']['reference'] is not None:
|
||||
data['ref'] = data['others']['reference']
|
||||
else:
|
||||
data['ref'] = '满足用户需求，言之有理即可'
|
||||
else:
|
||||
data['ref'] = '满足用户需求，言之有理即可'
|
||||
creation_dataset.append(data)
|
||||
dataset = Dataset.from_list(creation_dataset)
|
||||
return dataset
|
||||
@ -1,13 +1,16 @@
|
||||
# flake8: noqa
|
||||
import json
|
||||
import os.path as osp
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from datasets import Dataset
|
||||
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
from .utils import get_judgeanswer_and_reference
|
||||
|
||||
|
||||
@LOAD_DATASET.register_module()
|
||||
@ -36,3 +39,46 @@ class FofoDataset(BaseDataset):
|
||||
})
|
||||
dataset = Dataset.from_list(raw_data)
|
||||
return dataset
|
||||
|
||||
|
||||
def post_process_fofo(judgement: dict):
|
||||
"""Input a string like below:
|
||||
|
||||
xxx[[5]]xxx, and extract the score
|
||||
"""
|
||||
match = re.search(r"[\"']format_correctness[\"']:\s*([0-1]+)",
|
||||
judgement['prediction'])
|
||||
if match:
|
||||
score = int(match.group(1))
|
||||
else:
|
||||
return None
|
||||
|
||||
return {'score': score}
|
||||
|
||||
|
||||
@DICT_POSTPROCESSORS.register_module('fofo')
|
||||
def fofo_postprocess(output: dict, output_path: str) -> dict:
|
||||
judged_answers, references = get_judgeanswer_and_reference(
|
||||
output, output_path, post_process_fofo)
|
||||
|
||||
if len(judged_answers) == 0:
|
||||
scores = None
|
||||
|
||||
scores = defaultdict(list)
|
||||
for ans, ref in zip(judged_answers, references):
|
||||
domain = ref['domain']
|
||||
format_name = ref['format']
|
||||
format_type = ref['format_type']
|
||||
score = ans['score']
|
||||
if score is not None:
|
||||
scores['overall'].append(score)
|
||||
scores[domain].append(score)
|
||||
if format_type == 'general':
|
||||
scores[format_name].append(score)
|
||||
single_model_scores = {
|
||||
task: sum(score) / len(score)
|
||||
for task, score in scores.items()
|
||||
}
|
||||
results = single_model_scores
|
||||
results['details'] = output
|
||||
return results
|
||||
|
||||
@ -2,23 +2,17 @@
|
||||
import json
|
||||
import os.path as osp
|
||||
import re
|
||||
import statistics
|
||||
from typing import Optional
|
||||
|
||||
from datasets import Dataset
|
||||
|
||||
from opencompass.openicl.icl_evaluator import BaseEvaluator
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
|
||||
|
||||
def check_match(target, generation):
|
||||
pattern_str = target.replace('{{',
|
||||
'{').replace('}}',
|
||||
'}').replace('{answer}', '.*')
|
||||
pattern_str = re.escape(pattern_str).replace('\\.\\*', '.*')
|
||||
match = re.fullmatch(pattern_str, generation)
|
||||
return bool(match)
|
||||
from .utils import get_judgeanswer_and_reference
|
||||
|
||||
|
||||
@LOAD_DATASET.register_module()
|
||||
@ -39,11 +33,11 @@ class FollowBenchDataset(BaseDataset):
|
||||
'judge_prompt': item['judge_prompt'],
|
||||
'judge': item
|
||||
})
|
||||
elif cate == 'rule':
|
||||
raw_data.append({
|
||||
'instruction': item['instruction'],
|
||||
'judge': item
|
||||
})
|
||||
# elif cate == 'rule':
|
||||
# raw_data.append({
|
||||
# 'instruction': item['instruction'],
|
||||
# 'judge': item
|
||||
# })
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Category '{cate}' is not implemented.")
|
||||
@ -52,112 +46,201 @@ class FollowBenchDataset(BaseDataset):
|
||||
return dataset
|
||||
|
||||
|
||||
class FollowBenchEvaluator(BaseEvaluator):
|
||||
"""Evaluator for followbench rule-based eval."""
|
||||
def post_process_followbench(item):
|
||||
generation, level = item['prediction'], item['gold']['level']
|
||||
try:
|
||||
satisfy = generation.strip('```').strip().split('\n')[-1]
|
||||
|
||||
def __init__(self, num_workers=16) -> None:
|
||||
self.num_workers = num_workers
|
||||
if level == 1:
|
||||
if 'YES' in satisfy:
|
||||
return 1, 1
|
||||
elif 'NO' in satisfy:
|
||||
return 0, 0
|
||||
else:
|
||||
raise Exception('Invalid evaluation for level 1.')
|
||||
else:
|
||||
satisfy_list = re.search(r'\[.*\]', satisfy)
|
||||
if satisfy_list:
|
||||
satisfy_list = eval(satisfy_list.group())
|
||||
if len(satisfy_list) == level:
|
||||
num_true = 0
|
||||
for i in satisfy_list:
|
||||
if i == 'YES' or i == 'True':
|
||||
num_true += 1
|
||||
elif i in [
|
||||
'NO', 'False', 'PARTIAL', 'MAYBE', 'UNKNOWN',
|
||||
'N/A'
|
||||
]:
|
||||
num_true += 0
|
||||
else:
|
||||
raise Exception('Invalid element in the list.')
|
||||
return int(num_true == level), num_true / level
|
||||
else:
|
||||
raise Exception('Invalid number of elements in the list.')
|
||||
else:
|
||||
raise Exception('Invalid list that cannot be parsed.')
|
||||
|
||||
def score_single(self, pred, refer):
|
||||
generated_code = pred
|
||||
except Exception as e:
|
||||
return -1, -1
|
||||
|
||||
# get current dir because we will enter a temp dir to
|
||||
# execute generated code
|
||||
cwd = os.getcwd()
|
||||
|
||||
def chdir_return(cwd, return_value):
|
||||
os.chdir(cwd)
|
||||
return return_value
|
||||
def get_scores(judged_answers, references):
|
||||
results = [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]
|
||||
n_group = len(judged_answers) // 5
|
||||
n_groups = [n_group] * 5
|
||||
|
||||
# we create a tempdir to execute each generated program
|
||||
with tempfile.TemporaryDirectory() as tempdir_name:
|
||||
for judged_answer, reference in zip(judged_answers, references):
|
||||
if judged_answer[0] == -1:
|
||||
n_groups[reference['level'] - 1] -= 1
|
||||
else:
|
||||
results[0][reference['level'] - 1] += judged_answer[0]
|
||||
results[1][reference['level'] - 1] += judged_answer[1]
|
||||
|
||||
tempdir_name = Path(tempdir_name)
|
||||
# copy all files and data dependencies from
|
||||
shutil.copytree(refer['problem_path'],
|
||||
tempdir_name,
|
||||
dirs_exist_ok=True)
|
||||
# generated outputs will be put into `result`
|
||||
os.mkdir(tempdir_name / 'result')
|
||||
for i in range(len(results)):
|
||||
for j in range(len(results[i])):
|
||||
if n_groups[j] != 0:
|
||||
results[i][j] = results[i][j] / n_groups[j]
|
||||
else:
|
||||
results[i][j] = 0
|
||||
temp_dict = {
|
||||
'HSR_AVG': statistics.mean(results[0]),
|
||||
'SSR_AVG': statistics.mean(results[1])
|
||||
}
|
||||
for idx, s in enumerate(results[0]):
|
||||
temp_dict[f'HSR_L{idx+1}'] = s
|
||||
for idx, s in enumerate(results[1]):
|
||||
temp_dict[f'SSR_L{idx+1}'] = s
|
||||
|
||||
program = refer['code_context'].replace('[insert]', generated_code)
|
||||
with open(tempdir_name / 'program.py', 'w', encoding='UTF-8') as f:
|
||||
f.write(program)
|
||||
return temp_dict
|
||||
|
||||
# enter into the tempdir to execute
|
||||
os.chdir(tempdir_name)
|
||||
|
||||
execution_status = []
|
||||
# a question may not have test case but we can still execute and
|
||||
# see if there is error
|
||||
test_cnt = max(1, int(refer['test_case_cnt']))
|
||||
for i in range(1, test_cnt + 1):
|
||||
# notice this command, e.g., you may need to
|
||||
# replace `python` with `python3`
|
||||
cmd_text = f'python program.py --test_case {i}'
|
||||
time_limit = 60 # should not change the official time_limit
|
||||
cmd = Command(cmd_text, )
|
||||
exit_code = cmd.run(
|
||||
timeout=time_limit) # 0 if there is no error
|
||||
execution_status.append(exit_code)
|
||||
@DICT_POSTPROCESSORS.register_module('followbench')
|
||||
def followbench_postprocess(
|
||||
output: dict,
|
||||
output_path: str,
|
||||
) -> dict:
|
||||
judged_answers, references = get_judgeanswer_and_reference(
|
||||
output, output_path, post_process_followbench)
|
||||
|
||||
# return if execution error
|
||||
if sum(execution_status) > 0:
|
||||
return chdir_return(cwd, False)
|
||||
results = get_scores(judged_answers, references)
|
||||
results['details'] = output
|
||||
return results
|
||||
|
||||
# loading testing code as a module
|
||||
test_module = import_source_file(tempdir_name / 'test_code.py',
|
||||
'test_code')
|
||||
pass_flag = True
|
||||
|
||||
if int(refer['test_type']) == 3:
|
||||
# stringTest parses the generated code into AST
|
||||
# and check AST components
|
||||
# if there is static error, stringTest may raise an exception
|
||||
generated_code = generated_code.split('\n')
|
||||
for line in generated_code:
|
||||
if 'print' in line and '#' not in line.split('print'):
|
||||
generated_code.remove(line)
|
||||
generated_code = '\n'.join(generated_code)
|
||||
try:
|
||||
pass_flag = test_module.stringTest(generated_code)
|
||||
except Exception:
|
||||
# return False if stringTest error
|
||||
return chdir_return(cwd, False)
|
||||
# def check_match(target, generation):
|
||||
# pattern_str = target.replace('{{',
|
||||
# '{').replace('}}',
|
||||
# '}').replace('{answer}', '.*')
|
||||
# pattern_str = re.escape(pattern_str).replace('\\.\\*', '.*')
|
||||
# match = re.fullmatch(pattern_str, generation)
|
||||
# return bool(match)
|
||||
|
||||
test_cnt = max(int(refer['test_case_cnt']), 1)
|
||||
for i in range(1, test_cnt + 1):
|
||||
try:
|
||||
ans = pickle.load(open(f'ans/ans{i}.pkl', 'rb'))
|
||||
# loading the generated output might still raise Exception
|
||||
# if the generated code is not correct
|
||||
result = pickle.load(open(f'result/result_{i}.pkl', 'rb'))
|
||||
pass_flag = test_module.test(result, ans) == 1
|
||||
except Exception:
|
||||
# return False if stringTest error
|
||||
return chdir_return(cwd, False)
|
||||
# class FollowBenchEvaluator(BaseEvaluator):
|
||||
# """Evaluator for followbench rule-based eval."""
|
||||
|
||||
return chdir_return(cwd, pass_flag)
|
||||
# def __init__(self, num_workers=16) -> None:
|
||||
# self.num_workers = num_workers
|
||||
|
||||
def score(self, predictions, references):
|
||||
results = {'example': {'accuracy': [0, 0, 0, 0, 0], 'num': 0}}
|
||||
for prediction, reference in zip(predictions, references):
|
||||
if reference['category'] == 'example':
|
||||
results['example']['num'] += 1
|
||||
template = reference['target'].replace('{instruction}\n', '')
|
||||
match_result = check_match(template, prediction)
|
||||
print(match_result)
|
||||
results['example']['accuracy'][reference['level'] -
|
||||
1] += match_result
|
||||
results['example']['accuracy'] = [
|
||||
round(acc / (results['example']['num'] // 5), 2)
|
||||
for acc in results['example']['accuracy']
|
||||
]
|
||||
######## Still not finished for rule-based evaluation
|
||||
# def score_single(self, pred, refer):
|
||||
# generated_code = pred
|
||||
|
||||
# Each process changes cwd, need to use multi-processing
|
||||
with ProcessPoolExecutor(self.num_workers) as executor:
|
||||
passed = sum(
|
||||
list(executor.map(self.score_single, predictions, references)))
|
||||
# # get current dir because we will enter a temp dir to
|
||||
# # execute generated code
|
||||
# cwd = os.getcwd()
|
||||
|
||||
return {'accuracy': passed / total}
|
||||
# def chdir_return(cwd, return_value):
|
||||
# os.chdir(cwd)
|
||||
# return return_value
|
||||
|
||||
# # we create a tempdir to execute each generated program
|
||||
# with tempfile.TemporaryDirectory() as tempdir_name:
|
||||
|
||||
# tempdir_name = Path(tempdir_name)
|
||||
# # copy all files and data dependencies from
|
||||
# shutil.copytree(refer['problem_path'],
|
||||
# tempdir_name,
|
||||
# dirs_exist_ok=True)
|
||||
# # generated outputs will be put into `result`
|
||||
# os.mkdir(tempdir_name / 'result')
|
||||
|
||||
# program = refer['code_context'].replace('[insert]', generated_code)
|
||||
# with open(tempdir_name / 'program.py', 'w', encoding='UTF-8') as f:
|
||||
# f.write(program)
|
||||
|
||||
# # enter into the tempdir to execute
|
||||
# os.chdir(tempdir_name)
|
||||
|
||||
# execution_status = []
|
||||
# # a question may not have test case but we can still execute and
|
||||
# # see if there is error
|
||||
# test_cnt = max(1, int(refer['test_case_cnt']))
|
||||
# for i in range(1, test_cnt + 1):
|
||||
# # notice this command, e.g., you may need to
|
||||
# # replace `python` with `python3`
|
||||
# cmd_text = f'python program.py --test_case {i}'
|
||||
# time_limit = 60 # should not change the official time_limit
|
||||
# cmd = Command(cmd_text, )
|
||||
# exit_code = cmd.run(
|
||||
# timeout=time_limit) # 0 if there is no error
|
||||
# execution_status.append(exit_code)
|
||||
|
||||
# # return if execution error
|
||||
# if sum(execution_status) > 0:
|
||||
# return chdir_return(cwd, False)
|
||||
|
||||
# # loading testing code as a module
|
||||
# test_module = import_source_file(tempdir_name / 'test_code.py',
|
||||
# 'test_code')
|
||||
# pass_flag = True
|
||||
|
||||
# if int(refer['test_type']) == 3:
|
||||
# # stringTest parses the generated code into AST
|
||||
# # and check AST components
|
||||
# # if there is static error, stringTest may raise an exception
|
||||
# generated_code = generated_code.split('\n')
|
||||
# for line in generated_code:
|
||||
# if 'print' in line and '#' not in line.split('print'):
|
||||
# generated_code.remove(line)
|
||||
# generated_code = '\n'.join(generated_code)
|
||||
# try:
|
||||
# pass_flag = test_module.stringTest(generated_code)
|
||||
# except Exception:
|
||||
# # return False if stringTest error
|
||||
# return chdir_return(cwd, False)
|
||||
|
||||
# test_cnt = max(int(refer['test_case_cnt']), 1)
|
||||
# for i in range(1, test_cnt + 1):
|
||||
# try:
|
||||
# ans = pickle.load(open(f'ans/ans{i}.pkl', 'rb'))
|
||||
# # loading the generated output might still raise Exception
|
||||
# # if the generated code is not correct
|
||||
# result = pickle.load(open(f'result/result_{i}.pkl', 'rb'))
|
||||
# pass_flag = test_module.test(result, ans) == 1
|
||||
# except Exception:
|
||||
# # return False if stringTest error
|
||||
# return chdir_return(cwd, False)
|
||||
|
||||
# return chdir_return(cwd, pass_flag)
|
||||
|
||||
# def score(self, predictions, references):
|
||||
# results = {'example': {'accuracy': [0, 0, 0, 0, 0], 'num': 0}}
|
||||
# for prediction, reference in zip(predictions, references):
|
||||
# if reference['category'] == 'example':
|
||||
# results['example']['num'] += 1
|
||||
# template = reference['target'].replace('{instruction}\n', '')
|
||||
# match_result = check_match(template, prediction)
|
||||
# print(match_result)
|
||||
# results['example']['accuracy'][reference['level'] -
|
||||
# 1] += match_result
|
||||
# results['example']['accuracy'] = [
|
||||
# round(acc / (results['example']['num'] // 5), 2)
|
||||
# for acc in results['example']['accuracy']
|
||||
# ]
|
||||
# ######## Still not finished for rule-based evaluation
|
||||
|
||||
# # Each process changes cwd, need to use multi-processing
|
||||
# with ProcessPoolExecutor(self.num_workers) as executor:
|
||||
# passed = sum(
|
||||
# list(executor.map(self.score_single, predictions, references)))
|
||||
|
||||
# return {'accuracy': passed / total}
|
||||
|
||||
@ -1,88 +0,0 @@
|
||||
# flake8: noqa: E501
|
||||
import json
|
||||
import os.path as osp
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from datasets import Dataset, DatasetDict
|
||||
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
|
||||
from .subjective_cmp import SubjectiveCmpDataset
|
||||
|
||||
eng_base_prefix = """
|
||||
You are a judger. Please impartially judge whether an AI model's response to a question is correct based on the reference answers. You need to provide a conclusion of "correct" or "wrong," followed by the corresponding reasoning.
|
||||
|
||||
Note that since the reference answer is a candidate list, the AI model's response only needs to align with one item in the list to be deemed "correct."
|
||||
|
||||
Your judgment must strictly adhere to the following format:
|
||||
Conclusion: [[Correct]]
|
||||
Reasoning: xxx.
|
||||
|
||||
Conclusion: [[Wrong]]
|
||||
Reasoning: xxx.
|
||||
|
||||
[Question Start]
|
||||
{question}
|
||||
[Question End]
|
||||
|
||||
[Reference Answers Start]
|
||||
{ref}
|
||||
[Reference Answers End]
|
||||
|
||||
[Model Response Start]
|
||||
"""
|
||||
|
||||
chn_base_prefix = """
|
||||
你是一个评判者，请你基于参考答案，公正地评判一个AI模型对于问题的回答是否正确。你需要给出“对或错”的结论，然后再给出相应的理由。
|
||||
请注意，由于参考答案是一个候选列表，因此AI模型的回答只要符合列表中的某一项即可判断为“对”。
|
||||
你的评判必须严格遵守以下格式：
|
||||
结论：[[对]]
|
||||
理由：xxx。
|
||||
|
||||
结论：[[错]]
|
||||
理由：xxx。
|
||||
|
||||
[问题开始]
|
||||
{question}
|
||||
[问题结束]
|
||||
|
||||
[参考答案开始]
|
||||
{ref}
|
||||
[参考答案结束]
|
||||
|
||||
[模型回答开始]
|
||||
"""
|
||||
|
||||
|
||||
def prompt_construct(sample):
|
||||
lan = sample['others']['lan']
|
||||
question = sample['question']
|
||||
if lan == 'zh':
|
||||
prefix = chn_base_prefix.format(question=sample['question'],
|
||||
ref=str(sample['others']['answers']))
|
||||
suffix = '\n[模型回答结束]\n'
|
||||
elif lan == 'en':
|
||||
prefix = eng_base_prefix.format(question=sample['question'],
|
||||
ref=str(sample['others']['answers']))
|
||||
suffix = '\n[Model Response End]\n'
|
||||
return prefix, suffix
|
||||
|
||||
|
||||
@LOAD_DATASET.register_module()
|
||||
class IRDataset(SubjectiveCmpDataset):
|
||||
|
||||
def load(
|
||||
self,
|
||||
path: str,
|
||||
name: str,
|
||||
):
|
||||
dataset = list(super().load(path, name))
|
||||
subject_dataset = []
|
||||
for data in dataset:
|
||||
data['gpt4_prefix'], data['gpt4_suffix'] = prompt_construct(data)
|
||||
data['judge']['others'] = data['others']
|
||||
data['ref'] = str(data['others']['answers'])
|
||||
subject_dataset.append(data)
|
||||
dataset = Dataset.from_list(subject_dataset)
|
||||
return dataset
|
||||
513
opencompass/datasets/subjective/judgerbench.py
Normal file
513
opencompass/datasets/subjective/judgerbench.py
Normal file
@ -0,0 +1,513 @@
|
||||
# flake8: noqa
|
||||
import json
|
||||
import os.path as osp
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datasets import Dataset
|
||||
|
||||
from opencompass.openicl.icl_evaluator import BaseEvaluator
|
||||
from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
|
||||
#######################Judgerbench A Prompt#############################
|
||||
|
||||
base_prompt_prefix_en = """Please read the dialogue between the two assistants and the user to determine which assistant performed better during the conversation.
|
||||
Here is the dialogue content:
|
||||
[Dialogue Begin]\n"""
|
||||
### User:
|
||||
### Assistant A:
|
||||
### Assistant B:
|
||||
base_prompt_suffix_en = """\n[Dialogue End]
|
||||
If you believe Assistant A performed better, please output A directly.
|
||||
If you believe Assistant B performed better, please output B directly.
|
||||
Do not output any other content, just the option.
|
||||
Please output:"""
|
||||
|
||||
base_prompt_prefix_cn = """请阅读两个助手与用户之间的对话，以确定在对话过程中哪个助手表现更好。
|
||||
以下是对话内容：
|
||||
[对话开始]\n"""
|
||||
|
||||
base_prompt_suffix_cn = """\n[对话结束]
|
||||
如果你认为助手A表现更好，请直接输出A。
|
||||
如果你认为助手B表现更好，请直接输出B。
|
||||
不要输出任何其他内容，只需输出选项。
|
||||
请输出："""
|
||||
|
||||
#######################Judgerbench B Prompt#############################
|
||||
|
||||
alignbench_judge_prompt = """Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user's instructions and answers the user's question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any position biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better.\n
|
||||
[User Question]\n{question}\n[The Start of Assistant A's Answer]\n{prediction}\n[The End of Assistant A's Answer]\n\n[The Start of Assistant B's Answer]\n{ref}\n[The End of Assistant B's Answer]"""
|
||||
|
||||
alpacaeval_judge_prompt = """I require a leaderboard for various large language models. I'll provide you with prompts given to these models and their corresponding outputs. Your task is to assess these responses, and select the model that produces the best output from a human perspective.
|
||||
|
||||
## Instruction
|
||||
[[Instruction]]: {question}
|
||||
|
||||
## Model Outputs
|
||||
|
||||
Here are the unordered outputs from the models. Each output is associated with a specific model, identified by a unique model identifier.
|
||||
|
||||
[[model_identifier]]: "m",
|
||||
[[output]]: "{prediction}"
|
||||
|
||||
[[model_identifier]]: "M",
|
||||
[[output]]: "{ref}"
|
||||
|
||||
## Task
|
||||
|
||||
Evaluate the models based on the quality and relevance of their outputs, and select the model that generated the best output. Answer by providing the model identifier of the best model. We will use your output as the name of the best model, so make sure your output only contains one of the following model identifiers and nothing else (no quotes, no spaces, no new lines, ...): m or M.
|
||||
|
||||
## Best Model Identifier
|
||||
"""
|
||||
|
||||
arenahard_judge_prompt = """Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\".
|
||||
<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's Answer|>\n{prediction}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{ref}\n<|The End of Assistant B's Answer|>"""
|
||||
|
||||
wild_judge_prompt = """# Instruction
|
||||
|
||||
You are an expert evaluator. Your task is to evaluate the quality of the \
|
||||
responses generated by two AI models.
|
||||
We will provide you with the user query and a pair of AI-generated \
|
||||
responses (Response A and Response B).
|
||||
You should first read the user query and the conversation history \
|
||||
carefully for analyzing the task, and then evaluate the quality of the \
|
||||
responses based on and rules provided below.
|
||||
|
||||
# Conversation between User and AI
|
||||
|
||||
## History
|
||||
<|begin_of_history|>
|
||||
|
||||
{history}
|
||||
|
||||
<|end_of_history|>
|
||||
|
||||
## Current User Query
|
||||
<|begin_of_query|>
|
||||
|
||||
{user_query}
|
||||
|
||||
<|end_of_query|>
|
||||
|
||||
## Response A
|
||||
<|begin_of_response_A|>
|
||||
|
||||
{prediction}
|
||||
|
||||
<|end_of_response_A|>
|
||||
|
||||
## Response B
|
||||
<|begin_of_response_B|>
|
||||
|
||||
{ref}
|
||||
|
||||
<|end_of_response_B|>
|
||||
|
||||
# Evaluation
|
||||
|
||||
## Checklist
|
||||
|
||||
<|begin_of_checklist|>
|
||||
|
||||
{checklist}
|
||||
|
||||
<|end_of_checklist|>
|
||||
|
||||
Please use this checklist to guide your evaluation, but do not limit your \
|
||||
assessment to the checklist.
|
||||
|
||||
## Rules
|
||||
|
||||
You should compare the above two responses based on your analysis of the \
|
||||
user queries and the conversation history.
|
||||
You should first write down your analysis and the checklist that you used \
|
||||
for the evaluation, and then provide your assessment according to the \
|
||||
checklist.
|
||||
There are five choices to give your final assessment: ["A++", "A+", \
|
||||
"A=B", "B+", "B++"], which correspond to the following meanings:
|
||||
|
||||
- `A++`: Response A is much better than Response B.
|
||||
- `A+`: Response A is only slightly better than Response B.
|
||||
- `A=B`: Response A and B are of the same quality. Please use this \
|
||||
choice sparingly.
|
||||
- `B+`: Response B is only slightly better than Response A.
|
||||
- `B++`: Response B is much better than Response A.
|
||||
|
||||
|
||||
## Output Format
|
||||
First, please output your analysis for each model response, and \
|
||||
then summarize your assessment to three aspects: "reason A=B", \
|
||||
"reason A>B", and "reason B>A", and finally make your choice for \
|
||||
the final assessment.
|
||||
|
||||
"""
|
||||
|
||||
wildbench_suffix = """Please provide your evaluation results in the following json \
|
||||
format by filling in the placeholders in []:
|
||||
```
|
||||
{
|
||||
"analysis of A": "[analysis of Response A]",
|
||||
"analysis of B": "[analysis of Response B]",
|
||||
"reason of A=B": "[where Response A and B perform equally well]",
|
||||
"reason of A>B": "[where Response A is better than Response B]",
|
||||
"reason of B>A": "[where Response B is better than Response A]",
|
||||
"choice": "[A++ or A+ or A=B or B+ or B++]",
|
||||
}
|
||||
```"""
|
||||
|
||||
fofo_en_judge_prompt = """
|
||||
I would like you to create a leaderboard that evaluates the correctness of the format of answers from various large language models. To accomplish this, you will need to analyze the text prompts given to the models and their corresponding answers. Specifically, please ensure that your evaluation outputs are properly formatted as a json string. I will provide both the prompts and the responses for this purpose.
|
||||
|
||||
Here is the prompt:
|
||||
[[Instruction]]: {question}
|
||||
|
||||
Here are the outputs of the models:
|
||||
[[Model Name]]: 'model',
|
||||
[[Model Answer]]: '{prediction}'
|
||||
|
||||
|
||||
Please evaluate the formatting of the model's responses by checking if they comply with the format specifications stated in the prompt. Perform a thorough format check and provide a detailed explanation for why the format is correct or incorrect. Your feedback should include the name of the model, followed by the format correctness status represented as '1' for correct and '0' for incorrect. Present your reasoning as bullet points within a single string for each model assessed. In other words, you should produce the following output:
|
||||
|
||||
"""
|
||||
|
||||
fofo_en_suffix = """```json
|
||||
[
|
||||
{
|
||||
'model': <model-name>,
|
||||
'format_correctness': <correctness>,
|
||||
'reasons': <reasons-of-format-correctness>
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Please note that your response should be a properly formatted JSON string and should not contain any additional content. We will load it directly as a JSON string in Python."""
|
||||
|
||||
fofo_cn_judge_prompt = """
|
||||
我希望你创建一个排行榜，用于评估来自各种大型语言模型的回答格式的正确性。为了完成这个任务，你将需要分析给模型的文本提示以及它们对应的回答。具体来说，请确保你的评估输出正确地格式化为JSON字符串。我将为此提供提示和回答。
|
||||
|
||||
以下是提示内容：
|
||||
[[Instruction]]: {question}
|
||||
|
||||
以下是模型的输出结果：
|
||||
[[Model Name]]: 'model',
|
||||
[[Model Answer]]: '{prediction}'
|
||||
|
||||
请通过检查模型回答是否符合提示中声明的格式规范来评估模型回答的格式。进行彻底的格式检查，并提供格式正确或错误的详细解释。你的反馈应包括模型的名称，接着是格式正确性的状态，用'1'表示正确，'0'表示错误。将你的推理以每个评估模型的单个字符串中的 bullet 点形式呈现。换句话说，你应该生成以下输出：
|
||||
|
||||
"""
|
||||
|
||||
fofo_cn_suffix = """```json
|
||||
[
|
||||
{
|
||||
'model': <模型名称>,
|
||||
'format_correctness': <正确性>,
|
||||
'reasons': <格式正确性的原因>
|
||||
}
|
||||
]
|
||||
```
|
||||
请注意，你的回答应是一个正确格式化的JSON字符串，不应包含任何额外的内容。我们将在Python中直接将其作为JSON字符串加载。"""
|
||||
|
||||
|
||||
def parse_conversation(conversation):
|
||||
# parse conversation into chat dialogue
|
||||
role_dict = {'user': 'HUMAN', 'assistant': 'assistant', 'HUMAN': 'HUMAN'}
|
||||
chat_round = []
|
||||
history = ''
|
||||
if len(conversation) > 0:
|
||||
for x in conversation[:-1]:
|
||||
if x['role'] == 'user' or x['role'] == 'HUMAN':
|
||||
history += 'USER: ' + x['content'] + '\n\n'
|
||||
elif x['role'] == 'assistant':
|
||||
history += 'ASSISTANT: ' + x['content'] + '\n\n'
|
||||
else:
|
||||
print(conversation)
|
||||
exit()
|
||||
chat_round.append({
|
||||
'role': role_dict[x['role']],
|
||||
'content': x['content']
|
||||
})
|
||||
|
||||
last_query = conversation[-1]['content']
|
||||
chat_round.append({
|
||||
'role': role_dict[conversation[-1]['role']],
|
||||
'content': conversation[-1]['content']
|
||||
})
|
||||
chat_round.append({'role': 'assistant', 'content': ''})
|
||||
|
||||
return chat_round, last_query, history
|
||||
|
||||
|
||||
@LOAD_DATASET.register_module()
|
||||
class JudgerBenchDataset(BaseDataset):
|
||||
|
||||
def load(self, path: str, name: str, *args, **kwargs):
|
||||
|
||||
path = get_data_path(path, local_mode=True)
|
||||
filename = osp.join(path, f'{name}.json')
|
||||
raw_data = []
|
||||
with open(filename, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
if 'judgerbench_A' in name:
|
||||
for item in data:
|
||||
conversation_a = item['conversation_a']
|
||||
conversation_b = item['conversation_b']
|
||||
model_a = item['model_a']
|
||||
model_b = item['model_b']
|
||||
winner = item['winner']
|
||||
category = item['category']
|
||||
turn = item['turn']
|
||||
dialogue_en, dialogue_cn = '', ''
|
||||
for itema, itemb in zip(conversation_a, conversation_b):
|
||||
if itema['role'] == 'user':
|
||||
dialogue_en += '### User: ' + itema['content'] + '\n'
|
||||
dialogue_cn += '### 用户： ' + itema['content'] + '\n'
|
||||
elif itema['role'] == 'assistant':
|
||||
dialogue_en += '### Assistant A: ' + itema[
|
||||
'content'] + '\n'
|
||||
dialogue_en += '### Assistant B: ' + itemb[
|
||||
'content'] + '\n'
|
||||
dialogue_cn += '### 助手A： ' + itema['content'] + '\n'
|
||||
dialogue_cn += '### 助手B： ' + itemb['content'] + '\n'
|
||||
else:
|
||||
raise NotImplementedError
|
||||
if '_en' in name:
|
||||
prompt = base_prompt_prefix_en + dialogue_en + base_prompt_suffix_en
|
||||
lan = 'en'
|
||||
elif '_cn' in name:
|
||||
prompt = base_prompt_prefix_cn + dialogue_cn + base_prompt_suffix_cn
|
||||
lan = 'cn'
|
||||
raw_data.append({
|
||||
'judge_prompt': prompt,
|
||||
'judge': {
|
||||
'category': category,
|
||||
'turn': turn,
|
||||
'winner': winner,
|
||||
'model_a': model_a,
|
||||
'model_b': model_b,
|
||||
'dataset_name': 'judgerbench_A',
|
||||
'lan': lan
|
||||
}
|
||||
})
|
||||
elif 'judgerbench_B' in name:
|
||||
for item in data:
|
||||
dataset_name = item['dataset_name']
|
||||
question = item.get('question', None)
|
||||
prediction = item['prediction']
|
||||
others = item['others']
|
||||
ref = item['others'].get('ref', None)
|
||||
if dataset_name == 'alignment_bench_v1_1':
|
||||
judge_prompt = alignbench_judge_prompt.format(
|
||||
question=question, prediction=prediction, ref=ref)
|
||||
elif dataset_name == 'alpaca_eval':
|
||||
judge_prompt = alpacaeval_judge_prompt.format(
|
||||
question=question, prediction=prediction, ref=ref)
|
||||
elif dataset_name == 'arenahard':
|
||||
judge_prompt = arenahard_judge_prompt.format(
|
||||
question=question, prediction=prediction, ref=ref)
|
||||
elif dataset_name == 'wildbench':
|
||||
conversation = item['conversation']
|
||||
checklist = item['others']['checklist']
|
||||
chat_round, last_query, history = parse_conversation(
|
||||
conversation)
|
||||
judge_prompt = wild_judge_prompt.format(
|
||||
history=history,
|
||||
user_query=last_query,
|
||||
prediction=prediction,
|
||||
ref=ref,
|
||||
checklist=checklist) + wildbench_suffix
|
||||
elif dataset_name == 'fofo_test_prompts':
|
||||
judge_prompt = fofo_en_judge_prompt.format(
|
||||
question=question,
|
||||
prediction=prediction,
|
||||
) + fofo_en_suffix
|
||||
elif dataset_name == 'fofo_test_prompts_cn':
|
||||
judge_prompt = fofo_cn_judge_prompt.format(
|
||||
question=question,
|
||||
prediction=prediction,
|
||||
) + fofo_cn_suffix
|
||||
raw_data.append({
|
||||
'judge_prompt': judge_prompt,
|
||||
'judge': {
|
||||
'others': others,
|
||||
'meta_judge': item['gpt4o_pred'],
|
||||
'dataset_name': dataset_name
|
||||
}
|
||||
})
|
||||
else:
|
||||
pass
|
||||
dataset = Dataset.from_list(raw_data)
|
||||
return dataset
|
||||
|
||||
|
||||
class JudgerBenchEvaluator(BaseEvaluator):
|
||||
"""Evaluator for followbench rule-based eval."""
|
||||
|
||||
def __init__(self, num_workers=16) -> None:
|
||||
self.num_workers = num_workers
|
||||
|
||||
def get_judge_result(self, judge, dataset_name):
|
||||
if dataset_name == 'alignment_bench_v1_1':
|
||||
if '[[A]]' in judge:
|
||||
return 1
|
||||
elif '[[B]]' in judge:
|
||||
return -1
|
||||
else:
|
||||
return None
|
||||
elif dataset_name == 'alpaca_eval':
|
||||
if judge[0] == 'm':
|
||||
return 1
|
||||
elif judge[0] == 'M':
|
||||
return -1
|
||||
else:
|
||||
return None
|
||||
elif dataset_name == 'fofo_test_prompts_cn' or dataset_name == 'fofo_test_prompts':
|
||||
match = re.search(r"[\"']format_correctness[\"']:\s*([0-1]+)",
|
||||
judge)
|
||||
if match:
|
||||
score = int(match.group(1))
|
||||
return score
|
||||
else:
|
||||
return None
|
||||
elif dataset_name == 'wildbench':
|
||||
pattern = r'\"choice\": \"(.*?)\"'
|
||||
matched_result = re.findall(pattern, judge)
|
||||
if matched_result:
|
||||
if 'A++' in matched_result[0]:
|
||||
return 2
|
||||
elif 'A+' in matched_result[0]:
|
||||
return 1
|
||||
elif 'A=B' in matched_result[0]:
|
||||
return 0
|
||||
elif 'B+' in matched_result[0]:
|
||||
return -1
|
||||
elif 'B++' in matched_result[0]:
|
||||
return -2
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
elif dataset_name == 'arenahard':
|
||||
if result := re.findall('\[\[([AB<>=]+)\]\]', judge):
|
||||
if 'A>>B' in result[0]:
|
||||
return 2
|
||||
elif 'A>B' in result[0]:
|
||||
return 1
|
||||
elif 'A=B' in result[0]:
|
||||
return 0
|
||||
elif 'B>A' in result[0]:
|
||||
return -1
|
||||
elif 'B>>A' in result[0]:
|
||||
return -2
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
def score(self, predictions, references):
|
||||
if len(predictions) != len(references):
|
||||
return {'error': 'preds and refrs have different length.'}
|
||||
correlations = {}
|
||||
if references[0]['dataset_name'] == 'judgerbench_A':
|
||||
|
||||
correct, total = 0, 0
|
||||
category_stats = {}
|
||||
|
||||
for index, (pred, ref) in enumerate(zip(predictions, references)):
|
||||
category = ref['category']
|
||||
|
||||
if ref['winner'] == 'model_a':
|
||||
ref_winner = 'A'
|
||||
elif ref['winner'] == 'model_b':
|
||||
ref_winner = 'B'
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
if category not in category_stats:
|
||||
category_stats[category] = {'correct': 0, 'total': 0}
|
||||
if 'A' in pred:
|
||||
pred = 'A'
|
||||
elif 'B' in pred:
|
||||
pred = 'B'
|
||||
is_correct = pred == ref_winner
|
||||
if is_correct:
|
||||
category_stats[category]['correct'] += 1
|
||||
correct += 1
|
||||
category_stats[category]['total'] += 1
|
||||
total += 1
|
||||
else:
|
||||
correct, total = 0, 0
|
||||
category_stats = {}
|
||||
models_scores = {}
|
||||
|
||||
for index, (pred, ref) in enumerate(zip(predictions, references)):
|
||||
|
||||
test_model, swap = ref['others']['model'], ref['others'][
|
||||
'swap']
|
||||
|
||||
dataset_name = ref['dataset_name']
|
||||
meta_judge = self.get_judge_result(ref['meta_judge'],
|
||||
dataset_name)
|
||||
if meta_judge is None:
|
||||
continue
|
||||
else:
|
||||
model_judge = self.get_judge_result(pred, dataset_name)
|
||||
|
||||
#### Calculate absolute accuracy
|
||||
if dataset_name not in category_stats:
|
||||
category_stats[dataset_name] = {
|
||||
'correct': 0,
|
||||
'total': 0
|
||||
}
|
||||
is_correct = model_judge == meta_judge
|
||||
if is_correct:
|
||||
category_stats[dataset_name]['correct'] += 1
|
||||
correct += 1
|
||||
category_stats[dataset_name]['total'] += 1
|
||||
total += 1
|
||||
|
||||
#### Calculate similarity
|
||||
if dataset_name not in models_scores:
|
||||
models_scores[dataset_name] = {}
|
||||
if test_model not in models_scores[dataset_name]:
|
||||
models_scores[dataset_name][test_model] = {
|
||||
'meta': 0,
|
||||
'self': 0
|
||||
}
|
||||
if swap:
|
||||
models_scores[dataset_name][test_model][
|
||||
'meta'] += -1 * meta_judge
|
||||
if model_judge is not None:
|
||||
models_scores[dataset_name][test_model][
|
||||
'self'] += -1 * model_judge
|
||||
else:
|
||||
models_scores[dataset_name][test_model][
|
||||
'meta'] += meta_judge
|
||||
if model_judge is not None:
|
||||
models_scores[dataset_name][test_model][
|
||||
'self'] += model_judge
|
||||
|
||||
for dataset, models in models_scores.items():
|
||||
meta_scores = [model['meta'] for model in models.values()]
|
||||
self_scores = [model['self'] for model in models.values()]
|
||||
correlation = np.corrcoef(meta_scores, self_scores)[0, 1]
|
||||
correlations['corr_' + dataset] = round(correlation, 3)
|
||||
average_correlation = sum(
|
||||
correlations.values()) / len(correlations)
|
||||
|
||||
# 将平均值添加到字典的开始处
|
||||
correlations = {
|
||||
f'avg_corr': round(average_correlation, 3),
|
||||
**correlations
|
||||
}
|
||||
results = {'accuracy': round(correct / total, 3) if total > 0 else 0.0}
|
||||
|
||||
for category, stats in category_stats.items():
|
||||
category_accuracy = round(stats['correct'] / stats['total'],
|
||||
3) if stats['total'] > 0 else 0.0
|
||||
results[f'accuracy_{category}'] = category_accuracy
|
||||
results.update(correlations)
|
||||
return results
|
||||
@ -2,14 +2,21 @@
|
||||
import json
|
||||
import os.path as osp
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Optional
|
||||
|
||||
from datasets import Dataset, DatasetDict
|
||||
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
from .utils import get_judgeanswer_and_reference
|
||||
|
||||
COLUMNS = [
|
||||
'total', 'writing', 'roleplay', 'reasoning', 'math', 'coding',
|
||||
'extraction', 'stem', 'humanities'
|
||||
]
|
||||
NEED_REF_CATS = ['math', 'reasoning', 'coding', 'arena-hard-200']
|
||||
|
||||
pair_v2 = {
|
||||
@ -206,3 +213,56 @@ class MTBenchDataset(BaseDataset):
|
||||
})
|
||||
dataset = Dataset.from_list(raw_data)
|
||||
return dataset
|
||||
|
||||
|
||||
def post_process_mtbench(judgement: str):
|
||||
"""Input a string like below:
|
||||
|
||||
xxx[[5]]xxx, and extract the score
|
||||
"""
|
||||
judgement = judgement['prediction']
|
||||
pattern = r'Rating:\s*\[\[([\d.]+)\]\]'
|
||||
matched_result = re.findall(pattern, judgement)
|
||||
if matched_result:
|
||||
score = float(matched_result[0])
|
||||
else:
|
||||
return None
|
||||
return {'score': score}
|
||||
|
||||
|
||||
def get_capability_results(
|
||||
judged_answers,
|
||||
references,
|
||||
):
|
||||
columns = COLUMNS
|
||||
capability_ratings = defaultdict(int)
|
||||
capability_counts = defaultdict(int)
|
||||
capability_avg_ratings = defaultdict(float)
|
||||
if len(judged_answers) == 0:
|
||||
for column in columns:
|
||||
capability_avg_ratings[column] = ''
|
||||
else:
|
||||
for ans, ref in zip(judged_answers, references):
|
||||
capability_ratings['total'] += ans['score']
|
||||
capability_counts['total'] += 1
|
||||
capability_ratings[ref['capability']] += ans['score']
|
||||
capability_counts[ref['capability']] += 1
|
||||
|
||||
for capability, total_score in capability_ratings.items():
|
||||
s = total_score / capability_counts[capability]
|
||||
s = round(s, 2)
|
||||
capability_avg_ratings[capability] = s
|
||||
return capability_avg_ratings
|
||||
|
||||
|
||||
@DICT_POSTPROCESSORS.register_module('mtbench')
|
||||
def mtbench_postprocess(
|
||||
output: dict,
|
||||
output_path: str,
|
||||
) -> dict:
|
||||
judged_answers, references = get_judgeanswer_and_reference(
|
||||
output, output_path, post_process_mtbench)
|
||||
|
||||
results = get_capability_results(judged_answers, references)
|
||||
results['details'] = output
|
||||
return results
|
||||
|
||||
@ -4,12 +4,15 @@ import os.path as osp
|
||||
import re
|
||||
|
||||
from datasets import Dataset, DatasetDict
|
||||
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
from collections import defaultdict
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
from typing import Optional
|
||||
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
|
||||
from .utils import get_judgeanswer_and_reference
|
||||
skip_first_tasks = ['FR', 'CR', 'AR', 'SA', 'SC', 'CM']
|
||||
|
||||
need_ref_tasks = ['MR', 'GR']
|
||||
@ -325,3 +328,58 @@ class MTBench101Dataset(BaseDataset):
|
||||
|
||||
dataset = Dataset.from_list(raw_data)
|
||||
return dataset
|
||||
|
||||
|
||||
def post_process_mtbench101(judgement: str):
|
||||
"""Input a string like below:
|
||||
|
||||
xxx[[5]]xxx, and extract the score
|
||||
"""
|
||||
judgement = judgement['prediction']
|
||||
match = re.search(r'\[([0-9]+)\]', judgement)
|
||||
if match:
|
||||
score = int(match.group(1))
|
||||
|
||||
else:
|
||||
return None
|
||||
|
||||
return {'score': score, 'judgement': judgement}
|
||||
|
||||
|
||||
def get_final_results(judged_answers, references):
|
||||
|
||||
task_multi_id_scores = defaultdict(list)
|
||||
task_scores = defaultdict(list)
|
||||
|
||||
for ans, ref in zip(judged_answers, references):
|
||||
|
||||
task = ref['task']
|
||||
multi_id = ref['multi_id']
|
||||
score = ans['score']
|
||||
|
||||
task_multi_id_scores[(task, multi_id)].append(score)
|
||||
|
||||
for (task, multi_id), scores in task_multi_id_scores.items():
|
||||
min_score = min(scores)
|
||||
task_scores[task].append(min_score)
|
||||
|
||||
final_task_scores = {
|
||||
task: sum(scores) / len(scores) if scores else 0
|
||||
for task, scores in task_scores.items()
|
||||
}
|
||||
average_score = round(
|
||||
sum(final_task_scores.values()) / len(final_task_scores), 3)
|
||||
|
||||
return {f'avg': average_score, **final_task_scores}
|
||||
|
||||
|
||||
@DICT_POSTPROCESSORS.register_module('mtbench101')
|
||||
def mtbench101_postprocess(output: dict,
|
||||
output_path: str,
|
||||
) -> dict:
|
||||
judged_answers, references = get_judgeanswer_and_reference(
|
||||
output, output_path, post_process_mtbench101)
|
||||
|
||||
results = get_final_results(judged_answers, references)
|
||||
results['details'] = output
|
||||
return results
|
||||
30
opencompass/datasets/subjective/utils.py
Normal file
30
opencompass/datasets/subjective/utils.py
Normal file
@ -0,0 +1,30 @@
|
||||
# flake8: noqa: E501
|
||||
def get_judgeanswer_and_reference(result, filename, post_process):
|
||||
"""Extract judgements (scores) and references.
|
||||
|
||||
Args:
|
||||
dataset (ConfigDict): Dataset config.
|
||||
subdir_path (str): Model path in results dir.
|
||||
post_process (function): The pre-defined extract function.
|
||||
"""
|
||||
if len(result) == 0:
|
||||
print('*' * 100)
|
||||
print('There are no results for ' + filename)
|
||||
print('*' * 100)
|
||||
judged_answers = []
|
||||
references = []
|
||||
for k, v in result.items():
|
||||
processed_judge = post_process(v)
|
||||
if processed_judge is not None:
|
||||
judged_answers.append(processed_judge)
|
||||
references.append(v['gold'])
|
||||
# else:
|
||||
# print(v['prediction'])
|
||||
# print('-' * 128)
|
||||
if len(judged_answers) <= 0.95 * len(result):
|
||||
print('*' * 100)
|
||||
print(
|
||||
f'For your {filename} judge. Among {len(result)} judgements, successfully extracted {len(judged_answers)} judgements, please check!'
|
||||
)
|
||||
print('*' * 100)
|
||||
return judged_answers, references
|
||||
@ -1,11 +1,15 @@
|
||||
# flake8: noqa: F401, F403
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from datasets import Dataset, DatasetDict
|
||||
|
||||
from opencompass.registry import LOAD_DATASET
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET
|
||||
from opencompass.utils import get_data_path
|
||||
|
||||
from ..base import BaseDataset
|
||||
from .utils import get_judgeanswer_and_reference
|
||||
|
||||
score_prompt = """# Instruction
|
||||
|
||||
@ -249,3 +253,91 @@ class WildBenchDataset(BaseDataset):
|
||||
})
|
||||
dataset = Dataset.from_list(raw_data)
|
||||
return dataset
|
||||
|
||||
|
||||
task_group_new = {
|
||||
'Information seeking': 'Information/Advice seeking',
|
||||
'Creative Writing': 'Creative Tasks',
|
||||
'Coding & Debugging': 'Coding & Debugging',
|
||||
'Reasoning': 'Planning & Reasoning',
|
||||
'Editing': 'Creative Tasks',
|
||||
'Math': 'Math & Data Analysis',
|
||||
'Planning': 'Planning & Reasoning',
|
||||
'Brainstorming': 'Creative Tasks',
|
||||
'Role playing': 'Creative Tasks',
|
||||
'Advice seeking': 'Information/Advice seeking',
|
||||
'Data Analysis': 'Math & Data Analysis',
|
||||
'Others': 'Creative Tasks'
|
||||
}
|
||||
|
||||
|
||||
def post_process_wildbench_pair(judgement: dict):
|
||||
judgement = judgement['prediction']
|
||||
pattern = r'\"choice\": \"(.*?)\"'
|
||||
matched_result = re.findall(pattern, judgement)
|
||||
if matched_result:
|
||||
return matched_result[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def post_process_wildbench_single(judgement: dict):
|
||||
judgement = judgement['prediction']
|
||||
pattern = r'\"score\": \"(.*?)\"'
|
||||
matched_result = re.findall(pattern, judgement)
|
||||
try:
|
||||
score = float(matched_result[0])
|
||||
return {'score': score}
|
||||
except (ValueError, IndexError) as e:
|
||||
return None
|
||||
|
||||
# if matched_result:
|
||||
# score = float(matched_result[0])
|
||||
# else:
|
||||
# return None
|
||||
# return {'score': score}
|
||||
|
||||
|
||||
@DICT_POSTPROCESSORS.register_module('wildbench')
|
||||
def wildbench_postprocess(output: dict, output_path: str) -> dict:
|
||||
judged_answers, references = get_judgeanswer_and_reference(
|
||||
output, output_path, post_process_wildbench_pair)
|
||||
|
||||
win_base_model = defaultdict(float)
|
||||
win_compare_model = defaultdict(float)
|
||||
categories = defaultdict(float)
|
||||
|
||||
score_mapping = {'A++': 1, 'A+': 0.5, 'A=B': 0, 'B+': -0.5, 'B++': -1}
|
||||
for prediction, reference in zip(judged_answers, references):
|
||||
if prediction not in score_mapping:
|
||||
continue
|
||||
|
||||
flag = 1 if reference['answer1'] in [
|
||||
'HaiKu', 'gpt4-turbo', 'llama-2-70b-chat-hf'
|
||||
] else -1
|
||||
score_1 = score_mapping[prediction] * flag
|
||||
score_2 = -score_1
|
||||
|
||||
tags = [reference['primary_tag']] + reference['secondary_tag']
|
||||
for tag in tags:
|
||||
win_base_model[task_group_new[tag]] += score_1
|
||||
win_compare_model[task_group_new[tag]] += score_2
|
||||
categories[task_group_new[tag]] += 1
|
||||
|
||||
for capability in categories:
|
||||
win_base_model[capability] = win_base_model[capability] / categories[
|
||||
capability] * 100
|
||||
win_base_model[capability] = round(win_base_model[capability], 2)
|
||||
win_compare_model[capability] = win_compare_model[
|
||||
capability] / categories[capability] * 100
|
||||
win_compare_model[capability] = round(win_compare_model[capability], 2)
|
||||
|
||||
# Calculating the mean of the values
|
||||
average = sum(win_compare_model.values()) / len(win_compare_model)
|
||||
|
||||
# Adding the mean to the dictionary at the beginning
|
||||
win_compare_model['average'] = average
|
||||
|
||||
results = win_compare_model
|
||||
results['details'] = output
|
||||
return results
|
||||
|
||||
@ -11,7 +11,7 @@ from mmengine.config import ConfigDict
|
||||
|
||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||
from opencompass.registry import ICL_PROMPT_TEMPLATES
|
||||
from opencompass.registry import DICT_POSTPROCESSORS, ICL_PROMPT_TEMPLATES
|
||||
from opencompass.utils import build_dataset_from_cfg, build_model_from_cfg
|
||||
from opencompass.utils.logging import get_logger
|
||||
|
||||
@ -94,6 +94,7 @@ class LMEvaluator:
|
||||
pack_all_predictions: Optional[bool] = False,
|
||||
dataset_cfg: Optional[ConfigDict] = None,
|
||||
pred_postprocessor: Optional[ConfigDict] = None,
|
||||
dict_postprocessor: Optional[ConfigDict] = None,
|
||||
) -> None:
|
||||
self.output_path = output_path
|
||||
out_dir, out_name = osp.split(output_path)
|
||||
@ -115,6 +116,8 @@ class LMEvaluator:
|
||||
self.logger = get_logger()
|
||||
self.dataset_cfg = dataset_cfg
|
||||
self.pack_all_predictions = pack_all_predictions
|
||||
self.pred_postprocessor = pred_postprocessor
|
||||
self.dict_postprocessor = dict_postprocessor
|
||||
|
||||
def score(self,
|
||||
predictions,
|
||||
@ -226,11 +229,15 @@ class LMEvaluator:
|
||||
self.inferencer.inference(retriever=retriever, prompt_template=self.meta_review_prompt_tmpl)
|
||||
else:
|
||||
self.inferencer.inference(retriever=retriever, prompt_template=self.prompt_tmpl)
|
||||
|
||||
output = mmengine.load(self.output_path)
|
||||
return self.postprocess(output)
|
||||
|
||||
def postprocess(self, output: Dict) -> Dict:
|
||||
"""Postprocess output by adding necessary statistics or data into
|
||||
it."""
|
||||
return output
|
||||
if self.dict_postprocessor is None:
|
||||
return output
|
||||
else:
|
||||
kwargs = self.dict_postprocessor
|
||||
proc = DICT_POSTPROCESSORS.get(kwargs.pop('type'))
|
||||
return proc(output, self.output_path, **kwargs)
|
||||
|
||||
@ -23,6 +23,9 @@ MODELS = Registry('model', locations=['opencompass.models'])
|
||||
LOAD_DATASET = Registry('load_dataset', locations=['opencompass.datasets'])
|
||||
TEXT_POSTPROCESSORS = Registry(
|
||||
'text_postprocessors', locations=['opencompass.utils.text_postprocessors'])
|
||||
DICT_POSTPROCESSORS = Registry(
|
||||
'dict_postprocessors', locations=['opencompass.utils.dict_postprocessors'])
|
||||
|
||||
EVALUATORS = Registry('evaluators', locations=['opencompass.evaluators'])
|
||||
|
||||
ICL_INFERENCERS = Registry('icl_inferencers',
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
# flake8: noqa: F401, E501
|
||||
from .circular import CircularSummarizer # noqa: F401
|
||||
from .default import DefaultSummarizer # noqa: F401
|
||||
from .default_subjective import DefaultSubjectiveSummarizer # noqa: F401
|
||||
from .llm_compression import LLMCompressionSummarizer # noqa: F401
|
||||
from .multi_faceted import MultiFacetedSummarizer # noqa: F401
|
||||
from .subjective import * # noqa: F401
|
||||
|
||||
383
opencompass/summarizers/default_subjective.py
Normal file
383
opencompass/summarizers/default_subjective.py
Normal file
@ -0,0 +1,383 @@
|
||||
# flake8: noqa
|
||||
# yapf: disable
|
||||
import functools
|
||||
import getpass
|
||||
import math
|
||||
import os.path as osp
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import mmengine
|
||||
import tabulate
|
||||
from mmengine import ConfigDict
|
||||
|
||||
from opencompass.utils import (LarkReporter, dataset_abbr_from_cfg,
|
||||
get_infer_output_path, get_logger,
|
||||
model_abbr_from_cfg)
|
||||
from opencompass.utils.prompt import get_prompt_hash
|
||||
|
||||
METRIC_WHITELIST = ['score', 'auc_score', 'accuracy', 'humaneval_pass@1', 'rouge1', 'avg_toxicity_score', 'bleurt_diff', 'matthews_correlation', 'truth', 'f1', 'exact_match', 'extract_rate']
|
||||
METRIC_BLACKLIST = ['bp', 'sys_len', 'ref_len', 'type']
|
||||
|
||||
def model_abbr_from_cfg_used_in_summarizer(model):
|
||||
if model.get('summarizer_abbr', None):
|
||||
return model['summarizer_abbr']
|
||||
else:
|
||||
return model_abbr_from_cfg(model)
|
||||
|
||||
|
||||
class DefaultSubjectiveSummarizer:
|
||||
"""Default summarizer in OpenCompass.
|
||||
|
||||
Args:
|
||||
config (ConfigDict): The configuration object of the evaluation task. It's expected to be filled out at runtime.
|
||||
dataset_abbrs (list[str], optional): Dataset abbreviations to be listed in the summary.
|
||||
summary_groups (list): The dataset groups whose results need to be averaged out. For example, mmlu. Each item it a dict with
|
||||
'name' (str) and 'subsets' (list of dataset abbrs), and optionally
|
||||
'weights' if weighted average is needed.
|
||||
prompt_db: A deprecated field.
|
||||
"""
|
||||
|
||||
def __init__(self, config: ConfigDict, dataset_abbrs: Optional[List[str]] = None, summary_groups: List = [], prompt_db = None) -> None:
|
||||
self.tasks = []
|
||||
self.cfg = config
|
||||
self.eval_model_cfgs = self.cfg['eval']['partitioner']['models']
|
||||
self.eval_model_abbrs = [
|
||||
model_abbr_from_cfg(model) for model in self.eval_model_cfgs
|
||||
]
|
||||
self.judge_models = self.cfg.get('judge_models', None)
|
||||
self.logger = get_logger()
|
||||
self.summary_groups = summary_groups
|
||||
self.dataset_abbrs = dataset_abbrs
|
||||
if prompt_db:
|
||||
self.logger.warning('prompt_db is deprecated and no longer used. '
|
||||
'Please remove it from your config.')
|
||||
|
||||
# Enable lark bot if lark_url is presented
|
||||
self.lark_reporter = None
|
||||
if self.cfg.get('lark_bot_url', None):
|
||||
self.lark_reporter = LarkReporter(self.cfg['lark_bot_url'])
|
||||
|
||||
self.model_cfgs = self.cfg['models']
|
||||
self.dataset_cfgs = self.cfg['datasets']
|
||||
self.work_dir = self.cfg['work_dir']
|
||||
model_abbrs = []
|
||||
for model in self.model_cfgs:
|
||||
model_abbr = model_abbr_from_cfg_used_in_summarizer(model)
|
||||
if model_abbr in model_abbrs:
|
||||
continue
|
||||
model_abbrs.append(model_abbr)
|
||||
self.model_abbrs = model_abbrs
|
||||
|
||||
def _pick_up_results(self, judge_abbr):
|
||||
"""The function reads the numerical results of evaluations from the
|
||||
output folder based on the configuration file, and ultimately returns
|
||||
four dictionaries, each containing processed information in different
|
||||
formats. The contents of the four dictionaries are as follows:
|
||||
|
||||
- raw_results: contains the raw results of each model on each dataset (excluding details).
|
||||
- parsed_results: contains the results of each model on each dataset for each metric, with metrics in METRIC_BLACKLIST being ignored.
|
||||
- dataset_metrics: contains the list of metrics for each dataset, consistent with the metrics in parsed_results. The list is ordered according to the METRIC_WHITELIST,
|
||||
with metrics appearing earlier considered more important.
|
||||
- dataset_eval_mode: contains the evaluation mode for each dataset.
|
||||
"""
|
||||
# raw_results: {model_abbr: {dataset_abbr: result}}
|
||||
raw_results : Dict[str, Dict[str, Any]] = {}
|
||||
# parsed_results: {model_abbr: {dataset_abbr: {metric: score}}}
|
||||
parsed_results : Dict[str, Dict[str, Dict[str, float]]] = {}
|
||||
# dataset_metrics: {dataset_abbr: [metric]}
|
||||
dataset_metrics : Dict[str, List[str]] = {}
|
||||
|
||||
for model in self.model_cfgs:
|
||||
model_abbr = model_abbr_from_cfg_used_in_summarizer(model)
|
||||
parsed_results.setdefault(model_abbr, {})
|
||||
raw_results.setdefault(model_abbr, {})
|
||||
for dataset in self.dataset_cfgs:
|
||||
base_models = dataset.get('base_models', None)
|
||||
if base_models is None:
|
||||
base_models_list = ['']
|
||||
else:
|
||||
base_models_list = [item['abbr'] for item in base_models]
|
||||
|
||||
for base_model_abbr in base_models_list:
|
||||
dataset_abbr = dataset_abbr_from_cfg(dataset)
|
||||
origin_path = get_infer_output_path(model, dataset, osp.join(self.work_dir, 'results'))
|
||||
if base_model_abbr != '':
|
||||
temp_path, dataset_json_name = origin_path.rsplit('/', 1)[0], origin_path.rsplit('/', 1)[1]
|
||||
filepath = osp.join(temp_path.rsplit('/', 1)[0], base_model_abbr + '_' + temp_path.rsplit('/', 1)[1] + '_judged-by--' + judge_abbr, dataset_json_name)
|
||||
else:
|
||||
filepath = osp.join(origin_path.rsplit('/', 1)[0] + '_judged-by--' + judge_abbr, origin_path.rsplit('/', 1)[1])
|
||||
if not osp.exists(filepath):
|
||||
continue
|
||||
result = mmengine.load(filepath)
|
||||
result.pop('details', None)
|
||||
raw_results[model_abbr][dataset_abbr] = result
|
||||
if 'error' in result:
|
||||
self.logger.debug(f'error in {model_abbr} {dataset_abbr} {result["error"]}')
|
||||
continue
|
||||
_rst, _dm = {}, []
|
||||
for metric, score in result.items():
|
||||
if metric not in METRIC_BLACKLIST and isinstance(score, (int, float)):
|
||||
_rst[metric] = score
|
||||
_dm.append(metric)
|
||||
else:
|
||||
continue
|
||||
if len(_rst) == 0:
|
||||
self.logger.warning(f'unknown result format: {result}, continue')
|
||||
continue
|
||||
_dm = sorted(_dm, key=lambda i: METRIC_WHITELIST.index(i) if i in METRIC_WHITELIST else len(METRIC_WHITELIST))
|
||||
|
||||
if dataset_abbr in dataset_metrics:
|
||||
assert tuple(dataset_metrics[dataset_abbr]) == tuple(_dm), \
|
||||
f'{dataset_abbr} has different metrics: {dataset_metrics[dataset_abbr]} vs {_dm}'
|
||||
else:
|
||||
dataset_metrics[dataset_abbr] = _dm
|
||||
parsed_results[model_abbr][dataset_abbr] = _rst
|
||||
|
||||
# dataset_eval_mode: {dataset_abbr: eval_mode}
|
||||
dataset_eval_mode : Dict[str, str] = {}
|
||||
for dataset in self.dataset_cfgs:
|
||||
inferencer = dataset.get('infer_cfg', {}).get('inferencer', {}).get('type', '')
|
||||
inferencer = inferencer if isinstance(inferencer, str) else inferencer.__name__
|
||||
dataset_abbr = dataset_abbr_from_cfg(dataset)
|
||||
if 'GenInferencer' in inferencer:
|
||||
dataset_eval_mode[dataset_abbr] = 'gen'
|
||||
elif 'PPLInferencer' in inferencer:
|
||||
dataset_eval_mode[dataset_abbr] = 'ppl'
|
||||
elif 'LLInferencer' in inferencer:
|
||||
dataset_eval_mode[dataset_abbr] = 'll'
|
||||
else:
|
||||
dataset_eval_mode[dataset_abbr] = 'unknown'
|
||||
self.logger.warning(f'unknown inferencer: {inferencer} - {dataset_abbr}')
|
||||
return raw_results, parsed_results, dataset_metrics, dataset_eval_mode
|
||||
|
||||
def _calculate_group_metrics(self, raw_results, parsed_results, dataset_metrics, dataset_eval_mode):
|
||||
"""The function calculates the numerical results for each group based
|
||||
on the configuration in summary_groups, and updates the contents of
|
||||
each dictionary accordingly."""
|
||||
summary_groups = self.summary_groups
|
||||
for sg in summary_groups:
|
||||
for model_abbr in self.model_abbrs:
|
||||
available_metrics, missing_metrics = [], []
|
||||
for i in sg['subsets']:
|
||||
if isinstance(i, (list, tuple)):
|
||||
if i[0] in parsed_results[model_abbr] and i[1] in parsed_results[model_abbr][i[0]]:
|
||||
available_metrics.append(i)
|
||||
else:
|
||||
missing_metrics.append(i)
|
||||
else:
|
||||
if i in parsed_results[model_abbr]:
|
||||
available_metrics.append(i)
|
||||
else:
|
||||
missing_metrics.append(i)
|
||||
|
||||
if len(available_metrics) == 0:
|
||||
continue
|
||||
if len(missing_metrics) != 0:
|
||||
raw_results[model_abbr][sg['name']] = {'error': 'missing metrics: {}'.format(missing_metrics)}
|
||||
continue
|
||||
|
||||
if 'metric' in sg:
|
||||
default_metric = sg['metric']
|
||||
need_smart_metric = False
|
||||
else:
|
||||
need_smart_metric = True
|
||||
if sg.get('std', False):
|
||||
default_metric = 'standard_deviation'
|
||||
elif sg.get('sum', False):
|
||||
default_metric = 'sum'
|
||||
elif sg.get('weights', []):
|
||||
default_metric = 'weighted_average'
|
||||
else:
|
||||
default_metric = 'naive_average'
|
||||
|
||||
scores, eval_modes, group_metrics = {}, [], None
|
||||
if any(isinstance(dataset_abbr, (list, tuple)) for dataset_abbr in sg['subsets']) and \
|
||||
any(isinstance(dataset_abbr, str) for dataset_abbr in sg['subsets']):
|
||||
raise NotImplementedError('mixed dataset_abbr type is not supported')
|
||||
|
||||
if all(isinstance(dataset_abbr, (list, tuple)) for dataset_abbr in sg['subsets']):
|
||||
group_metrics = [default_metric]
|
||||
for dataset_abbr, metric in sg['subsets']:
|
||||
scores.setdefault(default_metric, {})[dataset_abbr + '@' + metric] = parsed_results[model_abbr][dataset_abbr][metric]
|
||||
eval_modes.append(dataset_eval_mode.get(dataset_abbr, 'unknown'))
|
||||
else:
|
||||
group_metrics = list(functools.reduce(lambda a, b: a & b, [set(dataset_metrics[dataset_abbr]) for dataset_abbr in sg['subsets']]))
|
||||
if need_smart_metric and len(group_metrics) > 1:
|
||||
for metric in group_metrics:
|
||||
for dataset_abbr in sg['subsets']:
|
||||
scores.setdefault(metric, {})[dataset_abbr + '@' + metric] = parsed_results[model_abbr][dataset_abbr][metric]
|
||||
eval_modes.append(dataset_eval_mode.get(sg['subsets'][0], 'unknown'))
|
||||
else:
|
||||
group_metrics = [default_metric]
|
||||
for dataset_abbr in sg['subsets']:
|
||||
metric = dataset_metrics[dataset_abbr][0]
|
||||
scores.setdefault(default_metric, {})[dataset_abbr + '@' + metric] = parsed_results[model_abbr][dataset_abbr][metric]
|
||||
eval_modes.append(dataset_eval_mode.get(dataset_abbr, 'unknown'))
|
||||
|
||||
result = {}
|
||||
for metric in scores:
|
||||
if default_metric == 'standard_deviation':
|
||||
avg = sum(scores[metric].values()) / len(scores[metric])
|
||||
variance = sum((scores[metric][k] - avg) ** 2 for k in scores[metric]) / len(scores[metric])
|
||||
scores[metric] = result[metric] = math.sqrt(variance)
|
||||
else:
|
||||
if sg.get('weights', []):
|
||||
# check sg['weights'][k] != 0 in case of scores[metric][k] is NaN
|
||||
try:
|
||||
numerator = sum(scores[metric][k] * sg['weights'][k] for k in sg['weights'] if sg['weights'][k] != 0)
|
||||
except KeyError:
|
||||
tmp_scores = {metric: {k.split('@')[0]: v for k, v in scores[metric].items()}}
|
||||
numerator = sum(tmp_scores[metric][k] * sg['weights'][k] for k in sg['weights'] if sg['weights'][k] != 0)
|
||||
denominator = sum(sg['weights'].values())
|
||||
else:
|
||||
numerator = sum(scores[metric].values())
|
||||
denominator = len(scores[metric])
|
||||
if default_metric == 'sum':
|
||||
scores[metric] = result[metric] = numerator
|
||||
else:
|
||||
scores[metric] = result[metric] = numerator / denominator
|
||||
eval_modes = list(set(eval_modes))
|
||||
eval_mode = eval_modes[0] if len(eval_modes) == 1 else 'mixed'
|
||||
|
||||
# add to global results
|
||||
raw_results[model_abbr].setdefault(sg['name'], {}).update(scores)
|
||||
parsed_results[model_abbr].setdefault(sg['name'], {}).update(result)
|
||||
dataset_metrics.setdefault(sg['name'], []).extend(group_metrics)
|
||||
dataset_eval_mode[sg['name']] = eval_mode
|
||||
|
||||
return raw_results, parsed_results, dataset_metrics, dataset_eval_mode
|
||||
|
||||
def _format_table(self, parsed_results, dataset_metrics, dataset_eval_mode, required_dataset_abbrs=None, skip_all_slash=False):
|
||||
dataset_abbrs = [dataset_abbr_from_cfg(dataset) for dataset in self.dataset_cfgs]
|
||||
prompt_version = {dataset_abbr_from_cfg(d): get_prompt_hash(d)[:6] for d in self.dataset_cfgs}
|
||||
|
||||
summarizer_dataset_abbrs = []
|
||||
if required_dataset_abbrs is None:
|
||||
# display all dataset metrics included in the config
|
||||
for dataset_abbr in dataset_abbrs:
|
||||
if dataset_abbr in dataset_metrics:
|
||||
for metric in dataset_metrics[dataset_abbr]:
|
||||
summarizer_dataset_abbrs.append((dataset_abbr, metric))
|
||||
else:
|
||||
summarizer_dataset_abbrs.append((dataset_abbr, None))
|
||||
# along with all possible group metrics
|
||||
for dataset_abbr in dataset_metrics:
|
||||
for metric in dataset_metrics[dataset_abbr]:
|
||||
if (dataset_abbr, metric) not in summarizer_dataset_abbrs:
|
||||
summarizer_dataset_abbrs.append((dataset_abbr, metric))
|
||||
else:
|
||||
# follow the required order
|
||||
for item in required_dataset_abbrs:
|
||||
if isinstance(item, str):
|
||||
summarizer_dataset_abbrs.append((item, None))
|
||||
elif isinstance(item, (list, tuple)):
|
||||
summarizer_dataset_abbrs.append((item[0], item[1]))
|
||||
|
||||
table = []
|
||||
header = ['dataset', 'version', 'metric', 'mode'] + self.model_abbrs
|
||||
table.append(header)
|
||||
for dataset_abbr, metric in summarizer_dataset_abbrs:
|
||||
if dataset_abbr not in dataset_metrics:
|
||||
if not skip_all_slash:
|
||||
table.append([dataset_abbr, '-', '-', '-'] + ['-'] * len(self.model_abbrs))
|
||||
continue
|
||||
if metric is None:
|
||||
metric = dataset_metrics[dataset_abbr][0]
|
||||
elif metric in dataset_metrics[dataset_abbr]:
|
||||
pass
|
||||
else:
|
||||
if not skip_all_slash:
|
||||
table.append([dataset_abbr, '-', '-', '-'] + ['-'] * len(self.model_abbrs))
|
||||
continue
|
||||
|
||||
row = [dataset_abbr, prompt_version.get(dataset_abbr, '-'), metric, dataset_eval_mode.get(dataset_abbr, '-')]
|
||||
for model_abbr in self.model_abbrs:
|
||||
if dataset_abbr in parsed_results[model_abbr]:
|
||||
row.append('{:.02f}'.format(parsed_results[model_abbr][dataset_abbr][metric]))
|
||||
else:
|
||||
row.append('-')
|
||||
table.append(row)
|
||||
return table
|
||||
|
||||
def _format_raw_txt(self, raw_results):
|
||||
raw_dataset_abbrs = []
|
||||
for model_abbr in self.model_abbrs:
|
||||
for dataset_abbr in raw_results[model_abbr]:
|
||||
if dataset_abbr not in raw_dataset_abbrs:
|
||||
raw_dataset_abbrs.append(dataset_abbr)
|
||||
raw_txts = []
|
||||
for model_abbr in self.model_abbrs:
|
||||
raw_txts.append('-------------------------------')
|
||||
raw_txts.append(f'Model: {model_abbr}')
|
||||
for dataset_abbr in raw_dataset_abbrs:
|
||||
result = raw_results[model_abbr].get(dataset_abbr, '{}')
|
||||
raw_txts.append(f'{dataset_abbr}: {result}')
|
||||
raw_txts = '\n'.join(raw_txts)
|
||||
return raw_txts
|
||||
|
||||
def _output_to_file(self, output_path, time_str, table, raw_txts, judge_abbr):
|
||||
# output to file
|
||||
if output_path is None:
|
||||
output_path = osp.join(self.work_dir, 'summary', f'summary_{time_str}.txt')
|
||||
output_csv_path = osp.join(self.work_dir, 'summary', f'summary_{time_str}.csv')
|
||||
else:
|
||||
output_csv_path = output_path.replace('.txt', '.csv')
|
||||
output_path = output_path.split('.txt')[0] + '_by_' + judge_abbr + '.txt'
|
||||
output_csv_path = output_csv_path.split('.csv')[0] + '_by_' + judge_abbr + '.csv'
|
||||
output_dir = osp.split(output_path)[0]
|
||||
mmengine.mkdir_or_exist(output_dir)
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
text = f'{time_str}\n' + \
|
||||
'tabulate format\n' + \
|
||||
'^' * 128 + '\n' + \
|
||||
tabulate.tabulate(table, headers='firstrow', floatfmt='.2f') + '\n' + \
|
||||
'$' * 128 + '\n\n' + \
|
||||
'-' * 128 + ' THIS IS A DIVIDER ' + '-' * 128 + '\n\n' + \
|
||||
'csv format\n' + \
|
||||
'^' * 128 + '\n' + \
|
||||
'\n'.join([','.join(row) for row in table]) + '\n' + \
|
||||
'$' * 128 + '\n\n' + \
|
||||
'-' * 128 + ' THIS IS A DIVIDER ' + '-' * 128 + '\n\n' + \
|
||||
'raw format\n' + \
|
||||
'^' * 128 + '\n' + \
|
||||
raw_txts + '\n' + \
|
||||
'$' * 128 + '\n'
|
||||
f.write(text)
|
||||
self.logger.info(f'write summary to {osp.abspath(output_path)}')
|
||||
|
||||
with open(output_csv_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join([','.join(row) for row in table]) + '\n')
|
||||
self.logger.info(f'write csv to {osp.abspath(output_csv_path)}')
|
||||
|
||||
def summarize(
|
||||
self,
|
||||
output_path: str = None,
|
||||
time_str: str = datetime.now().strftime('%Y%m%d_%H%M%S')): # noqa
|
||||
|
||||
for judge_model in self.judge_models:
|
||||
judge_abbr = model_abbr_from_cfg(judge_model)
|
||||
|
||||
# pick up results
|
||||
raw_results, parsed_results, dataset_metrics, dataset_eval_mode = self._pick_up_results(judge_abbr)
|
||||
|
||||
# calculate group metrics
|
||||
raw_results, parsed_results, dataset_metrics, dataset_eval_mode = \
|
||||
self._calculate_group_metrics(raw_results, parsed_results, dataset_metrics, dataset_eval_mode)
|
||||
|
||||
# format table
|
||||
table = self._format_table(parsed_results, dataset_metrics, dataset_eval_mode, required_dataset_abbrs=self.dataset_abbrs)
|
||||
|
||||
# format raw txt
|
||||
raw_txts = self._format_raw_txt(raw_results)
|
||||
|
||||
# output to screen
|
||||
print(tabulate.tabulate(table, headers='firstrow', floatfmt='.2f'))
|
||||
|
||||
# output to .text / .csv files
|
||||
self._output_to_file(output_path, time_str, table, raw_txts, judge_abbr)
|
||||
|
||||
if self.lark_reporter:
|
||||
content = f'{getpass.getuser()} 的'
|
||||
content += f'详细评测汇总已输出至 {osp.abspath(output_path)}'
|
||||
self.lark_reporter.post(content)
|
||||
@ -1,6 +1,6 @@
|
||||
# flake8: noqa
|
||||
# yapf: disable
|
||||
import copy
|
||||
import csv
|
||||
import os
|
||||
import os.path as osp
|
||||
import re
|
||||
@ -8,52 +8,34 @@ from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from itertools import product
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from mmengine import ConfigDict
|
||||
from tabulate import tabulate
|
||||
|
||||
from opencompass.partitioners.sub_naive import remove_duplicate_pairs
|
||||
from opencompass.summarizers.subjective.compass_arena import (
|
||||
check_position_bias, model_abbr_from_cfg_used_in_summarizer)
|
||||
from opencompass.summarizers.subjective.utils import (
|
||||
get_judgeanswer_and_reference, get_outdir)
|
||||
from opencompass.utils import dataset_abbr_from_cfg, model_abbr_from_cfg
|
||||
|
||||
from .utils import get_judgeanswer_and_reference, get_outdir
|
||||
|
||||
|
||||
def model_abbr_from_cfg_used_in_summarizer(model):
|
||||
if model.get('summarizer_abbr', None):
|
||||
return model['summarizer_abbr']
|
||||
else:
|
||||
return model_abbr_from_cfg(model)
|
||||
|
||||
def post_process_compass_arena(s):
|
||||
if result := re.findall(r'(?:选择：|Choice: )\[\[([ABC])\]\]', s):
|
||||
return result[0]
|
||||
def post_process_wildbench_pair(judgement: str):
|
||||
pattern = r'\"choice\": \"(.*?)\"'
|
||||
matched_result = re.findall(pattern, judgement)
|
||||
if matched_result:
|
||||
return matched_result[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
MAP = {'language':['总分','中文总分','英文总分','自然语言处理_cn','创作_cn','对话_cn','NLP_en','creation_en','chat_en'],
|
||||
'instruct':['总分','中文总分','英文总分',],
|
||||
'reasoning':['总分','中文总分','英文总分','Common Sense Reasoning_cn','Social Reasoning_cn','Humanities (History, Finance, etc.) Professional Reasoning_cn', 'Science and Engineering Professional Reasoning_cn',
|
||||
'Common Sense Reasoning_en','Social Reasoning_en','Humanities (History, Finance, etc.) Professional Reasoning_en', 'Science and Engineering Professional Reasoning_en',],
|
||||
'coding':['总分','中文总分','英文总分',]}
|
||||
|
||||
def check_position_bias(judged_answers, references, banned_choice=['C']):
|
||||
"""Check position bias for judgellm's judgement.
|
||||
|
||||
Args:
|
||||
judged_answers: The successfully extracted judgement.
|
||||
references: The references contains original question, which is used to located the same question for different position judgement.
|
||||
"""
|
||||
position_bias_flag = 0
|
||||
position_bias_dict = {}
|
||||
for judge, ref in zip(judged_answers, references):
|
||||
question = ref['question']
|
||||
question_hash = hash(question)
|
||||
if question_hash not in position_bias_dict:
|
||||
position_bias_dict[question_hash] = {
|
||||
'question': question,
|
||||
'judge': judge
|
||||
}
|
||||
else:
|
||||
first_judge = position_bias_dict[question_hash]['judge']
|
||||
if judge == first_judge and first_judge not in banned_choice and judge not in banned_choice:
|
||||
# If second choice is same with first choice, there has position bias.
|
||||
position_bias_flag += 1
|
||||
return position_bias_flag
|
||||
MAP = {'instruct':['总分','中文总分','英文总分',]}
|
||||
|
||||
|
||||
class CompassBenchSummarizer:
|
||||
@ -64,23 +46,16 @@ class CompassBenchSummarizer:
|
||||
It's expected to be filled out at runtime.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
config: ConfigDict,
|
||||
judge_type='general',
|
||||
check_pos_bias=True,
|
||||
summary_type='single') -> None:
|
||||
def __init__(self, config: ConfigDict, check_pos_bias=False) -> None:
|
||||
self.tasks = []
|
||||
self.cfg = config
|
||||
self.base_models = self.cfg['datasets'][0]['base_models']
|
||||
self.compare_models = self.cfg['eval']['partitioner']['models']
|
||||
self.judge_models = self.cfg.get('judge_models', None)
|
||||
self.meta_judge_model = self.cfg.eval.partitioner.get('meta_judge_model', None)
|
||||
self.judge_type = judge_type
|
||||
assert self.judge_type in ['general']
|
||||
self.judge_map = {'general': post_process_compass_arena}
|
||||
self.judge_function = self.judge_map[self.judge_type]
|
||||
self.judge_abbr = model_abbr_from_cfg(self.cfg['judge_models'][0])
|
||||
self.judge_function = post_process_wildbench_pair
|
||||
self.check_pos_bias = check_pos_bias
|
||||
self.summary_type = summary_type
|
||||
|
||||
def get_score(self, time_str):
|
||||
output_dir, results_folder = get_outdir(self.cfg, time_str)
|
||||
@ -91,88 +66,53 @@ class CompassBenchSummarizer:
|
||||
self.judge_models.append(self.meta_judge_model)
|
||||
|
||||
scores = {}
|
||||
|
||||
for idx, judge_model_cfg in enumerate(self.judge_models):
|
||||
judge_model = model_abbr_from_cfg(judge_model_cfg)
|
||||
scores[judge_model] = {}
|
||||
for dataset in self.cfg['datasets']:
|
||||
dataset_abbr = dataset_abbr_from_cfg(dataset)
|
||||
dataset_root, dataset_detail = dataset_abbr.split('/')[0], dataset_abbr.split('/')[1]
|
||||
scores[judge_model][dataset_abbr] = {}
|
||||
for model_pair in unique_combinations:
|
||||
model1 = model_pair[0]['abbr']
|
||||
model2 = model_pair[1]['abbr']
|
||||
base_model = model_pair[0]['abbr']
|
||||
compare_model = model_pair[1]['abbr']
|
||||
if idx == len(self.judge_models):
|
||||
subdir = model1 + '_' + model2 + '_summarized-by--' + judge_model
|
||||
subdir = base_model + '_' + compare_model + '_summarized-by--' + judge_model
|
||||
else:
|
||||
subdir = model1 + '_' + model2 + '_judged-by--' + judge_model
|
||||
subdir = base_model + '_' + compare_model + '_judged-by--' + judge_model
|
||||
subdir_path = os.path.join(results_folder, subdir)
|
||||
if not os.path.isdir(subdir_path):
|
||||
print(subdir_path + ' is not exist! please check!')
|
||||
scores[judge_model][dataset_abbr][compare_model] = None
|
||||
continue
|
||||
|
||||
judged_answers, references = get_judgeanswer_and_reference(dataset, subdir_path, self.judge_function)
|
||||
if len(judged_answers) == 0:
|
||||
scores[judge_model][dataset_abbr][model2] = {}
|
||||
continue
|
||||
if self.check_pos_bias:
|
||||
bias_num = check_position_bias(judged_answers, references)
|
||||
else:
|
||||
bias_num = 0
|
||||
win_model1 = defaultdict(float)
|
||||
win_model2 = defaultdict(float)
|
||||
categories = defaultdict(float)
|
||||
difficulties = defaultdict(float)
|
||||
languages = defaultdict(float)
|
||||
model1 = references[0]['answer1']
|
||||
model2 = references[0]['answer2']
|
||||
for prediction, reference in zip(judged_answers, references):
|
||||
categories[dataset_abbr] += 1
|
||||
categories[reference['category']] += 1
|
||||
difficulties['Level-' + str(reference['level'])] += 1
|
||||
languages['Lan-' + reference['lan']] += 1
|
||||
win_base_model = defaultdict(float)
|
||||
win_compare_model = defaultdict(float)
|
||||
score_mapping = {'A++': 1, 'A+': 0.5, 'A=B': 0, 'B+': -0.5, 'B++': -1}
|
||||
cnt = defaultdict(float)
|
||||
|
||||
if prediction == 'A':
|
||||
if reference['answer1'] == model1:
|
||||
score_1, score_2 = 1, 0
|
||||
else:
|
||||
score_1, score_2 = 0, 1
|
||||
elif prediction == 'B':
|
||||
if reference['answer1'] == model1:
|
||||
score_1, score_2 = 0, 1
|
||||
else:
|
||||
score_1, score_2 = 1, 0
|
||||
elif prediction == 'C':
|
||||
if self.summary_type == 'half_add':
|
||||
score_1, score_2 = 0.5, 0.5
|
||||
else:
|
||||
score_1, score_2 = 0, 0
|
||||
for judged_answer, reference in zip(judged_answers, references):
|
||||
if judged_answer not in score_mapping:
|
||||
continue
|
||||
else:
|
||||
flag = 1 if reference['answer1'] == base_model else -1
|
||||
score_1 = score_mapping[judged_answer]*flag
|
||||
score_2 = -score_1
|
||||
|
||||
win_model1[reference['category']] += score_1
|
||||
win_model1[dataset_abbr] += score_1
|
||||
win_model1['Level-' + str(reference['level'])] += score_1
|
||||
win_model1['Lan-' + reference['lan']] += score_1
|
||||
win_model2[reference['category']] += score_2
|
||||
win_model2[dataset_abbr] += score_2
|
||||
win_model2['Level-' + str(reference['level'])] += score_2
|
||||
win_model2['Lan-' + reference['lan']] += score_2
|
||||
for category in categories:
|
||||
win_model1[category] = win_model1[category] / categories[category] * 100
|
||||
win_model1[category] = round(win_model1[category], 2)
|
||||
win_model2[category] = win_model2[category] / categories[category] * 100
|
||||
win_model2[category] = round(win_model2[category], 2)
|
||||
win_model1['position_bias'] = bias_num
|
||||
win_model2['position_bias'] = bias_num
|
||||
for difficulty in difficulties:
|
||||
win_model1[difficulty] = win_model1[difficulty] / difficulties[difficulty] * 100
|
||||
win_model2[difficulty] = win_model2[difficulty] / difficulties[difficulty] * 100
|
||||
for language in languages:
|
||||
win_model1[language] = win_model1[language] / languages[language] * 100
|
||||
win_model2[language] = win_model2[language] / languages[language] * 100
|
||||
cnt[dataset_abbr] += 1
|
||||
win_compare_model[dataset_abbr] += score_2
|
||||
win_base_model[dataset_abbr] += score_1
|
||||
|
||||
if judge_model not in scores:
|
||||
scores[judge_model] = {}
|
||||
if dataset_abbr not in scores[judge_model]:
|
||||
scores[judge_model][dataset_abbr] = {}
|
||||
scores[judge_model][dataset_abbr][model2] = win_model2
|
||||
for key, value in cnt.items():
|
||||
win_base_model[key] = win_base_model[key] / value * 100
|
||||
win_base_model[key] = round(win_base_model[key], 2)
|
||||
win_compare_model[key] = win_compare_model[key] / value * 100
|
||||
win_compare_model[key ] = round(win_compare_model[key], 2)
|
||||
|
||||
return scores, difficulties, languages
|
||||
scores[judge_model][dataset_abbr][compare_model] = win_compare_model
|
||||
|
||||
return scores
|
||||
|
||||
def summarize(
|
||||
self,
|
||||
@ -186,63 +126,63 @@ class CompassBenchSummarizer:
|
||||
Returns:
|
||||
pd.DataFrame: The summary results.
|
||||
"""
|
||||
|
||||
|
||||
scores, difficulties, languages = self.get_score(time_str)
|
||||
# scores['win_' + model1] = win_model1
|
||||
scores = self.get_score(time_str)
|
||||
output_dir, results_folder = get_outdir(self.cfg, time_str)
|
||||
all_judge_file_list = []
|
||||
all_scores = {}
|
||||
for idx, judge_model in enumerate(self.judge_models):
|
||||
score_by_judgemodel = {}
|
||||
judge_abbr = model_abbr_from_cfg(judge_model)
|
||||
for dataset in self.cfg['datasets']:
|
||||
dataset_abbr = dataset_abbr_from_cfg(dataset)
|
||||
summarizer_model_abbrs = [model_abbr_from_cfg_used_in_summarizer(i) for i in self.compare_models]
|
||||
one_column = list(scores[judge_abbr][dataset_abbr].values())[0]
|
||||
detail_headers = [i for i in one_column.keys() if i not in [dataset_abbr, 'position_bias'] and i not in difficulties and i not in languages]
|
||||
row_headers = [dataset_abbr, 'position_bias']
|
||||
for difficulty in difficulties:
|
||||
row_headers += [difficulty]
|
||||
for language in languages:
|
||||
row_headers += [language]
|
||||
row_headers += detail_headers
|
||||
headers = [''] + summarizer_model_abbrs
|
||||
table = []
|
||||
for row_header in row_headers:
|
||||
row = [row_header]
|
||||
for model_cfg in self.compare_models:
|
||||
model_abbr = model_abbr_from_cfg(model_cfg)
|
||||
s = scores[judge_abbr][dataset_abbr][model_abbr].get(row_header, '')
|
||||
if isinstance(s, float):
|
||||
s = f'{s:.2f}'
|
||||
if isinstance(s, int):
|
||||
s = str(s)
|
||||
row.append(s)
|
||||
table.append(row)
|
||||
txt = tabulate(table, headers=headers)
|
||||
# print(txt)
|
||||
for judge_abbr, judge_scores in scores.items():
|
||||
new_score = {}
|
||||
for dataset_name, model_scores in judge_scores.items():
|
||||
dataset_root, dataset_detail = dataset_name.split('/')[0], dataset_name.split('/')[1]
|
||||
if dataset_root not in new_score:
|
||||
new_score[dataset_root] = {}
|
||||
if '_en_' in dataset_detail:
|
||||
for model_name, cate_score in model_scores.items():
|
||||
if model_name not in new_score[dataset_root]:
|
||||
new_score[dataset_root][model_name] = {}
|
||||
if len(cate_score) == 0:
|
||||
new_score[dataset_root][model_name]['英文总分'] = None
|
||||
else:
|
||||
new_score[dataset_root][model_name].update(cate_score)
|
||||
new_score[dataset_root][model_name]['英文总分'] = sum(cate_score.values()) / len(cate_score)
|
||||
elif '_cn_' in dataset_detail or '_zh_' in dataset_detail:
|
||||
for model_name, cate_score in model_scores.items():
|
||||
if model_name not in new_score[dataset_root]:
|
||||
new_score[dataset_root][model_name] = {}
|
||||
if len(cate_score) == 0:
|
||||
new_score[dataset_root][model_name]['中文总分'] = None
|
||||
else:
|
||||
new_score[dataset_root][model_name].update(cate_score)
|
||||
new_score[dataset_root][model_name]['中文总分'] = sum(cate_score.values()) / len(cate_score)
|
||||
for dataset, models in new_score.items():
|
||||
for model, details in models.items():
|
||||
if details['英文总分'] is not None and details['中文总分'] is not None:
|
||||
average_score = (details['英文总分'] + details['中文总分']) / 2
|
||||
else:
|
||||
average_score = None
|
||||
details['总分'] = average_score
|
||||
|
||||
if idx == len(self.judge_models):
|
||||
output_filename = osp.join(output_dir, dataset_abbr + '-summarized-by--' + judge_abbr + '-report.csv')
|
||||
else:
|
||||
output_filename = osp.join(output_dir, dataset_abbr + '-judged-by--' + judge_abbr + '-report.csv')
|
||||
|
||||
with open(output_filename, 'w') as f:
|
||||
f.write(','.join(headers) + '\n')
|
||||
for line in table:
|
||||
f.write(','.join(line) + '\n')
|
||||
all_judge_file_list.append(output_filename)
|
||||
for idx, model in enumerate(summarizer_model_abbrs):
|
||||
score_by_judgemodel[model] = {'overall': table[0][idx+1]}
|
||||
all_scores[judge_abbr]=score_by_judgemodel
|
||||
dfs = [pd.read_csv(file) for file in all_judge_file_list]
|
||||
df = pd.DataFrame()
|
||||
# Iterate over the MAP and new_score to populate the DataFrame
|
||||
for category, headers in MAP.items():
|
||||
category_data = []
|
||||
for model, scores in new_score[category].items():
|
||||
row_data = [model]
|
||||
for header in headers:
|
||||
# Append the score if available, otherwise append None
|
||||
row_data.append(scores.get(header, None))
|
||||
category_data.append(row_data)
|
||||
|
||||
if len(dfs) > 1:
|
||||
average_df = copy.deepcopy(dfs[0])
|
||||
for col in dfs[0].columns[1:]:
|
||||
for i in range(0, len(dfs[0])):
|
||||
average_df[col][i] = round(sum(df[col][i] for df in dfs) / len(dfs), 2)
|
||||
average_csv_path = osp.join(output_dir, 'CompassBench-Averaged-' + dataset_abbr + '-report.csv')
|
||||
average_df.to_csv(average_csv_path, index=False)
|
||||
return {'CompassBench': all_scores}
|
||||
# Create a DataFrame for the category and concatenate with the main DataFrame
|
||||
new_headers = [category+'_'+item for item in headers]
|
||||
category_df = pd.DataFrame(category_data, columns=[category] + new_headers)
|
||||
df = pd.concat([df, category_df.set_index(category)], axis=1)
|
||||
|
||||
df_transposed = df.T
|
||||
|
||||
|
||||
output_filename = osp.join(output_dir, 'summarized-by--' + judge_abbr + '-' + '-report.csv')
|
||||
|
||||
|
||||
transposed_csv_file_path = output_filename
|
||||
df_transposed.to_csv(transposed_csv_file_path)
|
||||
print(f'save to {output_filename}')
|
||||
|
||||
@ -4,6 +4,7 @@ from .build import * # noqa
|
||||
from .collect_env import * # noqa
|
||||
from .datasets import * # noqa
|
||||
from .dependency import * # noqa
|
||||
from .dict_postprocessors import * # noqa
|
||||
from .file import * # noqa
|
||||
from .fileio import * # noqa
|
||||
from .lark import * # noqa
|
||||
|
||||
6
opencompass/utils/dict_postprocessors.py
Normal file
6
opencompass/utils/dict_postprocessors.py
Normal file
@ -0,0 +1,6 @@
|
||||
from opencompass.registry import DICT_POSTPROCESSORS
|
||||
|
||||
|
||||
@DICT_POSTPROCESSORS.register_module('base')
|
||||
def base_postprocess(output: dict) -> dict:
|
||||
return output
|
||||
Loading…
Reference in New Issue
Block a user