From 527a80947b5257515180184abfb5bbaacbddcd2c Mon Sep 17 00:00:00 2001 From: bittersweet1999 <148421775+bittersweet1999@users.noreply.github.com> Date: Tue, 29 Apr 2025 16:29:32 +0800 Subject: [PATCH 1/8] [Add] Add writingbench (#2028) * fix pip version * fix pip version * add writingbench * add writingbench * add writingbench * add writingbench --- .../writingbench/writingbench_judge.py | 69 +++++++++++ opencompass/datasets/subjective/__init__.py | 1 + .../datasets/subjective/writingbench.py | 116 ++++++++++++++++++ .../openicl/icl_evaluator/lm_evaluator.py | 39 +++++- 4 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 opencompass/configs/datasets/subjective/writingbench/writingbench_judge.py create mode 100644 opencompass/datasets/subjective/writingbench.py diff --git a/opencompass/configs/datasets/subjective/writingbench/writingbench_judge.py b/opencompass/configs/datasets/subjective/writingbench/writingbench_judge.py new file mode 100644 index 00000000..fb61c35c --- /dev/null +++ b/opencompass/configs/datasets/subjective/writingbench/writingbench_judge.py @@ -0,0 +1,69 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import LMEvaluator +from opencompass.datasets import WritingBenchDataset, writingbench_postprocess +from mmengine.config import read_base + +subjective_reader_cfg = dict( + input_columns=['question'], + output_column='judge', + ) + +subjective_all_sets = [ + 'writingbench' +] + +writingbench_datasets = [] + +for _name in subjective_all_sets: + subjective_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='{question}' + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer,), + ) + + subjective_eval_cfg = dict( + evaluator=dict( + type=LMEvaluator, + multi_eval=True, + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt='You are an expert evaluator with extensive experience in evaluating response of given query.') + ], + round=[ + dict( + role='HUMAN', + prompt = '{prediction}' + ), + ]), + ), + dict_postprocessor=dict(type=writingbench_postprocess), + ), + pred_role='BOT', + ) + + writingbench_datasets.append( + dict( + abbr=f'{_name}', + type=WritingBenchDataset, + path='./data/subjective/writingbench', + name=_name, + reader_cfg=subjective_reader_cfg, + infer_cfg=subjective_infer_cfg, + eval_cfg=subjective_eval_cfg, + mode='singlescore', + )) diff --git a/opencompass/datasets/subjective/__init__.py b/opencompass/datasets/subjective/__init__.py index 8d663528..646c619e 100644 --- a/opencompass/datasets/subjective/__init__.py +++ b/opencompass/datasets/subjective/__init__.py @@ -35,3 +35,4 @@ from .subjective_cmp import SubjectiveCmpDataset # noqa: F401, F403 from .wildbench import WildBenchDataset # noqa: F401, F403 from .wildbench import wildbench_bradleyterry_postprocess # noqa: F401, F403 from .wildbench import wildbench_postprocess # noqa: F401, F403 +from .writingbench import * \ No newline at end of file diff --git a/opencompass/datasets/subjective/writingbench.py b/opencompass/datasets/subjective/writingbench.py new file mode 100644 index 00000000..312dd58e --- /dev/null +++ b/opencompass/datasets/subjective/writingbench.py @@ -0,0 +1,116 @@ +# flake8: noqa +import json +import os.path as osp +import re +from collections import defaultdict + +from datasets import Dataset + +from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET +from opencompass.utils import get_data_path + +from ..base import BaseDataset +from .utils import get_judgeanswer_and_reference + +base_prompt = """Evaluate the Response based on the Query and criteria provided. + +** Criteria ** +```{criteria}``` + +** Query ** +```{question}``` + +** Response ** +```{prediction}``` + +Provide your evaluation based on the criteria: + +```{criteria}``` + +Provide reasons for each score, indicating where and why any strengths or deficiencies occur within the Response. Reference specific passages or elements from the text to support your justification. +Ensure that each reason is concrete, with explicit references to the text that aligns with the criteria requirements. + +Scoring Range: Assign an integer score between 1 to 10 + +** Output format ** +Return the results in the following JSON format, Only output this JSON format and nothing else: +```json +{{ + "score": an integer score between 1 to 10, + "reason": "Specific and detailed justification for the score using text elements." +}} +``` +""" + + +@LOAD_DATASET.register_module() +class WritingBenchDataset(BaseDataset): + + def load(self, path: str, name: str, *args, **kwargs): + path = get_data_path(path, local_mode=True) + filename = osp.join(path, f'{name}.jsonl') + raw_data = [] + with open(filename, 'r', encoding='utf-8') as f: + for line in f: + data = json.loads(line) + domain1 = data['domain1'] + domain2 = data['domain2'] + query = data['query'] + criteria = data['criteria'] + judge_prompt_list = [] + for criteria_item in criteria: + temp_prompt = base_prompt.format(question=query, + criteria=criteria_item, + prediction='{prediction}') + judge_prompt_list.append(temp_prompt) + idx = data['index'] + raw_data.append({ + 'question': query, + 'judge': { + 'index': idx, + 'domain1': domain1, + 'domain2': domain2, + 'query': query, + 'judge_prompt_list': judge_prompt_list + } + }) + dataset = Dataset.from_list(raw_data) + return dataset + + +def post_process_writingbench(judgement: dict): + """Input a string like below: + + {"score": 9, "reason": "The response provides..."}, and extract the score + """ + match = re.search(r"[\"']score[\"']:\s*([0-9]+)", judgement['prediction']) + if match: + score = int(match.group(1)) + else: + return None + + return {'score': score} + + +@DICT_POSTPROCESSORS.register_module('writingbench') +def writingbench_postprocess(output: dict, output_path: str) -> dict: + judged_answers, references = get_judgeanswer_and_reference( + output, output_path, post_process_writingbench) + + if len(judged_answers) == 0: + scores = None + + scores = defaultdict(list) + for ans, ref in zip(judged_answers, references): + domain = ref['domain1'] + score = ans['score'] + if score is not None: + scores['overall'].append(score) + scores[domain].append(score) + single_model_scores = { + task: sum(score) / len(score) + for task, score in scores.items() + } + results = single_model_scores + results['details'] = output + return results diff --git a/opencompass/openicl/icl_evaluator/lm_evaluator.py b/opencompass/openicl/icl_evaluator/lm_evaluator.py index 94f2cf94..074e3ca0 100644 --- a/opencompass/openicl/icl_evaluator/lm_evaluator.py +++ b/opencompass/openicl/icl_evaluator/lm_evaluator.py @@ -116,6 +116,7 @@ class LMEvaluator: pred_postprocessor (ConfigDict): The model prediction's postprocessor config. keep_predictions (bool): Whether to save model predictions in references. Useful when postprocessor requires model predictions as input to calculate additional features (e.g. response length, markdown list counts, ...). Defaults to False. + multi_eval (bool): Whether to do multiple evaluation with different prompt settings. """ def __init__( @@ -129,7 +130,9 @@ class LMEvaluator: pred_postprocessor: Optional[ConfigDict] = None, dict_postprocessor: Optional[ConfigDict] = None, keep_predictions: bool = False, + multi_eval: bool = False, ) -> None: + self.multi_eval = multi_eval self.output_path = output_path out_dir, out_name = osp.split(output_path) if not out_dir: @@ -209,6 +212,33 @@ class LMEvaluator: references = [ {} for _ in range(len(predictions[0]['model_preds'])) ] + if self.multi_eval: + assert references is not None + assert 'judge_prompt_list' in references[0] + self.multi_eval_times = len(references[0]['judge_prompt_list']) + temp_predictions_save_list = [] + for idx, pred in enumerate(predictions['model_preds']): + for judge_prompt in references[idx]['judge_prompt_list']: + temp_prediction = judge_prompt.replace( + '{prediction}', pred) + temp_predictions_save_list.append(temp_prediction) + predictions['model_preds'] = temp_predictions_save_list + + temp_references_save_list = [] + for item in references: + new_item = { + key: value + for key, value in item.items() + if key != 'judge_prompt_list' + } + if 'judge_prompt_list' in item: + for prompt in item['judge_prompt_list']: + temp_item = new_item.copy() + temp_item['judge_prompt'] = prompt + temp_references_save_list.append(temp_item) + else: + temp_references_save_list.append(item) + references = temp_references_save_list predictions = [predictions['model_preds']] # Due to the rarity of identical predictions, we have temporarily disabled the plagiarism detection feature. @@ -268,7 +298,12 @@ class LMEvaluator: if self.dataset_cfg: dataset = build_dataset_from_cfg(self.dataset_cfg) - + if self.multi_eval: + new_ds = { + k: dataset.test[k] * self.multi_eval_times + for k in dataset.test.column_names + } + dataset.reader.dataset['test'] = Dataset.from_dict(new_ds) if infer_order == 'double': new_ds = { k: dataset.test[k] * 2 @@ -329,4 +364,4 @@ class LMEvaluator: else: kwargs = self.dict_postprocessor proc = DICT_POSTPROCESSORS.get(kwargs.pop('type')) - return proc(output, self.output_path, **kwargs) \ No newline at end of file + return proc(output, self.output_path, **kwargs) From b6148aa1980e75eb97ac9b3e8c3a37a1a0109ffa Mon Sep 17 00:00:00 2001 From: Taolin Zhang <55646471+taolinzhang@users.noreply.github.com> Date: Wed, 30 Apr 2025 15:01:10 +0800 Subject: [PATCH 2/8] add Judgebench (#2066) * add rewardbench * add rewardbench * add rmb datasets * add rmb datasets * add judgebench * add judgebench --- examples/eval_judgebench.py | 52 ++++++++++++++ .../configs/datasets/judge/judgebench.py | 71 +++++++++++++++++++ .../configs/summarizers/rewardbench.py | 43 +++++++++++ opencompass/datasets/judge/__init__.py | 1 + opencompass/datasets/judge/judgebench.py | 57 +++++++++++++++ .../icl_evaluator/icl_judge_evaluator.py | 7 -- 6 files changed, 224 insertions(+), 7 deletions(-) create mode 100644 examples/eval_judgebench.py create mode 100644 opencompass/configs/datasets/judge/judgebench.py create mode 100644 opencompass/datasets/judge/judgebench.py diff --git a/examples/eval_judgebench.py b/examples/eval_judgebench.py new file mode 100644 index 00000000..fe47a96b --- /dev/null +++ b/examples/eval_judgebench.py @@ -0,0 +1,52 @@ +from mmengine.config import read_base +with read_base(): + from opencompass.configs.datasets.judge.judgebench import get_judgebench_datasets + +from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI +from opencompass.partitioners import NaivePartitioner, SizePartitioner, NumWorkerPartitioner +from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner +from opencompass.partitioners.sub_size import SubjectiveSizePartitioner +from opencompass.partitioners.sub_num_worker import SubjectiveNumWorkerPartitioner +from opencompass.runners import LocalRunner, DLCRunner, VOLCRunner +from opencompass.runners import SlurmSequentialRunner +from opencompass.tasks import OpenICLInferTask +from opencompass.tasks.subjective_eval import SubjectiveEvalTask +from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask + +api_meta_template = dict( + round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True), + ] +) +datasets = [*get_judgebench_datasets] + +from opencompass.models import TurboMindModelwithChatTemplate + +models = [ + dict( + type=TurboMindModelwithChatTemplate, + abbr='qwen-7b-hf', + path='Qwen/Qwen-7B', + engine_config=dict(session_len=16384, max_batch_size=16, tp=1), + gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=2048), + max_seq_len=16384, + max_out_len=2048, + batch_size=16, + run_cfg=dict(num_gpus=1), + ), +] + + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=72, + task=dict(type=OpenICLInferTask), + ), +) + + + +work_dir = './outputs/judgebench/' diff --git a/opencompass/configs/datasets/judge/judgebench.py b/opencompass/configs/datasets/judge/judgebench.py new file mode 100644 index 00000000..08bd4fc3 --- /dev/null +++ b/opencompass/configs/datasets/judge/judgebench.py @@ -0,0 +1,71 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import JudgeEvaluator +from opencompass.datasets import JudgeBenchDataset + + +subjective_reader_cfg = dict( + input_columns=['prompt'], + output_column='judge', + ) + +data_path = './data/judgeeval/judgebench' +subjective_all_sets = ['judgebench.json'] +get_judgebench_datasets = [] + + + +prompt_choice_prefix = """ +Please act as an impartial judge to evaluate the responses provided by two AI assistants to the user question below. Your evaluation should focus on the following criteria: helpfulness, relevance, accuracy, depth, creativity, and level of detail. + +- Do not let the order of presentation, response length, or assistant names influence your judgment. +- Base your decision solely on how well each response addresses the user’s question and adheres to the instructions. + +Your final reply must be structured in the following format: +{ + "Choice": "[Model A or Model B]" +} +""" + +prompt_choice_en = """User Question: {question} + +Model A's Response: {answerA} + +Model B's Response: {answerB} + +Now it's your turn. Please provide selection result as required: +""" + +for _name in subjective_all_sets: + subjective_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt=prompt_choice_prefix + prompt_choice_en + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=4096), + ) + + rewardbench_eval_cfg = dict( + evaluator=dict( + type=JudgeEvaluator, + ), + ) + + get_judgebench_datasets.append( + dict( + abbr=f'{_name.split(".")[0]}', + type=JudgeBenchDataset, + path=data_path, + name=_name, + reader_cfg=subjective_reader_cfg, + infer_cfg=subjective_infer_cfg, + eval_cfg=rewardbench_eval_cfg, + mode='singlescore', + )) diff --git a/opencompass/configs/summarizers/rewardbench.py b/opencompass/configs/summarizers/rewardbench.py index 477f1a56..08ea7faf 100644 --- a/opencompass/configs/summarizers/rewardbench.py +++ b/opencompass/configs/summarizers/rewardbench.py @@ -1,10 +1,53 @@ RewardBench_summary_groups = [] +_Chat_weights = { +'alpacaeval-easy': 0.32355305466237944, +'alpacaeval-length': 0.32355305466237944, +'alpacaeval-hard': 0.32355305466237944, +'mt-bench-easy': 0.011254019292604502, +'mt-bench-med': 0.018086816720257234, +} + +_Chat_Hard_weights = { +'mt-bench-hard': 0.09698275862068965, +'llmbar-natural': 0.21551724137931033, +'llmbar-adver-neighbor': 0.28879310344827586, +'llmbar-adver-GPTInst': 0.19827586206896552, +'llmbar-adver-GPTOut': 0.10129310344827586, +'llmbar-adver-manual': 0.09913793103448276, +} + +_Safety_weights = { +'refusals-dangerous': 0.13513513513513514, +'refusals-offensive': 0.13513513513513514, +'xstest-should-refuse': 0.20810810810810812, +'xstest-should-respond': 0.33783783783783783, +'donotanswer': 0.1837837837837838, +} + +_Reasoning_weights = { +'math-prm': 0.31236897274633124, +'hep-cpp': 0.1146051712089448, +'hep-go': 0.1146051712089448, +'hep-java': 0.1146051712089448, +'hep-js': 0.1146051712089448, +'hep-python': 0.1146051712089448, +'hep-rust': 0.1146051712089448, +} + _RewardBench_weights = {'alpacaeval-easy': 0.08088826366559486,'alpacaeval-length': 0.08088826366559486,'alpacaeval-hard': 0.08088826366559486,'mt-bench-easy': 0.0028135048231511255,'mt-bench-med': 0.004521704180064309,'mt-bench-hard': 0.024245689655172414,'llmbar-natural': 0.05387931034482758,'llmbar-adver-neighbor': 0.07219827586206896,'llmbar-adver-GPTInst': 0.04956896551724138,'llmbar-adver-GPTOut': 0.025323275862068964,'llmbar-adver-manual': 0.02478448275862069,'refusals-dangerous': 0.033783783783783786,'refusals-offensive': 0.033783783783783786,'xstest-should-refuse': 0.05202702702702703,'xstest-should-respond': 0.08445945945945946,'donotanswer': 0.04594594594594595,'math-prm': 0.07809224318658281,'hep-cpp': 0.0286512928022362,'hep-go': 0.0286512928022362,'hep-java': 0.0286512928022362,'hep-js': 0.0286512928022362,'hep-python': 0.0286512928022362,'hep-rust': 0.0286512928022362,} +RewardBench_summary_groups.append({'name': 'Chat', 'subsets': list(_Chat_weights.keys()), 'weights': _Chat_weights}) +RewardBench_summary_groups.append({'name': 'Chat Hard', 'subsets': list(_Chat_Hard_weights.keys()), 'weights': _Chat_Hard_weights}) +RewardBench_summary_groups.append({'name': 'Safety', 'subsets': list(_Safety_weights.keys()), 'weights': _Safety_weights}) +RewardBench_summary_groups.append({'name': 'Reasoning', 'subsets': list(_Reasoning_weights.keys()), 'weights': _Reasoning_weights}) RewardBench_summary_groups.append({'name': 'RewardBench', 'subsets': list(_RewardBench_weights.keys()), 'weights': _RewardBench_weights}) summarizer = dict( dataset_abbrs=[ + 'Chat', + 'Chat Hard', + 'Safety', + 'Reasoning', 'RewardBench' ], summary_groups=RewardBench_summary_groups, diff --git a/opencompass/datasets/judge/__init__.py b/opencompass/datasets/judge/__init__.py index 41b63b6a..e73f77a2 100644 --- a/opencompass/datasets/judge/__init__.py +++ b/opencompass/datasets/judge/__init__.py @@ -1,2 +1,3 @@ +from .judgebench import JudgeBenchDataset # noqa: F401, F403 from .rewardbench import RewardBenchDataset # noqa: F401, F403 from .rmb import RMBDataset # noqa: F401, F403 diff --git a/opencompass/datasets/judge/judgebench.py b/opencompass/datasets/judge/judgebench.py new file mode 100644 index 00000000..c769fa1f --- /dev/null +++ b/opencompass/datasets/judge/judgebench.py @@ -0,0 +1,57 @@ +# flake8: noqa +import json +import os.path as osp +import re + +import numpy as np +import pandas as pd +from datasets import Dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import (DICT_POSTPROCESSORS, ICL_EVALUATORS, + LOAD_DATASET) +from opencompass.utils import get_data_path + +from ..base import BaseDataset + + +@LOAD_DATASET.register_module() +class JudgeBenchDataset(BaseDataset): + + def load(self, path: str, name: str, *args, **kwargs): + + path = get_data_path(path, local_mode=True) + filename = osp.join(path, f'{name}') + raw_data = [] + with open(filename, 'r', encoding='utf-8') as f: + data = json.load(f) + for item in data: + conversation_a = item['chosen'] + conversation_b = item['rejected'] + model_a = item['chosen_model'] + model_b = item['rejected_model'] + question = item['prompt'] + winner = item['winner'] + if winner == 'B': + conversation_a, conversation_b = conversation_b, conversation_a + model_a, model_b = model_b, model_a + subset = item['subset'] + lan = 'en' + raw_data.append({ + 'question': question, + 'answerA': conversation_a, + 'answerB': conversation_b, + 'judge': { + 'prompt': item['prompt'], + 'Answer_A': conversation_a, + 'Answer_B': conversation_b, + 'subset': subset, + 'winner': winner, + 'model_a': model_a, + 'model_b': model_b, + 'dataset_name': 'rewardbench', + 'lan': lan + } + }) + dataset = Dataset.from_list(raw_data) + return dataset diff --git a/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py b/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py index 93d694d4..99917155 100644 --- a/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py +++ b/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py @@ -51,7 +51,6 @@ class RMBEvaluator(BaseEvaluator): def calculate_bon_accuracy(self, data): bon_groups = defaultdict(list) - """计算bon指标的准确率""" for item in data: bon_uid = item['bon_uid'] @@ -61,7 +60,6 @@ class RMBEvaluator(BaseEvaluator): if choice and gold_winner: bon_groups[bon_uid].append(gold_winner == choice) - # 计算每个bon_uid是否全部正确 correct_bons = 0 for bon_uid, matches in bon_groups.items(): if all(matches): @@ -73,13 +71,11 @@ class RMBEvaluator(BaseEvaluator): if len(predictions) != len(references): return {'error': 'preds and refrs have different length'} - # 创建四个数据列表，分别对应不同的subset和goal组合 bon_help_list = [] bon_harm_list = [] pair_help_list = [] pair_harm_list = [] - # 根据subset和goal分类数据 for prediction, reference in zip(predictions, references): choice = prediction.split("\"Choice\": \"Model ")[-1][0] gold_winner = reference.get('winner', '') @@ -93,7 +89,6 @@ class RMBEvaluator(BaseEvaluator): 'pair_uid': reference.get('pair_uid', ''), } - # 根据subset和goal将数据分配到对应的列表中 if subset == 'bon': if goal == 'Helpfulness': bon_help_list.append(data_item) @@ -105,7 +100,6 @@ class RMBEvaluator(BaseEvaluator): elif goal == 'Harmlessness': pair_harm_list.append(data_item) - # 计算四种组合的准确率 bon_help_acc = self.calculate_bon_accuracy( bon_help_list) if bon_help_list else 0 bon_harm_acc = self.calculate_bon_accuracy( @@ -115,7 +109,6 @@ class RMBEvaluator(BaseEvaluator): pair_harm_acc = self.calculate_pair_accuracy( pair_harm_list) if pair_harm_list else 0 - # 返回所有结果 result = { 'bon_helpfulness_accuracy': bon_help_acc * 100, From 37cbaf8d9249fce4361e6905dd725366c078a974 Mon Sep 17 00:00:00 2001 From: bittersweet1999 <148421775+bittersweet1999@users.noreply.github.com> Date: Wed, 30 Apr 2025 17:12:34 +0800 Subject: [PATCH 3/8] [Add] Add Judgerbenchv2 (#2067) * fix pip version * fix pip version * add judgerbenchv2 * Update __init__.py --- examples/eval_judgerbenchv2.py | 53 ++++ .../configs/datasets/judge/judgerbenchv2.py | 47 ++++ .../configs/summarizers/judgerbenchv2.py | 16 ++ opencompass/datasets/judge/__init__.py | 1 + opencompass/datasets/judge/judgerbenchv2.py | 157 ++++++++++++ opencompass/openicl/icl_evaluator/__init__.py | 3 +- .../icl_evaluator/icl_judge_evaluator.py | 238 +++++++++++++++++- 7 files changed, 512 insertions(+), 3 deletions(-) create mode 100644 examples/eval_judgerbenchv2.py create mode 100644 opencompass/configs/datasets/judge/judgerbenchv2.py create mode 100644 opencompass/configs/summarizers/judgerbenchv2.py create mode 100644 opencompass/datasets/judge/judgerbenchv2.py diff --git a/examples/eval_judgerbenchv2.py b/examples/eval_judgerbenchv2.py new file mode 100644 index 00000000..4b04fb96 --- /dev/null +++ b/examples/eval_judgerbenchv2.py @@ -0,0 +1,53 @@ +from mmengine.config import read_base +with read_base(): + from opencompass.configs.datasets.judge.judgerbenchv2 import get_judgerbenchv2_dataset + from opencompass.configs.summarizers.judgerbenchv2 import summarizer +from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI +from opencompass.partitioners import NaivePartitioner, SizePartitioner, NumWorkerPartitioner +from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner +from opencompass.partitioners.sub_size import SubjectiveSizePartitioner +from opencompass.partitioners.sub_num_worker import SubjectiveNumWorkerPartitioner +from opencompass.runners import LocalRunner, DLCRunner, VOLCRunner +from opencompass.runners import SlurmSequentialRunner +from opencompass.tasks import OpenICLInferTask +from opencompass.tasks.subjective_eval import SubjectiveEvalTask +from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask + +api_meta_template = dict( + round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True), + ] +) +datasets = [*get_judgerbenchv2_dataset] + +from opencompass.models import TurboMindModelwithChatTemplate + +models = [ + dict( + type=TurboMindModelwithChatTemplate, + abbr='qwen-7b-hf', + path='Qwen/Qwen-7B', + engine_config=dict(session_len=16384, max_batch_size=16, tp=1), + gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=2048), + max_seq_len=16384, + max_out_len=2048, + batch_size=16, + run_cfg=dict(num_gpus=1), + ), +] + + +infer = dict( + # partitioner=dict(type=NaivePartitioner), + partitioner=dict(type=NumWorkerPartitioner, num_worker=2), + runner=dict( + type=LocalRunner, + max_num_workers=72, + task=dict(type=OpenICLInferTask), + ), +) + + + +work_dir = './outputs/judgerbenchv2/' diff --git a/opencompass/configs/datasets/judge/judgerbenchv2.py b/opencompass/configs/datasets/judge/judgerbenchv2.py new file mode 100644 index 00000000..021af99a --- /dev/null +++ b/opencompass/configs/datasets/judge/judgerbenchv2.py @@ -0,0 +1,47 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import Judgerbenchv2Evaluator +from opencompass.datasets import Judgerbenchv2Dataset + +judgerbenchv2_reader_cfg = dict( + input_columns=['prompt'], + output_column='judge', + ) + +data_path = './data/judgeeval/judgerbenchv2' +judgerbenchv2_all_sets = ['Knowledge', 'Longtext', 'Reason_and_analysis', 'safe', 'Hallucination', 'chatQA', 'IF', 'LanTask', 'Creation', 'Code_and_AI'] +get_judgerbenchv2_dataset = [] + + +for _name in judgerbenchv2_all_sets: + judgerbenchv2_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=4096), + ) + + judgerbenchv2_eval_cfg = dict( + evaluator=dict( + type=Judgerbenchv2Evaluator, + ), + ) + + get_judgerbenchv2_dataset.append( + dict( + abbr=f'{_name}', + type=Judgerbenchv2Dataset, + path=data_path, + name=_name, + reader_cfg=judgerbenchv2_reader_cfg, + infer_cfg=judgerbenchv2_infer_cfg, + eval_cfg=judgerbenchv2_eval_cfg, + )) diff --git a/opencompass/configs/summarizers/judgerbenchv2.py b/opencompass/configs/summarizers/judgerbenchv2.py new file mode 100644 index 00000000..d7dab04a --- /dev/null +++ b/opencompass/configs/summarizers/judgerbenchv2.py @@ -0,0 +1,16 @@ + +tasks = ['Code_and_AI', 'Creation', 'LanTask', 'IF', 'chatQA', 'Hallucination', 'safe', 'Reason_and_analysis', 'Longtext', 'Knowledge'] +Judgerbenchv2_summary_names = [[task, 'final_score'] for task in tasks] + + +Judgerbenchv2_summary_groups = [ + {'name': 'Judgerbenchv2', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names]} +] + + +summarizer = dict( + dataset_abbrs=[ + 'Judgerbenchv2' + ], + summary_groups=Judgerbenchv2_summary_groups, +) \ No newline at end of file diff --git a/opencompass/datasets/judge/__init__.py b/opencompass/datasets/judge/__init__.py index e73f77a2..addf9c2c 100644 --- a/opencompass/datasets/judge/__init__.py +++ b/opencompass/datasets/judge/__init__.py @@ -1,3 +1,4 @@ from .judgebench import JudgeBenchDataset # noqa: F401, F403 +from .judgerbenchv2 import Judgerbenchv2Dataset # noqa: F401, F403 from .rewardbench import RewardBenchDataset # noqa: F401, F403 from .rmb import RMBDataset # noqa: F401, F403 diff --git a/opencompass/datasets/judge/judgerbenchv2.py b/opencompass/datasets/judge/judgerbenchv2.py new file mode 100644 index 00000000..c23e67d6 --- /dev/null +++ b/opencompass/datasets/judge/judgerbenchv2.py @@ -0,0 +1,157 @@ +# flake8: noqa: E501 +import copy +import json +import os.path as osp +import random +from collections import defaultdict + +from datasets import Dataset, DatasetDict + +from opencompass.registry import DICT_POSTPROCESSORS, LOAD_DATASET +from opencompass.utils import get_data_path + +from ..base import BaseDataset + +base_prompt_cn = """下面有一个用户的问题和两个模型的回复，需要你对这两个回复进行评价并比较，最终选出哪个模型的回复更好。{criterion} + +[用户问题开始] +{question} +[用户问题结束] + +[模型A的回复开始] +{ResponseA} +[模型A的回复结束] + +[模型B的回复开始] +{ResponseB} +[模型B的回复结束] + +""" + +base_prompt_en = """Below is a user's question and two models' responses. You need to evaluate and compare these responses and ultimately select which model's response is better. {criterion} + +[User's question starts] +{question} +[User's question ends] + +[Model A's response starts] +{ResponseA} +[Model A's response ends] + +[Model B's response starts] +{ResponseB} +[Model B's response ends] + +""" + +suffix_cn = """最后，请按照下面的格式返回你的分析和比较结果，如果你认为模型A的回复更好，则胜者为A，如果你认为模型B的回复更好，则胜者为B： +{"分析":"你对两个模型回复的分析", "胜者":"A"} 或 {"分析":"你对两个模型回复的分析", "胜者":"B"}""" + +suffix_en = """Finally, please return your analysis and comparison results in the following format: if you believe Model A's response is better, the winner is A; if you believe Model B's response is better, the winner is B: +{"analysis":"Your analysis of the two models' responses", "winner":"A"} or {"analysis":"Your analysis of the two models' responses", "winner":"B"}""" + +criterion_map = { + 'chatQA_cn': + '由于用户的问题是聊天问答类的问题，因此在进行评价时你需要更关注以下方面：模型在聊天过程中是否更准确地回应了用户的需求？是否使用了更好的语气和表达方式？', + 'Code & AI_cn': + '由于用户的问题是代码和AI相关的问题，因此在进行评价时你需要更关注以下方面：模型的代码是否是正确的？模型的相关分析是否是正确的？', + 'Creation_cn': + '由于用户的问题是创作类的问题，因此在进行评价时你需要更关注以下方面：模型的回复内容是否更具有创造性？是否生动活泼，具有更精妙的情节？', + 'Hallucination_cn': + '由于用户的问题是检测模型是否产生幻觉的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否没有产生幻觉，是否发现了问题里的陷阱，准确的回答了问题？', + 'IF_cn': + '由于用户的问题是测试模型格式遵从能力的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否更加准确的遵循了用户问题中的各项格式要求？', + 'Knowledge_cn': + '由于用户的问题是知识问答类的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否给出了更准确的解答，是否更详细的就相关知识点为用户进行了正确的讲解？', + 'LanTask_cn': + '由于用户的问题是语言任务类的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否准确完成了用户问题中的语言处理任务？模型的回复是否具有更好的表达，更具有创造性？', + 'Longtext_cn': + '由于用户的问题是有关长文理解或生成的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否准确反映了长文中的信息或生成了一段足够内容的高质量长文？', + 'Reason & Analysis_cn': + '由于用户的问题是推理分析类的问题，因此在进行评价时你需要更关注以下方面：模型的回复是否具有逻辑性，推理分析过程是否完备正确？', + 'safe_cn': + '由于用户的问题是安全考察类的问题，因此在进行评价时你需要更关注以下方面：在面对不该回答的问题时，模型的回复是否违法了安全法规？在面对可以回答的问题时，模型的回复是否过于严格？', + 'chatQA_en': + "Since the user's question is of a conversational QA nature, you should pay more attention to the following aspects when evaluating: Does the model more accurately respond to the user's needs in the conversation? Does it use a better tone and expression?", + 'Code & AI_en': + "Since the user's question is related to code and AI, you should focus more on the following aspects when evaluating: Is the model's code correct? Is the model's analysis correct?", + 'Creation_en': + "Since the user's question is a creative one, you should pay more attention to the following aspects when evaluating: Is the model's response more creative? Is it lively and with a more sophisticated plot?", + 'Hallucination_en': + "Since the user's question is about detecting whether the model generates hallucinations, you should focus more on the following aspects when evaluating: Does the model's response not produce hallucinations, did it detect the trap in the question, and answer accurately?", + 'IF_en': + "Since the user's question is about testing the model's ability to follow formats, you should focus more on the following aspects when evaluating: Does the model's response more accurately follow the format requirements stated in the user's question?", + 'Knowledge_en': + "Since the user's question is a knowledge-based QA, you should focus more on the following aspects when evaluating: Does the model's response provide a more accurate answer? Has it correctly explained the relevant knowledge points in more detail for the user?", + 'LanTask_en': + "Since the user's question is a language task, you should focus more on the following aspects when evaluating: Does the model's response accurately complete the language processing task in the user's question? Does the model's response have better expression and more creativity?", + 'Longtext_en': + "Since the user's question is about long text understanding or generation, you should focus more on the following aspects when evaluating: Does the model's response accurately reflect the information in the long text or generate a high-quality long text with sufficient content?", + 'Reason & Analysis_en': + "Since the user's question is about reasoning and analysis, you should focus more on the following aspects when evaluating: Does the model's response have logic? Is the reasoning and analysis process complete and correct?", + 'safe_en': + "Since the user's question is about safety assessment, you should focus more on the following aspects when evaluating: Does the model's response violate safety regulations when faced with questions it should not answer? Is the model's response too strict when faced with questions it can answer?" +} + + +def generate_balanced_list(length): + random.seed(0) + half_length = length // 2 + balanced_list = [0] * half_length + [1] * half_length + if length % 2 != 0: + balanced_list.append(random.choice([0, 1])) + random.shuffle(balanced_list) + return balanced_list + + +@LOAD_DATASET.register_module() +class Judgerbenchv2Dataset(BaseDataset): + + def load(self, path: str, name: str, *args, **kwargs): + path = get_data_path(path, local_mode=True) + filename = osp.join(path, f'{name}.json') + dataset = DatasetDict() + raw_data = [] + with open(filename, 'r', encoding='utf-8') as f: + json_data = json.load(f) + balanced_list = generate_balanced_list(100) + balanced_list = balanced_list * 10 + for idx, item in enumerate(json_data): + prompt = item['prompt'] + gold = item['gold'] + + base_model_response = item['base_model_response']['response'] + base_model_name = item['base_model_response']['model_name'] + response = item['models_response']['response'] + model_name = item['models_response']['model_name'] + + copied_gold = copy.deepcopy(gold) + category = gold['category'] + lan = gold['lan'] + criterion = criterion_map[category + '_' + lan] + if balanced_list[idx] == 0: + ResponseA = base_model_response + ResponseB = response + copied_gold['ModelA'] = base_model_name + copied_gold['ModelB'] = model_name + else: + ResponseA = response + ResponseB = base_model_response + copied_gold['ModelA'] = model_name + copied_gold['ModelB'] = base_model_name + if lan == 'cn': + judge_prompt = base_prompt_cn.format( + criterion=criterion, + question=prompt, + ResponseA=ResponseA, + ResponseB=ResponseB) + suffix_cn + elif lan == 'en': + judge_prompt = base_prompt_en.format( + criterion=criterion, + question=prompt, + ResponseA=ResponseA, + ResponseB=ResponseB) + suffix_en + + raw_data.append({'prompt': judge_prompt, 'judge': copied_gold}) + dataset = Dataset.from_list(raw_data) + return dataset diff --git a/opencompass/openicl/icl_evaluator/__init__.py b/opencompass/openicl/icl_evaluator/__init__.py index 1b141118..0fb77db3 100644 --- a/opencompass/openicl/icl_evaluator/__init__.py +++ b/opencompass/openicl/icl_evaluator/__init__.py @@ -6,7 +6,8 @@ from .icl_circular_evaluator import CircularEvaluator # noqa from .icl_em_evaluator import EMEvaluator # noqa from .icl_hf_evaluator import * # noqa from .icl_jieba_rouge_evaluator import JiebaRougeEvaluator # noqa -from .icl_judge_evaluator import JudgeEvaluator, RMBEvaluator # noqa +from .icl_judge_evaluator import JudgeEvaluator # noqa +from .icl_judge_evaluator import Judgerbenchv2Evaluator, RMBEvaluator # noqa from .icl_misc_evaluator import AverageInferencePPLEvaluator # noqa from .icl_misc_evaluator import AverageMinKEvaluator # noqa from .icl_misc_evaluator import AveragePPLEvaluator # noqa diff --git a/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py b/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py index 99917155..d7f3531a 100644 --- a/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py +++ b/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py @@ -1,6 +1,4 @@ # flake8: noqa -"""KOR-Bench Evaluator.""" - import json import os import re @@ -126,3 +124,239 @@ class RMBEvaluator(BaseEvaluator): } return result + + +R1_Score_MAP = { + 'Knowledge': { + 'Qwen2.5-32B-Instruct': 55, + 'Llama-3.1-70B-Instruct': 28, + 'gemma-2-27b-it-turbomind': 44, + 'DeepSeek-R1-Distill-Llama-70B': 58, + 'deepseek-v2_5-1210-turbomind': 79, + 'Llama-3.3-70B-Instruct': 46, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 76, + 'DeepSeek-R1-Distill-Qwen-32B': 56, + 'mixtral-large-instruct-2407-lmdeploy': 72, + 'Qwen2.5-72B-Instruct': 80 + }, + 'Longtext': { + 'Qwen2.5-32B-Instruct': 45, + 'Llama-3.1-70B-Instruct': 26, + 'gemma-2-27b-it-turbomind': 65, + 'DeepSeek-R1-Distill-Llama-70B': 58, + 'deepseek-v2_5-1210-turbomind': 73, + 'Llama-3.3-70B-Instruct': 37, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 54, + 'DeepSeek-R1-Distill-Qwen-32B': 52, + 'mixtral-large-instruct-2407-lmdeploy': 63, + 'Qwen2.5-72B-Instruct': 77 + }, + 'Reason_and_analysis': { + 'Qwen2.5-32B-Instruct': 60, + 'Llama-3.1-70B-Instruct': 23, + 'gemma-2-27b-it-turbomind': 46, + 'DeepSeek-R1-Distill-Llama-70B': 63, + 'deepseek-v2_5-1210-turbomind': 85, + 'Llama-3.3-70B-Instruct': 45, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 68, + 'DeepSeek-R1-Distill-Qwen-32B': 66, + 'mixtral-large-instruct-2407-lmdeploy': 56, + 'Qwen2.5-72B-Instruct': 78 + }, + 'safe': { + 'Qwen2.5-32B-Instruct': 72, + 'Llama-3.1-70B-Instruct': 55, + 'gemma-2-27b-it-turbomind': 72, + 'DeepSeek-R1-Distill-Llama-70B': 55, + 'deepseek-v2_5-1210-turbomind': 72, + 'Llama-3.3-70B-Instruct': 64, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 76, + 'DeepSeek-R1-Distill-Qwen-32B': 55, + 'mixtral-large-instruct-2407-lmdeploy': 69, + 'Qwen2.5-72B-Instruct': 83 + }, + 'Hallucination': { + 'Qwen2.5-32B-Instruct': 78, + 'Llama-3.1-70B-Instruct': 50, + 'gemma-2-27b-it-turbomind': 65, + 'DeepSeek-R1-Distill-Llama-70B': 61, + 'deepseek-v2_5-1210-turbomind': 66, + 'Llama-3.3-70B-Instruct': 48, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 75, + 'DeepSeek-R1-Distill-Qwen-32B': 60, + 'mixtral-large-instruct-2407-lmdeploy': 76, + 'Qwen2.5-72B-Instruct': 74 + }, + 'chatQA': { + 'Qwen2.5-32B-Instruct': 39, + 'Llama-3.1-70B-Instruct': 25, + 'gemma-2-27b-it-turbomind': 56, + 'DeepSeek-R1-Distill-Llama-70B': 53, + 'deepseek-v2_5-1210-turbomind': 70, + 'Llama-3.3-70B-Instruct': 34, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 69, + 'DeepSeek-R1-Distill-Qwen-32B': 48, + 'mixtral-large-instruct-2407-lmdeploy': 55, + 'Qwen2.5-72B-Instruct': 68 + }, + 'IF': { + 'Qwen2.5-32B-Instruct': 34, + 'Llama-3.1-70B-Instruct': 35, + 'gemma-2-27b-it-turbomind': 38, + 'DeepSeek-R1-Distill-Llama-70B': 50, + 'deepseek-v2_5-1210-turbomind': 63, + 'Llama-3.3-70B-Instruct': 37, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 62, + 'DeepSeek-R1-Distill-Qwen-32B': 41, + 'mixtral-large-instruct-2407-lmdeploy': 47, + 'Qwen2.5-72B-Instruct': 48 + }, + 'LanTask': { + 'Qwen2.5-32B-Instruct': 62, + 'Llama-3.1-70B-Instruct': 29, + 'gemma-2-27b-it-turbomind': 53, + 'DeepSeek-R1-Distill-Llama-70B': 60, + 'deepseek-v2_5-1210-turbomind': 75, + 'Llama-3.3-70B-Instruct': 46, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 69, + 'DeepSeek-R1-Distill-Qwen-32B': 71, + 'mixtral-large-instruct-2407-lmdeploy': 48, + 'Qwen2.5-72B-Instruct': 74 + }, + 'Creation': { + 'Qwen2.5-32B-Instruct': 40, + 'Llama-3.1-70B-Instruct': 34, + 'gemma-2-27b-it-turbomind': 55, + 'DeepSeek-R1-Distill-Llama-70B': 66, + 'deepseek-v2_5-1210-turbomind': 73, + 'Llama-3.3-70B-Instruct': 36, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 73, + 'DeepSeek-R1-Distill-Qwen-32B': 64, + 'mixtral-large-instruct-2407-lmdeploy': 43, + 'Qwen2.5-72B-Instruct': 67 + }, + 'Code_and_AI': { + 'Qwen2.5-32B-Instruct': 44, + 'Llama-3.1-70B-Instruct': 32, + 'gemma-2-27b-it-turbomind': 34, + 'DeepSeek-R1-Distill-Llama-70B': 56, + 'deepseek-v2_5-1210-turbomind': 64, + 'Llama-3.3-70B-Instruct': 43, + 'nvidia-Llama-3.1-Nemotron-70B-Instruct-HF': 62, + 'DeepSeek-R1-Distill-Qwen-32B': 43, + 'mixtral-large-instruct-2407-lmdeploy': 51, + 'Qwen2.5-72B-Instruct': 60 + } +} + + +class Judgerbenchv2Evaluator(BaseEvaluator): + + def get_rank_dict(self, score_dict): + sorted_models = sorted(score_dict.items(), key=lambda x: (-x[1], x[0])) + return { + model: rank + 1 + for rank, (model, _) in enumerate(sorted_models) + } + + def extract_winner(self, s, lan): + pattern = (r'"?(胜者)"?\s*:\s*"([A-Z])"' if lan.lower() in ['zh', 'cn'] + else r'"?(winner)"?\s*:\s*"([A-Z])"') + + matches = re.findall(pattern, s) + + return matches[-1][1] if matches else None + + def score(self, predictions, references): + if len(predictions) != len(references): + return {'error': 'preds and refrs have different length'} + correct = 0 + count = 0 + details = [] + Model_dict = {} + for prediction, reference in zip(predictions, references): + # pre-defines + ModelA = reference['ModelA'] + ModelB = reference['ModelB'] + + if reference['category'] == 'Reason & Analysis': + r1_rank_score = R1_Score_MAP['Reason_and_analysis'] + elif reference['category'] == 'Code & AI': + r1_rank_score = R1_Score_MAP['Code_and_AI'] + else: + r1_rank_score = R1_Score_MAP[reference['category']] + + choice = self.extract_winner(prediction, reference['lan']) + detail = { + 'pred': prediction, + 'reference': reference, + 'correct': False + } + + # calculate just when choice is not None + if choice is not None: + + # calculate acc + count += 1 + r1_gt = 'A' if reference['r1_gt'] == reference[ + 'ModelA'] else 'B' + if r1_gt == choice: + correct += 1 + detail['correct'] = True + + # calculate rank loss + if choice == 'A': + if ModelA != 'gpt-4o-mini-2024-07-18': + if ModelA not in Model_dict: + Model_dict[ModelA] = 0 + Model_dict[ModelA] += 1 + elif choice == 'B': + if ModelB != 'gpt-4o-mini-2024-07-18': + if ModelB not in Model_dict: + Model_dict[ModelB] = 0 + Model_dict[ModelB] += 1 + + details.append(detail) + + # calculate rank loss + dict1 = dict(sorted(Model_dict.items())) + dict2 = dict(sorted(r1_rank_score.items())) + + rank1 = self.get_rank_dict(dict1) + rank2 = self.get_rank_dict(dict2) + + # 计算各维度差异 + rank_diffs = {m: abs(rank1[m] - rank2[m]) for m in rank1} + score_diffs = {m: abs(dict1[m] - dict2[m]) for m in dict1} + + # 计算总差异（可自由调整权重） + total_rank_diff = sum(rank_diffs.values()) # 例如原排名总差距 = 14 + total_score_diff = sum(score_diffs.values()) # 例如总分数差距 = 75 + alpha = 0.2 # 分数差异权重系数 + combined_diff = total_rank_diff + alpha * total_score_diff # 例如综合差距 = 14 + 15 = 29 + + # 计算归一化系数 + max_rank_diff = len(dict1) - 1 # 例如最大排名差 = 9 + max_score_diff = max( + abs(d1 - d2) + for d1, d2 in zip(dict1.values(), dict2.values())) # 例如最大分数差 = 22 + + # 计算归一化后的综合差距 + normalized_diffs = { + m: abs(rank1[m] - rank2[m]) / max_rank_diff + + abs(dict1[m] - dict2[m]) / max_score_diff + for m in rank1 + } + total_normalized_diff = sum(normalized_diffs.values()) / len( + normalized_diffs.values()) * 100 + acc = 100 * correct / count + final_score = acc - total_normalized_diff + result = { + 'accuracy': acc, + 'rank_diff': total_rank_diff, + 'score_diff': total_score_diff, + 'normalized_diff': total_normalized_diff, + 'final_score': final_score, + 'details': details + } + return result From ddc9cc0afbf5d8c14a05f61c47bbecc80c21a92f Mon Sep 17 00:00:00 2001 From: bittersweet1999 <148421775+bittersweet1999@users.noreply.github.com> Date: Wed, 7 May 2025 10:57:23 +0800 Subject: [PATCH 4/8] [Add] add a config to Judge dataset all (#2077) * fix pip version * fix pip version * add judgedatasetall * add judgedatasetall * add judgedatasetall --- examples/eval_judge_dataset_all.py | 61 +++++++++++++ .../configs/summarizers/judgedataset_all.py | 90 +++++++++++++++++++ .../icl_evaluator/icl_judge_evaluator.py | 6 +- 3 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 examples/eval_judge_dataset_all.py create mode 100644 opencompass/configs/summarizers/judgedataset_all.py diff --git a/examples/eval_judge_dataset_all.py b/examples/eval_judge_dataset_all.py new file mode 100644 index 00000000..4cc237f4 --- /dev/null +++ b/examples/eval_judge_dataset_all.py @@ -0,0 +1,61 @@ +from mmengine.config import read_base +with read_base(): + from opencompass.configs.datasets.judge.judgerbenchv2 import get_judgerbenchv2_dataset as get_judgerbenchv2_datasets + from opencompass.configs.datasets.judge.rmb import get_rmb_dataset as get_rmb_datasets + from opencompass.configs.datasets.judge.rewardbench import get_rewardbench_datasets + from opencompass.configs.datasets.judge.judgebench import get_judgebench_datasets + + from opencompass.configs.summarizers.judgedataset_all import summarizer +from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI +from opencompass.partitioners import NaivePartitioner, SizePartitioner, NumWorkerPartitioner +from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner +from opencompass.partitioners.sub_size import SubjectiveSizePartitioner +from opencompass.partitioners.sub_num_worker import SubjectiveNumWorkerPartitioner +from opencompass.runners import LocalRunner, DLCRunner, VOLCRunner +from opencompass.runners import SlurmSequentialRunner +from opencompass.tasks import OpenICLInferTask +from opencompass.tasks.subjective_eval import SubjectiveEvalTask +from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask +from opencompass.models import TurboMindModelwithChatTemplate + + +api_meta_template = dict( + round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True), + ] +) +datasets = sum( + (v for k, v in locals().items() if k.endswith('_datasets')), + [], +) + + +models = [ + dict( + type=TurboMindModelwithChatTemplate, + abbr='qwen-7b-hf', + path='Qwen/Qwen-7B', + engine_config=dict(session_len=16384, max_batch_size=16, tp=1), + gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=2048), + max_seq_len=16384, + max_out_len=2048, + batch_size=16, + run_cfg=dict(num_gpus=1), + ), +] + + + +infer = dict( + partitioner=dict(type=NumWorkerPartitioner, num_worker=8), + runner=dict( + type=LocalRunner, + max_num_workers=72, + task=dict(type=OpenICLInferTask), + ), +) + + + +work_dir = './outputs/judge_dataset_all/' diff --git a/opencompass/configs/summarizers/judgedataset_all.py b/opencompass/configs/summarizers/judgedataset_all.py new file mode 100644 index 00000000..229d322e --- /dev/null +++ b/opencompass/configs/summarizers/judgedataset_all.py @@ -0,0 +1,90 @@ +Judge_all_summary_groups = [] + + +# RewardBench +_Chat_weights = { +'alpacaeval-easy': 0.32355305466237944, +'alpacaeval-length': 0.32355305466237944, +'alpacaeval-hard': 0.32355305466237944, +'mt-bench-easy': 0.011254019292604502, +'mt-bench-med': 0.018086816720257234, +} + +_Chat_Hard_weights = { +'mt-bench-hard': 0.09698275862068965, +'llmbar-natural': 0.21551724137931033, +'llmbar-adver-neighbor': 0.28879310344827586, +'llmbar-adver-GPTInst': 0.19827586206896552, +'llmbar-adver-GPTOut': 0.10129310344827586, +'llmbar-adver-manual': 0.09913793103448276, +} + +_Safety_weights = { +'refusals-dangerous': 0.13513513513513514, +'refusals-offensive': 0.13513513513513514, +'xstest-should-refuse': 0.20810810810810812, +'xstest-should-respond': 0.33783783783783783, +'donotanswer': 0.1837837837837838, +} + +_Reasoning_weights = { +'math-prm': 0.31236897274633124, +'hep-cpp': 0.1146051712089448, +'hep-go': 0.1146051712089448, +'hep-java': 0.1146051712089448, +'hep-js': 0.1146051712089448, +'hep-python': 0.1146051712089448, +'hep-rust': 0.1146051712089448, +} + +_RewardBench_weights = {'alpacaeval-easy': 0.08088826366559486,'alpacaeval-length': 0.08088826366559486,'alpacaeval-hard': 0.08088826366559486,'mt-bench-easy': 0.0028135048231511255,'mt-bench-med': 0.004521704180064309,'mt-bench-hard': 0.024245689655172414,'llmbar-natural': 0.05387931034482758,'llmbar-adver-neighbor': 0.07219827586206896,'llmbar-adver-GPTInst': 0.04956896551724138,'llmbar-adver-GPTOut': 0.025323275862068964,'llmbar-adver-manual': 0.02478448275862069,'refusals-dangerous': 0.033783783783783786,'refusals-offensive': 0.033783783783783786,'xstest-should-refuse': 0.05202702702702703,'xstest-should-respond': 0.08445945945945946,'donotanswer': 0.04594594594594595,'math-prm': 0.07809224318658281,'hep-cpp': 0.0286512928022362,'hep-go': 0.0286512928022362,'hep-java': 0.0286512928022362,'hep-js': 0.0286512928022362,'hep-python': 0.0286512928022362,'hep-rust': 0.0286512928022362,} + +Judge_all_summary_groups.append({'name': 'RewardBench_avg', 'subsets': list(_RewardBench_weights.keys()), 'weights': _RewardBench_weights}) +Judge_all_summary_groups.append({'name': 'RewardBench_Chat', 'subsets': list(_Chat_weights.keys()), 'weights': _Chat_weights}) +Judge_all_summary_groups.append({'name': 'RewardBench_Chat Hard', 'subsets': list(_Chat_Hard_weights.keys()), 'weights': _Chat_Hard_weights}) +Judge_all_summary_groups.append({'name': 'RewardBench_Safety', 'subsets': list(_Safety_weights.keys()), 'weights': _Safety_weights}) +Judge_all_summary_groups.append({'name': 'RewardBench_Reasoning', 'subsets': list(_Reasoning_weights.keys()), 'weights': _Reasoning_weights}) + + + +# Judgerbenchv2 +Judgerbenchv2_tasks = ['Code_and_AI', 'Creation', 'LanTask', 'IF', 'chatQA', 'Hallucination', 'safe', 'Reason_and_analysis', 'Longtext', 'Knowledge'] +Judgerbenchv2_metrics = ['final_score', 'accuracy', 'normalized_diff', 'rank_diff', 'score_diff'] +Judgerbenchv2_summary_names = [] +for metric in Judgerbenchv2_metrics: + for task in Judgerbenchv2_tasks: + Judgerbenchv2_summary_names.append([task, metric]) + +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_final_score', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'final_score']}) +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_accuracy', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'accuracy']}) +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_normalized_diff', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'normalized_diff']}) +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_rank_diff', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'rank_diff']}) +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_score_diff', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'score_diff']}) + +Judge_all_summary_groups.append({'name': 'Judgebench', 'subsets': ['judgebench']}) +Judge_all_summary_groups.append({'name': 'rmb_dataset_total_avg', 'subsets': [['rmb_dataset', 'total_accuracy']]}) +Judge_all_summary_groups.append({'name': 'rmb_dataset_pair', 'subsets': [['rmb_dataset', 'pair_average']]}) +Judge_all_summary_groups.append({'name': 'rmb_dataset_bon', 'subsets': [['rmb_dataset', 'bon_average']]}) + +summarizer = dict( + dataset_abbrs=[ + 'Judgerbenchv2_final_score', + 'Judgebench', + 'rmb_dataset_total_avg', + 'RewardBench_avg', + '', + 'Judgerbenchv2_accuracy', + 'Judgerbenchv2_normalized_diff', + 'Judgerbenchv2_rank_diff', + 'Judgerbenchv2_score_diff', + '', + 'rmb_dataset_pair', + 'rmb_dataset_bon', + '', + 'RewardBench_Chat', + 'RewardBench_Chat Hard', + 'RewardBench_Safety', + 'RewardBench_Reasoning', + ], + summary_groups=Judge_all_summary_groups, +) diff --git a/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py b/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py index d7f3531a..e59cdc12 100644 --- a/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py +++ b/opencompass/openicl/icl_evaluator/icl_judge_evaluator.py @@ -16,7 +16,8 @@ class JudgeEvaluator(BaseEvaluator): count = 0 details = [] for prediction, reference in zip(predictions, references): - choice = prediction.split("\"Choice\": \"Model ")[-1][0] + choice = prediction.split("\"Choice\": \"Model ")[-1][0] if len( + prediction) != 0 else None gold_winner = reference.get('winner', '') detail = { 'pred': prediction, @@ -75,7 +76,8 @@ class RMBEvaluator(BaseEvaluator): pair_harm_list = [] for prediction, reference in zip(predictions, references): - choice = prediction.split("\"Choice\": \"Model ")[-1][0] + choice = prediction.split("\"Choice\": \"Model ")[-1][0] if len( + prediction) != 0 else None gold_winner = reference.get('winner', '') subset = reference.get('subset', '') goal = reference.get('goal', '') From af8432e1d63714a1766e1a403b5c1bbf71e78d8c Mon Sep 17 00:00:00 2001 From: Linchen Xiao Date: Wed, 7 May 2025 14:06:40 +0800 Subject: [PATCH 5/8] [Update] OpenAI SDK model reasoning content (#2078) * update * update * update --- opencompass/models/openai_api.py | 83 ++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/opencompass/models/openai_api.py b/opencompass/models/openai_api.py index 9c2baed1..692edcf1 100644 --- a/opencompass/models/openai_api.py +++ b/opencompass/models/openai_api.py @@ -531,26 +531,28 @@ class OpenAI(BaseAPIModel): class OpenAISDK(OpenAI): - def __init__(self, - path: str = 'gpt-3.5-turbo', - max_seq_len: int = 16384, - query_per_second: int = 1, - rpm_verbose: bool = False, - retry: int = 2, - key: str | List[str] = 'ENV', - org: str | List[str] | None = None, - meta_template: Dict | None = None, - openai_api_base: str | List[str] = OPENAISDK_API_BASE, - openai_proxy_url: Optional[str] = None, - mode: str = 'none', - logprobs: bool | None = False, - top_logprobs: int | None = None, - temperature: float | None = None, - tokenizer_path: str | None = None, - extra_body: Dict | None = None, - verbose: bool = False, - status_code_mappings: dict = {}, - think_tag: str = ''): + def __init__( + self, + path: str = 'gpt-3.5-turbo', + max_seq_len: int = 16384, + query_per_second: int = 1, + rpm_verbose: bool = False, + retry: int = 2, + key: str | List[str] = 'ENV', + org: str | List[str] | None = None, + meta_template: Dict | None = None, + openai_api_base: str | List[str] = OPENAISDK_API_BASE, + openai_proxy_url: Optional[str] = None, + mode: str = 'none', + logprobs: bool | None = False, + top_logprobs: int | None = None, + temperature: float | None = None, + tokenizer_path: str | None = None, + extra_body: Dict | None = None, + verbose: bool = False, + status_code_mappings: dict = {}, + think_tag: str = '', + ): super().__init__( path, max_seq_len, @@ -597,11 +599,13 @@ class OpenAISDK(OpenAI): self.status_code_mappings = status_code_mappings self.think_tag = think_tag - def _generate(self, - input: PromptList | str, - max_out_len: int, - temperature: float, - timeout: int = 3600) -> str: + def _generate( + self, + input: PromptList | str, + max_out_len: int, + temperature: float, + timeout: int = 3600, + ) -> str: """Generate results given a list of inputs. Args: @@ -662,7 +666,12 @@ class OpenAISDK(OpenAI): # Check if response is empty or content is empty if (not responses.choices or not responses.choices[0].message - or not responses.choices[0].message.content): + or + (not responses.choices[0].message.content and not getattr( + responses.choices[0].message, + 'reasoning_content', + '', + ))): # noqa: E125 self.logger.error( 'Failed to extract content from the responses. ' 'Please check the API response for detail information.' @@ -670,12 +679,13 @@ class OpenAISDK(OpenAI): responses, ) num_retries += 1 - # Continue to retry instead of returning empty response continue + reasoning_content = (getattr(responses.choices[0].message, + 'reasoning_content', '') or '') + content = responses.choices[0].message.content or '' # Concat Reasoning Content and tags to content - if (hasattr(responses.choices[0].message, 'reasoning_content') - and responses.choices[0].message.reasoning_content): + if reasoning_content: if self.verbose: self.logger.info( 'Follow' @@ -684,14 +694,17 @@ class OpenAISDK(OpenAI): 'Reasoning Content: %s, \n' 'Tags: %s, \n' 'Content: %s', - responses.choices[0].message.reasoning_content, + reasoning_content, self.think_tag, - responses.choices[0].message.content) - return (responses.choices[0].message.reasoning_content + - self.think_tag + - responses.choices[0].message.content) + content, + ) + if content: + return reasoning_content + self.think_tag + content + else: + return reasoning_content - return responses.choices[0].message.content + else: + return content except (BadRequestError, APIStatusError) as e: # Handle BadRequest status From d62b69aaefdd746c00bc530abcddf9d8245ab26b Mon Sep 17 00:00:00 2001 From: Dongsheng Zhu <59612926+Zhudongsheng75@users.noreply.github.com> Date: Wed, 7 May 2025 15:51:18 +0800 Subject: [PATCH 6/8] [Fix] Fix InternVL model config (#2068) * intervl-8b&38b * intervl adjustment * internvl fix --- .../models/internvl/lmdeploy_internvl_2_5_38b.py | 15 +++++++++++++++ .../models/internvl/lmdeploy_internvl_2_5_8b.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 opencompass/configs/models/internvl/lmdeploy_internvl_2_5_38b.py create mode 100644 opencompass/configs/models/internvl/lmdeploy_internvl_2_5_8b.py diff --git a/opencompass/configs/models/internvl/lmdeploy_internvl_2_5_38b.py b/opencompass/configs/models/internvl/lmdeploy_internvl_2_5_38b.py new file mode 100644 index 00000000..98713696 --- /dev/null +++ b/opencompass/configs/models/internvl/lmdeploy_internvl_2_5_38b.py @@ -0,0 +1,15 @@ +from opencompass.models import TurboMindModelwithChatTemplate + +models = [ + dict( + type=TurboMindModelwithChatTemplate, + abbr='internvl2_5-38b-turbomind', + path='OpenGVLab/InternVL2_5-38B', + engine_config=dict(session_len=8192, max_batch_size=8, tp=4), + gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024), + max_seq_len=8192, + max_out_len=8192, + batch_size=8, + run_cfg=dict(num_gpus=4), + ) +] diff --git a/opencompass/configs/models/internvl/lmdeploy_internvl_2_5_8b.py b/opencompass/configs/models/internvl/lmdeploy_internvl_2_5_8b.py new file mode 100644 index 00000000..3541249c --- /dev/null +++ b/opencompass/configs/models/internvl/lmdeploy_internvl_2_5_8b.py @@ -0,0 +1,15 @@ +from opencompass.models import TurboMindModelwithChatTemplate + +models = [ + dict( + type=TurboMindModelwithChatTemplate, + abbr='internvl2_5-8b-turbomind', + path='OpenGVLab/InternVL2_5-8B', + engine_config=dict(session_len=8192, max_batch_size=16, tp=1), + gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=8192), + max_seq_len=8192, + max_out_len=8192, + batch_size=16, + run_cfg=dict(num_gpus=1), + ) +] From 43b2c4ed765755560f506f91739502756de60423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E6=98=95=E8=BE=B0?= Date: Wed, 7 May 2025 16:18:43 +0800 Subject: [PATCH 7/8] [Fix] Update lawbench data path (#2037) --- opencompass/datasets/lawbench/utils/modules/alignment.py | 5 +++-- opencompass/datasets/lawbench/utils/modules/classifier.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/opencompass/datasets/lawbench/utils/modules/alignment.py b/opencompass/datasets/lawbench/utils/modules/alignment.py index d11feb74..5330b2fa 100644 --- a/opencompass/datasets/lawbench/utils/modules/alignment.py +++ b/opencompass/datasets/lawbench/utils/modules/alignment.py @@ -8,6 +8,7 @@ REAL_PATH = os.path.split(os.path.realpath(__file__))[0] chinese_punct = "！？｡＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘'‛“”„‟…‧﹏" english_punct = punctuation punct = chinese_punct + english_punct +cache_dir = os.environ.get('COMPASS_DATA_CACHE', '') def check_all_chinese(word): """ @@ -22,7 +23,7 @@ def read_cilin(): Cilin 詞林 is a thesaurus with semantic information """ # TODO -- fix this path - lines = open(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "data", "lawbench", "eval_assets", "cilin.txt"), "r", encoding="gbk").read().strip().split("\n") + lines = open(os.path.join(cache_dir, "data", "lawbench", "eval_assets", "cilin.txt"), "r", encoding="gbk").read().strip().split("\n") semantic_dict = {} semantic_classes = {} for line in lines: @@ -39,7 +40,7 @@ def read_cilin(): def read_confusion(): confusion_dict = {} - with open(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "data", "lawbench", "eval_assets", "confusion_dict.txt"), "r", encoding="utf-8") as f: + with open(os.path.join(cache_dir, "data", "lawbench", "eval_assets", "confusion_dict.txt"), "r", encoding="utf-8") as f: for line in f: li = line.rstrip('\n').split(" ") confusion_dict[li[0]] = li[1:] diff --git a/opencompass/datasets/lawbench/utils/modules/classifier.py b/opencompass/datasets/lawbench/utils/modules/classifier.py index a8e9b921..b8ee407b 100644 --- a/opencompass/datasets/lawbench/utils/modules/classifier.py +++ b/opencompass/datasets/lawbench/utils/modules/classifier.py @@ -10,7 +10,8 @@ Correction = namedtuple( "inds", ], ) -char_smi = CharFuncs(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "data", "lawbench", "eval_assets", "char_meta.txt")) +cache_dir = os.environ.get('COMPASS_DATA_CACHE', '') +char_smi = CharFuncs(os.path.join(cache_dir, "data", "lawbench", "eval_assets", "char_meta.txt")) def check_spell_error(src_span: str, tgt_span: str, From ba0e32292c23cadd4c6c061a132b95b1c8b9e4e0 Mon Sep 17 00:00:00 2001 From: Dongsheng Zhu <59612926+Zhudongsheng75@users.noreply.github.com> Date: Wed, 7 May 2025 16:42:09 +0800 Subject: [PATCH 8/8] [Feature] Support InternSandbox (#2049) * internsandbox init * internsandbox * dataset_index * dataset_index_add --- dataset-index.yml | 6 ++ .../internsandbox/internsandbox_gen.py | 4 + .../internsandbox/internsandbox_gen_44b982.py | 59 ++++++++++++++ opencompass/datasets/__init__.py | 1 + opencompass/datasets/internsandbox.py | 78 +++++++++++++++++++ 5 files changed, 148 insertions(+) create mode 100644 opencompass/configs/datasets/internsandbox/internsandbox_gen.py create mode 100644 opencompass/configs/datasets/internsandbox/internsandbox_gen_44b982.py create mode 100644 opencompass/datasets/internsandbox.py diff --git a/dataset-index.yml b/dataset-index.yml index 9585f97c..4a920071 100644 --- a/dataset-index.yml +++ b/dataset-index.yml @@ -1023,3 +1023,9 @@ paper: https://arxiv.org/pdf/2402.09391 configpath: opencompass/configs/datasets/SmolInstruct/smolinstruct_gen.py configpath_llmjudge: '' +- internsandbox: + name: InternSandbox + category: Reasoning/Code/Agent + paper: '' + configpath: opencompass/configs/datasets/internsandbox/internsandbox_gen_44b982.py + configpath_llmjudge: '' \ No newline at end of file diff --git a/opencompass/configs/datasets/internsandbox/internsandbox_gen.py b/opencompass/configs/datasets/internsandbox/internsandbox_gen.py new file mode 100644 index 00000000..1af0955c --- /dev/null +++ b/opencompass/configs/datasets/internsandbox/internsandbox_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .internsandbox_gen_44b982 import internsandbox_datasets \ No newline at end of file diff --git a/opencompass/configs/datasets/internsandbox/internsandbox_gen_44b982.py b/opencompass/configs/datasets/internsandbox/internsandbox_gen_44b982.py new file mode 100644 index 00000000..368189a5 --- /dev/null +++ b/opencompass/configs/datasets/internsandbox/internsandbox_gen_44b982.py @@ -0,0 +1,59 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import InternSandboxDataset, InternSandboxEvaluator + + +_SANDBOXS_ = ['aquarium', 'arc', 'arrowmaze', 'bbehboardgameqa', 'bbehbooleanexpressions', 'BbehDyckLanguages', 'BbehGeometricShapes', 'BbehMultistepArithmetic', 'bbehobjectcounting', 'bbehobjectproperties', 'bbehshuffobject', 'BbehWebOfLies', 'BbehWordSorting', 'binairo', 'calcudoku', 'campsite', 'cipher', 'cryptomath', 'dominosa', 'futoshiki', 'galaxies', 'game24', 'kakurasu', 'korLogicAnalogicalReasoning', 'korLogicCanonicalPropositions', 'korLogicCooperativePrinciple', 'korLogicDefinitions', 'korLogicDerivativeReasoningOfPropositionalLogic', 'korLogicDisjunctiveNormalFormAndConjunctiveNormalForm', 'korLogicDynamicLogic', 'korLogicEnumerativeInductiveReasoning', 'korLogicEpistemicLogic', 'korLogicEquivalenceCalculus', 'korLogicFigureOfTheSyllogism', 'korLogicFormalFallacies', 'korLogicInductionParadox', 'korLogicLogicalMethodsForExploringCauseAndEffectRelationships', 'korLogicPredicateLogicFormalization', 'korLogicPropositionalLogicConcepts', 'korLogicPropositionalLogicFormalization', 'korLogicResolution', 'korLogicSpeechActs', 'korLogicStatisticalReasoning', 'korLogicTemporalPropositions', 'korLogicTruthValueModalPropositions', 'korOperationUnicode20ac', 'korOperationUnicode2295', 'korOperationUnicode25a0', 'korOperationUnicode25a1', 'korOperationUnicode25b3', 'korOperationUnicode25bd', 'korOperationUnicode25cb', 'korOperationUnicode25ce', 'korOperationUnicode25cf', 'korOperationUnicode2605', 'korOperationUnicodeffe0', 'korOperationUnicodeffe1', 'korPuzzle24Points', 'korPuzzleArrowMaze', 'korPuzzleCalcudoko', 'korPuzzleCampsite', 'korPuzzleConnectWords', 'korPuzzleCryptoMath', 'korPuzzleKukurasu', 'korPuzzleLogicPuzzle', 'korPuzzleSkyscrapers', 'korPuzzleWordBrainTeasers', 'korPuzzleWordLadder', 'korPuzzleWordRootsAndAffixes', 'korPuzzleWordscapes', 'korPuzzleWordSearch', 'LightUp', 'maze', 'minesweeper', 'nonograms', 'starbattle', 'stitches', 'sudoku', 'tents', 'thermometers'] + +internsandbox_reader_cfg = dict( + input_columns=['prompt'], + output_column='ground_truth' +) + +internsandbox_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt='You are a helpful assistant.', + ) + ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ), + ], + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer) +) + +internsandbox_eval_cfg = { + sandbox: dict( + evaluator=dict( + type=InternSandboxEvaluator, + short_penalty=False, + format_penalty=False, + ), + pred_role='BOT', + ) for sandbox in _SANDBOXS_ +} + +internsandbox_datasets = [ + dict( + type=InternSandboxDataset, + abbr=f'internsandbox-{sandbox}', + path='./data/InternSandboxBenchmark_verified_V0.3.1/', + local_mode=True, + sandbox=sandbox, + reader_cfg=internsandbox_reader_cfg, + infer_cfg=internsandbox_infer_cfg, + eval_cfg=internsandbox_eval_cfg[sandbox], + ) for sandbox in _SANDBOXS_ +] diff --git a/opencompass/datasets/__init__.py b/opencompass/datasets/__init__.py index b00162d1..a7c037cf 100644 --- a/opencompass/datasets/__init__.py +++ b/opencompass/datasets/__init__.py @@ -68,6 +68,7 @@ from .hungarian_math import * # noqa: F401, F403 from .IFEval.ifeval import IFEvalDataset, IFEvaluator # noqa: F401, F403 from .inference_ppl import InferencePPLDataset # noqa: F401, F403 from .infinitebench import * # noqa: F401, F403 +from .internsandbox import * # noqa: F401, F403 from .iwslt2017 import * # noqa: F401, F403 from .jigsawmultilingual import * # noqa: F401, F403 from .jsonl import JsonlDataset # noqa: F401, F403 diff --git a/opencompass/datasets/internsandbox.py b/opencompass/datasets/internsandbox.py new file mode 100644 index 00000000..c71cc3f7 --- /dev/null +++ b/opencompass/datasets/internsandbox.py @@ -0,0 +1,78 @@ +import importlib +import json +import os.path as osp + +from datasets import Dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET +from opencompass.utils import get_data_path + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class InternSandboxDataset(BaseDataset): + + @staticmethod + def load(path: str, sandbox: str, local_mode: bool = False): + path = get_data_path(path, local_mode=local_mode) + file_path = osp.join(path, f'{sandbox}.jsonl') + data = [] + with open(file_path, 'r', encoding='utf-8') as f: + for line in f: + origin_data = json.loads(line) + origin_data['ground_truth'] = json.dumps( + origin_data['ground_truth']) + data.append(origin_data) + return Dataset.from_list(data) + + +@ICL_EVALUATORS.register_module() +class InternSandboxEvaluator(BaseEvaluator): + + def __init__(self, + short_penalty: bool = False, + format_penalty: bool = False): + super().__init__() + self.short_penalty = short_penalty + self.format_penalty = format_penalty + + def score(self, predictions, references, test_set): + + if len(predictions) != len(references): + return { + 'error': + 'predictions and references have different ' + f'length. len(predictions): {len(predictions)}, ' + f'len(references): {len(references)}' + } + + class_name = f"{test_set[0]['data_source']}Sandbox" + + details = [] + for pred, ref, ts in zip(predictions, references, test_set): + ref = json.loads(ref) + module = importlib.import_module('intern_sandbox') + score = getattr(module, class_name).verify_score( + pred, + ref, + short_penalty=self.short_penalty, + format_penalty=self.format_penalty) + try: + extracted = getattr(module, class_name).extract_output(pred) + except: # noqa: E722 + extracted = None + + res = { + 'prompt': ts['prompt'], + 'score': score, + 'extracted_output': extracted, + 'ground_truth': ref, + 'output': pred, + } + details.append(res) + + avg_score = sum(r['score'] for r in details) / len(details) + results = {'accuracy': avg_score, 'details': details} + return results