From a77a040ba78bb8a4eb8a4043a7f1482189206793 Mon Sep 17 00:00:00 2001 From: bittersweet1999 <1487910649@qq.com> Date: Wed, 7 May 2025 02:44:58 +0000 Subject: [PATCH] add judgedatasetall --- examples/eval_judge_dataset_all.py | 61 +++++++++++++ .../configs/summarizers/judgedataset_all.py | 90 +++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 examples/eval_judge_dataset_all.py create mode 100644 opencompass/configs/summarizers/judgedataset_all.py diff --git a/examples/eval_judge_dataset_all.py b/examples/eval_judge_dataset_all.py new file mode 100644 index 00000000..4cc237f4 --- /dev/null +++ b/examples/eval_judge_dataset_all.py @@ -0,0 +1,61 @@ +from mmengine.config import read_base +with read_base(): + from opencompass.configs.datasets.judge.judgerbenchv2 import get_judgerbenchv2_dataset as get_judgerbenchv2_datasets + from opencompass.configs.datasets.judge.rmb import get_rmb_dataset as get_rmb_datasets + from opencompass.configs.datasets.judge.rewardbench import get_rewardbench_datasets + from opencompass.configs.datasets.judge.judgebench import get_judgebench_datasets + + from opencompass.configs.summarizers.judgedataset_all import summarizer +from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI +from opencompass.partitioners import NaivePartitioner, SizePartitioner, NumWorkerPartitioner +from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner +from opencompass.partitioners.sub_size import SubjectiveSizePartitioner +from opencompass.partitioners.sub_num_worker import SubjectiveNumWorkerPartitioner +from opencompass.runners import LocalRunner, DLCRunner, VOLCRunner +from opencompass.runners import SlurmSequentialRunner +from opencompass.tasks import OpenICLInferTask +from opencompass.tasks.subjective_eval import SubjectiveEvalTask +from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask +from opencompass.models import TurboMindModelwithChatTemplate + + +api_meta_template = dict( + round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True), + ] +) +datasets = sum( + (v for k, v in locals().items() if k.endswith('_datasets')), + [], +) + + +models = [ + dict( + type=TurboMindModelwithChatTemplate, + abbr='qwen-7b-hf', + path='Qwen/Qwen-7B', + engine_config=dict(session_len=16384, max_batch_size=16, tp=1), + gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=2048), + max_seq_len=16384, + max_out_len=2048, + batch_size=16, + run_cfg=dict(num_gpus=1), + ), +] + + + +infer = dict( + partitioner=dict(type=NumWorkerPartitioner, num_worker=8), + runner=dict( + type=LocalRunner, + max_num_workers=72, + task=dict(type=OpenICLInferTask), + ), +) + + + +work_dir = './outputs/judge_dataset_all/' diff --git a/opencompass/configs/summarizers/judgedataset_all.py b/opencompass/configs/summarizers/judgedataset_all.py new file mode 100644 index 00000000..229d322e --- /dev/null +++ b/opencompass/configs/summarizers/judgedataset_all.py @@ -0,0 +1,90 @@ +Judge_all_summary_groups = [] + + +# RewardBench +_Chat_weights = { +'alpacaeval-easy': 0.32355305466237944, +'alpacaeval-length': 0.32355305466237944, +'alpacaeval-hard': 0.32355305466237944, +'mt-bench-easy': 0.011254019292604502, +'mt-bench-med': 0.018086816720257234, +} + +_Chat_Hard_weights = { +'mt-bench-hard': 0.09698275862068965, +'llmbar-natural': 0.21551724137931033, +'llmbar-adver-neighbor': 0.28879310344827586, +'llmbar-adver-GPTInst': 0.19827586206896552, +'llmbar-adver-GPTOut': 0.10129310344827586, +'llmbar-adver-manual': 0.09913793103448276, +} + +_Safety_weights = { +'refusals-dangerous': 0.13513513513513514, +'refusals-offensive': 0.13513513513513514, +'xstest-should-refuse': 0.20810810810810812, +'xstest-should-respond': 0.33783783783783783, +'donotanswer': 0.1837837837837838, +} + +_Reasoning_weights = { +'math-prm': 0.31236897274633124, +'hep-cpp': 0.1146051712089448, +'hep-go': 0.1146051712089448, +'hep-java': 0.1146051712089448, +'hep-js': 0.1146051712089448, +'hep-python': 0.1146051712089448, +'hep-rust': 0.1146051712089448, +} + +_RewardBench_weights = {'alpacaeval-easy': 0.08088826366559486,'alpacaeval-length': 0.08088826366559486,'alpacaeval-hard': 0.08088826366559486,'mt-bench-easy': 0.0028135048231511255,'mt-bench-med': 0.004521704180064309,'mt-bench-hard': 0.024245689655172414,'llmbar-natural': 0.05387931034482758,'llmbar-adver-neighbor': 0.07219827586206896,'llmbar-adver-GPTInst': 0.04956896551724138,'llmbar-adver-GPTOut': 0.025323275862068964,'llmbar-adver-manual': 0.02478448275862069,'refusals-dangerous': 0.033783783783783786,'refusals-offensive': 0.033783783783783786,'xstest-should-refuse': 0.05202702702702703,'xstest-should-respond': 0.08445945945945946,'donotanswer': 0.04594594594594595,'math-prm': 0.07809224318658281,'hep-cpp': 0.0286512928022362,'hep-go': 0.0286512928022362,'hep-java': 0.0286512928022362,'hep-js': 0.0286512928022362,'hep-python': 0.0286512928022362,'hep-rust': 0.0286512928022362,} + +Judge_all_summary_groups.append({'name': 'RewardBench_avg', 'subsets': list(_RewardBench_weights.keys()), 'weights': _RewardBench_weights}) +Judge_all_summary_groups.append({'name': 'RewardBench_Chat', 'subsets': list(_Chat_weights.keys()), 'weights': _Chat_weights}) +Judge_all_summary_groups.append({'name': 'RewardBench_Chat Hard', 'subsets': list(_Chat_Hard_weights.keys()), 'weights': _Chat_Hard_weights}) +Judge_all_summary_groups.append({'name': 'RewardBench_Safety', 'subsets': list(_Safety_weights.keys()), 'weights': _Safety_weights}) +Judge_all_summary_groups.append({'name': 'RewardBench_Reasoning', 'subsets': list(_Reasoning_weights.keys()), 'weights': _Reasoning_weights}) + + + +# Judgerbenchv2 +Judgerbenchv2_tasks = ['Code_and_AI', 'Creation', 'LanTask', 'IF', 'chatQA', 'Hallucination', 'safe', 'Reason_and_analysis', 'Longtext', 'Knowledge'] +Judgerbenchv2_metrics = ['final_score', 'accuracy', 'normalized_diff', 'rank_diff', 'score_diff'] +Judgerbenchv2_summary_names = [] +for metric in Judgerbenchv2_metrics: + for task in Judgerbenchv2_tasks: + Judgerbenchv2_summary_names.append([task, metric]) + +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_final_score', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'final_score']}) +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_accuracy', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'accuracy']}) +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_normalized_diff', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'normalized_diff']}) +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_rank_diff', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'rank_diff']}) +Judge_all_summary_groups.append({'name': 'Judgerbenchv2_score_diff', 'subsets': [[name, metric] for name, metric in Judgerbenchv2_summary_names if metric == 'score_diff']}) + +Judge_all_summary_groups.append({'name': 'Judgebench', 'subsets': ['judgebench']}) +Judge_all_summary_groups.append({'name': 'rmb_dataset_total_avg', 'subsets': [['rmb_dataset', 'total_accuracy']]}) +Judge_all_summary_groups.append({'name': 'rmb_dataset_pair', 'subsets': [['rmb_dataset', 'pair_average']]}) +Judge_all_summary_groups.append({'name': 'rmb_dataset_bon', 'subsets': [['rmb_dataset', 'bon_average']]}) + +summarizer = dict( + dataset_abbrs=[ + 'Judgerbenchv2_final_score', + 'Judgebench', + 'rmb_dataset_total_avg', + 'RewardBench_avg', + '', + 'Judgerbenchv2_accuracy', + 'Judgerbenchv2_normalized_diff', + 'Judgerbenchv2_rank_diff', + 'Judgerbenchv2_score_diff', + '', + 'rmb_dataset_pair', + 'rmb_dataset_bon', + '', + 'RewardBench_Chat', + 'RewardBench_Chat Hard', + 'RewardBench_Safety', + 'RewardBench_Reasoning', + ], + summary_groups=Judge_all_summary_groups, +)