diff --git a/configs/datasets/compassbench_v1_3/compassbench_v1_3_objective_gen.py b/configs/datasets/compassbench_v1_3/compassbench_v1_3_objective_gen.py new file mode 100644 index 00000000..dd960fcc --- /dev/null +++ b/configs/datasets/compassbench_v1_3/compassbench_v1_3_objective_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .compassbench_v1_3_objective_gen_068af0 import compassbench_aug_datasets # noqa: F401, F403 diff --git a/configs/datasets/compassbench_v1_3/compassbench_v1_3_objective_gen_068af0.py b/configs/datasets/compassbench_v1_3/compassbench_v1_3_objective_gen_068af0.py new file mode 100644 index 00000000..a2e777ec --- /dev/null +++ b/configs/datasets/compassbench_v1_3/compassbench_v1_3_objective_gen_068af0.py @@ -0,0 +1,74 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator +from opencompass.datasets.compassbench_obj import CompassBenchObjectiveV1_3, compassbench_objective_v1_3_postprocess +from opencompass.utils.text_postprocessors import first_option_postprocess + + +prompt_cn = { + 'single_choice_cn': '以下是一道单项选择题，请你根据你了解的知识给出正确的答案选项。请你一步步推理并在最后用“答案选项为X”来回答，其中X是ABCD中你认为正确的选项序号\n下面是你要回答的题目：\n{question}\n让我们一步步解决这个问题：', + 'cloze_cn': '以下是一道填空题，请你根据你了解的知识一步步思考后把你的最终答案放到\\boxed{}中。\n下面是你要回答的题目：\n{question}\n让我们一步步解决这个问题：', +} + +prompt_en = { + 'single_choice_en': "Here is a single-choice question. Please give the correct answer based on your knowledge. Please reason step by step and answer with 'The answer is X' at the end, where X is the option number you think is correct.\nHere is the question you need to answer:\n{question}\nLet's solve this problem step by step:", + 'cloze_en': "Here is a fill-in-the-blank question. Please think step by step based on your knowledge and put your final answer in \\boxed{}. Here is the question you need to answer:\n{question}\nLet's solve this problem step by step:", +} + + +douknow_sets = { + 'wiki': ['single_choice_cn'], + 'math': ['single_choice_cn'], +} + +# Set up the prompts +CircularEval = True + + +compassbench_aug_datasets = [] + +for _split in list(douknow_sets.keys()): + for _name in douknow_sets[_split]: + if 'cn' in _name: + single_choice_prompts = prompt_cn + cloze_prompts = prompt_cn + else: + single_choice_prompts = prompt_en + cloze_prompts = prompt_en + douknow_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template=dict( + begin='', + round=[ + dict( + role='HUMAN', + prompt= single_choice_prompts[_name], + ), + dict(role='BOT', prompt='{answer}'),] if 'choice' in _name else cloze_prompts[_name], + ), + ice_token='', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ) + douknow_eval_cfg = dict( + evaluator=dict(type=CircularEvaluator if CircularEval else AccEvaluator) if 'single_choice' in _name else dict(type=AccEvaluator), + pred_postprocessor=dict(type=first_option_postprocess, options='ABCD' ) if 'single_choice' in _name else dict(type=compassbench_objective_v1_3_postprocess, name=_name)) + + compassbench_aug_datasets.append( + dict( + type=CompassBenchObjectiveV1_3, + path=f'./data/compassbench_objective_v1_3/{_split}/{_name}.jsonl', + name='circular_' + _name if CircularEval else _name, + abbr='compassbench-' + _split + '-' + _name + 'circular'if CircularEval else '', + reader_cfg=dict( + input_columns=['question'], + output_column='answer' + ), + infer_cfg=douknow_infer_cfg, + eval_cfg=douknow_eval_cfg, + )) + +del _split, _name diff --git a/opencompass/datasets/compassbench_obj.py b/opencompass/datasets/compassbench_obj.py new file mode 100644 index 00000000..3f51492b --- /dev/null +++ b/opencompass/datasets/compassbench_obj.py @@ -0,0 +1,71 @@ +import copy +import json +import re + +from datasets import Dataset + +from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS + +from .base import BaseDataset + + +def get_number(options): + + result_string = '' + for i, option in enumerate(options, start=65): + result_string += f'{chr(i)}. {option}\n' + return result_string + + +@LOAD_DATASET.register_module() +class CompassBenchObjectiveV1_3(BaseDataset): + + @staticmethod + def load(path: str, name: str): + + circular_patterns = ['ABCD', 'BCDA', 'CDAB', 'DABC'] + + data = [] + with open(path, 'r') as infile: + for id, line in enumerate(infile): + entry = json.loads(line) + if 'cloze' in name: + data.append({ + 'question': entry['question'].strip(), + 'answer': entry['answer'].strip() + }) + elif 'circular' in name: + for c in circular_patterns: + line = copy.deepcopy(entry) + options = [] + for i in range(4): + options.append(line['options'][ord(c[i]) - + ord('A')]) + line['options'] = options + line['answer'] = { + c[0]: 'A', + c[1]: 'B', + c[2]: 'C', + c[3]: 'D' + }[line['answer']] + line['answer'] = str( + id) + '--' + line['answer'] + '--' + c + line['question'] = line['question'].strip( + ) + '\n' + get_number(line['options']) + data.append(line) + else: + # treat as normal single choice question + entry['question'] = entry['question'].strip( + ) + '\n' + get_number(entry['options']) + data.append(entry) + + dataset = Dataset.from_list(data) + return dataset + + +@TEXT_POSTPROCESSORS.register_module() +def compassbench_objective_v1_3_postprocess(text: str) -> str: + pattern = r'\\boxed{(.*?)}' + match = re.search(pattern, text) + + return match.group(1) if match else ''