diff --git a/configs/datasets/SVAMP/svamp_gen.py b/configs/datasets/SVAMP/svamp_gen.py new file mode 100644 index 00000000..efd5c708 --- /dev/null +++ b/configs/datasets/SVAMP/svamp_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .svamp_gen_fb25e4 import svamp_datasets # noqa: F401, F403 diff --git a/configs/datasets/SVAMP/svamp_gen_fb25e4.py b/configs/datasets/SVAMP/svamp_gen_fb25e4.py new file mode 100644 index 00000000..814414b5 --- /dev/null +++ b/configs/datasets/SVAMP/svamp_gen_fb25e4.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import SVAMPDataset, gsm8k_postprocess, Gsm8kEvaluator + +svamp_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt="Question: There are 87 oranges and 290 bananas in Philip's collection. If the bananas are organized into 2 groups and oranges are organized into 93 groups How big is each group of bananas?\nLet's think step by step\nAnswer:"), + dict(role='BOT', prompt="To find the size of each group of bananas, we divide the total number of bananas (290) by the number of groups (2): 290 / 2 = 145. Therefore, each group of bananas contains 145 bananas. The answer is 145.\n"), + dict(role='HUMAN', prompt="Question: Marco and his dad went strawberry picking. Marco's dad's strawberries weighed 11 pounds. If together their strawberries weighed 30 pounds. How much did Marco's strawberries weigh?\nLet's think step by step\nAnswer:"), + dict(role='BOT', prompt="To find Marco's strawberries' weight, we subtract his dad's strawberries' weight (11 pounds) from the total weight of their strawberries (30 pounds): 30 - 11 = 19. Therefore, Marco's strawberries weighed 19 pounds. The answer is 19.\n"), + dict(role='HUMAN', prompt="Question: Edward spent $ 6 to buy 2 books each book costing him the same amount of money. Now he has $ 12. How much did each book cost?\nLet's think step by step\nAnswer:"), + dict(role='BOT', prompt="To find the cost of each book, we subtract the initial amount of money Edward had ($6) from the current amount of money he has ($12) and divide it by the number of books (2): (12 - 6) / 2 = 6 / 2 = 3 Therefore, each book cost $3. The answer is 3.\n"), + dict(role='HUMAN', prompt="Question: Frank was reading through his favorite book. The book had 3 chapters, each with the same number of pages. It has a total of 594 pages. It took Frank 607 days to finish the book. How many pages are in each chapter?\nLet's think step by step\nAnswer:"), + dict(role='BOT', prompt="To find the number of pages in each chapter, we divide the total number of pages in the book (594) by the number of chapters (3): 594 / 3 = 198. Therefore, each chapter has 198 pages. The answer is 198.\n"), + dict(role='HUMAN', prompt="Question: {question}\nLet's think step by step\nAnswer:"), + ], + )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +svamp_eval_cfg = dict(evaluator=dict(type=Gsm8kEvaluator), + pred_postprocessor=dict(type=gsm8k_postprocess)) + +svamp_datasets = [ + dict( + abbr='svamp', + type=SVAMPDataset, + path='./data/svamp/test.jsonl', + reader_cfg=dict(input_columns=['question'], output_column='answer'), + infer_cfg=svamp_infer_cfg, + eval_cfg=svamp_eval_cfg) +] diff --git a/opencompass/datasets/__init__.py b/opencompass/datasets/__init__.py index 8b113c2f..f63eac86 100644 --- a/opencompass/datasets/__init__.py +++ b/opencompass/datasets/__init__.py @@ -72,6 +72,7 @@ from .strategyqa import * # noqa: F401, F403 from .subjective_cmp import SubjectiveCmpDataset # noqa: F401, F403 from .summedits import * # noqa: F401, F403 from .summscreen import * # noqa: F401, F403 +from .svamp import * # noqa: F401, F403 from .tabmwp import * # noqa: F401, F403 from .TheoremQA import * # noqa: F401, F403 from .tnews import * # noqa: F401, F403 diff --git a/opencompass/datasets/svamp.py b/opencompass/datasets/svamp.py new file mode 100644 index 00000000..801c26db --- /dev/null +++ b/opencompass/datasets/svamp.py @@ -0,0 +1,23 @@ +import json + +from datasets import Dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class SVAMPDataset(BaseDataset): + + @staticmethod + def load(path): + dataset = [] + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = json.loads(line.strip()) + question = line['Body'] + ' ' + line['Question'] + answer = str(int(line['Answer'])) + dataset.append({'question': question, 'answer': answer}) + dataset = Dataset.from_list(dataset) + return dataset