commit 7d346000bb8f1f7611f88dc8e003bdf8c9ae3ece Author: gaotongxiao Date: Tue Jul 4 21:34:55 2023 +0800 initial commit diff --git a/README_zh-CN.md b/README_zh-CN.md new file mode 100644 index 00000000..48e8ef12 --- /dev/null +++ b/README_zh-CN.md @@ -0,0 +1,66 @@ +
+ + +[![docs](https://readthedocs.org/projects/opencompass/badge/?version=dev-1.x)](https://opencompass.readthedocs.io/en/dev-1.x/?badge=dev-1.x) +[![license](https://img.shields.io/github/license/IntenLM/opencompass.svg)](https://github.com/InternLM/opencompass/blob/main/LICENSE) +[![PyPI](https://badge.fury.io/py/opencompass.svg)](https://pypi.org/project/opencompass/) + +[📘Documentation](https://opencompass.readthedocs.io/en/latest/) | +[🛠️Installation](https://opencompass.readthedocs.io/en/latest/get_started/install.html) | +[🤔Reporting Issues](https://github.com/InternLM/opencompass/issues/new/choose) + +[English](/README.md) | 简体中文 + +
+ +## 介绍 + +OpenCompass 是面向大模型评测的一站式平台，旨在提供一套公平、公开、可复现的大模型评测基准方案。其主要特点如下： + +- **模型及数据集的全方位支持**：预支持 20+ HuggingFace 及 API 模型，并提供 50+ 个数据集约 30 万题的的模型评测方案，6 大维度的能力全面评测。 + +- **高效分布式评测**：一行命令实现任务分割和分布式评测，数小时即可完成千亿模型全量评测\*。 + +- **多样化评测范式**：支持零样本、小样本及思维链评测，结合标准型或对话型提示词模板，轻松激发各种模型最大性能。 + +- **易于扩展的模块化设计**：想增加新模型或数据集？想要自定义更高级的任务分割策略，甚至接入新的集群管理系统？OpenCompass 的一切均可轻松扩展！ + +- **完善的实验记录及上报机制**：使用配置文件完整记录每一次实验，关键信息有迹可循；结果实时上报飞书机器人，第一时间知晓实验情况。 + +## 模型能力排名 + +## 能力维度 & 模型支持 + +## 安装 + +下面展示了快速安装的步骤。有部分第三方功能可能需要额外步骤才能正常运行，详细步骤请参考[安装指南](https://opencompass.readthedocs.io/zh_cn/latest/get_started.html)。 + +```Python +conda create --name opencompass python=3.8 pytorch torchvision -c pytorch -y +conda activate opencompass +git clone https://github.com/InternLM/opencompass opencompass +cd opencompass +pip install -r requirements/runtime.txt +pip install -e . +# 下载数据集到 data/ 处 +# TODO: .... +``` + +## 评测 + +请阅读[快速上手](https://opencompass.readthedocs.io/zh_cn/latest/get_started.html)了解如何运行一个评测任务。 + +## 致谢 + +该项目部分的代码引用并修改自 [OpenICL](https://github.com/Shark-NLP/OpenICL)。 + +## 引用 + +```bibtex +@misc{2023opencompass, + title={OpenCompass: A Universal Evaluation Platform for Foundation Models}, + author={OpenCompass Contributors}, + howpublished = {\url{https://github.com/InternLM/OpenCompass}}, + year={2023} +} +``` diff --git a/configs/datasets/ARC_c/ARC_c_ppl.py b/configs/datasets/ARC_c/ARC_c_ppl.py new file mode 100644 index 00000000..9467f346 --- /dev/null +++ b/configs/datasets/ARC_c/ARC_c_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .ARC_c_ppl_ba951c import ARC_c_datasets # noqa: F401, F403 diff --git a/configs/datasets/ARC_c/ARC_c_ppl_ba951c.py b/configs/datasets/ARC_c/ARC_c_ppl_ba951c.py new file mode 100644 index 00000000..cd124c99 --- /dev/null +++ b/configs/datasets/ARC_c/ARC_c_ppl_ba951c.py @@ -0,0 +1,53 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import ARCDataset + +ARC_c_reader_cfg = dict( + input_columns=['question', 'textA', 'textB', 'textC', 'textD'], + output_column='answerKey') + +ARC_c_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + "A": + dict( + round=[ + dict(role="HUMAN", prompt="Question: {question}\nAnswer: "), + dict(role="BOT", prompt="{textA}") + ], ), + "B": + dict( + round=[ + dict(role="HUMAN", prompt="Question: {question}\nAnswer: "), + dict(role="BOT", prompt="{textB}") + ], ), + "C": + dict( + round=[ + dict(role="HUMAN", prompt="Question: {question}\nAnswer: "), + dict(role="BOT", prompt="{textC}") + ], ), + "D": + dict( + round=[ + dict(role="HUMAN", prompt="Question: {question}\nAnswer: "), + dict(role="BOT", prompt="{textD}") + ], ), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +ARC_c_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +ARC_c_datasets = [ + dict( + type=ARCDataset, + abbr='ARC-c', + path='./data/ARC/ARC-c/ARC-Challenge-Dev.jsonl', + reader_cfg=ARC_c_reader_cfg, + infer_cfg=ARC_c_infer_cfg, + eval_cfg=ARC_c_eval_cfg) +] diff --git a/configs/datasets/ARC_e/ARC_e_gen.py b/configs/datasets/ARC_e/ARC_e_gen.py new file mode 100644 index 00000000..43c771d1 --- /dev/null +++ b/configs/datasets/ARC_e/ARC_e_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .ARC_e_gen_0a29bf import ARC_e_datasets # noqa: F401, F403 diff --git a/configs/datasets/CLUE_C3/CLUE_C3_ppl.py b/configs/datasets/CLUE_C3/CLUE_C3_ppl.py new file mode 100644 index 00000000..70f4fe7f --- /dev/null +++ b/configs/datasets/CLUE_C3/CLUE_C3_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .CLUE_C3_ppl_588820 import C3_datasets # noqa: F401, F403 diff --git a/configs/datasets/CLUE_C3/CLUE_C3_ppl_20320d.py b/configs/datasets/CLUE_C3/CLUE_C3_ppl_20320d.py new file mode 100644 index 00000000..e0879417 --- /dev/null +++ b/configs/datasets/CLUE_C3/CLUE_C3_ppl_20320d.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import C3Dataset + +C3_reader_cfg = dict( + input_columns=[ + 'question', 'content', 'choice0', 'choice1', 'choice2', 'choice3', + 'choices' + ], + output_column='label') + +C3_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: "文章：{content}\n问题：{question}\n答案：{choice0}", + 1: "文章：{content}\n问题：{question}\n答案：{choice1}", + 2: "文章：{content}\n问题：{question}\n答案：{choice2}", + 3: "文章：{content}\n问题：{question}\n答案：{choice3}" + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +C3_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +C3_datasets = [ + dict( + type=C3Dataset, + abbr='C3', + path='./data/CLUE/C3/dev_0.json', + reader_cfg=C3_reader_cfg, + infer_cfg=C3_infer_cfg, + eval_cfg=C3_eval_cfg) +] diff --git a/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_698c27.py b/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_698c27.py new file mode 100644 index 00000000..f9fd6fab --- /dev/null +++ b/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_698c27.py @@ -0,0 +1,27 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import EMEvaluator +from opencompass.datasets import DRCDDataset + +DRCD_reader_cfg = dict( + input_columns=['question', 'context'], output_column='answers') + +DRCD_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template="文章：{context}\n根据上文，回答如下问题： {question}\n答："), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +DRCD_eval_cfg = dict(evaluator=dict(type=EMEvaluator), ) + +DRCD_datasets = [ + dict( + type=DRCDDataset, + abbr='DRCD_dev', + path='./data/CLUE/DRCD/dev.json', + reader_cfg=DRCD_reader_cfg, + infer_cfg=DRCD_infer_cfg, + eval_cfg=DRCD_eval_cfg), +] diff --git a/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_9b30c2.py b/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_9b30c2.py new file mode 100644 index 00000000..43599682 --- /dev/null +++ b/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_9b30c2.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import EMEvaluator +from opencompass.datasets import DRCDDataset + +DRCD_reader_cfg = dict( + input_columns=['question', 'context'], output_column='answers') + +DRCD_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role="HUMAN", prompt="文章：{context}\n根据上文，回答如下问题：{question}"), + dict(role="BOT", prompt="答："), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +DRCD_eval_cfg = dict( + evaluator=dict(type=EMEvaluator), + pred_role="BOT", +) + +DRCD_datasets = [ + dict( + type=DRCDDataset, + abbr='DRCD_dev', + path='./data/CLUE/DRCD/dev.json', + reader_cfg=DRCD_reader_cfg, + infer_cfg=DRCD_infer_cfg, + eval_cfg=DRCD_eval_cfg), +] diff --git a/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_2ea62b.py b/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_2ea62b.py new file mode 100644 index 00000000..1d56a12b --- /dev/null +++ b/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_2ea62b.py @@ -0,0 +1,42 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import cmnliDataset_V2 + +cmnli_reader_cfg = dict( + input_columns=["sentence1", "sentence2"], + output_column="label", + test_split="train") + +cmnli_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}\nA. 对\nB. 错\nC. 可能\n请从“A”，“B”，“C”中进行选择。\n答：" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +cmnli_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +cmnli_datasets = [ + dict( + abbr="cmnli", + type=cmnliDataset_V2, + path="./data/CLUE/cmnli/cmnli_public/dev.json", + reader_cfg=cmnli_reader_cfg, + infer_cfg=cmnli_infer_cfg, + eval_cfg=cmnli_eval_cfg, + ) +] diff --git a/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_316313.py b/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_316313.py new file mode 100644 index 00000000..c86e2c29 --- /dev/null +++ b/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_316313.py @@ -0,0 +1,42 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import cmnliDataset_V2 + +cmnli_reader_cfg = dict( + input_columns=["sentence1", "sentence2"], + output_column="label", + test_split="train") + +cmnli_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？\nA. 蕴含\nB. 矛盾\nC. 无关\n请从“A”，“B”，“C”中进行选择。\n答：" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +cmnli_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +cmnli_datasets = [ + dict( + abbr="cmnli", + type=cmnliDataset_V2, + path="./data/CLUE/cmnli/cmnli_public/dev.json", + reader_cfg=cmnli_reader_cfg, + infer_cfg=cmnli_infer_cfg, + eval_cfg=cmnli_eval_cfg, + ) +] diff --git a/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_1c652a.py b/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_1c652a.py new file mode 100644 index 00000000..eb051898 --- /dev/null +++ b/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_1c652a.py @@ -0,0 +1,56 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +cmnli_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], + output_column='label', + test_split='train') + +cmnli_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'contradiction': + dict(round=[ + dict( + role="HUMAN", + prompt="语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？" + ), + dict(role="BOT", prompt="矛盾") + ]), + 'entailment': + dict(round=[ + dict( + role="HUMAN", + prompt="语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？" + ), + dict(role="BOT", prompt="蕴含") + ]), + 'neutral': + dict(round=[ + dict( + role="HUMAN", + prompt="语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？" + ), + dict(role="BOT", prompt="无关") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +cmnli_datasets = [ + dict( + type=HFDataset, + abbr='cmnli', + path='json', + split='train', + data_files='./data/CLUE/cmnli/cmnli_public/dev.json', + reader_cfg=cmnli_reader_cfg, + infer_cfg=cmnli_infer_cfg, + eval_cfg=cmnli_eval_cfg) +] diff --git a/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen_7c44b0.py b/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen_7c44b0.py new file mode 100644 index 00000000..71518795 --- /dev/null +++ b/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen_7c44b0.py @@ -0,0 +1,43 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import cmnliDataset_V2 + +ocnli_reader_cfg = dict( + input_columns=["sentence1", "sentence2"], + output_column="label", +) + +# TODO: two prompt templates for ocnli +ocnli_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？\nA. 蕴含\n B. 矛盾\n C. 无关\n请从“A”，“B”，“C”中进行选择。\n答：" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +ocnli_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +ocnli_datasets = [ + dict( + abbr="ocnli", + type=cmnliDataset_V2, # ocnli share the same format with cmnli + path="./data/CLUE/OCNLI/dev.json", + reader_cfg=ocnli_reader_cfg, + infer_cfg=ocnli_infer_cfg, + eval_cfg=ocnli_eval_cfg, + ) +] diff --git a/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_eaa2be.py b/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_eaa2be.py new file mode 100644 index 00000000..e9074329 --- /dev/null +++ b/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_eaa2be.py @@ -0,0 +1,51 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +ocnli_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], output_column='label') + +# TODO: two prompt templates for ocnli +ocnli_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'contradiction': + dict(round=[ + dict( + role="HUMAN", + prompt="阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？"), + dict(role="BOT", prompt="错") + ]), + 'entailment': + dict(round=[ + dict( + role="HUMAN", + prompt="阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？"), + dict(role="BOT", prompt="对") + ]), + 'neutral': + dict(round=[ + dict( + role="HUMAN", prompt="如果{sentence1}为真，那么{sentence2}也为真吗？"), + dict(role="BOT", prompt="可能") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +ocnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator), ) + +ocnli_datasets = [ + dict( + type=HFDataset, + abbr='ocnli', + path='json', + split='train', + data_files='./data/CLUE/OCNLI/dev.json', + reader_cfg=ocnli_reader_cfg, + infer_cfg=ocnli_infer_cfg, + eval_cfg=ocnli_eval_cfg) +] diff --git a/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_f103ab.py b/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_f103ab.py new file mode 100644 index 00000000..e6d42db3 --- /dev/null +++ b/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_f103ab.py @@ -0,0 +1,55 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +ocnli_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], output_column='label') + +# TODO: two prompt templates for ocnli +ocnli_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'contradiction': + dict(round=[ + dict( + role="HUMAN", + prompt="语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？" + ), + dict(role="BOT", prompt="矛盾") + ]), + 'entailment': + dict(round=[ + dict( + role="HUMAN", + prompt="语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？" + ), + dict(role="BOT", prompt="蕴含") + ]), + 'neutral': + dict(round=[ + dict( + role="HUMAN", + prompt="语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？" + ), + dict(role="BOT", prompt="无关") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +ocnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator), ) + +ocnli_datasets = [ + dict( + type=HFDataset, + abbr='ocnli', + path='json', + split='train', + data_files='./data/CLUE/OCNLI/dev.json', + reader_cfg=ocnli_reader_cfg, + infer_cfg=ocnli_infer_cfg, + eval_cfg=ocnli_eval_cfg) +] diff --git a/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_ppl_332a41.py b/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_ppl_332a41.py new file mode 100644 index 00000000..d93c40fd --- /dev/null +++ b/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_ppl_332a41.py @@ -0,0 +1,65 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +bustm_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], + output_column='label', + test_split='train') + +bustm_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt="请判断以下两句话说的是否是一个意思：") + ], + round=[ + dict(role="HUMAN", prompt="{sentence1}，{sentence2}"), + dict(role="BOT", prompt="两句话说的毫不相关。") + ]), + 1: + dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt="请判断以下两句话说的是否是一个意思：") + ], + round=[ + dict(role="HUMAN", prompt="{sentence1}，{sentence2}"), + dict(role="BOT", prompt="两句话说是的一个意思。") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +bustm_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +bustm_datasets = [ + dict( + type=HFDataset, + abbr='bustm-dev', + path='json', + data_files='./data/FewCLUE/bustm/dev_few_all.json', + split='train', + reader_cfg=bustm_reader_cfg, + infer_cfg=bustm_infer_cfg, + eval_cfg=bustm_eval_cfg), + dict( + type=HFDataset, + abbr='bustm-test', + path='json', + data_files='./data/FewCLUE/bustm/test_public.json', + split='train', + reader_cfg=bustm_reader_cfg, + infer_cfg=bustm_infer_cfg, + eval_cfg=bustm_eval_cfg) +] diff --git a/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_ppl_47f2ab.py b/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_ppl_47f2ab.py new file mode 100644 index 00000000..1d27385f --- /dev/null +++ b/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_ppl_47f2ab.py @@ -0,0 +1,59 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +bustm_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], + output_column='label', + test_split='train') + +bustm_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[ + dict( + role="HUMAN", + prompt= + "语句一：“{sentence1}”\n语句二：“{sentence2}”\n请判断语句一和语句二说的是否是一个意思？" + ), + dict(role="BOT", prompt="两句话说的毫不相关。") + ]), + 1: + dict(round=[ + dict( + role="HUMAN", + prompt= + "语句一：“{sentence1}”\n语句二：“{sentence2}”\n请判断语句一和语句二说的是否是一个意思？" + ), + dict(role="BOT", prompt="两句话说是的一个意思。") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +bustm_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +bustm_datasets = [ + dict( + type=HFDataset, + abbr='bustm-dev', + path='json', + data_files='./data/FewCLUE/bustm/dev_few_all.json', + split='train', + reader_cfg=bustm_reader_cfg, + infer_cfg=bustm_infer_cfg, + eval_cfg=bustm_eval_cfg), + dict( + type=HFDataset, + abbr='bustm-test', + path='json', + data_files='./data/FewCLUE/bustm/test_public.json', + split='train', + reader_cfg=bustm_reader_cfg, + infer_cfg=bustm_infer_cfg, + eval_cfg=bustm_eval_cfg) +] diff --git a/configs/datasets/FewCLUE_chid/FewCLUE_chid_gen_686c63.py b/configs/datasets/FewCLUE_chid/FewCLUE_chid_gen_686c63.py new file mode 100644 index 00000000..6c638558 --- /dev/null +++ b/configs/datasets/FewCLUE_chid/FewCLUE_chid_gen_686c63.py @@ -0,0 +1,50 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CHIDDataset_V2 + +chid_reader_cfg = dict( + input_columns=["content","A","B","C","D","E","F","G"], + output_column="answer", +) + +chid_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role="HUMAN", + prompt= + "{content}\n请选择______处所填的词\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nE. {E}\nF. {F}\nG. {G}\n请从”A“，”B“，”C“，”D“，”E“，”F“，”G“中进行选择。答：", + ), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +chid_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +chid_datasets = [ + dict( + abbr="chid-dev", + type=CHIDDataset_V2, + path="./data/FewCLUE/chid/dev_few_all.json", + reader_cfg=chid_reader_cfg, + infer_cfg=chid_infer_cfg, + eval_cfg=chid_eval_cfg, + ), + dict( + abbr="chid-test", + type=CHIDDataset_V2, + path="./data/FewCLUE/chid/test_public.json", + reader_cfg=chid_reader_cfg, + infer_cfg=chid_infer_cfg, + eval_cfg=chid_eval_cfg, + ), +] diff --git a/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_2a9e61.py b/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_2a9e61.py new file mode 100644 index 00000000..b6b83762 --- /dev/null +++ b/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_2a9e61.py @@ -0,0 +1,54 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CluewscDataset + +cluewsc_reader_cfg = dict( + input_columns=['span1', 'span2', 'text', 'new_text'], + output_column='answer') + +cluewsc_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[ + dict( + role="HUMAN", + prompt="{text}\n此处，代词“{span2}“被用于指代“{span1}“吗?"), + dict(role="BOT", prompt="否") + ]), + 1: + dict(round=[ + dict( + role="HUMAN", + prompt="{text}\n此处，代词“{span2}“被用于指代“{span1}“吗?"), + dict(role="BOT", prompt="是") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +cluewsc_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +cluewsc_datasets = [ + dict( + type=CluewscDataset, + path='json', + abbr='cluewsc-dev', + data_files='./data/FewCLUE/cluewsc/dev_few_all.json', + split='train', + reader_cfg=cluewsc_reader_cfg, + infer_cfg=cluewsc_infer_cfg, + eval_cfg=cluewsc_eval_cfg), + dict( + type=CluewscDataset, + path='json', + abbr='cluewsc-test', + data_files='./data/FewCLUE/cluewsc/test_public.json', + split='train', + reader_cfg=cluewsc_reader_cfg, + infer_cfg=cluewsc_infer_cfg, + eval_cfg=cluewsc_eval_cfg), +] diff --git a/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_d335d5.py b/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_d335d5.py new file mode 100644 index 00000000..458616a3 --- /dev/null +++ b/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_ppl_d335d5.py @@ -0,0 +1,44 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CluewscDataset + +cluewsc_reader_cfg = dict( + input_columns=['span1', 'span2', 'text', 'new_text'], + output_column='answer') + +cluewsc_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + "{text}\nHere, is the pronoun \"{span2}\" used to mean \"{span1}\"? No.", + 1: + "{text}\nHere, is the pronoun \"{span2}\" used to mean \"{span1}\"? Yes.", + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +cluewsc_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +cluewsc_datasets = [ + dict( + type=CluewscDataset, + path='json', + abbr='cluewsc-dev', + data_files='./data/FewCLUE/cluewsc/dev_few_all.json', + split='train', + reader_cfg=cluewsc_reader_cfg, + infer_cfg=cluewsc_infer_cfg, + eval_cfg=cluewsc_eval_cfg), + dict( + type=CluewscDataset, + path='json', + abbr='cluewsc-test', + data_files='./data/FewCLUE/cluewsc/test_public.json', + split='train', + reader_cfg=cluewsc_reader_cfg, + infer_cfg=cluewsc_infer_cfg, + eval_cfg=cluewsc_eval_cfg), +] diff --git a/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl.py b/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl.py new file mode 100644 index 00000000..e2d6a4ad --- /dev/null +++ b/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .FewCLUE_csl_ppl_8eee08 import csl_datasets # noqa: F401, F403 diff --git a/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl_f99d7a.py b/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl_f99d7a.py new file mode 100644 index 00000000..f4856114 --- /dev/null +++ b/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl_f99d7a.py @@ -0,0 +1,45 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CslDataset + +csl_reader_cfg = dict( + input_columns=["abst", "keywords"], output_column='label') + +csl_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[dict(role="HUMAN", prompt="摘要：{abst}")]), + 1: + dict( + round=[dict(role="HUMAN", prompt="摘要：{abst}\n关键词：{keywords}") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +csl_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +csl_datasets = [ + dict( + type=CslDataset, + path='json', + abbr='csl_dev', + data_files='./data/FewCLUE/csl/dev_few_all.json', + split='train', + reader_cfg=csl_reader_cfg, + infer_cfg=csl_infer_cfg, + eval_cfg=csl_eval_cfg), + dict( + type=CslDataset, + path='json', + abbr='csl_test', + data_files='./data/FewCLUE/csl/test_public.json', + split='train', + reader_cfg=csl_reader_cfg, + infer_cfg=csl_infer_cfg, + eval_cfg=csl_eval_cfg) +] diff --git a/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_ppl_b59c1f.py b/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_ppl_b59c1f.py new file mode 100644 index 00000000..b640eb25 --- /dev/null +++ b/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_ppl_b59c1f.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +eprstmt_reader_cfg = dict( + input_columns=['sentence'], output_column='label', test_split='train') + +eprstmt_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'Negative': ' 内容： "{sentence}"。情绪分类：消极。', + 'Positive': ' 内容： "{sentence}"。情绪分类：积极。', + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +eprstmt_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +eprstmt_datasets = [ + dict( + type=HFDataset, + abbr='eprstmt-dev', + path='json', + data_files='./data/FewCLUE/eprstmt/dev_few_all.json', + split='train', + reader_cfg=eprstmt_reader_cfg, + infer_cfg=eprstmt_infer_cfg, + eval_cfg=eprstmt_eval_cfg), + dict( + type=HFDataset, + abbr='eprstmt-test', + path='json', + data_files='./data/FewCLUE/eprstmt/test_public.json', + split='train', + reader_cfg=eprstmt_reader_cfg, + infer_cfg=eprstmt_infer_cfg, + eval_cfg=eprstmt_eval_cfg) +] diff --git a/configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_ppl_29abd6.py b/configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_ppl_29abd6.py new file mode 100644 index 00000000..96faddd1 --- /dev/null +++ b/configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_ppl_29abd6.py @@ -0,0 +1,60 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +ocnli_fc_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], + output_column='label', + test_split='train') + +ocnli_fc_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'contradiction': + dict(round=[ + dict( + role="HUMAN", + prompt="阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？"), + dict(role="BOT", prompt="错") + ]), + 'entailment': + dict(round=[ + dict( + role="HUMAN", + prompt="阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？"), + dict(role="BOT", prompt="对") + ]), + 'neutral': + dict(round=[ + dict( + role="HUMAN", prompt="如果{sentence1}为真，那么{sentence2}也为真吗？"), + dict(role="BOT", prompt="可能") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) +ocnli_fc_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +ocnli_fc_datasets = [ + dict( + type=HFDataset, + abbr='ocnli_fc-dev', + path='json', + split='train', + data_files='./data/FewCLUE/ocnli/dev_few_all.json', + reader_cfg=ocnli_fc_reader_cfg, + infer_cfg=ocnli_fc_infer_cfg, + eval_cfg=ocnli_fc_eval_cfg), + dict( + type=HFDataset, + abbr='ocnli_fc-test', + path='json', + split='train', + data_files='./data/FewCLUE/ocnli/test_public.json', + reader_cfg=ocnli_fc_reader_cfg, + infer_cfg=ocnli_fc_infer_cfg, + eval_cfg=ocnli_fc_eval_cfg) +] diff --git a/configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_ppl_b828fc.py b/configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_ppl_b828fc.py new file mode 100644 index 00000000..a842885d --- /dev/null +++ b/configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_ppl_b828fc.py @@ -0,0 +1,44 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +ocnli_fc_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], + output_column='label', + test_split='train') + +ocnli_fc_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'contradiction': + '阅读文章：{sentence1}\n根据上文，回答如下问题： {sentence2}？\n答：错', + 'entailment': '阅读文章：{sentence1}\n根据上文，回答如下问题： {sentence2}？\n答：对', + 'neutral': '如果{sentence1}为真，那么{sentence2}也为真吗?可能' + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) +ocnli_fc_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +ocnli_fc_datasets = [ + dict( + type=HFDataset, + abbr='ocnli_fc-dev', + path='json', + split='train', + data_files='./data/FewCLUE/ocnli/dev_few_all.json', + reader_cfg=ocnli_fc_reader_cfg, + infer_cfg=ocnli_fc_infer_cfg, + eval_cfg=ocnli_fc_eval_cfg), + dict( + type=HFDataset, + abbr='ocnli_fc-test', + path='json', + split='train', + data_files='./data/FewCLUE/ocnli/test_public.json', + reader_cfg=ocnli_fc_reader_cfg, + infer_cfg=ocnli_fc_infer_cfg, + eval_cfg=ocnli_fc_eval_cfg) +] diff --git a/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_gen_8d59ba.py b/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_gen_8d59ba.py new file mode 100644 index 00000000..9257b72e --- /dev/null +++ b/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_gen_8d59ba.py @@ -0,0 +1,74 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import TNewsDataset_V2 + +tnews_reader_cfg = dict( + input_columns="sentence", + output_column="label_desc2", +) + +tnews_labels = [ + "农业新闻", # news_agriculture + "旅游新闻", # news_travel + "游戏新闻", # news_game + "科技类别公司新闻", # news_tech + "体育类别新闻", # news_sports + "初升高教育新闻", # news_edu + "娱乐圈新闻", # news_entertainment + "投资资讯", # news_finance + "军事类别常识", # news_military + "车辆新闻", # news_car + "楼市新闻", # news_house + "环球不含中国类别新闻", # news_world + "书籍文化历史类别新闻", # news_culture + "故事类别新闻", # news_story + "股票市场类别新闻", # news_stock +] +_tnews_options_list_str = "\n".join(f'{chr(ord("A") + i)}. {tnews_labels[i]}' + for i in range(len(tnews_labels))) +_tnews_options_range_str = "，".join(f'“{chr(ord("A") + i)}”' + for i in range(len(tnews_labels))) + +tnews_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + f"{{sentence}}\n请判断上述内容属于什么新闻？\n{_tnews_options_list_str}\n请从{_tnews_options_range_str}中进行选择。\n答：", + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +tnews_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +tnews_datasets = [ + dict( + abbr="tnews-dev", + type=TNewsDataset_V2, + path="./data/FewCLUE/tnews/dev_few_all.json", + reader_cfg=tnews_reader_cfg, + infer_cfg=tnews_infer_cfg, + eval_cfg=tnews_eval_cfg, + ), + dict( + abbr="tnews-test", + type=TNewsDataset_V2, + path="./data/FewCLUE/tnews/test_public.json", + reader_cfg=tnews_reader_cfg, + infer_cfg=tnews_infer_cfg, + eval_cfg=tnews_eval_cfg, + ), +] + +del _tnews_options_list_str, _tnews_options_range_str diff --git a/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_ppl.py b/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_ppl.py new file mode 100644 index 00000000..d542ea26 --- /dev/null +++ b/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .FewCLUE_tnews_ppl_784b9e import tnews_datasets # noqa: F401, F403 diff --git a/configs/datasets/GaokaoBench/GaokaoBench_gen_aed980.py b/configs/datasets/GaokaoBench/GaokaoBench_gen_aed980.py new file mode 100644 index 00000000..399b1b6a --- /dev/null +++ b/configs/datasets/GaokaoBench/GaokaoBench_gen_aed980.py @@ -0,0 +1,301 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import GaokaoBenchDataset + +_MCQ_prompts = [ + { + "type": "single_choice", + "keyword": "2010-2022_Math_II_MCQs", + "prefix_prompt": + "请你做一道数学选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：", + "comment": "" + }, + { + "type": "single_choice", + "keyword": "2010-2022_Math_I_MCQs", + "prefix_prompt": + "请你做一道数学选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：", + "comment": "" + }, + { + "type": + "single_choice", + "keyword": + "2010-2022_History_MCQs", + "prefix_prompt": + "请你做一道历史选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "single_choice", + "keyword": + "2010-2022_Biology_MCQs", + "prefix_prompt": + "请你做一道生物选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "single_choice", + "keyword": + "2010-2022_Political_Science_MCQs", + "prefix_prompt": + "请你做一道政治选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "multi_choice", + "keyword": + "2010-2022_Physics_MCQs", + "prefix_prompt": + "请你做一道物理选择题。\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出所有符合题意的答案，并写在【答案】和之间。\n例如：【答案】 AB \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】... \n请你严格按照上述格式作答。\n" + }, + { + "type": + "single_choice", + "keyword": + "2010-2022_Chemistry_MCQs", + "prefix_prompt": + "请你做一道化学选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "single_choice", + "keyword": + "2010-2013_English_MCQs", + "prefix_prompt": + "请你做一道英语选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_Chinese_Modern_Lit", + "prefix_prompt": + "请你做一道语文阅读理解题，其中包含三个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_English_Fill_in_Blanks", + "prefix_prompt": + "请你做一道英语完形填空题,其中包含二十个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n" + }, + { + "type": + "five_out_of_seven", + "keyword": + "2012-2022_English_Cloze_Test", + "prefix_prompt": + "请回答下面的问题，将符合题意的五个选项的字母写在【答案】和之间，例如“【答案】 A B C D E \n请严格按照上述格式作答。\n" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_Geography_MCQs", + "prefix_prompt": + "请你做一道地理选择题，其中包含两到三个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_English_Reading_Comp", + "prefix_prompt": + "请你做一道英语阅读理解题，其中包含三到五个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_Chinese_Lang_and_Usage_MCQs", + "prefix_prompt": + "请你做一道语文选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n（1）【解析】 ... \n【答案】 ... \n（2）【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。如果不止一道题，请分别作答\n题目如下：" + }, +] +_FBQ_prompts = [{ + "type": "cloze", + "keyword": "2010-2022_Math_I_Fill-in-the-Blank", + "prefix_prompt": + "请解答下面的数学填空题\n仔细阅读题目，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间。\n完整的题目回答格式如下：\n【解析】 ...\n【答案】...\n请你严格按照上述格式作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": "2010-2022_Math_II_Fill-in-the-Blank", + "prefix_prompt": + "请解答下面的数学填空题\n仔细阅读题目，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间。\n完整的题目回答格式如下：\n【解析】 ...\n【答案】...\n请你严格按照上述格式作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": + "2010-2022_Chinese_Language_Famous_Passages_and_Sentences_Dictation", + "prefix_prompt": + "请回答下面的语文填空题\n请你仔细阅读题目，先找到题目对应的中国名篇，再从名篇中找到合适的句子填写到题目的空白处。请你将思考过程写在【解析】和之间，将最终答案写在【答案】和之间。\n完整的题目回答格式如下：\n（1）【解析】 ...\n【答案】...\n（2）【解析】 ...\n【答案】...\n请严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": "2014-2022_English_Language_Cloze_Passage", + "prefix_prompt": + "请回答下面的英语短文填词题\n仔细阅读题目，空白处请填入一个适当单词或者括号内单词的正确形式。请你一步步思考，将思考过程写在【解析】和之间，将最终答案写在【答案】和之间。\n完整的题目回答格式如下：\n（1）【解析】 ...\n【答案】...\n（2）【解析】 ...\n【答案】...\n请严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" +}] +_OEQ_prompts = [ + { + "type": "subjective", + "keyword": "2010-2022_Geography_Open-ended_Questions", + "prefix_prompt": + "请解答下面的地理解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。你的答案请写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下：", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chemistry_Open-ended_Questions", + "prefix_prompt": + "请解答下面的化学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Math_I_Open-ended_Questions", + "prefix_prompt": + "请解答下面的数学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间，答案需要有完整的解题步骤。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_History_Open-ended_Questions", + "prefix_prompt": + "请解答下面的历史解答题\n仔细阅读材料和题目，并充分结合你已有的知识，解答其中的问题。请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Biology_Open-ended_Questions", + "prefix_prompt": + "请解答下面的生物解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间,同一小题的答案用\t分隔开。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\t...\n (2)【解析】 ...\n【答案】...\t...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Math_II_Open-ended_Questions", + "prefix_prompt": + "请解答下面的数学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间，答案需要有完整的解题步骤。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Physics_Open-ended_Questions", + "prefix_prompt": + "请解答下面的物理解答题，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将思考过程写在【解析】和之间。请把你的最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【解析】 ...\n【答案】 ...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答。如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Political_Science_Open-ended_Questions", + "prefix_prompt": + "请解答下面的政治解答题\n仔细阅读材料和题目，并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "correction", + "keyword": "2012-2022_English_Language_Error_Correction", + "prefix_prompt": + "请解答下面的英语短文改错题，仔细阅读题目并充分结合你你已有的知识，找出其中10处需要改动的地方。请你一步步思考，把修改后的短文写在【答案】和之间。\n完整的题目回答格式如下：【答案】 ...\n 请你严格按照上述格式作答。\n题目如下:", + # "prefix_prompt": [ + # "请解答下面的英语短文改错题，仔细阅读题目并充分结合你你已有的知识，找出其中10处需要改动的地方。请你一步步思考，把修改后的短文写在【答案】和之间。\n完整的题目回答格式如下：【答案】 ...\n 请你严格按照上述格式作答。\n题目如下:", + # "请比较下面两篇短文，找到第二篇和第一篇的10处不同，每处不同只涉及一个单词，请将结果写在【答案】和之间。例如：【答案】1. 将play改为plays\n 2.增加了the\n ... \n 完整的题目回答格式如下：【答案】(1) ... \n (2) ...\n ...(10) ...\n\n请你严格按照上述格式作答。\n短文如下:" + # ], + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Ancient_Poetry_Reading", + "prefix_prompt": + "请解答下面的语文古代诗歌阅读题，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Practical_Text_Reading", + "prefix_prompt": + "请解答下面的语文实用类文本阅读，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Literary_Text_Reading", + "prefix_prompt": + "请解答下面的语文文学类文本阅读，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答,如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Classical_Chinese_Reading", + "prefix_prompt": + "请解答下面的语文文言文阅读，仔细阅读题目，前三题是单选题，最后一题要将文言文翻译为现代汉语。请你一步步思考并把最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。翻译题把翻译后的现代汉语句子写在【答案】后面，例如”【答案】今天天气很好 ”\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": + "2010-2022_Chinese_Language_Language_and_Writing_Skills_Open-ended_Questions", + "prefix_prompt": + "请解答下面的语文解答题，仔细阅读题目，注意其中可能含有选择题。请你一步步思考并将思考过程写在【解析】和之间。请把你的最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【解析】 ...\n【答案】 ...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答。如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + } +] + +GaokaoBench_datasets = [] +for _folder, _prompts in [ + ("Multiple-choice_Questions", _MCQ_prompts), + ("Fill-in-the-blank_Questions", _FBQ_prompts), + ("Open-ended_Questions", _OEQ_prompts), +]: + for _p in _prompts: + _reader_cfg = { + "input_columns": ['question'], + "output_column": 'answer', + } + _infer_cfg = { + "ice_template": { + "type": PromptTemplate, + "template": { + "round": [{ + "role": "HUMAN", + "prompt": _p['prefix_prompt'] + '{question}' + }] + }, + "ice_token": "" + }, + "retriever": { + "type": ZeroRetriever + }, + "inferencer": { + "type": GenInferencer, + "max_out_len": 1024, + } + } + _eval_cfg = { + "evaluator": { + "type": "GaokaoBenchEvaluator" + "_" + _p['type'], + }, + "pred_role": "BOT", + } + _base_path = './data/GAOKAO-BENCH/data' + _dataset = { + "type": GaokaoBenchDataset, + "abbr": "GaokaoBench_" + _p['keyword'], + "path": _base_path + '/' + _folder + '/' + _p['keyword'] + ".json", + "reader_cfg": _reader_cfg, + "infer_cfg": _infer_cfg, + "eval_cfg": _eval_cfg, + } + + GaokaoBench_datasets.append(_dataset) + +_temporary_variables = [k for k in globals() if k.startswith('_')] +for _t in _temporary_variables: + del globals()[_t] +del _temporary_variables, _t diff --git a/configs/datasets/GaokaoBench/GaokaoBench_mixed.py b/configs/datasets/GaokaoBench/GaokaoBench_mixed.py new file mode 100644 index 00000000..baeccf45 --- /dev/null +++ b/configs/datasets/GaokaoBench/GaokaoBench_mixed.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .GaokaoBench_mixed_f2038e import GaokaoBench_datasets # noqa: F401, F403 diff --git a/configs/datasets/GaokaoBench/GaokaoBench_mixed_f2038e.py b/configs/datasets/GaokaoBench/GaokaoBench_mixed_f2038e.py new file mode 100644 index 00000000..b8e18cb7 --- /dev/null +++ b/configs/datasets/GaokaoBench/GaokaoBench_mixed_f2038e.py @@ -0,0 +1,353 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer, PPLInferencer +from opencompass.datasets import GaokaoBenchDataset + +_MCQ_prompts = [ + { + "type": "single_choice", + "keyword": "2010-2022_Math_II_MCQs", + "prefix_prompt": + "请你做一道数学选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：", + "comment": "" + }, + { + "type": "single_choice", + "keyword": "2010-2022_Math_I_MCQs", + "prefix_prompt": + "请你做一道数学选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：", + "comment": "" + }, + { + "type": + "single_choice", + "keyword": + "2010-2022_History_MCQs", + "prefix_prompt": + "请你做一道历史选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "single_choice", + "keyword": + "2010-2022_Biology_MCQs", + "prefix_prompt": + "请你做一道生物选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "single_choice", + "keyword": + "2010-2022_Political_Science_MCQs", + "prefix_prompt": + "请你做一道政治选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "multi_choice", + "keyword": + "2010-2022_Physics_MCQs", + "prefix_prompt": + "请你做一道物理选择题。\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出所有符合题意的答案，并写在【答案】和之间。\n例如：【答案】 AB \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】... \n请你严格按照上述格式作答。\n" + }, + { + "type": + "single_choice", + "keyword": + "2010-2022_Chemistry_MCQs", + "prefix_prompt": + "请你做一道化学选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "single_choice", + "keyword": + "2010-2013_English_MCQs", + "prefix_prompt": + "请你做一道英语选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。\n题目如下：" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_Chinese_Modern_Lit", + "prefix_prompt": + "请你做一道语文阅读理解题，其中包含三个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_English_Fill_in_Blanks", + "prefix_prompt": + "请你做一道英语完形填空题,其中包含二十个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n" + }, + { + "type": + "five_out_of_seven", + "keyword": + "2012-2022_English_Cloze_Test", + "prefix_prompt": + "请回答下面的问题，将符合题意的五个选项的字母写在【答案】和之间，例如“【答案】 A B C D E \n请严格按照上述格式作答。\n" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_Geography_MCQs", + "prefix_prompt": + "请你做一道地理选择题，其中包含两到三个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_English_Reading_Comp", + "prefix_prompt": + "请你做一道英语阅读理解题，其中包含三到五个小题。\n请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：（1）【答案】 A \n（2）【答案】 B \n请你严格按照上述格式作答。\n" + }, + { + "type": + "multi_question_choice", + "keyword": + "2010-2022_Chinese_Lang_and_Usage_MCQs", + "prefix_prompt": + "请你做一道语文选择题\n请你一步一步思考并将思考过程写在【解析】和之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和之间。\n例如：【答案】: A \n完整的题目回答的格式如下：\n（1）【解析】 ... \n【答案】 ... \n（2）【解析】 ... \n【答案】 ... \n请你严格按照上述格式作答。如果不止一道题，请分别作答\n题目如下：" + }, +] +_FBQ_prompts = [{ + "type": "cloze", + "keyword": "2010-2022_Math_I_Fill-in-the-Blank", + "prefix_prompt": + "请解答下面的数学填空题\n仔细阅读题目，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间。\n完整的题目回答格式如下：\n【解析】 ...\n【答案】...\n请你严格按照上述格式作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": "2010-2022_Math_II_Fill-in-the-Blank", + "prefix_prompt": + "请解答下面的数学填空题\n仔细阅读题目，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间。\n完整的题目回答格式如下：\n【解析】 ...\n【答案】...\n请你严格按照上述格式作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": + "2010-2022_Chinese_Language_Famous_Passages_and_Sentences_Dictation", + "prefix_prompt": + "请回答下面的语文填空题\n请你仔细阅读题目，先找到题目对应的中国名篇，再从名篇中找到合适的句子填写到题目的空白处。请你将思考过程写在【解析】和之间，将最终答案写在【答案】和之间。\n完整的题目回答格式如下：\n（1）【解析】 ...\n【答案】...\n（2）【解析】 ...\n【答案】...\n请严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": "2014-2022_English_Language_Cloze_Passage", + "prefix_prompt": + "请回答下面的英语短文填词题\n仔细阅读题目，空白处请填入一个适当单词或者括号内单词的正确形式。请你一步步思考，将思考过程写在【解析】和之间，将最终答案写在【答案】和之间。\n完整的题目回答格式如下：\n（1）【解析】 ...\n【答案】...\n（2）【解析】 ...\n【答案】...\n请严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" +}] +_OEQ_prompts = [ + { + "type": "subjective", + "keyword": "2010-2022_Geography_Open-ended_Questions", + "prefix_prompt": + "请解答下面的地理解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。你的答案请写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下：", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chemistry_Open-ended_Questions", + "prefix_prompt": + "请解答下面的化学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Math_I_Open-ended_Questions", + "prefix_prompt": + "请解答下面的数学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间，答案需要有完整的解题步骤。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_History_Open-ended_Questions", + "prefix_prompt": + "请解答下面的历史解答题\n仔细阅读材料和题目，并充分结合你已有的知识，解答其中的问题。请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Biology_Open-ended_Questions", + "prefix_prompt": + "请解答下面的生物解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间,同一小题的答案用\t分隔开。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\t...\n (2)【解析】 ...\n【答案】...\t...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Math_II_Open-ended_Questions", + "prefix_prompt": + "请解答下面的数学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间，答案需要有完整的解题步骤。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Physics_Open-ended_Questions", + "prefix_prompt": + "请解答下面的物理解答题，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将思考过程写在【解析】和之间。请把你的最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【解析】 ...\n【答案】 ...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答。如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Political_Science_Open-ended_Questions", + "prefix_prompt": + "请解答下面的政治解答题\n仔细阅读材料和题目，并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "correction", + "keyword": "2012-2022_English_Language_Error_Correction", + "prefix_prompt": + "请解答下面的英语短文改错题，仔细阅读题目并充分结合你你已有的知识，找出其中10处需要改动的地方。请你一步步思考，把修改后的短文写在【答案】和之间。\n完整的题目回答格式如下：【答案】 ...\n 请你严格按照上述格式作答。\n题目如下:", + # "prefix_prompt": [ + # "请解答下面的英语短文改错题，仔细阅读题目并充分结合你你已有的知识，找出其中10处需要改动的地方。请你一步步思考，把修改后的短文写在【答案】和之间。\n完整的题目回答格式如下：【答案】 ...\n 请你严格按照上述格式作答。\n题目如下:", + # "请比较下面两篇短文，找到第二篇和第一篇的10处不同，每处不同只涉及一个单词，请将结果写在【答案】和之间。例如：【答案】1. 将play改为plays\n 2.增加了the\n ... \n 完整的题目回答格式如下：【答案】(1) ... \n (2) ...\n ...(10) ...\n\n请你严格按照上述格式作答。\n短文如下:" + # ], + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Ancient_Poetry_Reading", + "prefix_prompt": + "请解答下面的语文古代诗歌阅读题，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Practical_Text_Reading", + "prefix_prompt": + "请解答下面的语文实用类文本阅读，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Literary_Text_Reading", + "prefix_prompt": + "请解答下面的语文文学类文本阅读，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答,如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Classical_Chinese_Reading", + "prefix_prompt": + "请解答下面的语文文言文阅读，仔细阅读题目，前三题是单选题，最后一题要将文言文翻译为现代汉语。请你一步步思考并把最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。翻译题把翻译后的现代汉语句子写在【答案】后面，例如”【答案】今天天气很好 ”\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": + "2010-2022_Chinese_Language_Language_and_Writing_Skills_Open-ended_Questions", + "prefix_prompt": + "请解答下面的语文解答题，仔细阅读题目，注意其中可能含有选择题。请你一步步思考并将思考过程写在【解析】和之间。请把你的最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【解析】 ...\n【答案】 ...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答。如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + } +] + +GaokaoBench_datasets = [] +for _folder, _prompts in [ + ("Multiple-choice_Questions", _MCQ_prompts), + ("Fill-in-the-blank_Questions", _FBQ_prompts), + ("Open-ended_Questions", _OEQ_prompts), +]: + for _p in _prompts: + if _p['type'] == "single_choice": + continue + _reader_cfg = { + "input_columns": ['question'], + "output_column": 'answer', + } + _infer_cfg = { + "ice_template": { + "type": PromptTemplate, + "template": { + "round": [{ + "role": "HUMAN", + "prompt": _p['prefix_prompt'] + '{question}' + }] + }, + "ice_token": "" + }, + "retriever": { + "type": ZeroRetriever + }, + "inferencer": { + "type": GenInferencer, + "max_out_len": 1024, + } + } + _eval_cfg = { + "evaluator": { + "type": "GaokaoBenchEvaluator" + "_" + _p['type'], + }, + "pred_role": "BOT", + } + _base_path = './data/GAOKAO-BENCH/data' + _dataset = { + "type": GaokaoBenchDataset, + "abbr": "GaokaoBench_" + _p['keyword'], + "path": _base_path + '/' + _folder + '/' + _p['keyword'] + ".json", + "reader_cfg": _reader_cfg, + "infer_cfg": _infer_cfg, + "eval_cfg": _eval_cfg, + } + + GaokaoBench_datasets.append(_dataset) + +_folder = "Multiple-choice_Questions" +for _p in _MCQ_prompts: + if _p['type'] != "single_choice": + continue + _reader_cfg = { + "input_columns": ['question'], + "output_column": 'answer', + } + _infer_cfg = { + "ice_template": { + "type": PromptTemplate, + "template": { + answer: { + "round": [{ + "role": "HUMAN", + "prompt": _p['prefix_prompt'] + '{question}' + }, { + "role": "BOT", + "prompt": f"【答案】{answer} " + }] + } + for answer in ['A', 'B', 'C', 'D'] + }, + "ice_token": "" + }, + "retriever": { + "type": ZeroRetriever + }, + "inferencer": { + "type": PPLInferencer + } + } + _eval_cfg = { + "evaluator": { + "type": "GaokaoBenchEvaluator" + "_" + _p['type'], + }, + "pred_role": "BOT", + } + _base_path = './data/GAOKAO-BENCH/data' + _dataset = { + "type": GaokaoBenchDataset, + "abbr": "GaokaoBench_" + _p['keyword'], + "path": _base_path + '/' + _folder + '/' + _p['keyword'] + ".json", + "reader_cfg": _reader_cfg, + "infer_cfg": _infer_cfg, + "eval_cfg": _eval_cfg, + } + + GaokaoBench_datasets.append(_dataset) + +_temporary_variables = [k for k in globals() if k.startswith('_')] +for _t in _temporary_variables: + del globals()[_t] +del _temporary_variables, _t diff --git a/configs/datasets/PJExam/PJExam_gen.py b/configs/datasets/PJExam/PJExam_gen.py new file mode 100644 index 00000000..962dc3cc --- /dev/null +++ b/configs/datasets/PJExam/PJExam_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .PJExam_gen_785c37 import PJExam_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen.py b/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen.py new file mode 100644 index 00000000..a59cfe6b --- /dev/null +++ b/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .SuperGLUE_AX_g_gen_7a5dee import AX_g_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen.py b/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen.py new file mode 100644 index 00000000..178bb1e9 --- /dev/null +++ b/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .SuperGLUE_BoolQ_gen_8525d1 import BoolQ_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_094411.py b/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_094411.py new file mode 100644 index 00000000..5eaeee9c --- /dev/null +++ b/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_094411.py @@ -0,0 +1,45 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import BoolQDataset + +BoolQ_reader_cfg = dict( + input_columns=["question", "passage"], + output_column="answer", + test_split="train") + +BoolQ_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[ + dict(role="HUMAN", prompt="{passage}\nQuestion: {question}"), + dict(role="BOT", prompt="No."), + ]), + 1: + dict(round=[ + dict(role="HUMAN", prompt="{passage}\nQuestion: {question}"), + dict(role="BOT", prompt="Yes."), + ]), + }, + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), +) + +BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +BoolQ_datasets = [ + dict( + type=BoolQDataset, + abbr="BoolQ", + path="json", + data_files="./data/SuperGLUE/BoolQ/val.jsonl", + split="train", + reader_cfg=BoolQ_reader_cfg, + infer_cfg=BoolQ_infer_cfg, + eval_cfg=BoolQ_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl.py b/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl.py new file mode 100644 index 00000000..2688f70c --- /dev/null +++ b/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .SuperGLUE_CB_ppl_32adbb import CB_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl_32adbb.py b/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl_32adbb.py new file mode 100644 index 00000000..c01db79b --- /dev/null +++ b/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl_32adbb.py @@ -0,0 +1,62 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +CB_reader_cfg = dict( + input_columns=["premise", "hypothesis"], + output_column="label", +) + +CB_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + "contradiction": + dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nWhat is the relation between the two sentences?" + ), + dict(role="BOT", prompt="Contradiction"), + ]), + "entailment": + dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nWhat is the relation between the two sentences?" + ), + dict(role="BOT", prompt="Entailment"), + ]), + "neutral": + dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nWhat is the relation between the two sentences?" + ), + dict(role="BOT", prompt="Neutral"), + ]), + }, + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), +) + +CB_eval_cfg = dict(evaluator=dict(type=AccEvaluator), ) + +CB_datasets = [ + dict( + type=HFDataset, + abbr="CB", + path="json", + split="train", + data_files="./data/SuperGLUE/CB/val.jsonl", + reader_cfg=CB_reader_cfg, + infer_cfg=CB_infer_cfg, + eval_cfg=CB_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_ddb78c.py b/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_ddb78c.py new file mode 100644 index 00000000..10dda696 --- /dev/null +++ b/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_ddb78c.py @@ -0,0 +1,49 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +COPA_reader_cfg = dict( + input_columns=["question", "premise", "choice1", "choice2"], + output_column="label", + test_split="train") + +COPA_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: + dict(round=[ + dict( + role="HUMAN", + prompt="{premise}\nQuestion: What may be the {question}?\nAnswer:"), + dict(role="BOT", prompt="{choice1}"), + ]), + 1: + dict(round=[ + dict( + role="HUMAN", + prompt="{premise}\nQuestion: What may be the {question}?\nAnswer:"), + dict(role="BOT", prompt="{choice2}"), + ]), + }, + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), +) + +COPA_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +COPA_datasets = [ + dict( + type=HFDataset, + abbr="COPA", + path="json", + data_files="./data/SuperGLUE/COPA/val.jsonl", + split="train", + reader_cfg=COPA_reader_cfg, + infer_cfg=COPA_infer_cfg, + eval_cfg=COPA_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl.py b/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl.py new file mode 100644 index 00000000..de46be78 --- /dev/null +++ b/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .SuperGLUE_MultiRC_ppl_83a304 import MultiRC_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_gen_ce346a.py b/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_gen_ce346a.py new file mode 100644 index 00000000..f1696437 --- /dev/null +++ b/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_gen_ce346a.py @@ -0,0 +1,42 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AXDataset_V2 + +RTE_reader_cfg = dict( + input_columns=["hypothesis", "premise"], + output_column="label", +) + +RTE_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +RTE_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +RTE_datasets = [ + dict( + abbr="RTE", + type=AXDataset_V2, # rte share the same format with ax + path="./data/SuperGLUE/RTE/val.jsonl", + reader_cfg=RTE_reader_cfg, + infer_cfg=RTE_infer_cfg, + eval_cfg=RTE_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_ppl_29a22c.py b/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_ppl_29a22c.py new file mode 100644 index 00000000..e992ddc7 --- /dev/null +++ b/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_ppl_29a22c.py @@ -0,0 +1,53 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +RTE_reader_cfg = dict( + input_columns=["hypothesis", "premise"], + output_column="label", + test_split="train") + +RTE_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + "entailment": + dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?" + ), + dict(role="BOT", prompt="Yes"), + ]), + "not_entailment": + dict(round=[ + dict( + role="HUMAN", + prompt= + "{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?" + ), + dict(role="BOT", prompt="No"), + ]) + }, + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), +) + +RTE_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +RTE_datasets = [ + dict( + type=HFDataset, + abbr="RTE", + path="json", + data_files="./data/SuperGLUE/RTE/val.jsonl", + split="train", + reader_cfg=RTE_reader_cfg, + infer_cfg=RTE_infer_cfg, + eval_cfg=RTE_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen_90d5b6.py b/configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen_90d5b6.py new file mode 100644 index 00000000..06dbc502 --- /dev/null +++ b/configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen_90d5b6.py @@ -0,0 +1,29 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import EMEvaluator +from opencompass.datasets import ReCoRDDataset + +ReCoRD_reader_cfg = dict( + input_columns=['question', 'text'], output_column='answers') + +ReCoRD_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template= + "Passage:{text}\nResult:{question}\nQuestion: What entity does ____ refer to in the result?Give me the entity name:"), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +ReCoRD_eval_cfg = dict( + evaluator=dict(type=EMEvaluator), pred_postprocessor=dict(type='ReCoRD')) + +ReCoRD_datasets = [ + dict( + type=ReCoRDDataset, + abbr='ReCoRD', + path='./data/SuperGLUE/ReCoRD/val.jsonl', + reader_cfg=ReCoRD_reader_cfg, + infer_cfg=ReCoRD_infer_cfg, + eval_cfg=ReCoRD_eval_cfg) +] diff --git a/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl.py b/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl.py new file mode 100644 index 00000000..0ac78582 --- /dev/null +++ b/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .SuperGLUE_WSC_ppl_85f45f import WSC_datasets # noqa: F401, F403 diff --git a/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_89e489.py b/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_89e489.py new file mode 100644 index 00000000..16645eec --- /dev/null +++ b/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_89e489.py @@ -0,0 +1,34 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import WSCDataset + +WSC_reader_cfg = dict( + input_columns=['span1', 'span2', 'text', 'new_text'], + output_column='answer') + +WSC_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: "{text}", + 1: "{new_text}" + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +WSC_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +WSC_datasets = [ + dict( + type=WSCDataset, + path='json', + abbr='WSC', + data_files='./data/SuperGLUE/WSC/val.jsonl', + split='train', + reader_cfg=WSC_reader_cfg, + infer_cfg=WSC_infer_cfg, + eval_cfg=WSC_eval_cfg, + ) +] diff --git a/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_9ed37a.py b/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_9ed37a.py new file mode 100644 index 00000000..68f0954f --- /dev/null +++ b/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_ppl_9ed37a.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import WSCDataset + +WSC_reader_cfg = dict( + input_columns=["span1", "span2", "text", "new_text"], + output_column="answer", +) + +WSC_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: dict(round=[ + dict(role="HUMAN", prompt="{text}"), + ]), + 1: dict(round=[ + dict(role="HUMAN", prompt="{new_text}"), + ]), + }, + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer), +) + +WSC_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +WSC_datasets = [ + dict( + type=WSCDataset, + path="json", + abbr="WSC", + data_files="./data/SuperGLUE/WSC/val.jsonl", + split="train", + reader_cfg=WSC_reader_cfg, + infer_cfg=WSC_infer_cfg, + eval_cfg=WSC_eval_cfg, + ) +] diff --git a/configs/datasets/TheoremQA/TheoremQA_gen.py b/configs/datasets/TheoremQA/TheoremQA_gen.py new file mode 100644 index 00000000..d26e9d27 --- /dev/null +++ b/configs/datasets/TheoremQA/TheoremQA_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .TheoremQA_gen_891fcf import TheoremQA_datasets # noqa: F401, F403 diff --git a/configs/datasets/TheoremQA/TheoremQA_gen_24bc13.py b/configs/datasets/TheoremQA/TheoremQA_gen_24bc13.py new file mode 100644 index 00000000..9651c019 --- /dev/null +++ b/configs/datasets/TheoremQA/TheoremQA_gen_24bc13.py @@ -0,0 +1,37 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import TheoremQADataset + +TheoremQA_reader_cfg = dict( + input_columns=['Question', 'Answer_type'], + output_column='Answer', + train_split='test') + +TheoremQA_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt= + """You are a mathematician, you are supposed to answer the given question. You need to output the answer in your final sentence like "Therefore, the answer is ...". The answer can only be one of the following forms:\n1. a numerical value like 0.1, no symbol and no unit at all.\n2. a list of number like [2, 3, 4].\n3. True/False.\n4. an option like (a), (b), (c), (d)\nQuestion: {Question}\nLet\'s think step by step.""" + ), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +TheoremQA_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='TheoremQA')) + +TheoremQA_datasets = [ + dict( + abbr='TheoremQA', + type=TheoremQADataset, + path="./data/TheoremQA/test.csv", + reader_cfg=TheoremQA_reader_cfg, + infer_cfg=TheoremQA_infer_cfg, + eval_cfg=TheoremQA_eval_cfg) +] diff --git a/configs/datasets/TheoremQA/TheoremQA_gen_891fcf.py b/configs/datasets/TheoremQA/TheoremQA_gen_891fcf.py new file mode 100644 index 00000000..bbbd6b24 --- /dev/null +++ b/configs/datasets/TheoremQA/TheoremQA_gen_891fcf.py @@ -0,0 +1,48 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import TheoremQADataset + +TheoremQA_reader_cfg = dict( + input_columns=['Question', 'Answer_type'], + output_column='Answer', + train_split='test') + +TheoremQA_prompt1 = """You are a mathematician, you are supposed to answer the given question. You need to output the answer in your final sentence like "Therefore, the answer is ...". The answer can only be one of the following forms: +1. a numerical value like 0.1, no symbol and no unit at all. +2. a list of number like [2, 3, 4]. +3. True/False. +4. an option like (a), (b), (c), (d) +""" +TheoremQA_prompt2 = 'Question: {Question}\nLet\'s think step by step.' + +TheoremQA_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt=TheoremQA_prompt1), + ], + round=[ + dict(role='HUMAN', prompt=TheoremQA_prompt2), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +TheoremQA_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='TheoremQA')) + +TheoremQA_datasets = [ + dict( + abbr='TheoremQA', + type=TheoremQADataset, + path="./data/TheoremQA/test.csv", + reader_cfg=TheoremQA_reader_cfg, + infer_cfg=TheoremQA_infer_cfg, + eval_cfg=TheoremQA_eval_cfg) +] diff --git a/configs/datasets/Xsum/Xsum_gen.py b/configs/datasets/Xsum/Xsum_gen.py new file mode 100644 index 00000000..d7d3307f --- /dev/null +++ b/configs/datasets/Xsum/Xsum_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .Xsum_gen_d2126e import Xsum_datasets # noqa: F401, F403 diff --git a/configs/datasets/Xsum/Xsum_gen_bb6e7b.py b/configs/datasets/Xsum/Xsum_gen_bb6e7b.py new file mode 100644 index 00000000..8ef0ef34 --- /dev/null +++ b/configs/datasets/Xsum/Xsum_gen_bb6e7b.py @@ -0,0 +1,30 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.datasets import XsumDataset + +Xsum_reader_cfg = dict(input_columns=['dialogue'], output_column='summary') + +Xsum_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='Document：{dialogue}\n' + 'Based on the previous text, provide a brief single summary:'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +Xsum_eval_cfg = dict( + evaluator=dict(type=RougeEvaluator), + pred_postprocessor=dict(type='Xsum'), +) + +Xsum_datasets = [ + dict( + type=XsumDataset, + abbr='Xsum', + path='./data/Xsum/dev.jsonl', + reader_cfg=Xsum_reader_cfg, + infer_cfg=Xsum_infer_cfg, + eval_cfg=Xsum_eval_cfg) +] diff --git a/configs/datasets/Xsum/Xsum_gen_d2126e.py b/configs/datasets/Xsum/Xsum_gen_d2126e.py new file mode 100644 index 00000000..5da91c7d --- /dev/null +++ b/configs/datasets/Xsum/Xsum_gen_d2126e.py @@ -0,0 +1,39 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.datasets import XsumDataset + +Xsum_reader_cfg = dict(input_columns=["dialogue"], output_column="summary") + +Xsum_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "Document：{dialogue}\nBased on the previous text, provide a brief single summary:" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +Xsum_eval_cfg = dict( + evaluator=dict(type=RougeEvaluator), + pred_role='BOT', + pred_postprocessor=dict(type="Xsum"), +) + +Xsum_datasets = [ + dict( + type=XsumDataset, + abbr="Xsum", + path="./data/Xsum/dev.jsonl", + reader_cfg=Xsum_reader_cfg, + infer_cfg=Xsum_infer_cfg, + eval_cfg=Xsum_eval_cfg, + ) +] diff --git a/configs/datasets/agieval/agieval_gen_508849.py b/configs/datasets/agieval/agieval_gen_508849.py new file mode 100644 index 00000000..42019307 --- /dev/null +++ b/configs/datasets/agieval/agieval_gen_508849.py @@ -0,0 +1,84 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AGIEvalDataset, AGIEvalEvaluator + +agieval_reader_cfg = dict( + input_columns=['problem_input'], output_column='label') + +agieval_single_choice_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', + 'logiqa-zh', + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', +] +agieval_multiple_choices_sets = [ + # 'jec-qa-kd', # 数据需要额外处理 + # 'jec-qa-ca', # 数据需要额外处理 +] +agieval_cloze_sets = ['gaokao-mathcloze', 'math'] + +agieval_datasets = [] +for name in agieval_single_choice_sets: + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[dict(role='HUMAN', prompt='{problem_input}')])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, labels=['A', 'B', 'C', 'D'])) + + agieval_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='first-capital')) + + agieval_datasets.append( + dict( + type=AGIEvalDataset, + path='./data/AGIEval/data/v1/', + name=name, + abbr='agieval-' + name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for name in agieval_cloze_sets: + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[dict(role="HUMAN", prompt='{problem_input}')])), + retriever=dict(type=ZeroRetriever + ), # retriver 不起作用，以输入参数为准 (zero-shot / few-shot) + inferencer=dict(type=GenInferencer)) + + agieval_eval_cfg = dict( + evaluator=dict(type=AGIEvalEvaluator), pred_role="BOT") + + agieval_datasets.append( + dict( + type=AGIEvalDataset, + path='./data/AGIEval/data/v1/', + name=name, + abbr='agieval-' + name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +del name, agieval_infer_cfg, agieval_eval_cfg diff --git a/configs/datasets/agieval/agieval_gen_dc7dae.py b/configs/datasets/agieval/agieval_gen_dc7dae.py new file mode 100644 index 00000000..0df8f90a --- /dev/null +++ b/configs/datasets/agieval/agieval_gen_dc7dae.py @@ -0,0 +1,203 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AGIEvalDataset_v2, AGIEvalEvaluator + +agieval_reader_cfg = dict( + input_columns=['question', 'options'], output_column='label') + +agieval_single_choice_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', + 'logiqa-zh', + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', +] +agieval_multiple_choices_sets = [ + 'jec-qa-kd', + 'jec-qa-ca', +] +agieval_cloze_sets = ['gaokao-mathcloze', 'math'] +agieval_chinese_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', + 'logiqa-zh', + 'gaokao-mathcloze', +] +agieval_english_sets = [ + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', + 'math', +] +agieval_gaokao_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', +] + +agieval_datasets = [] +for _name in agieval_single_choice_sets: + if _name in agieval_chinese_sets: + _hint = '答案是： ' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}') + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='first-capital')) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _name in agieval_multiple_choices_sets: + if _name in agieval_chinese_sets: + _hint = '答案是： ' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}') + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='first-capital-multi')) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _name in agieval_cloze_sets: + if _name in agieval_chinese_sets: + _hint = '答案是： ' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[dict(role='HUMAN', prompt=f'{{question}}\n{_hint}')])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict(evaluator=dict(type=AGIEvalEvaluator)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _item in agieval_datasets: + _name = _item['name'] + _intro = { + 'gaokao-chinese': + '以下是一道中国高考语文选择题，请选择正确的答案。', + 'gaokao-english': + '以下是一道中国高考英语选择题，请选择正确的答案。', + 'gaokao-geography': + '以下是一道中国高考地理选择题，请选择正确的答案。', + 'gaokao-history': + '以下是一道中国高考历史选择题，请选择正确的答案。', + 'gaokao-biology': + '以下是一道中国高考生物选择题，请选择正确的答案。', + 'gaokao-chemistry': + '以下是一道中国高考化学选择题，请选择正确的答案。', + 'gaokao-physics': + '以下是一道中国高考物理选择题，请选择正确的答案。', + 'gaokao-mathqa': + '以下是一道中国高考数学选择题，请选择正确的答案。', + 'logiqa-zh': + '以下是一道中国公务员考试题，请选择正确的答案。', + 'lsat-ar': + 'The following is a LSAT Analytical Reasoning question. Please select the correct answer.', + 'lsat-lr': + 'The following is a LSAT Logical Reasoning question. Please select the correct answer.', + 'lsat-rc': + 'The following is a LSAT Reading Comprehension question. Please select the correct answer.', + 'logiqa-en': + 'The following is a Logic Reasoning question. Please select the correct answer.', + 'sat-math': + 'The following is a SAT Math question. Please select the correct answer.', + 'sat-en': + 'The following is a SAT English question. Please select the correct answer.', + 'sat-en-without-passage': + 'The following is a SAT English question. Please select the correct answer.', + 'aqua-rat': + 'The following is a AQUA-RAT question. Please select the correct answer.', + 'jec-qa-kd': + '以下是一道中国司法考试基础知识题，请选择正确的答案。', + 'jec-qa-ca': + '以下是一道中国司法考试案例分析题，请选择正确的答案。', + 'gaokao-mathcloze': + '以下是一道中国高考数学填空题，请填入正确的答案。', + 'math': + 'The following is a Math question. Please select the correct answer.', + }[_name] + _templates = _item['infer_cfg']['prompt_template']['template'] + _templates['round'][0][ + 'prompt'] = _intro + '\n' + _templates['round'][0]['prompt'] + +del _item, _intro, _templates, _name, _hint, agieval_infer_cfg, agieval_eval_cfg diff --git a/configs/datasets/agieval/agieval_mixed_2f14ad.py b/configs/datasets/agieval/agieval_mixed_2f14ad.py new file mode 100644 index 00000000..555ca0da --- /dev/null +++ b/configs/datasets/agieval/agieval_mixed_2f14ad.py @@ -0,0 +1,217 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer, GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AGIEvalDataset_v2, AGIEvalEvaluator + +agieval_single_choice_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', + 'logiqa-zh', + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', +] +agieval_multiple_choices_sets = [ + 'jec-qa-kd', + 'jec-qa-ca', +] +agieval_cloze_sets = ['gaokao-mathcloze', 'math'] +agieval_chinese_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', + 'logiqa-zh', + 'gaokao-mathcloze', +] +agieval_english_sets = [ + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', + 'math', +] +agieval_gaokao_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', +] + +agieval_datasets = [] +for _name in agieval_single_choice_sets: + if _name in ['lsat-ar', 'lsat-lr', 'lsat-rc', 'aqua-rat']: + _options = ['A', 'B', 'C', 'D', 'E'] + else: + _options = ['A', 'B', 'C', 'D'] + if _name in agieval_chinese_sets: + _hint = '答案是：' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + label: dict(round=[ + dict(role='HUMAN', prompt='{question}\n{options}'), + dict(role='BOT', prompt=f'{_hint}{label}') + ]) + for label in _options + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer, labels=_options)) + + agieval_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=dict( + input_columns=['question', 'options'] + _options, + output_column='label'), + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _name in agieval_multiple_choices_sets: + if _name in agieval_chinese_sets: + _hint = '答案是： ' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}') + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='first-capital-multi')) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=dict( + input_columns=['question', 'options'], output_column='label'), + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _name in agieval_cloze_sets: + if _name in agieval_chinese_sets: + _hint = '答案是：' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[dict(role='HUMAN', prompt=f'{{question}}{_hint}')])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict(evaluator=dict(type=AGIEvalEvaluator)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=dict( + input_columns=['question', 'options'], output_column='label'), + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _item in agieval_datasets: + _name = _item['name'] + _intro = { + 'gaokao-chinese': + '以下是一道中国高考语文选择题，请选择正确的答案。', + 'gaokao-english': + '以下是一道中国高考英语选择题，请选择正确的答案。', + 'gaokao-geography': + '以下是一道中国高考地理选择题，请选择正确的答案。', + 'gaokao-history': + '以下是一道中国高考历史选择题，请选择正确的答案。', + 'gaokao-biology': + '以下是一道中国高考生物选择题，请选择正确的答案。', + 'gaokao-chemistry': + '以下是一道中国高考化学选择题，请选择正确的答案。', + 'gaokao-physics': + '以下是一道中国高考物理选择题，请选择正确的答案。', + 'gaokao-mathqa': + '以下是一道中国高考数学选择题，请选择正确的答案。', + 'logiqa-zh': + '以下是一道中国公务员考试题，请选择正确的答案。', + 'lsat-ar': + 'The following is a LSAT Analytical Reasoning question. Please select the correct answer.', + 'lsat-lr': + 'The following is a LSAT Logical Reasoning question. Please select the correct answer.', + 'lsat-rc': + 'The following is a LSAT Reading Comprehension question. Please select the correct answer.', + 'logiqa-en': + 'The following is a Logic Reasoning question. Please select the correct answer.', + 'sat-math': + 'The following is a SAT Math question. Please select the correct answer.', + 'sat-en': + 'The following is a SAT English question. Please select the correct answer.', + 'sat-en-without-passage': + 'The following is a SAT English question. Please select the correct answer.', + 'aqua-rat': + 'The following is a AQUA-RAT question. Please select the correct answer.', + 'jec-qa-kd': + '以下是一道中国司法考试基础知识题，请选择正确的答案。', + 'jec-qa-ca': + '以下是一道中国司法考试案例分析题，请选择正确的答案。', + 'gaokao-mathcloze': + '以下是一道中国高考数学填空题，请填入正确的答案。', + 'math': + 'The following is a Math question. Please select the correct answer.', + }[_name] + _templates = _item['infer_cfg']['prompt_template']['template'] + + if _item['infer_cfg']['inferencer']['type'] == PPLInferencer: + for _label in _templates: + _templates[_label]['round'][0][ + 'prompt'] = _intro + '\n' + _templates[_label]['round'][0][ + 'prompt'] + else: + _templates['round'][0][ + 'prompt'] = _intro + '\n' + _templates['round'][0]['prompt'] + +del _item, _intro, _templates, _label, _name, _options, _hint, agieval_infer_cfg, agieval_eval_cfg diff --git a/configs/datasets/apps/apps_gen_193ec0.py b/configs/datasets/apps/apps_gen_193ec0.py new file mode 100644 index 00000000..481a4e05 --- /dev/null +++ b/configs/datasets/apps/apps_gen_193ec0.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HFDataset, HumanEvaluator + +apps_reader_cfg = dict( + input_columns=['question'], output_column='problem_id', train_split='test') + +# TODO: allow empty output-column +apps_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt='Write a python program:'), + ], + round=[ + dict(role='HUMAN', prompt='{question}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +apps_eval_cfg = dict( + evaluator=dict(type=HumanEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type='humaneval'), +) + +apps_datasets = [ + dict( + type=HFDataset, + path='codeparrot/apps', + reader_cfg=apps_reader_cfg, + infer_cfg=apps_infer_cfg, + eval_cfg=apps_eval_cfg) +] diff --git a/configs/datasets/apps/apps_gen_36c409.py b/configs/datasets/apps/apps_gen_36c409.py new file mode 100644 index 00000000..2f510e11 --- /dev/null +++ b/configs/datasets/apps/apps_gen_36c409.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HFDataset, HumanEvaluator + +apps_reader_cfg = dict( + input_columns=['question'], output_column='problem_id', train_split='test') + +# TODO: allow empty output-column +apps_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role='HUMAN', prompt='Write a python program:\n{question}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +apps_eval_cfg = dict( + evaluator=dict(type=HumanEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type='humaneval'), +) + +apps_datasets = [ + dict( + type=HFDataset, + path='codeparrot/apps', + reader_cfg=apps_reader_cfg, + infer_cfg=apps_infer_cfg, + eval_cfg=apps_eval_cfg) +] diff --git a/configs/datasets/apps/apps_gen_e85947.py b/configs/datasets/apps/apps_gen_e85947.py new file mode 100644 index 00000000..4eec64fa --- /dev/null +++ b/configs/datasets/apps/apps_gen_e85947.py @@ -0,0 +1,30 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HFDataset, HumanEvaluator + +apps_reader_cfg = dict( + input_columns=['question'], output_column='problem_id', train_split='test') + +# TODO: allow empty output-column +apps_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='{question}'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +apps_eval_cfg = dict( + evaluator=dict(type=HumanEvaluator), + k=[1, 10, 100], + pred_postprocessor=dict(type='humaneval'), +) + +apps_datasets = [ + dict( + type=HFDataset, + path='codeparrot/apps', + reader_cfg=apps_reader_cfg, + infer_cfg=apps_infer_cfg, + eval_cfg=apps_eval_cfg) +] diff --git a/configs/datasets/bbh/bbh_gen_58abc3.py b/configs/datasets/bbh/bbh_gen_58abc3.py new file mode 100644 index 00000000..a8ce9b54 --- /dev/null +++ b/configs/datasets/bbh/bbh_gen_58abc3.py @@ -0,0 +1,100 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import BBHDataset, BBHEvaluator + +bbh_reader_cfg = dict(input_columns=["input"], output_column="target") + +_path_prefix = "./data/BBH" + +bbh_multiple_choice_sets = [ + 'temporal_sequences', + 'disambiguation_qa', + 'date_understanding', + 'tracking_shuffled_objects_three_objects', + 'penguins_in_a_table', + 'geometric_shapes', + 'snarks', + 'ruin_names', + 'tracking_shuffled_objects_seven_objects', + 'tracking_shuffled_objects_five_objects', + 'logical_deduction_three_objects', + 'hyperbaton', + 'logical_deduction_five_objects', + 'logical_deduction_seven_objects', + 'movie_recommendation', + 'salient_translation_error_detection', + 'reasoning_about_colored_objects', +] +bbh_free_form_sets = [ + 'multistep_arithmetic_two', + 'navigate', + 'dyck_languages', + 'word_sorting', + 'sports_understanding', + 'boolean_expressions', + 'object_counting', + 'formal_fallacies', + 'causal_judgement', + 'web_of_lies', +] + +bbh_datasets = [] +for _name in bbh_multiple_choice_sets: + _hint = open(f"{_path_prefix}/lib_prompt/{_name}.txt", 'r').read() + bbh_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + f"Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: Let's think step by step." + ) + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + bbh_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type='bbh-mcq'), + dataset_postprocessor=dict(type='bbh-mcq')) + + bbh_datasets.append( + dict( + type=BBHDataset, + path=f"{_path_prefix}/data", + name=_name, + abbr='bbh-' + _name, + reader_cfg=bbh_reader_cfg, + infer_cfg=bbh_infer_cfg.copy(), + eval_cfg=bbh_eval_cfg.copy())) + +for _name in bbh_free_form_sets: + _hint = open(f"{_path_prefix}/lib_prompt/{_name}.txt", 'r').read() + bbh_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + f"Follow the given examples and answer the question.\n{_hint}\n\nQ: {{input}}\nA: Let's think step by step." + ) + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + bbh_eval_cfg = dict(evaluator=dict(type=BBHEvaluator), pred_role="BOT") + + bbh_datasets.append( + dict( + type=BBHDataset, + path=f"{_path_prefix}/data", + name=_name, + abbr='bbh-' + _name, + reader_cfg=bbh_reader_cfg, + infer_cfg=bbh_infer_cfg.copy(), + eval_cfg=bbh_eval_cfg.copy())) + +del _name, _hint, _path_prefix diff --git a/configs/datasets/ceval/ceval_gen_c24a21.py b/configs/datasets/ceval/ceval_gen_c24a21.py new file mode 100644 index 00000000..c87cd3bd --- /dev/null +++ b/configs/datasets/ceval/ceval_gen_c24a21.py @@ -0,0 +1,187 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import FixKRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CEvalDataset + +ceval_subject_mapping = { + "computer_network": + ["Computer Network", "\u8ba1\u7b97\u673a\u7f51\u7edc", "STEM"], + "operating_system": + ["Operating System", "\u64cd\u4f5c\u7cfb\u7edf", "STEM"], + "computer_architecture": + ["Computer Architecture", "\u8ba1\u7b97\u673a\u7ec4\u6210", "STEM"], + "college_programming": + ["College Programming", "\u5927\u5b66\u7f16\u7a0b", "STEM"], + "college_physics": ["College Physics", "\u5927\u5b66\u7269\u7406", "STEM"], + "college_chemistry": + ["College Chemistry", "\u5927\u5b66\u5316\u5b66", "STEM"], + "advanced_mathematics": + ["Advanced Mathematics", "\u9ad8\u7b49\u6570\u5b66", "STEM"], + "probability_and_statistics": + ["Probability and Statistics", "\u6982\u7387\u7edf\u8ba1", "STEM"], + "discrete_mathematics": + ["Discrete Mathematics", "\u79bb\u6563\u6570\u5b66", "STEM"], + "electrical_engineer": [ + "Electrical Engineer", "\u6ce8\u518c\u7535\u6c14\u5de5\u7a0b\u5e08", + "STEM" + ], + "metrology_engineer": + ["Metrology Engineer", "\u6ce8\u518c\u8ba1\u91cf\u5e08", "STEM"], + "high_school_mathematics": + ["High School Mathematics", "\u9ad8\u4e2d\u6570\u5b66", "STEM"], + "high_school_physics": + ["High School Physics", "\u9ad8\u4e2d\u7269\u7406", "STEM"], + "high_school_chemistry": + ["High School Chemistry", "\u9ad8\u4e2d\u5316\u5b66", "STEM"], + "high_school_biology": [ + "High School Biology", "\u9ad8\u4e2d\u751f\u7269", "STEM" + ], + "middle_school_mathematics": [ + "Middle School Mathematics", "\u521d\u4e2d\u6570\u5b66", "STEM" + ], + "middle_school_biology": [ + "Middle School Biology", "\u521d\u4e2d\u751f\u7269", "STEM" + ], + "middle_school_physics": [ + "Middle School Physics", "\u521d\u4e2d\u7269\u7406", "STEM" + ], + "middle_school_chemistry": [ + "Middle School Chemistry", "\u521d\u4e2d\u5316\u5b66", "STEM" + ], + "veterinary_medicine": [ + "Veterinary Medicine", "\u517d\u533b\u5b66", "STEM" + ], + "college_economics": [ + "College Economics", "\u5927\u5b66\u7ecf\u6d4e\u5b66", "Social Science" + ], + "business_administration": [ + "Business Administration", "\u5de5\u5546\u7ba1\u7406", "Social Science" + ], + "marxism": [ + "Marxism", "\u9a6c\u514b\u601d\u4e3b\u4e49\u57fa\u672c\u539f\u7406", + "Social Science" + ], + "mao_zedong_thought": [ + "Mao Zedong Thought", + "\u6bdb\u6cfd\u4e1c\u601d\u60f3\u548c\u4e2d\u56fd\u7279\u8272\u793e\u4f1a\u4e3b\u4e49\u7406\u8bba\u4f53\u7cfb\u6982\u8bba", + "Social Science" + ], + "education_science": [ + "Education Science", "\u6559\u80b2\u5b66", "Social Science" + ], + "teacher_qualification": [ + "Teacher Qualification", "\u6559\u5e08\u8d44\u683c", "Social Science" + ], + "high_school_politics": [ + "High School Politics", "\u9ad8\u4e2d\u653f\u6cbb", "Social Science" + ], + "high_school_geography": [ + "High School Geography", "\u9ad8\u4e2d\u5730\u7406", "Social Science" + ], + "middle_school_politics": [ + "Middle School Politics", "\u521d\u4e2d\u653f\u6cbb", "Social Science" + ], + "middle_school_geography": [ + "Middle School Geography", "\u521d\u4e2d\u5730\u7406", "Social Science" + ], + "modern_chinese_history": + ["Modern Chinese History", "\u8fd1\u4ee3\u53f2\u7eb2\u8981", "Humanities"], + "ideological_and_moral_cultivation": [ + "Ideological and Moral Cultivation", + "\u601d\u60f3\u9053\u5fb7\u4fee\u517b\u4e0e\u6cd5\u5f8b\u57fa\u7840", + "Humanities" + ], + "logic": ["Logic", "\u903b\u8f91\u5b66", "Humanities"], + "law": ["Law", "\u6cd5\u5b66", "Humanities"], + "chinese_language_and_literature": [ + "Chinese Language and Literature", + "\u4e2d\u56fd\u8bed\u8a00\u6587\u5b66", "Humanities" + ], + "art_studies": ["Art Studies", "\u827a\u672f\u5b66", "Humanities"], + "professional_tour_guide": [ + "Professional Tour Guide", "\u5bfc\u6e38\u8d44\u683c", "Humanities" + ], + "legal_professional": [ + "Legal Professional", "\u6cd5\u5f8b\u804c\u4e1a\u8d44\u683c", + "Humanities" + ], + "high_school_chinese": [ + "High School Chinese", "\u9ad8\u4e2d\u8bed\u6587", "Humanities" + ], + "high_school_history": [ + "High School History", "\u9ad8\u4e2d\u5386\u53f2", "Humanities" + ], + "middle_school_history": [ + "Middle School History", "\u521d\u4e2d\u5386\u53f2", "Humanities" + ], + "civil_servant": ["Civil Servant", "\u516c\u52a1\u5458", "Other"], + "sports_science": ["Sports Science", "\u4f53\u80b2\u5b66", "Other"], + "plant_protection": [ + "Plant Protection", "\u690d\u7269\u4fdd\u62a4", "Other" + ], + "basic_medicine": ["Basic Medicine", "\u57fa\u7840\u533b\u5b66", "Other"], + "clinical_medicine": [ + "Clinical Medicine", "\u4e34\u5e8a\u533b\u5b66", "Other" + ], + "urban_and_rural_planner": [ + "Urban and Rural Planner", + "\u6ce8\u518c\u57ce\u4e61\u89c4\u5212\u5e08", "Other" + ], + "accountant": ["Accountant", "\u6ce8\u518c\u4f1a\u8ba1\u5e08", "Other"], + "fire_engineer": [ + "Fire Engineer", "\u6ce8\u518c\u6d88\u9632\u5de5\u7a0b\u5e08", "Other" + ], + "environmental_impact_assessment_engineer": [ + "Environmental Impact Assessment Engineer", + "\u73af\u5883\u5f71\u54cd\u8bc4\u4ef7\u5de5\u7a0b\u5e08", "Other" + ], + "tax_accountant": ["Tax Accountant", "\u7a0e\u52a1\u5e08", "Other"], + "physician": ["Physician", "\u533b\u5e08\u8d44\u683c", "Other"] +} +ceval_all_sets = list(ceval_subject_mapping.keys()) + +ceval_datasets = [] +for _split in ["val", "test"]: + for _name in ceval_all_sets: + _ch_name = ceval_subject_mapping[_name][1] + ceval_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template=dict( + begin="", + round=[ + dict( + role="HUMAN", + prompt= + f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: " + ), + dict(role="BOT", prompt="{answer}"), + ]), + ice_token="", + ), + retriever=dict(type=FixKRetriever), + inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]), + ) + + ceval_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='first-capital')) + + ceval_datasets.append( + dict( + type=CEvalDataset, + path="./data/ceval/formal_ceval", + name=_name, + abbr="ceval-" + _name if _split == "val" else "ceval-test-" + + _name, + reader_cfg=dict( + input_columns=["question", "A", "B", "C", "D"], + output_column="answer", + train_split="dev", + test_split=_split), + infer_cfg=ceval_infer_cfg, + eval_cfg=ceval_eval_cfg, + )) + +del _split, _name, _ch_name diff --git a/configs/datasets/ceval/ceval_gen_ee2cb0.py b/configs/datasets/ceval/ceval_gen_ee2cb0.py new file mode 100644 index 00000000..323cbafa --- /dev/null +++ b/configs/datasets/ceval/ceval_gen_ee2cb0.py @@ -0,0 +1,185 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import FixKRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CEvalDataset + +ceval_subject_mapping = { + "computer_network": + ["Computer Network", "\u8ba1\u7b97\u673a\u7f51\u7edc", "STEM"], + "operating_system": + ["Operating System", "\u64cd\u4f5c\u7cfb\u7edf", "STEM"], + "computer_architecture": + ["Computer Architecture", "\u8ba1\u7b97\u673a\u7ec4\u6210", "STEM"], + "college_programming": + ["College Programming", "\u5927\u5b66\u7f16\u7a0b", "STEM"], + "college_physics": ["College Physics", "\u5927\u5b66\u7269\u7406", "STEM"], + "college_chemistry": + ["College Chemistry", "\u5927\u5b66\u5316\u5b66", "STEM"], + "advanced_mathematics": + ["Advanced Mathematics", "\u9ad8\u7b49\u6570\u5b66", "STEM"], + "probability_and_statistics": + ["Probability and Statistics", "\u6982\u7387\u7edf\u8ba1", "STEM"], + "discrete_mathematics": + ["Discrete Mathematics", "\u79bb\u6563\u6570\u5b66", "STEM"], + "electrical_engineer": [ + "Electrical Engineer", "\u6ce8\u518c\u7535\u6c14\u5de5\u7a0b\u5e08", + "STEM" + ], + "metrology_engineer": + ["Metrology Engineer", "\u6ce8\u518c\u8ba1\u91cf\u5e08", "STEM"], + "high_school_mathematics": + ["High School Mathematics", "\u9ad8\u4e2d\u6570\u5b66", "STEM"], + "high_school_physics": + ["High School Physics", "\u9ad8\u4e2d\u7269\u7406", "STEM"], + "high_school_chemistry": + ["High School Chemistry", "\u9ad8\u4e2d\u5316\u5b66", "STEM"], + "high_school_biology": [ + "High School Biology", "\u9ad8\u4e2d\u751f\u7269", "STEM" + ], + "middle_school_mathematics": [ + "Middle School Mathematics", "\u521d\u4e2d\u6570\u5b66", "STEM" + ], + "middle_school_biology": [ + "Middle School Biology", "\u521d\u4e2d\u751f\u7269", "STEM" + ], + "middle_school_physics": [ + "Middle School Physics", "\u521d\u4e2d\u7269\u7406", "STEM" + ], + "middle_school_chemistry": [ + "Middle School Chemistry", "\u521d\u4e2d\u5316\u5b66", "STEM" + ], + "veterinary_medicine": [ + "Veterinary Medicine", "\u517d\u533b\u5b66", "STEM" + ], + "college_economics": [ + "College Economics", "\u5927\u5b66\u7ecf\u6d4e\u5b66", "Social Science" + ], + "business_administration": [ + "Business Administration", "\u5de5\u5546\u7ba1\u7406", "Social Science" + ], + "marxism": [ + "Marxism", "\u9a6c\u514b\u601d\u4e3b\u4e49\u57fa\u672c\u539f\u7406", + "Social Science" + ], + "mao_zedong_thought": [ + "Mao Zedong Thought", + "\u6bdb\u6cfd\u4e1c\u601d\u60f3\u548c\u4e2d\u56fd\u7279\u8272\u793e\u4f1a\u4e3b\u4e49\u7406\u8bba\u4f53\u7cfb\u6982\u8bba", + "Social Science" + ], + "education_science": [ + "Education Science", "\u6559\u80b2\u5b66", "Social Science" + ], + "teacher_qualification": [ + "Teacher Qualification", "\u6559\u5e08\u8d44\u683c", "Social Science" + ], + "high_school_politics": [ + "High School Politics", "\u9ad8\u4e2d\u653f\u6cbb", "Social Science" + ], + "high_school_geography": [ + "High School Geography", "\u9ad8\u4e2d\u5730\u7406", "Social Science" + ], + "middle_school_politics": [ + "Middle School Politics", "\u521d\u4e2d\u653f\u6cbb", "Social Science" + ], + "middle_school_geography": [ + "Middle School Geography", "\u521d\u4e2d\u5730\u7406", "Social Science" + ], + "modern_chinese_history": + ["Modern Chinese History", "\u8fd1\u4ee3\u53f2\u7eb2\u8981", "Humanities"], + "ideological_and_moral_cultivation": [ + "Ideological and Moral Cultivation", + "\u601d\u60f3\u9053\u5fb7\u4fee\u517b\u4e0e\u6cd5\u5f8b\u57fa\u7840", + "Humanities" + ], + "logic": ["Logic", "\u903b\u8f91\u5b66", "Humanities"], + "law": ["Law", "\u6cd5\u5b66", "Humanities"], + "chinese_language_and_literature": [ + "Chinese Language and Literature", + "\u4e2d\u56fd\u8bed\u8a00\u6587\u5b66", "Humanities" + ], + "art_studies": ["Art Studies", "\u827a\u672f\u5b66", "Humanities"], + "professional_tour_guide": [ + "Professional Tour Guide", "\u5bfc\u6e38\u8d44\u683c", "Humanities" + ], + "legal_professional": [ + "Legal Professional", "\u6cd5\u5f8b\u804c\u4e1a\u8d44\u683c", + "Humanities" + ], + "high_school_chinese": [ + "High School Chinese", "\u9ad8\u4e2d\u8bed\u6587", "Humanities" + ], + "high_school_history": [ + "High School History", "\u9ad8\u4e2d\u5386\u53f2", "Humanities" + ], + "middle_school_history": [ + "Middle School History", "\u521d\u4e2d\u5386\u53f2", "Humanities" + ], + "civil_servant": ["Civil Servant", "\u516c\u52a1\u5458", "Other"], + "sports_science": ["Sports Science", "\u4f53\u80b2\u5b66", "Other"], + "plant_protection": [ + "Plant Protection", "\u690d\u7269\u4fdd\u62a4", "Other" + ], + "basic_medicine": ["Basic Medicine", "\u57fa\u7840\u533b\u5b66", "Other"], + "clinical_medicine": [ + "Clinical Medicine", "\u4e34\u5e8a\u533b\u5b66", "Other" + ], + "urban_and_rural_planner": [ + "Urban and Rural Planner", + "\u6ce8\u518c\u57ce\u4e61\u89c4\u5212\u5e08", "Other" + ], + "accountant": ["Accountant", "\u6ce8\u518c\u4f1a\u8ba1\u5e08", "Other"], + "fire_engineer": [ + "Fire Engineer", "\u6ce8\u518c\u6d88\u9632\u5de5\u7a0b\u5e08", "Other" + ], + "environmental_impact_assessment_engineer": [ + "Environmental Impact Assessment Engineer", + "\u73af\u5883\u5f71\u54cd\u8bc4\u4ef7\u5de5\u7a0b\u5e08", "Other" + ], + "tax_accountant": ["Tax Accountant", "\u7a0e\u52a1\u5e08", "Other"], + "physician": ["Physician", "\u533b\u5e08\u8d44\u683c", "Other"] +} +ceval_all_sets = list(ceval_subject_mapping.keys()) + +ceval_datasets = [] +for _split in ["val"]: + for _name in ceval_all_sets: + _ch_name = ceval_subject_mapping[_name][1] + ceval_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template=dict( + begin="", + round=[ + dict( + role="HUMAN", + prompt= + f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: " + ), + dict(role="BOT", prompt="{answer}"), + ]), + ice_token="", + ), + retriever=dict(type=FixKRetriever), + inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]), + ) + + ceval_eval_cfg = dict(evaluator=dict(type=AccEvaluator), ) + + ceval_datasets.append( + dict( + type=CEvalDataset, + path="./data/ceval/formal_ceval", + name=_name, + abbr="ceval-" + _name if _split == "val" else "ceval-test-" + + _name, + reader_cfg=dict( + input_columns=["question", "A", "B", "C", "D"], + output_column="answer", + train_split="dev", + test_split=_split), + infer_cfg=ceval_infer_cfg, + eval_cfg=ceval_eval_cfg, + )) + +del _split, _name, _ch_name diff --git a/configs/datasets/ceval/ceval_ppl.py b/configs/datasets/ceval/ceval_ppl.py new file mode 100644 index 00000000..f2df3b7c --- /dev/null +++ b/configs/datasets/ceval/ceval_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .ceval_ppl_275812 import ceval_datasets # noqa: F401, F403 diff --git a/configs/datasets/ceval/ceval_ppl_162686.py b/configs/datasets/ceval/ceval_ppl_162686.py new file mode 100644 index 00000000..56a6bb64 --- /dev/null +++ b/configs/datasets/ceval/ceval_ppl_162686.py @@ -0,0 +1,188 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import FixKRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CEvalDataset + +ceval_subject_mapping = { + "computer_network": + ["Computer Network", "\u8ba1\u7b97\u673a\u7f51\u7edc", "STEM"], + "operating_system": + ["Operating System", "\u64cd\u4f5c\u7cfb\u7edf", "STEM"], + "computer_architecture": + ["Computer Architecture", "\u8ba1\u7b97\u673a\u7ec4\u6210", "STEM"], + "college_programming": + ["College Programming", "\u5927\u5b66\u7f16\u7a0b", "STEM"], + "college_physics": ["College Physics", "\u5927\u5b66\u7269\u7406", "STEM"], + "college_chemistry": + ["College Chemistry", "\u5927\u5b66\u5316\u5b66", "STEM"], + "advanced_mathematics": + ["Advanced Mathematics", "\u9ad8\u7b49\u6570\u5b66", "STEM"], + "probability_and_statistics": + ["Probability and Statistics", "\u6982\u7387\u7edf\u8ba1", "STEM"], + "discrete_mathematics": + ["Discrete Mathematics", "\u79bb\u6563\u6570\u5b66", "STEM"], + "electrical_engineer": [ + "Electrical Engineer", "\u6ce8\u518c\u7535\u6c14\u5de5\u7a0b\u5e08", + "STEM" + ], + "metrology_engineer": + ["Metrology Engineer", "\u6ce8\u518c\u8ba1\u91cf\u5e08", "STEM"], + "high_school_mathematics": + ["High School Mathematics", "\u9ad8\u4e2d\u6570\u5b66", "STEM"], + "high_school_physics": + ["High School Physics", "\u9ad8\u4e2d\u7269\u7406", "STEM"], + "high_school_chemistry": + ["High School Chemistry", "\u9ad8\u4e2d\u5316\u5b66", "STEM"], + "high_school_biology": [ + "High School Biology", "\u9ad8\u4e2d\u751f\u7269", "STEM" + ], + "middle_school_mathematics": [ + "Middle School Mathematics", "\u521d\u4e2d\u6570\u5b66", "STEM" + ], + "middle_school_biology": [ + "Middle School Biology", "\u521d\u4e2d\u751f\u7269", "STEM" + ], + "middle_school_physics": [ + "Middle School Physics", "\u521d\u4e2d\u7269\u7406", "STEM" + ], + "middle_school_chemistry": [ + "Middle School Chemistry", "\u521d\u4e2d\u5316\u5b66", "STEM" + ], + "veterinary_medicine": [ + "Veterinary Medicine", "\u517d\u533b\u5b66", "STEM" + ], + "college_economics": [ + "College Economics", "\u5927\u5b66\u7ecf\u6d4e\u5b66", "Social Science" + ], + "business_administration": [ + "Business Administration", "\u5de5\u5546\u7ba1\u7406", "Social Science" + ], + "marxism": [ + "Marxism", "\u9a6c\u514b\u601d\u4e3b\u4e49\u57fa\u672c\u539f\u7406", + "Social Science" + ], + "mao_zedong_thought": [ + "Mao Zedong Thought", + "\u6bdb\u6cfd\u4e1c\u601d\u60f3\u548c\u4e2d\u56fd\u7279\u8272\u793e\u4f1a\u4e3b\u4e49\u7406\u8bba\u4f53\u7cfb\u6982\u8bba", + "Social Science" + ], + "education_science": [ + "Education Science", "\u6559\u80b2\u5b66", "Social Science" + ], + "teacher_qualification": [ + "Teacher Qualification", "\u6559\u5e08\u8d44\u683c", "Social Science" + ], + "high_school_politics": [ + "High School Politics", "\u9ad8\u4e2d\u653f\u6cbb", "Social Science" + ], + "high_school_geography": [ + "High School Geography", "\u9ad8\u4e2d\u5730\u7406", "Social Science" + ], + "middle_school_politics": [ + "Middle School Politics", "\u521d\u4e2d\u653f\u6cbb", "Social Science" + ], + "middle_school_geography": [ + "Middle School Geography", "\u521d\u4e2d\u5730\u7406", "Social Science" + ], + "modern_chinese_history": + ["Modern Chinese History", "\u8fd1\u4ee3\u53f2\u7eb2\u8981", "Humanities"], + "ideological_and_moral_cultivation": [ + "Ideological and Moral Cultivation", + "\u601d\u60f3\u9053\u5fb7\u4fee\u517b\u4e0e\u6cd5\u5f8b\u57fa\u7840", + "Humanities" + ], + "logic": ["Logic", "\u903b\u8f91\u5b66", "Humanities"], + "law": ["Law", "\u6cd5\u5b66", "Humanities"], + "chinese_language_and_literature": [ + "Chinese Language and Literature", + "\u4e2d\u56fd\u8bed\u8a00\u6587\u5b66", "Humanities" + ], + "art_studies": ["Art Studies", "\u827a\u672f\u5b66", "Humanities"], + "professional_tour_guide": [ + "Professional Tour Guide", "\u5bfc\u6e38\u8d44\u683c", "Humanities" + ], + "legal_professional": [ + "Legal Professional", "\u6cd5\u5f8b\u804c\u4e1a\u8d44\u683c", + "Humanities" + ], + "high_school_chinese": [ + "High School Chinese", "\u9ad8\u4e2d\u8bed\u6587", "Humanities" + ], + "high_school_history": [ + "High School History", "\u9ad8\u4e2d\u5386\u53f2", "Humanities" + ], + "middle_school_history": [ + "Middle School History", "\u521d\u4e2d\u5386\u53f2", "Humanities" + ], + "civil_servant": ["Civil Servant", "\u516c\u52a1\u5458", "Other"], + "sports_science": ["Sports Science", "\u4f53\u80b2\u5b66", "Other"], + "plant_protection": [ + "Plant Protection", "\u690d\u7269\u4fdd\u62a4", "Other" + ], + "basic_medicine": ["Basic Medicine", "\u57fa\u7840\u533b\u5b66", "Other"], + "clinical_medicine": [ + "Clinical Medicine", "\u4e34\u5e8a\u533b\u5b66", "Other" + ], + "urban_and_rural_planner": [ + "Urban and Rural Planner", + "\u6ce8\u518c\u57ce\u4e61\u89c4\u5212\u5e08", "Other" + ], + "accountant": ["Accountant", "\u6ce8\u518c\u4f1a\u8ba1\u5e08", "Other"], + "fire_engineer": [ + "Fire Engineer", "\u6ce8\u518c\u6d88\u9632\u5de5\u7a0b\u5e08", "Other" + ], + "environmental_impact_assessment_engineer": [ + "Environmental Impact Assessment Engineer", + "\u73af\u5883\u5f71\u54cd\u8bc4\u4ef7\u5de5\u7a0b\u5e08", "Other" + ], + "tax_accountant": ["Tax Accountant", "\u7a0e\u52a1\u5e08", "Other"], + "physician": ["Physician", "\u533b\u5e08\u8d44\u683c", "Other"] +} +ceval_all_sets = list(ceval_subject_mapping.keys()) + +ceval_datasets = [] +for _split in ["val", "test"]: + for _name in ceval_all_sets: + _ch_name = ceval_subject_mapping[_name][1] + ceval_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template={ + answer: dict( + begin="", + round=[ + dict( + role="HUMAN", + prompt= + f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: " + ), + dict(role="BOT", prompt=answer), + ]) + for answer in ["A", "B", "C", "D"] + }, + ice_token="", + ), + retriever=dict(type=FixKRetriever), + inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]), + ) + + ceval_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + + ceval_datasets.append( + dict( + type=CEvalDataset, + path="./data/ceval/formal_ceval", + name=_name, + abbr="ceval-" + _name if _split == "val" else "ceval-test-" + + _name, + reader_cfg=dict( + input_columns=["question", "A", "B", "C", "D"], + output_column="answer", + train_split="dev", + test_split=_split), + infer_cfg=ceval_infer_cfg, + eval_cfg=ceval_eval_cfg, + )) + +del _split, _name, _ch_name diff --git a/configs/datasets/ceval/ceval_ppl_275812.py b/configs/datasets/ceval/ceval_ppl_275812.py new file mode 100644 index 00000000..212b5b33 --- /dev/null +++ b/configs/datasets/ceval/ceval_ppl_275812.py @@ -0,0 +1,188 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import FixKRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CEvalDataset + +ceval_subject_mapping = { + "computer_network": + ["Computer Network", "\u8ba1\u7b97\u673a\u7f51\u7edc", "STEM"], + "operating_system": + ["Operating System", "\u64cd\u4f5c\u7cfb\u7edf", "STEM"], + "computer_architecture": + ["Computer Architecture", "\u8ba1\u7b97\u673a\u7ec4\u6210", "STEM"], + "college_programming": + ["College Programming", "\u5927\u5b66\u7f16\u7a0b", "STEM"], + "college_physics": ["College Physics", "\u5927\u5b66\u7269\u7406", "STEM"], + "college_chemistry": + ["College Chemistry", "\u5927\u5b66\u5316\u5b66", "STEM"], + "advanced_mathematics": + ["Advanced Mathematics", "\u9ad8\u7b49\u6570\u5b66", "STEM"], + "probability_and_statistics": + ["Probability and Statistics", "\u6982\u7387\u7edf\u8ba1", "STEM"], + "discrete_mathematics": + ["Discrete Mathematics", "\u79bb\u6563\u6570\u5b66", "STEM"], + "electrical_engineer": [ + "Electrical Engineer", "\u6ce8\u518c\u7535\u6c14\u5de5\u7a0b\u5e08", + "STEM" + ], + "metrology_engineer": + ["Metrology Engineer", "\u6ce8\u518c\u8ba1\u91cf\u5e08", "STEM"], + "high_school_mathematics": + ["High School Mathematics", "\u9ad8\u4e2d\u6570\u5b66", "STEM"], + "high_school_physics": + ["High School Physics", "\u9ad8\u4e2d\u7269\u7406", "STEM"], + "high_school_chemistry": + ["High School Chemistry", "\u9ad8\u4e2d\u5316\u5b66", "STEM"], + "high_school_biology": [ + "High School Biology", "\u9ad8\u4e2d\u751f\u7269", "STEM" + ], + "middle_school_mathematics": [ + "Middle School Mathematics", "\u521d\u4e2d\u6570\u5b66", "STEM" + ], + "middle_school_biology": [ + "Middle School Biology", "\u521d\u4e2d\u751f\u7269", "STEM" + ], + "middle_school_physics": [ + "Middle School Physics", "\u521d\u4e2d\u7269\u7406", "STEM" + ], + "middle_school_chemistry": [ + "Middle School Chemistry", "\u521d\u4e2d\u5316\u5b66", "STEM" + ], + "veterinary_medicine": [ + "Veterinary Medicine", "\u517d\u533b\u5b66", "STEM" + ], + "college_economics": [ + "College Economics", "\u5927\u5b66\u7ecf\u6d4e\u5b66", "Social Science" + ], + "business_administration": [ + "Business Administration", "\u5de5\u5546\u7ba1\u7406", "Social Science" + ], + "marxism": [ + "Marxism", "\u9a6c\u514b\u601d\u4e3b\u4e49\u57fa\u672c\u539f\u7406", + "Social Science" + ], + "mao_zedong_thought": [ + "Mao Zedong Thought", + "\u6bdb\u6cfd\u4e1c\u601d\u60f3\u548c\u4e2d\u56fd\u7279\u8272\u793e\u4f1a\u4e3b\u4e49\u7406\u8bba\u4f53\u7cfb\u6982\u8bba", + "Social Science" + ], + "education_science": [ + "Education Science", "\u6559\u80b2\u5b66", "Social Science" + ], + "teacher_qualification": [ + "Teacher Qualification", "\u6559\u5e08\u8d44\u683c", "Social Science" + ], + "high_school_politics": [ + "High School Politics", "\u9ad8\u4e2d\u653f\u6cbb", "Social Science" + ], + "high_school_geography": [ + "High School Geography", "\u9ad8\u4e2d\u5730\u7406", "Social Science" + ], + "middle_school_politics": [ + "Middle School Politics", "\u521d\u4e2d\u653f\u6cbb", "Social Science" + ], + "middle_school_geography": [ + "Middle School Geography", "\u521d\u4e2d\u5730\u7406", "Social Science" + ], + "modern_chinese_history": + ["Modern Chinese History", "\u8fd1\u4ee3\u53f2\u7eb2\u8981", "Humanities"], + "ideological_and_moral_cultivation": [ + "Ideological and Moral Cultivation", + "\u601d\u60f3\u9053\u5fb7\u4fee\u517b\u4e0e\u6cd5\u5f8b\u57fa\u7840", + "Humanities" + ], + "logic": ["Logic", "\u903b\u8f91\u5b66", "Humanities"], + "law": ["Law", "\u6cd5\u5b66", "Humanities"], + "chinese_language_and_literature": [ + "Chinese Language and Literature", + "\u4e2d\u56fd\u8bed\u8a00\u6587\u5b66", "Humanities" + ], + "art_studies": ["Art Studies", "\u827a\u672f\u5b66", "Humanities"], + "professional_tour_guide": [ + "Professional Tour Guide", "\u5bfc\u6e38\u8d44\u683c", "Humanities" + ], + "legal_professional": [ + "Legal Professional", "\u6cd5\u5f8b\u804c\u4e1a\u8d44\u683c", + "Humanities" + ], + "high_school_chinese": [ + "High School Chinese", "\u9ad8\u4e2d\u8bed\u6587", "Humanities" + ], + "high_school_history": [ + "High School History", "\u9ad8\u4e2d\u5386\u53f2", "Humanities" + ], + "middle_school_history": [ + "Middle School History", "\u521d\u4e2d\u5386\u53f2", "Humanities" + ], + "civil_servant": ["Civil Servant", "\u516c\u52a1\u5458", "Other"], + "sports_science": ["Sports Science", "\u4f53\u80b2\u5b66", "Other"], + "plant_protection": [ + "Plant Protection", "\u690d\u7269\u4fdd\u62a4", "Other" + ], + "basic_medicine": ["Basic Medicine", "\u57fa\u7840\u533b\u5b66", "Other"], + "clinical_medicine": [ + "Clinical Medicine", "\u4e34\u5e8a\u533b\u5b66", "Other" + ], + "urban_and_rural_planner": [ + "Urban and Rural Planner", + "\u6ce8\u518c\u57ce\u4e61\u89c4\u5212\u5e08", "Other" + ], + "accountant": ["Accountant", "\u6ce8\u518c\u4f1a\u8ba1\u5e08", "Other"], + "fire_engineer": [ + "Fire Engineer", "\u6ce8\u518c\u6d88\u9632\u5de5\u7a0b\u5e08", "Other" + ], + "environmental_impact_assessment_engineer": [ + "Environmental Impact Assessment Engineer", + "\u73af\u5883\u5f71\u54cd\u8bc4\u4ef7\u5de5\u7a0b\u5e08", "Other" + ], + "tax_accountant": ["Tax Accountant", "\u7a0e\u52a1\u5e08", "Other"], + "physician": ["Physician", "\u533b\u5e08\u8d44\u683c", "Other"] +} +ceval_all_sets = list(ceval_subject_mapping.keys()) + +ceval_datasets = [] +for _split in ["val"]: + for _name in ceval_all_sets: + _ch_name = ceval_subject_mapping[_name][1] + ceval_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template={ + answer: dict( + begin="", + round=[ + dict( + role="HUMAN", + prompt= + f"以下是中国关于{_ch_name}考试的单项选择题，请选出其中的正确答案。\n{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: " + ), + dict(role="BOT", prompt=answer), + ]) + for answer in ["A", "B", "C", "D"] + }, + ice_token="", + ), + retriever=dict(type=FixKRetriever), + inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]), + ) + + ceval_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + + ceval_datasets.append( + dict( + type=CEvalDataset, + path="./data/ceval/formal_ceval", + name=_name, + abbr="ceval-" + _name if _split == "val" else "ceval-test-" + + _name, + reader_cfg=dict( + input_columns=["question", "A", "B", "C", "D"], + output_column="answer", + train_split="dev", + test_split=_split), + infer_cfg=ceval_infer_cfg, + eval_cfg=ceval_eval_cfg, + )) + +del _split, _name, _ch_name diff --git a/configs/datasets/commonsenseqa/commonsenseqa_ppl_2ca33c.py b/configs/datasets/commonsenseqa/commonsenseqa_ppl_2ca33c.py new file mode 100644 index 00000000..26324929 --- /dev/null +++ b/configs/datasets/commonsenseqa/commonsenseqa_ppl_2ca33c.py @@ -0,0 +1,50 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import MDLRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import commonsenseqaDataset + +commonsenseqa_reader_cfg = dict( + input_columns=['question', 'A', 'B', 'C', 'D', 'E'], + output_column='answerKey', + test_split='validation') + +_ice_template = dict( + type=PromptTemplate, + template={ + ans: dict( + begin='', + round=[ + dict(role="HUMAN", prompt="Question: {question}\nAnswer: "), + dict(role="BOT", prompt=ans_token), + ]) + for ans, ans_token in [["A", "{A}"], ["B", "{B}"], + ["C", "{C}"], ["D", "{D}"], + ["E", "{E}"]] + }, + ice_token='') + +commonsenseqa_infer_cfg = dict( + ice_template=_ice_template, + retriever=dict( + type=MDLRetriever, + ice_num=8, + candidate_num=30, + select_time=10, + seed=1, + batch_size=12, + ice_template=_ice_template), + inferencer=dict(type=PPLInferencer)) + +commonsenseqa_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +commonsenseqa_datasets = [ + dict( + type=commonsenseqaDataset, + path='commonsense_qa', + reader_cfg=commonsenseqa_reader_cfg, + infer_cfg=commonsenseqa_infer_cfg, + eval_cfg=commonsenseqa_eval_cfg) +] + +del _ice_template diff --git a/configs/datasets/crowspairs/crowspairs_ppl.py b/configs/datasets/crowspairs/crowspairs_ppl.py new file mode 100644 index 00000000..532ff292 --- /dev/null +++ b/configs/datasets/crowspairs/crowspairs_ppl.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .crowspairs_ppl_f60797 import crowspairs_datasets # noqa: F401, F403 diff --git a/configs/datasets/drop/drop_gen_e54fe7.py b/configs/datasets/drop/drop_gen_e54fe7.py new file mode 100644 index 00000000..01b8d763 --- /dev/null +++ b/configs/datasets/drop/drop_gen_e54fe7.py @@ -0,0 +1,42 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import EMEvaluator +from opencompass.datasets import dropDataset + +drop_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template= + '''Text: In the county, the population was spread out with 23.50% under the age of 18, 8.70% from 18 to 24, 29.70% from 25 to 44, 24.70% from 45 to 64, and 13.30% who were 65 years of age or older. +Question: How many more percent are under the age of 18 compared to the 18 to 24 group? +Anawer: According to the text, 23.5% are under the age of 18, and 8.7% are from ages 18 to 24. 23.5%-8.7%=14.8%. So the answer is 14.8. + +Text: Playing in their second straight Thanksgiving game, the Eagles struggled especially on defense, where they were unable to stop the much-hyped Lions offense. The worst of it all was how unproven rookie Eric Rowe was tasked with covering wide receiver Calvin Johnson, leading to Johnson catching 3 touchdowns. Stafford’s five passing touchdowns, including three of them to Johnson was too much for the Eagles to overcome and for the second consecutive time this season, the Eagles gave up 45 points in a game. With the loss, the Eagles drop to 4-7 on the season and 6-1 when playing on Thanksgiving. +Question: How many TD passes did Stafford throw other than to Johnson? +Anawer: According to the text, Stafford threw 5 TD passes, 3 of which were to Johnson. 5-3=2. So the answer is 2. + +Text: {prompt} +Question: {question} +Anawer:'''), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +drop_eval_cfg = dict( + evaluator=dict(type=EMEvaluator), pred_postprocessor=dict( + type='gsm8k')) # use the same processor to find answer + +drop_datasets = [ + dict( + abbr='drop', + type=dropDataset, + path='drop', + reader_cfg=dict( + input_columns=['prompt'], + output_column='answers', + train_split='validation', + test_split='validation', + ), + infer_cfg=drop_infer_cfg, + eval_cfg=drop_eval_cfg) +] diff --git a/configs/datasets/flores/flores_gen_45df8b.py b/configs/datasets/flores/flores_gen_45df8b.py new file mode 100644 index 00000000..d340d3b2 --- /dev/null +++ b/configs/datasets/flores/flores_gen_45df8b.py @@ -0,0 +1,154 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import TopkRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import BleuEvaluator +from opencompass.datasets import FloresFirst100Dataset + +_flores_lang_map = [ + ["eng", "eng_Latn", "English", "Indo-European-Germanic"], + ["afr", "afr_Latn", "Afrikaans", "Indo-European-Germanic"], + ["dan", "dan_Latn", "Danish", "Indo-European-Germanic"], + ["deu", "deu_Latn", "German", "Indo-European-Germanic"], + ["isl", "isl_Latn", "Icelandic", "Indo-European-Germanic"], + ["ltz", "ltz_Latn", "Luxembourgish", "Indo-European-Germanic"], + ["nld", "nld_Latn", "Dutch", "Indo-European-Germanic"], + ["nob", "nob_Latn", "Norwegian", "Indo-European-Germanic"], + ["swe", "swe_Latn", "Swedish", "Indo-European-Germanic"], + ["ast", "ast_Latn", "Asturian", "Indo-European-Romance"], + ["cat", "cat_Latn", "Catalan", "Indo-European-Romance"], + ["fra", "fra_Latn", "French", "Indo-European-Romance"], + ["glg", "glg_Latn", "Galician", "Indo-European-Romance"], + ["oci", "oci_Latn", "Occitan", "Indo-European-Romance"], + ["por", "por_Latn", "Portuguese", "Indo-European-Romance"], + ["ron", "ron_Latn", "Romanian", "Indo-European-Romance"], + ["spa", "spa_Latn", "Spanish", "Indo-European-Romance"], + ["bel", "bel_Cyrl", "Belarusian", "Indo-European-Slavic"], + ["bos", "bos_Latn", "Bosnian", "Indo-European-Slavic"], + ["bul", "bul_Cyrl", "Bulgarian", "Indo-European-Slavic"], + ["ces", "ces_Latn", "Czech", "Indo-European-Slavic"], + ["hrv", "hrv_Latn", "Croatian", "Indo-European-Slavic"], + ["mkd", "mkd_Cyrl", "Macedonian", "Indo-European-Slavic"], + ["pol", "pol_Latn", "Polish", "Indo-European-Slavic"], + ["rus", "rus_Cyrl", "Russian", "Indo-European-Slavic"], + ["slk", "slk_Latn", "Slovak", "Indo-European-Slavic"], + ["slv", "slv_Latn", "Slovenian", "Indo-European-Slavic"], + ["srp", "srp_Cyrl", "Serbian", "Indo-European-Slavic"], + ["ukr", "ukr_Cyrl", "Ukrainian", "Indo-European-Slavic"], + ["asm", "asm_Beng", "Assamese", "Indo-European-Indo-Aryan"], + ["ben", "ben_Beng", "Bengali", "Indo-European-Indo-Aryan"], + ["guj", "guj_Gujr", "Gujarati", "Indo-European-Indo-Aryan"], + ["hin", "hin_Deva", "Hindi", "Indo-European-Indo-Aryan"], + ["mar", "mar_Deva", "Marathi", "Indo-European-Indo-Aryan"], + ["npi", "npi_Deva", "Nepali", "Indo-European-Indo-Aryan"], + ["ory", "ory_Orya", "Oriya", "Indo-European-Indo-Aryan"], + ["pan", "pan_Guru", "Punjabi", "Indo-European-Indo-Aryan"], + ["snd", "snd_Arab", "Sindhi", "Indo-European-Indo-Aryan"], + ["urd", "urd_Arab", "Urdu", "Indo-European-Indo-Aryan"], + ["ckb", "ckb_Arab", "Kurdish", "Indo-European-Other"], + ["cym", "cym_Latn", "Welsh", "Indo-European-Other"], + ["ell", "ell_Grek", "Greek", "Indo-European-Other"], + ["fas", "pes_Arab", "Persian", "Indo-European-Other"], + ["gle", "gle_Latn", "Irish", "Indo-European-Other"], + ["hye", "hye_Armn", "Armenian", "Indo-European-Other"], + ["ita", "ita_Latn", "Italian", "Indo-European-Other"], + ["lav", "lvs_Latn", "Latvian", "Indo-European-Other"], + ["lit", "lit_Latn", "Lithuanian", "Indo-European-Other"], + ["pus", "pbt_Arab", "Pashto", "Indo-European-Other"], + ["tgk", "tgk_Cyrl", "Tajik", "Indo-European-Other"], + ["ceb", "ceb_Latn", "Cebuano", "Austronesian"], + ["ind", "ind_Latn", "Indonesian", "Austronesian"], + ["jav", "jav_Latn", "Javanese", "Austronesian"], + ["mri", "mri_Latn", "Maori", "Austronesian"], + ["msa", "zsm_Latn", "Malay", "Austronesian"], + ["tgl", "tgl_Latn", "Tagalog", "Austronesian"], + ["ibo", "ibo_Latn", "Igbo", "Atlantic-Congo"], + ["kam", "kam_Latn", "Kamba", "Atlantic-Congo"], + ["kea", "kea_Latn", "Kabuverdianu", "Atlantic-Congo"], + ["lin", "lin_Latn", "Lingala", "Atlantic-Congo"], + ["lug", "lug_Latn", "Luganda", "Atlantic-Congo"], + ["nso", "nso_Latn", "Northern Sotho", "Atlantic-Congo"], + ["nya", "nya_Latn", "Nyanja", "Atlantic-Congo"], + ["sna", "sna_Latn", "Shona", "Atlantic-Congo"], + ["swh", "swh_Latn", "Swahili", "Atlantic-Congo"], + ["umb", "umb_Latn", "Umbundu", "Atlantic-Congo"], + ["wol", "wol_Latn", "Wolof", "Atlantic-Congo"], + ["xho", "xho_Latn", "Xhosa", "Atlantic-Congo"], + ["yor", "yor_Latn", "Yoruba", "Atlantic-Congo"], + ["zul", "zul_Latn", "Zulu", "Atlantic-Congo"], + ["amh", "amh_Ethi", "Amharic", "Afro-Asiatic"], + ["ara", "arb_Arab", "Arabic", "Afro-Asiatic"], + ["ful", "fuv_Latn", "Fulah", "Afro-Asiatic"], + ["mlt", "mlt_Latn", "Maltese", "Afro-Asiatic"], + ["orm", "gaz_Latn", "Oromo", "Afro-Asiatic"], + ["som", "som_Latn", "Somali", "Afro-Asiatic"], + ["azj", "azj_Latn", "Azerbaijani", "Turkic"], + ["kaz", "kaz_Cyrl", "Kazakh", "Turkic"], + ["kir", "kir_Cyrl", "Kyrgyz", "Turkic"], + ["tur", "tur_Latn", "Turkish", "Turkic"], + ["uzb", "uzn_Latn", "Uzbek", "Turkic"], + ["kan", "kan_Knda", "Kannada", "Dravidian"], + ["mal", "mal_Mlym", "Malayalam", "Dravidian"], + ["tam", "tam_Taml", "Tamil", "Dravidian"], + ["tel", "tel_Telu", "Telugu", "Dravidian"], + ["mya", "mya_Mymr", "Burmese", "Sino-Tibetan"], + ["zho_simpl", "zho_Hans", "Chinese (Simpl)", "Sino-Tibetan"], + ["zho_trad", "zho_Hant", "Chinese (Trad)", "Sino-Tibetan"], + ["est", "est_Latn", "Estonian", "Other"], + ["fin", "fin_Latn", "Finnish", "Other"], + ["hau", "hau_Latn", "Hausa", "Other"], + ["heb", "heb_Hebr", "Hebrew", "Other"], + ["hun", "hun_Latn", "Hungarian", "Other"], + ["jpn", "jpn_Jpan", "Japanese", "Other"], + ["kat", "kat_Geor", "Georgian", "Other"], + ["khm", "khm_Khmr", "Khmer", "Other"], + ["kor", "kor_Hang", "Korean", "Other"], + ["lao", "lao_Laoo", "Lao", "Other"], + ["luo", "luo_Latn", "Luo", "Other"], + ["mon", "khk_Cyrl", "Mongolian", "Other"], + ["tha", "tha_Thai", "Thai", "Other"], + ["vie", "vie_Latn", "Vietnamese", "Other"], +] +flores_lang_map = {i[0]: i for i in _flores_lang_map} +_flores_subtasks = [f"eng-{i}" for i in flores_lang_map if i != "eng" + ] + [f"{i}-eng" for i in flores_lang_map if i != "eng"] + +flores_datasets = [] +for _flores_subtask in _flores_subtasks: + _src, _tgt = _flores_subtask.split("-") + _, _flores_source, _src_inst, _ = flores_lang_map[_src] + _, _flores_target, _tgt_inst, _ = flores_lang_map[_tgt] + + flores_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template=f"{{sentence_{_flores_source}}} = {{sentence_{_flores_target}}}" if _flores_subtask != "zho_simpl-eng" + else f"Chinese: {{sentence_{_flores_source}}}\nEnglish: {{sentence_{_flores_target}}}", + ice_token="", + ), + retriever=dict(type=TopkRetriever, ice_num=8), + inferencer=dict(type=GenInferencer), + ) + flores_eval_cfg = dict( + evaluator=dict(type=BleuEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="flores"), + dataset_postprocessor=dict(type="flores"), + ) + if _tgt == "zho_simpl": + flores_eval_cfg["pred_postprocessor"] = dict(type="flores-chinese") + flores_eval_cfg["dataset_postprocessor"] = dict(type="flores-chinese") + flores_datasets.append( + dict( + type=FloresFirst100Dataset, + abbr=f"flores_100_{_src}-{_tgt}", + name=f"{_flores_source}-{_flores_target}", + reader_cfg=dict( + input_columns=f"sentence_{_flores_source}", + output_column=f"sentence_{_flores_target}", + train_split="dev", + test_split="devtest"), + infer_cfg=flores_infer_cfg.copy(), + eval_cfg=flores_eval_cfg.copy(), + )) + +del _flores_lang_map, _flores_subtask, _src, _tgt, _, _flores_source, _src_inst, _flores_target, _tgt_inst diff --git a/configs/datasets/flores/flores_gen_8eb9ca.py b/configs/datasets/flores/flores_gen_8eb9ca.py new file mode 100644 index 00000000..8e76644b --- /dev/null +++ b/configs/datasets/flores/flores_gen_8eb9ca.py @@ -0,0 +1,163 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import TopkRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import BleuEvaluator +from opencompass.datasets import FloresFirst100Dataset + +_flores_lang_map = [ + ["eng", "eng_Latn", "English", "Indo-European-Germanic"], + ["afr", "afr_Latn", "Afrikaans", "Indo-European-Germanic"], + ["dan", "dan_Latn", "Danish", "Indo-European-Germanic"], + ["deu", "deu_Latn", "German", "Indo-European-Germanic"], + ["isl", "isl_Latn", "Icelandic", "Indo-European-Germanic"], + ["ltz", "ltz_Latn", "Luxembourgish", "Indo-European-Germanic"], + ["nld", "nld_Latn", "Dutch", "Indo-European-Germanic"], + ["nob", "nob_Latn", "Norwegian", "Indo-European-Germanic"], + ["swe", "swe_Latn", "Swedish", "Indo-European-Germanic"], + ["ast", "ast_Latn", "Asturian", "Indo-European-Romance"], + ["cat", "cat_Latn", "Catalan", "Indo-European-Romance"], + ["fra", "fra_Latn", "French", "Indo-European-Romance"], + ["glg", "glg_Latn", "Galician", "Indo-European-Romance"], + ["oci", "oci_Latn", "Occitan", "Indo-European-Romance"], + ["por", "por_Latn", "Portuguese", "Indo-European-Romance"], + ["ron", "ron_Latn", "Romanian", "Indo-European-Romance"], + ["spa", "spa_Latn", "Spanish", "Indo-European-Romance"], + ["bel", "bel_Cyrl", "Belarusian", "Indo-European-Slavic"], + ["bos", "bos_Latn", "Bosnian", "Indo-European-Slavic"], + ["bul", "bul_Cyrl", "Bulgarian", "Indo-European-Slavic"], + ["ces", "ces_Latn", "Czech", "Indo-European-Slavic"], + ["hrv", "hrv_Latn", "Croatian", "Indo-European-Slavic"], + ["mkd", "mkd_Cyrl", "Macedonian", "Indo-European-Slavic"], + ["pol", "pol_Latn", "Polish", "Indo-European-Slavic"], + ["rus", "rus_Cyrl", "Russian", "Indo-European-Slavic"], + ["slk", "slk_Latn", "Slovak", "Indo-European-Slavic"], + ["slv", "slv_Latn", "Slovenian", "Indo-European-Slavic"], + ["srp", "srp_Cyrl", "Serbian", "Indo-European-Slavic"], + ["ukr", "ukr_Cyrl", "Ukrainian", "Indo-European-Slavic"], + ["asm", "asm_Beng", "Assamese", "Indo-European-Indo-Aryan"], + ["ben", "ben_Beng", "Bengali", "Indo-European-Indo-Aryan"], + ["guj", "guj_Gujr", "Gujarati", "Indo-European-Indo-Aryan"], + ["hin", "hin_Deva", "Hindi", "Indo-European-Indo-Aryan"], + ["mar", "mar_Deva", "Marathi", "Indo-European-Indo-Aryan"], + ["npi", "npi_Deva", "Nepali", "Indo-European-Indo-Aryan"], + ["ory", "ory_Orya", "Oriya", "Indo-European-Indo-Aryan"], + ["pan", "pan_Guru", "Punjabi", "Indo-European-Indo-Aryan"], + ["snd", "snd_Arab", "Sindhi", "Indo-European-Indo-Aryan"], + ["urd", "urd_Arab", "Urdu", "Indo-European-Indo-Aryan"], + ["ckb", "ckb_Arab", "Kurdish", "Indo-European-Other"], + ["cym", "cym_Latn", "Welsh", "Indo-European-Other"], + ["ell", "ell_Grek", "Greek", "Indo-European-Other"], + ["fas", "pes_Arab", "Persian", "Indo-European-Other"], + ["gle", "gle_Latn", "Irish", "Indo-European-Other"], + ["hye", "hye_Armn", "Armenian", "Indo-European-Other"], + ["ita", "ita_Latn", "Italian", "Indo-European-Other"], + ["lav", "lvs_Latn", "Latvian", "Indo-European-Other"], + ["lit", "lit_Latn", "Lithuanian", "Indo-European-Other"], + ["pus", "pbt_Arab", "Pashto", "Indo-European-Other"], + ["tgk", "tgk_Cyrl", "Tajik", "Indo-European-Other"], + ["ceb", "ceb_Latn", "Cebuano", "Austronesian"], + ["ind", "ind_Latn", "Indonesian", "Austronesian"], + ["jav", "jav_Latn", "Javanese", "Austronesian"], + ["mri", "mri_Latn", "Maori", "Austronesian"], + ["msa", "zsm_Latn", "Malay", "Austronesian"], + ["tgl", "tgl_Latn", "Tagalog", "Austronesian"], + ["ibo", "ibo_Latn", "Igbo", "Atlantic-Congo"], + ["kam", "kam_Latn", "Kamba", "Atlantic-Congo"], + ["kea", "kea_Latn", "Kabuverdianu", "Atlantic-Congo"], + ["lin", "lin_Latn", "Lingala", "Atlantic-Congo"], + ["lug", "lug_Latn", "Luganda", "Atlantic-Congo"], + ["nso", "nso_Latn", "Northern Sotho", "Atlantic-Congo"], + ["nya", "nya_Latn", "Nyanja", "Atlantic-Congo"], + ["sna", "sna_Latn", "Shona", "Atlantic-Congo"], + ["swh", "swh_Latn", "Swahili", "Atlantic-Congo"], + ["umb", "umb_Latn", "Umbundu", "Atlantic-Congo"], + ["wol", "wol_Latn", "Wolof", "Atlantic-Congo"], + ["xho", "xho_Latn", "Xhosa", "Atlantic-Congo"], + ["yor", "yor_Latn", "Yoruba", "Atlantic-Congo"], + ["zul", "zul_Latn", "Zulu", "Atlantic-Congo"], + ["amh", "amh_Ethi", "Amharic", "Afro-Asiatic"], + ["ara", "arb_Arab", "Arabic", "Afro-Asiatic"], + ["ful", "fuv_Latn", "Fulah", "Afro-Asiatic"], + ["mlt", "mlt_Latn", "Maltese", "Afro-Asiatic"], + ["orm", "gaz_Latn", "Oromo", "Afro-Asiatic"], + ["som", "som_Latn", "Somali", "Afro-Asiatic"], + ["azj", "azj_Latn", "Azerbaijani", "Turkic"], + ["kaz", "kaz_Cyrl", "Kazakh", "Turkic"], + ["kir", "kir_Cyrl", "Kyrgyz", "Turkic"], + ["tur", "tur_Latn", "Turkish", "Turkic"], + ["uzb", "uzn_Latn", "Uzbek", "Turkic"], + ["kan", "kan_Knda", "Kannada", "Dravidian"], + ["mal", "mal_Mlym", "Malayalam", "Dravidian"], + ["tam", "tam_Taml", "Tamil", "Dravidian"], + ["tel", "tel_Telu", "Telugu", "Dravidian"], + ["mya", "mya_Mymr", "Burmese", "Sino-Tibetan"], + ["zho_simpl", "zho_Hans", "Chinese (Simpl)", "Sino-Tibetan"], + ["zho_trad", "zho_Hant", "Chinese (Trad)", "Sino-Tibetan"], + ["est", "est_Latn", "Estonian", "Other"], + ["fin", "fin_Latn", "Finnish", "Other"], + ["hau", "hau_Latn", "Hausa", "Other"], + ["heb", "heb_Hebr", "Hebrew", "Other"], + ["hun", "hun_Latn", "Hungarian", "Other"], + ["jpn", "jpn_Jpan", "Japanese", "Other"], + ["kat", "kat_Geor", "Georgian", "Other"], + ["khm", "khm_Khmr", "Khmer", "Other"], + ["kor", "kor_Hang", "Korean", "Other"], + ["lao", "lao_Laoo", "Lao", "Other"], + ["luo", "luo_Latn", "Luo", "Other"], + ["mon", "khk_Cyrl", "Mongolian", "Other"], + ["tha", "tha_Thai", "Thai", "Other"], + ["vie", "vie_Latn", "Vietnamese", "Other"], +] +flores_lang_map = {i[0]: i for i in _flores_lang_map} +_flores_subtasks = [f"eng-{i}" for i in flores_lang_map if i != "eng" + ] + [f"{i}-eng" for i in flores_lang_map if i != "eng"] + +flores_datasets = [] +for _flores_subtask in _flores_subtasks: + _src, _tgt = _flores_subtask.split("-") + _, _flores_source, _src_inst, _ = flores_lang_map[_src] + _, _flores_target, _tgt_inst, _ = flores_lang_map[_tgt] + + flores_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template=dict( + begin="", + round=[ + dict( + role="HUMAN", + prompt= + f"Translate the following {_src_inst} statements to {_tgt_inst}.\n{{sentence_{_flores_source}}}" + ), + dict(role="BOT", prompt=f"{{sentence_{_flores_target}}}"), + ], + ), + ice_token="", + ), + retriever=dict(type=TopkRetriever, ice_num=8), + inferencer=dict(type=GenInferencer), + ) + flores_eval_cfg = dict( + evaluator=dict(type=BleuEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="flores"), + dataset_postprocessor=dict(type="flores"), + ) + if _tgt == "zho_simpl": + flores_eval_cfg["pred_postprocessor"] = dict(type="flores-chinese") + flores_eval_cfg["dataset_postprocessor"] = dict(type="flores-chinese") + flores_datasets.append( + dict( + type=FloresFirst100Dataset, + abbr=f"flores_100_{_src}-{_tgt}", + name=f"{_flores_source}-{_flores_target}", + reader_cfg=dict( + input_columns=f"sentence_{_flores_source}", + output_column=f"sentence_{_flores_target}", + train_split="dev", + test_split="devtest"), + infer_cfg=flores_infer_cfg.copy(), + eval_cfg=flores_eval_cfg.copy(), + )) + +del _flores_lang_map, _flores_subtask, _src, _tgt, _, _flores_source, _src_inst, _flores_target, _tgt_inst diff --git a/configs/datasets/glm/GaokaoBench.py b/configs/datasets/glm/GaokaoBench.py new file mode 100644 index 00000000..f91def26 --- /dev/null +++ b/configs/datasets/glm/GaokaoBench.py @@ -0,0 +1,368 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer, GLMChoiceInferencer +from opencompass.datasets import GaokaoBenchDataset + +MCQ_TMPL = """\ +请你做一道{type}。 +请你一步一步思考并将思考过程写在[解析]和之间。你将从A，B，C，D中选出正确的答案，并写在【答案]和之间。 +例如：[答案]: A +完整的题目回答的格式如下： +回答：[解析] ... +[答案] ... +请你严格按照上述格式作答。 +题目如下：{{question}} +回答：""" + +MULTI_MCQ_TMPL = """\ +请你做一道{type}。 +请你一步一步思考。每一题你将从A，B，C，D中选出正确的答案，并写在[答案]和之间。 +例如：（1）[答案] A +(2)[答案] B +请你严格按照上述格式作答。 +题目如下：{{question}} +回答：""" + +CLOZE_TMPL = """\ +请你做一道{type}。 +请你一步一步思考。将符合题意的五个选项的字母写在[答案]和之间。 +例如：[答案] A B C D E +请严格按照上述格式作答。 +题目如下：{{question}} +回答：""" + +_MCQ_prompts = [ + { + "type": "single_choice", + "keyword": "2010-2022_Math_II_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='数学选择题'), + "comment": "" + }, + { + "type": "single_choice", + "keyword": "2010-2022_Math_I_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='数学选择题'), + "comment": "" + }, + { + "type": "single_choice", + "keyword": "2010-2022_History_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='历史选择题'), + }, + { + "type": "single_choice", + "keyword": "2010-2022_Biology_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='生物选择题'), + }, + { + "type": "single_choice", + "keyword": "2010-2022_Political_Science_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='政治选择题'), + }, + { + "type": "multi_choice", + "keyword": "2010-2022_Physics_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='物理选择题'), + }, + { + "type": "single_choice", + "keyword": "2010-2022_Chemistry_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='化学选择题'), + }, + { + "type": "single_choice", + "keyword": "2010-2013_English_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='英语选择题'), + }, + { + "type": "multi_question_choice", + "keyword": "2010-2022_Chinese_Modern_Lit", + "prefix_prompt": MULTI_MCQ_TMPL.format(type='语文阅读理解题，其中包含三个小题'), + }, + { + "type": "multi_question_choice", + "keyword": "2010-2022_English_Fill_in_Blanks", + "prefix_prompt": MULTI_MCQ_TMPL.format(type='英语完形填空题,其中包含二十个小题'), + }, + { + "type": "five_out_of_seven", + "keyword": "2012-2022_English_Cloze_Test", + "prefix_prompt": CLOZE_TMPL.format(type='英语完形填空题'), + }, + { + "type": "multi_question_choice", + "keyword": "2010-2022_Geography_MCQs", + "prefix_prompt": MULTI_MCQ_TMPL.format(type='地理选择题'), + }, + { + "type": "multi_question_choice", + "keyword": "2010-2022_English_Reading_Comp", + "prefix_prompt": MULTI_MCQ_TMPL.format(type='英语阅读理解题，其中包含三到五个小题。'), + }, + { + "type": "multi_question_choice", + "keyword": "2010-2022_Chinese_Lang_and_Usage_MCQs", + "prefix_prompt": MCQ_TMPL.format(type='语文选择题'), + }, +] + +_FBQ_prompts = [{ + "type": "cloze", + "keyword": "2010-2022_Math_I_Fill-in-the-Blank", + "prefix_prompt": + "请解答下面的数学填空题\n仔细阅读题目，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间。\n完整的题目回答格式如下：\n【解析】 ...\n【答案】...\n请你严格按照上述格式作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": "2010-2022_Math_II_Fill-in-the-Blank", + "prefix_prompt": + "请解答下面的数学填空题\n仔细阅读题目，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间。\n完整的题目回答格式如下：\n【解析】 ...\n【答案】...\n请你严格按照上述格式作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": + "2010-2022_Chinese_Language_Famous_Passages_and_Sentences_Dictation", + "prefix_prompt": + "请回答下面的语文填空题\n请你仔细阅读题目，先找到题目对应的中国名篇，再从名篇中找到合适的句子填写到题目的空白处。请你将思考过程写在【解析】和之间，将最终答案写在【答案】和之间。\n完整的题目回答格式如下：\n（1）【解析】 ...\n【答案】...\n（2）【解析】 ...\n【答案】...\n请严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" +}, { + "type": "cloze", + "keyword": "2014-2022_English_Language_Cloze_Passage", + "prefix_prompt": + "请回答下面的英语短文填词题\n仔细阅读题目，空白处请填入一个适当单词或者括号内单词的正确形式。请你一步步思考，将思考过程写在【解析】和之间，将最终答案写在【答案】和之间。\n完整的题目回答格式如下：\n（1）【解析】 ...\n【答案】...\n（2）【解析】 ...\n【答案】...\n请严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" +}] +_OEQ_prompts = [ + { + "type": "subjective", + "keyword": "2010-2022_Geography_Open-ended_Questions", + "prefix_prompt": + "请解答下面的地理解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。你的答案请写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下：", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chemistry_Open-ended_Questions", + "prefix_prompt": + "请解答下面的化学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Math_I_Open-ended_Questions", + "prefix_prompt": + "请解答下面的数学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间，答案需要有完整的解题步骤。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_History_Open-ended_Questions", + "prefix_prompt": + "请解答下面的历史解答题\n仔细阅读材料和题目，并充分结合你已有的知识，解答其中的问题。请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Biology_Open-ended_Questions", + "prefix_prompt": + "请解答下面的生物解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间,同一小题的答案用\t分隔开。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\t...\n (2)【解析】 ...\n【答案】...\t...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Math_II_Open-ended_Questions", + "prefix_prompt": + "请解答下面的数学解答题\n仔细阅读题目并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间，答案需要有完整的解题步骤。\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Physics_Open-ended_Questions", + "prefix_prompt": + "请解答下面的物理解答题，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将思考过程写在【解析】和之间。请把你的最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【解析】 ...\n【答案】 ...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答。如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Political_Science_Open-ended_Questions", + "prefix_prompt": + "请解答下面的政治解答题\n仔细阅读材料和题目，并充分结合你已有的知识，解答其中的问题，请你一步步思考并将思考过程写在【解析】和之间。请把你的答案写在【答案】和之间\n完整的题目回答格式如下：\n(1)【解析】 ...\n【答案】...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "correction", + "keyword": "2012-2022_English_Language_Error_Correction", + "prefix_prompt": + "请解答下面的英语短文改错题，仔细阅读题目并充分结合你你已有的知识，找出其中10处需要改动的地方。请你一步步思考，把修改后的短文写在【答案】和之间。\n完整的题目回答格式如下：【答案】 ...\n 请你严格按照上述格式作答。\n题目如下:", + # "prefix_prompt": [ + # "请解答下面的英语短文改错题，仔细阅读题目并充分结合你你已有的知识，找出其中10处需要改动的地方。请你一步步思考，把修改后的短文写在【答案】和之间。\n完整的题目回答格式如下：【答案】 ...\n 请你严格按照上述格式作答。\n题目如下:", + # "请比较下面两篇短文，找到第二篇和第一篇的10处不同，每处不同只涉及一个单词，请将结果写在【答案】和之间。例如：【答案】1. 将play改为plays\n 2.增加了the\n ... \n 完整的题目回答格式如下：【答案】(1) ... \n (2) ...\n ...(10) ...\n\n请你严格按照上述格式作答。\n短文如下:" + # ], + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Ancient_Poetry_Reading", + "prefix_prompt": + "请解答下面的语文古代诗歌阅读题，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Practical_Text_Reading", + "prefix_prompt": + "请解答下面的语文实用类文本阅读，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Literary_Text_Reading", + "prefix_prompt": + "请解答下面的语文文学类文本阅读，仔细阅读题目，注意其中可能含有单选题和多选题。请你一步步思考并将最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答,如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": "2010-2022_Chinese_Language_Classical_Chinese_Reading", + "prefix_prompt": + "请解答下面的语文文言文阅读，仔细阅读题目，前三题是单选题，最后一题要将文言文翻译为现代汉语。请你一步步思考并把最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。翻译题把翻译后的现代汉语句子写在【答案】后面，例如”【答案】今天天气很好 ”\n完整的题目回答格式如下：（1）[答案】 ...\n (2)【答案】...\n请你严格按照上述格式作答，如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + }, + { + "type": "subjective", + "keyword": + "2010-2022_Chinese_Language_Language_and_Writing_Skills_Open-ended_Questions", + "prefix_prompt": + "请解答下面的语文解答题，仔细阅读题目，注意其中可能含有选择题。请你一步步思考并将思考过程写在【解析】和之间。请把你的最终答案写在【答案】和之间。选择题你要从选项中选出符合题意的答案，例如“【答案】A ”。\n完整的题目回答格式如下：（1）【解析】 ...\n【答案】 ...\n (2)【解析】 ...\n【答案】...\n请你严格按照上述格式作答。如果不止一道题，请分别作答。\n题目如下:", + "comment": "" + } +] + +gaokao_bench_datasets = [] +for _folder, _prompts in [ + ("Multiple-choice_Questions", _MCQ_prompts), + ("Fill-in-the-blank_Questions", _FBQ_prompts), + ("Open-ended_Questions", _OEQ_prompts), +]: + for _p in _prompts: + if _p['type'] == "single_choice": + continue + _reader_cfg = { + "input_columns": ['question'], + "output_column": 'answer', + } + _infer_cfg = { + "ice_template": { + "type": PromptTemplate, + "template": { + "round": [{ + "role": + "HUMAN", + "prompt": + _p['prefix_prompt'].format(question='') + }] + }, + "column_token_map": { + "question": "" + }, + "ice_token": "" + }, + "retriever": { + "type": ZeroRetriever + }, + "inferencer": { + "type": GenInferencer + } + } + _eval_cfg = { + "evaluator": { + "type": "GaokaoBenchEvaluator" + "_" + _p['type'], + }, + "pred_role": "BOT", + } + _base_path = './data/GAOKAO-BENCH/data' + _dataset = { + "type": GaokaoBenchDataset, + "abbr": "GaokaoBench_" + _p['keyword'], + "path": _base_path + '/' + _folder + '/' + _p['keyword'] + ".json", + "reader_cfg": _reader_cfg, + "infer_cfg": _infer_cfg, + "eval_cfg": _eval_cfg, + } + + gaokao_bench_datasets.append(_dataset) + +# Single choice dataset +_folder = "Multiple-choice_Questions" +for _p in _MCQ_prompts: + if _p['type'] != "single_choice": + continue + _reader_cfg = { + "input_columns": ['question'], + "output_column": 'answer', + } + _infer_cfg = { + "ice_template": { + "type": PromptTemplate, + "template": { + answer: { + "round": [{ + "role": "HUMAN", + "prompt": _p['prefix_prompt'] + '' + }, { + "role": "BOT", + "prompt": f"【答案】{answer} " + }] + } + for answer in ['A', 'B', 'C', 'D'] + }, + "column_token_map": { + "question": "" + }, + "ice_token": "" + }, + "prompt_template": { + "type": PromptTemplate, + "template": { + "round": [{ + "role": "HUMAN", + "prompt": _p['prefix_prompt'] + '' + }, { + "role": "BOT", + "prompt": "【答案】(" + }] + }, + "column_token_map": { + "question": "" + }, + }, + "retriever": { + "type": ZeroRetriever + }, + "inferencer": { + "type": GLMChoiceInferencer, + "choices": ['A', 'B', 'C', 'D'], + } + } + _eval_cfg = { + "evaluator": { + "type": "GaokaoBenchEvaluator" + "_" + _p['type'], + }, + "pred_role": "BOT", + } + _base_path = './data/GAOKAO-BENCH/data' + _dataset = { + "type": GaokaoBenchDataset, + "abbr": "GaokaoBench_" + _p['keyword'], + "path": _base_path + '/' + _folder + '/' + _p['keyword'] + ".json", + "reader_cfg": _reader_cfg, + "infer_cfg": _infer_cfg, + "eval_cfg": _eval_cfg, + } + + gaokao_bench_datasets.append(_dataset) + +_temporary_variables = [k for k in globals() if k.startswith('_')] +for _t in _temporary_variables: + del globals()[_t] +del _temporary_variables, _t diff --git a/configs/datasets/glm/_all_.py b/configs/datasets/glm/_all_.py new file mode 100644 index 00000000..69eb3620 --- /dev/null +++ b/configs/datasets/glm/_all_.py @@ -0,0 +1,32 @@ +_base_ = [ + 'bustm.py', + 'afqmc.py', + 'eprstmt.py', + 'ocnli_fc.py', + 'ocnli.py', + 'cmnli.py', + 'csl.py', + 'chid.py', + 'cluewsc.py', + 'tnews.py', + 'C3.py', + 'CMRC.py', + 'DRCD.py', + 'lcsts.py', + 'piqa.py', + 'commonsenseqa.py', + 'gsm8k.py', + 'flores.py', + 'humaneval.py', + 'mbpp.py', + 'triviaqa.py', + 'nq.py', + 'agieval.py', + 'mmlu.py', + 'ceval.py', +] + +datasets = [] +for k, v in _base_.items(): + if k.endswith("_datasets"): + datasets += v diff --git a/configs/datasets/glm/afqmc.py b/configs/datasets/glm/afqmc.py new file mode 100644 index 00000000..dc020743 --- /dev/null +++ b/configs/datasets/glm/afqmc.py @@ -0,0 +1,38 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +afqmc_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], + output_column='label', + test_split='train') + +afqmc_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + # 0: "{sentence1},{sentence2}不同。", + # 1: "{sentence1},{sentence2}相似。" + 0: + "I received the questions \"{sentence1}\" and \"{sentence2}\". Are they duplicates?[MASK]no", + 1: + "I received the questions \"{sentence1}\" and \"{sentence2}\". Are they duplicates?[MASK]yes", + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +afqmc_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +afqmc_datasets = [ + dict( + type=HFDataset, + abbr='afqmc', + path='json', + data_files='./data/CLUE/AFQMC/test_public.json', + split='train', + reader_cfg=afqmc_reader_cfg, + infer_cfg=afqmc_infer_cfg, + eval_cfg=afqmc_eval_cfg) +] diff --git a/configs/datasets/glm/agieval.py b/configs/datasets/glm/agieval.py new file mode 100644 index 00000000..b481f6e6 --- /dev/null +++ b/configs/datasets/glm/agieval.py @@ -0,0 +1,110 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GLMChoiceInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AGIEvalDataset + +agieval_reader_cfg = dict( + input_columns=['problem_input'], output_column='label') + +agieval_single_choice_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', + 'logiqa-zh', + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', +] +agieval_multiple_choices_sets = [ + 'jec-qa-kd', # 数据需要额外处理 + 'jec-qa-ca', # 数据需要额外处理 +] +agieval_cloze_sets = ['gaokao-mathcloze', 'math'] + +agieval_datasets = [] +for name in agieval_single_choice_sets: + agieval_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template={ + label: f'{{problem_input}} {label}' + for label in ['A', 'B', 'C', 'D'] + }), + retriever=dict(type=ZeroRetriever + ), # retriver 不起作用，以输入参数为准 (zero-shot / few-shot) + inferencer=dict( + type=GLMChoiceInferencer, choices=['A', 'B', 'C', 'D'])) + + agieval_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset, + path='./data/AGIEval/data/v1/', + name=name, + abbr='agieval-' + name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for name in agieval_multiple_choices_sets: + _hint = '答案是： ' + agieval_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}') + ]), + ice_token=''), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type='GenInferencer')) + agieval_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='first-capital-multi')) + agieval_datasets.append( + dict( + type='AGIEvalDataset_v2', + path='./data/AGIEval/data/v1/', + name=name, + abbr='agieval-' + name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for name in agieval_cloze_sets: + agieval_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template='{problem_input}', + ice_token=''), + retriever=dict(type=ZeroRetriever + ), # retriver 不起作用，以输入参数为准 (zero-shot / few-shot) + inferencer=dict(type='GenInferencer')) + + agieval_eval_cfg = dict(evaluator=dict(type='AGIEvalEvaluator')) + + agieval_datasets.append( + dict( + type=AGIEvalDataset, + path='./data/AGIEval/data/v1/', + name=name, + abbr='agieval-' + name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +del name, agieval_infer_cfg, agieval_eval_cfg diff --git a/configs/datasets/glm/cmnli.py b/configs/datasets/glm/cmnli.py new file mode 100644 index 00000000..462e8013 --- /dev/null +++ b/configs/datasets/glm/cmnli.py @@ -0,0 +1,38 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +cmnli_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], + output_column='label', + test_split='train') + +cmnli_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'contradiction': + '{sentence1}\nKeeping in mind the above text, consider: {sentence2}？\nIs this "always", "sometimes", or "never" correct? [MASK]never', + 'entailment': + '{sentence1}\nKeeping in mind the above text, consider: {sentence2}？\nIs this "always", "sometimes", or "never" correct? [MASK]always', + 'neutral': + '{sentence1}\nKeeping in mind the above text, consider: {sentence2}？\nIs this "always", "sometimes", or "never" correct? [MASK]sometimes' + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +cmnli_datasets = [ + dict( + type=HFDataset, + abbr='cmnli', + path='json', + split='train', + data_files='./data/CLUE/cmnli/cmnli_public/dev.json', + reader_cfg=cmnli_reader_cfg, + infer_cfg=cmnli_infer_cfg, + eval_cfg=cmnli_eval_cfg) +] diff --git a/configs/datasets/glm/csl.py b/configs/datasets/glm/csl.py new file mode 100644 index 00000000..ee8560b2 --- /dev/null +++ b/configs/datasets/glm/csl.py @@ -0,0 +1,46 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GLMChoiceInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import CslDataset + +csl_reader_cfg = dict( + input_columns=["abst", "keywords"], output_column='label') + +csl_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template={ + 0: "摘要：", + 1: "摘要：关键词：" + }, + column_token_map={ + "abst": '', + 'keywords': '' + }, + ice_token=''), + prompt_template=dict( + type=PromptTemplate, + template= + 'Abstract: \nKeyword: \n Does all keywords come from the given abstract? (Yes or No)', + column_token_map={ + "abst": '', + 'keywords': '' + }, + ice_token=''), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GLMChoiceInferencer, choices=['No', 'Yes'])) + +csl_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +csl_datasets = [ + dict( + type=CslDataset, + path='json', + abbr='csl', + data_files='./data/FewCLUE/csl/test_public.json', + split='train', + reader_cfg=csl_reader_cfg, + infer_cfg=csl_infer_cfg, + eval_cfg=csl_eval_cfg) +] diff --git a/configs/datasets/glm/humaneval.py b/configs/datasets/glm/humaneval.py new file mode 100644 index 00000000..ecc99087 --- /dev/null +++ b/configs/datasets/glm/humaneval.py @@ -0,0 +1,30 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HFDataset, HumanEvaluator + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='{prompt}'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvaluator), + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type='humaneval'), +) + +humaneval_datasets = [ + dict( + type=HFDataset, + path='openai_humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/configs/datasets/glm/ocnli.py b/configs/datasets/glm/ocnli.py new file mode 100644 index 00000000..3c4abd43 --- /dev/null +++ b/configs/datasets/glm/ocnli.py @@ -0,0 +1,37 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +ocnli_reader_cfg = dict( + input_columns=['sentence1', 'sentence2'], output_column='label') + +# TODO: two prompt templates for ocnli +ocnli_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'contradiction': + '{sentence1}\nKeeping in mind the above text, consider: {sentence2}？\nIs this "always", "sometimes", or "never" correct? [MASK]never', + 'entailment': + '{sentence1}\nKeeping in mind the above text, consider: {sentence2}？\nIs this "always", "sometimes", or "never" correct? [MASK]always', + 'neutral': + '{sentence1}\nKeeping in mind the above text, consider: {sentence2}？\nIs this "always", "sometimes", or "never" correct? [MASK]sometimes' + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +ocnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator), ) + +ocnli_datasets = [ + dict( + type=HFDataset, + abbr='ocnli', + path='json', + split='train', + data_files='./data/CLUE/OCNLI/dev.json', + reader_cfg=ocnli_reader_cfg, + infer_cfg=ocnli_infer_cfg, + eval_cfg=ocnli_eval_cfg) +] diff --git a/configs/datasets/gsm8k/gsm8k_gen_6a4ce6.py b/configs/datasets/gsm8k/gsm8k_gen_6a4ce6.py new file mode 100644 index 00000000..858ee28b --- /dev/null +++ b/configs/datasets/gsm8k/gsm8k_gen_6a4ce6.py @@ -0,0 +1,50 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +gsm8k_reader_cfg = dict(input_columns=['question'], output_column='answer') + +gsm8k_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?'), + dict(role='BOT', prompt='A: We start with 15 trees. Later we have 21 trees. The difference must be the number of trees they planted. So, they must have planted 21 - 15 = 6 trees. The answer is 6.\n'), + dict(role='HUMAN', prompt='Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?'), + dict(role='BOT', prompt='A: There are 3 cars in the parking lot already. 2 more arrive. Now there are 3 + 2 = 5 cars. The answer is 5.\n'), + dict(role='HUMAN', prompt='Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?'), + dict(role='BOT', prompt="A: Leah had 32 chocolates and Leah's sister had 42. That means there were originally 32 + 42 = 74 chocolates. 35 have been eaten. So in total they still have 74 - 35 = 39 chocolates. The answer is 39.\n"), + dict(role='HUMAN', prompt='Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?'), + dict(role='BOT', prompt='A: Jason had 20 lollipops. Since he only has 12 now, he must have given the rest to Denny. The number of lollipops he has given to Denny must have been 20 - 12 = 8 lollipops. The answer is 8.\n'), + dict(role='HUMAN', prompt='Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?'), + dict(role='BOT', prompt='A: He has 5 toys. He got 2 from mom, so after that he has 5 + 2 = 7 toys. Then he got 2 more from dad, so in total he has 7 + 2 = 9 toys. The answer is 9.\n'), + dict(role='HUMAN', prompt='Q: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?'), + dict(role='BOT', prompt='A: There are 4 days from monday to thursday. 5 computers were added each day. That means in total 4 * 5 = 20 computers were added. There were 9 computers in the beginning, so now there are 9 + 20 = 29 computers. The answer is 29.\n'), + dict(role='HUMAN', prompt='Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?'), + dict(role='BOT', prompt='A: Michael initially had 58 balls. He lost 23 on Tuesday, so after that he has 58 - 23 = 35 balls. On Wednesday he lost 2 more so now he has 35 - 2 = 33 balls. The answer is 33.\n'), + dict(role='HUMAN', prompt='Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?'), + dict(role='BOT', prompt='A: She bought 5 bagels for $3 each. This means she spent 5 * $3 = $15 on the bagels. She had $23 in beginning, so now she has $23 - $15 = $8. The answer is 8.\n'), + dict(role='HUMAN', prompt='Q: {question}'), + dict(role='BOT', prompt='A: {answer}\n'), + ], + )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +gsm8k_eval_cfg = dict(evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type='gsm8k'), + dataset_postprocessor=dict(type='gsm8k_dataset')) + +gsm8k_datasets = [ + dict( + type=HFDataset, + path='gsm8k', + name='main', + reader_cfg=gsm8k_reader_cfg, + infer_cfg=gsm8k_infer_cfg, + eval_cfg=gsm8k_eval_cfg) +] diff --git a/configs/datasets/gsm8k/gsm8k_gen_76914c.py b/configs/datasets/gsm8k/gsm8k_gen_76914c.py new file mode 100644 index 00000000..697c63ac --- /dev/null +++ b/configs/datasets/gsm8k/gsm8k_gen_76914c.py @@ -0,0 +1,51 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +gsm8k_reader_cfg = dict(input_columns=['question'], output_column='answer') + +gsm8k_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?'), + dict(role='BOT', prompt='A: We start with 15 trees. Later we have 21 trees. The difference must be the number of trees they planted. So, they must have planted 21 - 15 = 6 trees. The answer is 6.\n'), + dict(role='HUMAN', prompt='Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?'), + dict(role='BOT', prompt='A: There are 3 cars in the parking lot already. 2 more arrive. Now there are 3 + 2 = 5 cars. The answer is 5.\n'), + dict(role='HUMAN', prompt='Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?'), + dict(role='BOT', prompt="A: Leah had 32 chocolates and Leah's sister had 42. That means there were originally 32 + 42 = 74 chocolates. 35 have been eaten. So in total they still have 74 - 35 = 39 chocolates. The answer is 39.\n"), + dict(role='HUMAN', prompt='Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?'), + dict(role='BOT', prompt='A: Jason had 20 lollipops. Since he only has 12 now, he must have given the rest to Denny. The number of lollipops he has given to Denny must have been 20 - 12 = 8 lollipops. The answer is 8.\n'), + dict(role='HUMAN', prompt='Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?'), + dict(role='BOT', prompt='A: He has 5 toys. He got 2 from mom, so after that he has 5 + 2 = 7 toys. Then he got 2 more from dad, so in total he has 7 + 2 = 9 toys. The answer is 9.\n'), + dict(role='HUMAN', prompt='Q: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?'), + dict(role='BOT', prompt='A: There are 4 days from monday to thursday. 5 computers were added each day. That means in total 4 * 5 = 20 computers were added. There were 9 computers in the beginning, so now there are 9 + 20 = 29 computers. The answer is 29.\n'), + dict(role='HUMAN', prompt='Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?'), + dict(role='BOT', prompt='A: Michael initially had 58 balls. He lost 23 on Tuesday, so after that he has 58 - 23 = 35 balls. On Wednesday he lost 2 more so now he has 35 - 2 = 33 balls. The answer is 33.\n'), + dict(role='HUMAN', prompt='Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?'), + dict(role='BOT', prompt='A: She bought 5 bagels for $3 each. This means she spent 5 * $3 = $15 on the bagels. She had $23 in beginning, so now she has $23 - $15 = $8. The answer is 8.\n'), + dict(role='HUMAN', prompt='Q: {question}'), + dict(role='BOT', prompt='A: {answer}\n'), + ], + )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +gsm8k_eval_cfg = dict(evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type='gsm8k'), + dataset_postprocessor=dict(type='gsm8k_dataset')) + +gsm8k_datasets = [ + dict( + abbr='gsm8k', + type=HFDataset, + path='gsm8k', + name='main', + reader_cfg=gsm8k_reader_cfg, + infer_cfg=gsm8k_infer_cfg, + eval_cfg=gsm8k_eval_cfg) +] diff --git a/configs/datasets/gsm8k/gsm8k_gen_aa9688.py b/configs/datasets/gsm8k/gsm8k_gen_aa9688.py new file mode 100644 index 00000000..36052772 --- /dev/null +++ b/configs/datasets/gsm8k/gsm8k_gen_aa9688.py @@ -0,0 +1,88 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +gsm8k_reader_cfg = dict(input_columns=['question'], output_column='answer') + +gsm8k_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template= + '''Question: Angelo and Melanie want to plan how many hours over the next week they should study together for their test next week. They have 2 chapters of their textbook to study and 4 worksheets to memorize. They figure out that they should dedicate 3 hours to each chapter of their textbook and 1.5 hours for each worksheet. If they plan to study no more than 4 hours each day, how many days should they plan to study total over the next week if they take a 10-minute break every hour, include 3 10-minute snack breaks each day, and 30 minutes for lunch each day? +Let's think step by step +Answer: +Angelo and Melanie think they should dedicate 3 hours to each of the 2 chapters, 3 hours x 2 chapters = 6 hours total. +For the worksheets they plan to dedicate 1.5 hours for each worksheet, 1.5 hours x 4 worksheets = 6 hours total. +Angelo and Melanie need to start with planning 12 hours to study, at 4 hours a day, 12 / 4 = 3 days. +However, they need to include time for breaks and lunch. Every hour they want to include a 10-minute break, so 12 total hours x 10 minutes = 120 extra minutes for breaks. +They also want to include 3 10-minute snack breaks, 3 x 10 minutes = 30 minutes. +And they want to include 30 minutes for lunch each day, so 120 minutes for breaks + 30 minutes for snack breaks + 30 minutes for lunch = 180 minutes, or 180 / 60 minutes per hour = 3 extra hours. +So Angelo and Melanie want to plan 12 hours to study + 3 hours of breaks = 15 hours total. +They want to study no more than 4 hours each day, 15 hours / 4 hours each day = 3.75 +They will need to plan to study 4 days to allow for all the time they need. +The answer is 4 + +Question: Mark's basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws. Their opponents score double the 2 pointers but half the 3 pointers and free throws. What's the total number of points scored by both teams added together? +Let's think step by step +Answer: +Mark's team scores 25 2 pointers, meaning they scored 25*2= 50 points in 2 pointers. +His team also scores 6 3 pointers, meaning they scored 8*3= 24 points in 3 pointers +They scored 10 free throws, and free throws count as one point so they scored 10*1=10 points in free throws. +All together his team scored 50+24+10= 84 points +Mark's opponents scored double his team's number of 2 pointers, meaning they scored 50*2=100 points in 2 pointers. +His opponents scored half his team's number of 3 pointers, meaning they scored 24/2= 12 points in 3 pointers. +They also scored half Mark's team's points in free throws, meaning they scored 10/2=5 points in free throws. +All together Mark's opponents scored 100+12+5=117 points +The total score for the game is both team's scores added together, so it is 84+117=201 points +The answer is 201 + +Question: Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles? +Let's think step by step +Answer: +When Bella buys 2/5 times more marbles, she'll have increased the number of marbles by 2/5*60 = 24 +The total number of marbles she'll have is 60+24 = 84 +If Bella currently has 60 marbles, and she has two times as many marbles as frisbees, she has 60/2 = 30 frisbees. +If Bella buys 2/5 times more frisbees, she'll have 2/5*30 = 12 more frisbees. +The total number of frisbees she'll have will increase to 30+12 = 42 +Bella also has 20 more frisbees than deck cards, meaning she has 30-20 = 10 deck cards +If she buys 2/5 times more deck cards, she'll have 2/5*10 = 4 more deck cards. +The total number of deck cards she'll have is 10+4 = 14 +Together, Bella will have a total of 14+42+84 = 140 items +The answer is 140 + +Question: A group of 4 fruit baskets contains 9 apples, 15 oranges, and 14 bananas in the first three baskets and 2 less of each fruit in the fourth basket. How many fruits are there? +Let's think step by step +Answer: +For the first three baskets, the number of apples and oranges in one basket is 9+15=24 +In total, together with bananas, the number of fruits in one basket is 24+14=38 for the first three baskets. +Since there are three baskets each having 38 fruits, there are 3*38=114 fruits in the first three baskets. +The number of apples in the fourth basket is 9-2=7 +There are also 15-2=13 oranges in the fourth basket +The combined number of oranges and apples in the fourth basket is 13+7=20 +The fourth basket also contains 14-2=12 bananas. +In total, the fourth basket has 20+12=32 fruits. +The four baskets together have 32+114=146 fruits. +The answer is 146 + +Question: {question}{answer} +'''), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +gsm8k_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='gsm8k'), + dataset_postprocessor=dict(type='gsm8k_dataset')) + +gsm8k_datasets = [ + dict( + abbr='gsm8k', + type=HFDataset, + path='gsm8k', + name='main', + reader_cfg=gsm8k_reader_cfg, + infer_cfg=gsm8k_infer_cfg, + eval_cfg=gsm8k_eval_cfg) +] diff --git a/configs/datasets/gsm8k/gsm8k_gen_d19ade.py b/configs/datasets/gsm8k/gsm8k_gen_d19ade.py new file mode 100644 index 00000000..1d39f46b --- /dev/null +++ b/configs/datasets/gsm8k/gsm8k_gen_d19ade.py @@ -0,0 +1,87 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +gsm8k_reader_cfg = dict(input_columns=['question'], output_column='answer') + +gsm8k_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template= + '''Question: Angelo and Melanie want to plan how many hours over the next week they should study together for their test next week. They have 2 chapters of their textbook to study and 4 worksheets to memorize. They figure out that they should dedicate 3 hours to each chapter of their textbook and 1.5 hours for each worksheet. If they plan to study no more than 4 hours each day, how many days should they plan to study total over the next week if they take a 10-minute break every hour, include 3 10-minute snack breaks each day, and 30 minutes for lunch each day? +Let's think step by step +Answer: +Angelo and Melanie think they should dedicate 3 hours to each of the 2 chapters, 3 hours x 2 chapters = 6 hours total. +For the worksheets they plan to dedicate 1.5 hours for each worksheet, 1.5 hours x 4 worksheets = 6 hours total. +Angelo and Melanie need to start with planning 12 hours to study, at 4 hours a day, 12 / 4 = 3 days. +However, they need to include time for breaks and lunch. Every hour they want to include a 10-minute break, so 12 total hours x 10 minutes = 120 extra minutes for breaks. +They also want to include 3 10-minute snack breaks, 3 x 10 minutes = 30 minutes. +And they want to include 30 minutes for lunch each day, so 120 minutes for breaks + 30 minutes for snack breaks + 30 minutes for lunch = 180 minutes, or 180 / 60 minutes per hour = 3 extra hours. +So Angelo and Melanie want to plan 12 hours to study + 3 hours of breaks = 15 hours total. +They want to study no more than 4 hours each day, 15 hours / 4 hours each day = 3.75 +They will need to plan to study 4 days to allow for all the time they need. +The answer is 4 + +Question: Mark's basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws. Their opponents score double the 2 pointers but half the 3 pointers and free throws. What's the total number of points scored by both teams added together? +Let's think step by step +Answer: +Mark's team scores 25 2 pointers, meaning they scored 25*2= 50 points in 2 pointers. +His team also scores 6 3 pointers, meaning they scored 8*3= 24 points in 3 pointers +They scored 10 free throws, and free throws count as one point so they scored 10*1=10 points in free throws. +All together his team scored 50+24+10= 84 points +Mark's opponents scored double his team's number of 2 pointers, meaning they scored 50*2=100 points in 2 pointers. +His opponents scored half his team's number of 3 pointers, meaning they scored 24/2= 12 points in 3 pointers. +They also scored half Mark's team's points in free throws, meaning they scored 10/2=5 points in free throws. +All together Mark's opponents scored 100+12+5=117 points +The total score for the game is both team's scores added together, so it is 84+117=201 points +The answer is 201 + +Question: Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles? +Let's think step by step +Answer: +When Bella buys 2/5 times more marbles, she'll have increased the number of marbles by 2/5*60 = 24 +The total number of marbles she'll have is 60+24 = 84 +If Bella currently has 60 marbles, and she has two times as many marbles as frisbees, she has 60/2 = 30 frisbees. +If Bella buys 2/5 times more frisbees, she'll have 2/5*30 = 12 more frisbees. +The total number of frisbees she'll have will increase to 30+12 = 42 +Bella also has 20 more frisbees than deck cards, meaning she has 30-20 = 10 deck cards +If she buys 2/5 times more deck cards, she'll have 2/5*10 = 4 more deck cards. +The total number of deck cards she'll have is 10+4 = 14 +Together, Bella will have a total of 14+42+84 = 140 items +The answer is 140 + +Question: A group of 4 fruit baskets contains 9 apples, 15 oranges, and 14 bananas in the first three baskets and 2 less of each fruit in the fourth basket. How many fruits are there? +Let's think step by step +Answer: +For the first three baskets, the number of apples and oranges in one basket is 9+15=24 +In total, together with bananas, the number of fruits in one basket is 24+14=38 for the first three baskets. +Since there are three baskets each having 38 fruits, there are 3*38=114 fruits in the first three baskets. +The number of apples in the fourth basket is 9-2=7 +There are also 15-2=13 oranges in the fourth basket +The combined number of oranges and apples in the fourth basket is 13+7=20 +The fourth basket also contains 14-2=12 bananas. +In total, the fourth basket has 20+12=32 fruits. +The four baskets together have 32+114=146 fruits. +The answer is 146 + +Question: {question}{answer} +'''), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +gsm8k_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='gsm8k'), + dataset_postprocessor=dict(type='gsm8k_dataset')) + +gsm8k_datasets = [ + dict( + type=HFDataset, + path='gsm8k', + name='main', + reader_cfg=gsm8k_reader_cfg, + infer_cfg=gsm8k_infer_cfg, + eval_cfg=gsm8k_eval_cfg) +] diff --git a/configs/datasets/hellaswag/hellaswag_ppl_8e07d6.py b/configs/datasets/hellaswag/hellaswag_ppl_8e07d6.py new file mode 100644 index 00000000..fa6294f8 --- /dev/null +++ b/configs/datasets/hellaswag/hellaswag_ppl_8e07d6.py @@ -0,0 +1,35 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import hellaswagDataset + +hellaswag_reader_cfg = dict( + input_columns=['ctx', 'A', 'B', 'C', 'D'], + output_column='label', + train_split='validation', + test_split='validation') + +hellaswag_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + i: dict(round=[ + dict(role="HUMAN", prompt="{ctx}"), + dict(role="BOT", prompt=f"{{{chr(ord('A') + i)}}}"), + ]) + for i in range(4) + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +hellaswag_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +hellaswag_datasets = [ + dict( + type=hellaswagDataset, + path='hellaswag', + reader_cfg=hellaswag_reader_cfg, + infer_cfg=hellaswag_infer_cfg, + eval_cfg=hellaswag_eval_cfg) +] diff --git a/configs/datasets/hellaswag/hellaswag_ppl_e57e0f.py b/configs/datasets/hellaswag/hellaswag_ppl_e57e0f.py new file mode 100644 index 00000000..e90a7556 --- /dev/null +++ b/configs/datasets/hellaswag/hellaswag_ppl_e57e0f.py @@ -0,0 +1,34 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import hellaswagDataset + +hellaswag_reader_cfg = dict( + input_columns=['ctx', 'A', 'B', 'C', 'D'], + output_column='label', + train_split='validation', + test_split='validation') + +hellaswag_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: "{ctx} {A}", + 1: "{ctx} {B}", + 2: "{ctx} {C}", + 3: "{ctx} {D}", + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +hellaswag_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +hellaswag_datasets = [ + dict( + type=hellaswagDataset, + path='hellaswag', + reader_cfg=hellaswag_reader_cfg, + infer_cfg=hellaswag_infer_cfg, + eval_cfg=hellaswag_eval_cfg) +] diff --git a/configs/datasets/humaneval/humaneval_gen_bce65c.py b/configs/datasets/humaneval/humaneval_gen_bce65c.py new file mode 100644 index 00000000..68dc6d36 --- /dev/null +++ b/configs/datasets/humaneval/humaneval_gen_bce65c.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HFDataset, HumanEvaluator + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt='Complete the following python code:'), + ], + round=[ + dict(role='HUMAN', prompt='{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type='humaneval'), +) + +humaneval_datasets = [ + dict( + type=HFDataset, + path='openai_humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/configs/datasets/iwslt2017/iwslt2017_gen.py b/configs/datasets/iwslt2017/iwslt2017_gen.py new file mode 100644 index 00000000..d44eb91a --- /dev/null +++ b/configs/datasets/iwslt2017/iwslt2017_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .iwslt2017_gen_02ea0b import iwslt2017_datasets # noqa: F401, F403 diff --git a/configs/datasets/iwslt2017/iwslt2017_gen_66a5bb.py b/configs/datasets/iwslt2017/iwslt2017_gen_66a5bb.py new file mode 100644 index 00000000..1d50d397 --- /dev/null +++ b/configs/datasets/iwslt2017/iwslt2017_gen_66a5bb.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import BM25Retriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import BleuEvaluator +from opencompass.datasets import IWSLT2017Dataset + +iwslt2017_reader_cfg = dict( + input_columns='en', output_column='de', train_split='validation') + +iwslt2017_infer_cfg = dict( + ice_template=dict(type='PromptTemplate', + template=dict( + begin=[ + dict(role='SYSTEM', fallback_role="HUMAN", prompt='Please translate the following English statements to German:'), + '', + ], + round=[ + dict(role='HUMAN', prompt='{en}'), + dict(role='BOT', prompt='{de}'), + ] + ), + ice_token=''), + retriever=dict(type=BM25Retriever, ice_num=1), + inferencer=dict(type=GenInferencer)) + +iwslt2017_eval_cfg = dict( + evaluator=dict(type=BleuEvaluator), + pred_role='BOT', + pred_postprocessor=dict(type='general_cn'), + dataset_postprocessor=dict(type='general_cn')) + +iwslt2017_datasets = [ + dict( + type=IWSLT2017Dataset, + path='iwslt2017', + name='iwslt2017-en-de', + reader_cfg=iwslt2017_reader_cfg, + infer_cfg=iwslt2017_infer_cfg, + eval_cfg=iwslt2017_eval_cfg) +] \ No newline at end of file diff --git a/configs/datasets/lambada/lambada_gen_8d0a67.py b/configs/datasets/lambada/lambada_gen_8d0a67.py new file mode 100644 index 00000000..d798045c --- /dev/null +++ b/configs/datasets/lambada/lambada_gen_8d0a67.py @@ -0,0 +1,29 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import lambadaDataset, LambadaEvaluator + +lambada_reader_cfg = dict( + input_columns=['prompt'], + output_column='label', + train_split='test', + test_split='test') + +lambada_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='Please complete the following sentence: {prompt}'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=5)) + +lambada_eval_cfg = dict(evaluator=dict(type=LambadaEvaluator)) + +lambada_datasets = [ + dict( + abbr='lambada', + type=lambadaDataset, + path='craffel/openai_lambada', + reader_cfg=lambada_reader_cfg, + infer_cfg=lambada_infer_cfg, + eval_cfg=lambada_eval_cfg) +] diff --git a/configs/datasets/math/math_gen_2f9b1c.py b/configs/datasets/math/math_gen_2f9b1c.py new file mode 100644 index 00000000..9e3a6616 --- /dev/null +++ b/configs/datasets/math/math_gen_2f9b1c.py @@ -0,0 +1,53 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import MATHDataset, MATHEvaluator + +math_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='''Problem: +Find the domain of the expression $\frac{{\sqrt{{x-2}}}}{{\sqrt{{5-x}}}}$.}} +Solution: +The expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\boxed{{[2,5)}}$. +Final Answer: The final answer is $[2,5)$. I hope it is correct. + +Problem: +If $\det \mathbf{{A}} = 2$ and $\det \mathbf{{B}} = 12,$ then find $\det (\mathbf{{A}} \mathbf{{B}}).$ +Solution: +We have that $\det (\mathbf{{A}} \mathbf{{B}}) = (\det \mathbf{{A}})(\det \mathbf{{B}}) = (2)(12) = \boxed{{24}}.$ +Final Answer: The final answer is $24$. I hope it is correct. + +Problem: +Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight? +Solution: +If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \begin{{align*}} 30n&=480\\ \Rightarrow\qquad n&=480/30=\boxed{{16}} \end{{align*}} +Final Answer: The final answer is $16$. I hope it is correct. + +Problem: +If the system of equations: \begin{{align*}} 6x-4y&=a,\\ 6y-9x &=b. \end{{align*}}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\frac{{a}}{{b}},$ assuming $b$ is nonzero. +Solution: +If we multiply the first equation by $-\frac{{3}}{{2}}$, we obtain $$6y-9x=-\frac{{3}}{{2}}a.$$Since we also know that $6y-9x=b$, we have $$-\frac{{3}}{{2}}a=b\Rightarrow\frac{{a}}{{b}}=\boxed{{-\frac{{2}}{{3}}}}.$$ +Final Answer: The final answer is $-\frac{{2}}{{3}}$. I hope it is correct. + +Problem: +{problem}Solution: +{solution}'''), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +math_eval_cfg = dict( + evaluator=dict(type=MATHEvaluator), pred_postprocessor=dict(type='math')) + +math_datasets = [ + dict( + type=MATHDataset, + abbr='math', + path='./data/math/math.json', + reader_cfg=dict( + input_columns=['problem'], + output_column='solution', + ), + infer_cfg=math_infer_cfg, + eval_cfg=math_eval_cfg) +] diff --git a/configs/datasets/mbpp/mbpp_gen_4104e4.py b/configs/datasets/mbpp/mbpp_gen_4104e4.py new file mode 100644 index 00000000..234ffd96 --- /dev/null +++ b/configs/datasets/mbpp/mbpp_gen_4104e4.py @@ -0,0 +1,64 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import MBPPDataset, MBPPEvaluator + +mbpp_reader_cfg = dict( + input_columns=['text', 'test_list'], output_column='code') + +mbpp_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role="HUMAN", + prompt= + "You are an expert Python programmer, and here is your task: Write a function to find the similar elements from the given two tuple lists. Your code should pass these tests:\n\n assert similar_elements((3, 4, 5, 6),(5, 7, 4, 10)) == (4, 5)\n assert similar_elements((1, 2, 3, 4),(5, 4, 3, 7)) == (3, 4) \n assert similar_elements((11, 12, 14, 13),(17, 15, 14, 13)) == (13, 14) \n" + ), + dict( + role="BOT", + prompt= + "[BEGIN]\n 'def similar_elements(test_tup1, test_tup2):\r\n res = tuple(set(test_tup1) & set(test_tup2))\r\n return (res)' \n[DONE] \n\n " + ), + dict( + role="HUMAN", + prompt= + "You are an expert Python programmer, and here is your task: Write a python function to identify non-prime numbers. Your code should pass these tests:\n\n assert is_not_prime(2) == False \n assert is_not_prime(10) == True \n assert is_not_prime(35) == True \n" + ), + dict( + role="BOT", + prompt= + "[BEGIN]\n 'import math\r\ndef is_not_prime(n):\r\n result = False\r\n for i in range(2,int(math.sqrt(n)) + 1):\r\n if n % i == 0:\r\n result = True\r\n return result' \n[DONE] \n\n " + ), + dict( + role="HUMAN", + prompt= + "You are an expert Python programmer, and here is your task: Write a function to find the largest integers from a given list of numbers using heap queue algorithm. Your code should pass these tests:\n\n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],3)==[85, 75, 65] \n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],2)==[85, 75] \n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],5)==[85, 75, 65, 58, 35] \n" + ), + dict( + role="BOT", + prompt= + "[BEGIN]\n 'import heapq as hq\r\ndef heap_queue_largest(nums,n):\r\n largest_nums = hq.nlargest(n, nums)\r\n return largest_nums' \n[DONE] \n\n " + ), + dict( + role="HUMAN", + prompt= + "You are an expert Python programmer, and here is your task: {text} Your code should pass these tests:\n\n {test_list} \n" + ), + dict(role="BOT", prompt="[BEGIN]\n"), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +mbpp_eval_cfg = dict(evaluator=dict(type=MBPPEvaluator), pred_role="BOT") + +mbpp_datasets = [ + dict( + type=MBPPDataset, + abbr='mbpp', + path='./data/mbpp/mbpp.jsonl', + reader_cfg=mbpp_reader_cfg, + infer_cfg=mbpp_infer_cfg, + eval_cfg=mbpp_eval_cfg) +] diff --git a/configs/datasets/mmlu/mmlu_gen_c3ca20.py b/configs/datasets/mmlu/mmlu_gen_c3ca20.py new file mode 100644 index 00000000..4a079d34 --- /dev/null +++ b/configs/datasets/mmlu/mmlu_gen_c3ca20.py @@ -0,0 +1,123 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import FixKRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import MMLUDataset + +# None of the mmlu dataset in huggingface is correctly parsed, so we use our own dataset reader +# Please download the dataset from https://people.eecs.berkeley.edu/~hendrycks/data.tar + +mmlu_reader_cfg = dict( + input_columns=["input", "A", "B", "C", "D"], + output_column="target", + train_split='dev') + +mmlu_prompt_template = dict( + type='PromptTemplate', + template=None, + ice_token='') + +mmlu_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: ' + ), + dict(role='BOT', prompt='{target}\n') + ])), + prompt_template=mmlu_prompt_template, + retriever=dict(type=FixKRetriever), + inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4])) + +mmlu_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='first-capital')) + +mmlu_all_sets = [ + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_physics", + "electrical_engineering", + "astronomy", + "anatomy", + "abstract_algebra", + "machine_learning", + "clinical_knowledge", + "global_facts", + "management", + "nutrition", + "marketing", + "professional_accounting", + "high_school_geography", + "international_law", + "moral_scenarios", + "computer_security", + "high_school_microeconomics", + "professional_law", + "medical_genetics", + "professional_psychology", + "jurisprudence", + "world_religions", + "philosophy", + "virology", + "high_school_chemistry", + "public_relations", + "high_school_macroeconomics", + "human_sexuality", + "elementary_mathematics", + "high_school_physics", + "high_school_computer_science", + "high_school_european_history", + "business_ethics", + "moral_disputes", + "high_school_statistics", + "miscellaneous", + "formal_logic", + "high_school_government_and_politics", + "prehistory", + "security_studies", + "high_school_biology", + "logical_fallacies", + "high_school_world_history", + "professional_medicine", + "high_school_mathematics", + "college_medicine", + "high_school_us_history", + "sociology", + "econometrics", + "high_school_psychology", + "human_aging", + "us_foreign_policy", + "conceptual_physics", +] + +mmlu_datasets = [] +for _name in mmlu_all_sets: + mmlu_datasets.append( + dict( + abbr=f"lukaemon_mmlu_{_name}", + type=MMLUDataset, + path="./data/mmlu/", + name=_name, + reader_cfg=mmlu_reader_cfg, + infer_cfg=mmlu_infer_cfg.copy(), + eval_cfg=mmlu_eval_cfg)) + + mmlu_datasets[-1]['infer_cfg'][ + 'prompt_template'] = mmlu_prompt_template.copy() + mmlu_datasets[-1]['infer_cfg']['prompt_template']['template'] = \ + dict( + begin=[ + dict(role='SYSTEM', fallback_role='HUMAN', prompt=f'The following are multiple choice questions (with answers) about {_name.replace("_", " ")}.'), + '', + ], + round=[ + dict(role='HUMAN', prompt='{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: '), + ] + ) + +del _name diff --git a/configs/datasets/mmlu/mmlu_ppl_c6bbe6.py b/configs/datasets/mmlu/mmlu_ppl_c6bbe6.py new file mode 100644 index 00000000..900c1eb9 --- /dev/null +++ b/configs/datasets/mmlu/mmlu_ppl_c6bbe6.py @@ -0,0 +1,113 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import FixKRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import MMLUDataset + +# None of the mmlu dataset in huggingface is correctly parsed, so we use our own dataset reader +# Please download the dataset from https://people.eecs.berkeley.edu/~hendrycks/data.tar + +mmlu_reader_cfg = dict( + input_columns=["input", "A", "B", "C", "D"], + output_column="target", + train_split='dev') + +mmlu_all_sets = [ + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_physics", + "electrical_engineering", + "astronomy", + "anatomy", + "abstract_algebra", + "machine_learning", + "clinical_knowledge", + "global_facts", + "management", + "nutrition", + "marketing", + "professional_accounting", + "high_school_geography", + "international_law", + "moral_scenarios", + "computer_security", + "high_school_microeconomics", + "professional_law", + "medical_genetics", + "professional_psychology", + "jurisprudence", + "world_religions", + "philosophy", + "virology", + "high_school_chemistry", + "public_relations", + "high_school_macroeconomics", + "human_sexuality", + "elementary_mathematics", + "high_school_physics", + "high_school_computer_science", + "high_school_european_history", + "business_ethics", + "moral_disputes", + "high_school_statistics", + "miscellaneous", + "formal_logic", + "high_school_government_and_politics", + "prehistory", + "security_studies", + "high_school_biology", + "logical_fallacies", + "high_school_world_history", + "professional_medicine", + "high_school_mathematics", + "college_medicine", + "high_school_us_history", + "sociology", + "econometrics", + "high_school_psychology", + "human_aging", + "us_foreign_policy", + "conceptual_physics", +] + +mmlu_datasets = [] +for _name in mmlu_all_sets: + _hint = f'The following are multiple choice questions (with answers) about {_name.replace("_", " ")}.\n\n' + mmlu_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template={ + opt: + f"{{input}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nAnswer: {opt}\n" + for opt in ["A", "B", "C", "D"] + }, + ), + prompt_template=dict( + type=PromptTemplate, + template={ + opt: + f"{_hint}{{input}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nAnswer: {opt}" + for opt in ["A", "B", "C", "D"] + }, + ice_token="", + ), + retriever=dict(type=FixKRetriever), + inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]), + ) + + mmlu_eval_cfg = dict(evaluator=dict(type=AccEvaluator), ) + + mmlu_datasets.append( + dict( + abbr=f"lukaemon_mmlu_{_name}", + type=MMLUDataset, + path="./data/mmlu/", + name=_name, + reader_cfg=mmlu_reader_cfg, + infer_cfg=mmlu_infer_cfg, + eval_cfg=mmlu_eval_cfg, + )) + +del _name, _hint diff --git a/configs/datasets/nq/nq_gen_f491e3.py b/configs/datasets/nq/nq_gen_f491e3.py new file mode 100644 index 00000000..5d782b6f --- /dev/null +++ b/configs/datasets/nq/nq_gen_f491e3.py @@ -0,0 +1,27 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import NaturalQuestionDataset, NQEvaluator + +nq_reader_cfg = dict( + input_columns=['question'], output_column='answer', train_split='test') + +nq_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template="Answer these questions:\nQ: {question}?\nA:{answer}", + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role="BOT") + +nq_datasets = [ + dict( + type=NaturalQuestionDataset, + abbr='nq', + path='./data/nq/', + reader_cfg=nq_reader_cfg, + infer_cfg=nq_infer_cfg, + eval_cfg=nq_eval_cfg) +] diff --git a/configs/datasets/obqa/obqa_gen.py b/configs/datasets/obqa/obqa_gen.py new file mode 100644 index 00000000..0886d001 --- /dev/null +++ b/configs/datasets/obqa/obqa_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .obqa_gen_b2cde9 import obqa_datasets # noqa: F401, F403 diff --git a/configs/datasets/obqa/obqa_gen_b2cde9.py b/configs/datasets/obqa/obqa_gen_b2cde9.py new file mode 100644 index 00000000..a9ef4f41 --- /dev/null +++ b/configs/datasets/obqa/obqa_gen_b2cde9.py @@ -0,0 +1,62 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import OBQADataset + +_input_columns = [ + ["question_stem", "A", "B", "C", "D"], + ["question_stem", "A", "B", "C", "D", "fact1"], +] +_template = [ + dict( + round=[ + dict( + role="HUMAN", + prompt="Question: {question_stem}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer:" + ), + ], ), + dict( + round=[ + dict( + role="HUMAN", + prompt="Given the fact: {fact1}\nQuestion: {question_stem}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer:", + ), + ], ), +] + +obqa_datasets = [ + dict( + abbr="openbookqa", + type=OBQADataset, + path="openbookqa", + split="test", + ), + dict( + abbr="openbookqa_fact", + type=OBQADataset, + path="openbookqa", + name="additional", + split="test", + ), +] + +for _i in range(2): + obqa_reader_cfg = dict( + input_columns=_input_columns[_i], output_column="answerKey") + obqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=_template[_i]), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ) + obqa_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), + ) + + obqa_datasets[_i]["reader_cfg"] = obqa_reader_cfg + obqa_datasets[_i]["infer_cfg"] = obqa_infer_cfg + obqa_datasets[_i]["eval_cfg"] = obqa_eval_cfg diff --git a/configs/datasets/qasper/qasper_gen_1230f2.py b/configs/datasets/qasper/qasper_gen_1230f2.py new file mode 100644 index 00000000..fd7403e2 --- /dev/null +++ b/configs/datasets/qasper/qasper_gen_1230f2.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import QASPERDataset, TriviaQAEvaluator + +qasper_reader_cfg = dict( + input_columns=['question', 'evidence'], + output_column='answer', + train_split='dev', + test_split='dev') + +qasper_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role='HUMAN', + prompt='{evidence}\nAnswer these questions:\nQ: {question}?A:'), + dict(role='BOT', prompt=''), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict( + type=GenInferencer, max_out_len=50, max_seq_len=8192, batch_size=4)) + +qasper_eval_cfg = dict(evaluator=dict(type=TriviaQAEvaluator), pred_role='BOT') + +qasper_datasets = [ + dict( + type=QASPERDataset, + abbr='QASPER', + path='./data/QASPER/', + reader_cfg=qasper_reader_cfg, + infer_cfg=qasper_infer_cfg, + eval_cfg=qasper_eval_cfg) +] diff --git a/configs/datasets/qasper/qasper_gen_bf7900.py b/configs/datasets/qasper/qasper_gen_bf7900.py new file mode 100644 index 00000000..1d828858 --- /dev/null +++ b/configs/datasets/qasper/qasper_gen_bf7900.py @@ -0,0 +1,30 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import QASPERDataset, TriviaQAEvaluator + +qasper_reader_cfg = dict( + input_columns=['question', 'evidence'], + output_column='answer', + train_split='dev', + test_split='dev') + +qasper_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template="{evidence}\nAnswer these questions:\nQ: {question}?\nA:"), + retriever=dict(type=ZeroRetriever), + inferencer=dict( + type=GenInferencer, max_out_len=50, max_seq_len=8192, batch_size=4)) + +qasper_eval_cfg = dict(evaluator=dict(type=TriviaQAEvaluator)) + +qasper_datasets = [ + dict( + type=QASPERDataset, + abbr='QASPER', + path='./data/QASPER/', + reader_cfg=qasper_reader_cfg, + infer_cfg=qasper_infer_cfg, + eval_cfg=qasper_eval_cfg) +] diff --git a/configs/datasets/race/race_ppl_04e06a.py b/configs/datasets/race/race_ppl_04e06a.py new file mode 100644 index 00000000..20f4d32b --- /dev/null +++ b/configs/datasets/race/race_ppl_04e06a.py @@ -0,0 +1,48 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import RaceDataset + +race_reader_cfg = dict( + input_columns=['article', 'question', 'A', 'B', 'C', 'D'], + output_column='answer') + +race_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + ans: dict(round=[ + dict( + role="HUMAN", + prompt= + "Read the article, and answer the question by replying A, B, C or D.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}" + ), + dict(role="BOT", prompt=ans_token), + ]) + for ans, ans_token in [["A", "{A}"], ["B", "{B}"], ["C", "{C}"], + ["D", "{D}"]] + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +race_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +race_datasets = [ + dict( + type=RaceDataset, + abbr='race-middle', + path='race', + name='middle', + reader_cfg=race_reader_cfg, + infer_cfg=race_infer_cfg, + eval_cfg=race_eval_cfg), + dict( + type=RaceDataset, + abbr='race-high', + path='race', + name='high', + reader_cfg=race_reader_cfg, + infer_cfg=race_infer_cfg, + eval_cfg=race_eval_cfg) +] diff --git a/configs/datasets/race/race_ppl_d816a7.py b/configs/datasets/race/race_ppl_d816a7.py new file mode 100644 index 00000000..e98e997d --- /dev/null +++ b/configs/datasets/race/race_ppl_d816a7.py @@ -0,0 +1,46 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import RaceDataset + +race_reader_cfg = dict( + input_columns=['article', 'question', 'A', 'B', 'C', 'D'], + output_column='answer') + +race_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 'A': + 'Read the article, and answer the question by replying A, B, C or D.\n\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n\nAnswer: A', + 'B': + 'Read the article, and answer the question by replying A, B, C or D.\n\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n\nAnswer: B', + 'C': + 'Read the article, and answer the question by replying A, B, C or D.\n\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n\nAnswer: C', + 'D': + 'Read the article, and answer the question by replying A, B, C or D.\n\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n\nAnswer: D', + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +race_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +race_datasets = [ + dict( + type=RaceDataset, + abbr='race-middle', + path='race', + name='middle', + reader_cfg=race_reader_cfg, + infer_cfg=race_infer_cfg, + eval_cfg=race_eval_cfg), + dict( + type=RaceDataset, + abbr='race-high', + path='race', + name='high', + reader_cfg=race_reader_cfg, + infer_cfg=race_infer_cfg, + eval_cfg=race_eval_cfg) +] diff --git a/configs/datasets/realtoxicprompts/realtoxicprompts_gen_3ea730.py b/configs/datasets/realtoxicprompts/realtoxicprompts_gen_3ea730.py new file mode 100644 index 00000000..07246639 --- /dev/null +++ b/configs/datasets/realtoxicprompts/realtoxicprompts_gen_3ea730.py @@ -0,0 +1,34 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import ToxicEvaluator +from opencompass.datasets import RealToxicPromptsDataset + +realtoxicprompts_reader_cfg = dict( + input_columns=['prompt_text'], + output_column='filename', + train_split='train', + test_split='train') + +# TODO: allow empty output-column +realtoxicprompts_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='{prompt_text}'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +# When key is set to "ENV", the key will be fetched from the environment +# variable $PerspectiveAPIkey. Otherwise, set key in here directly. +realtoxicprompts_eval_cfg = dict( + evaluator=dict(type=ToxicEvaluator, key='ENV'), ) + +realtoxicprompts_datasets = [ + dict( + type=RealToxicPromptsDataset, + path='allenai/real-toxicity-prompts', + challenging_subset=True, + reader_cfg=realtoxicprompts_reader_cfg, + infer_cfg=realtoxicprompts_infer_cfg, + eval_cfg=realtoxicprompts_eval_cfg) +] diff --git a/configs/datasets/siqa/siqa_ppl_049da0.py b/configs/datasets/siqa/siqa_ppl_049da0.py new file mode 100644 index 00000000..24ea1b6e --- /dev/null +++ b/configs/datasets/siqa/siqa_ppl_049da0.py @@ -0,0 +1,45 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +siqa_reader_cfg = dict( + input_columns=['context', 'question', 'answerA', 'answerB', 'answerC'], + output_column='label', + test_split='validation') + +siqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 1: + dict(round=[ + dict(role='HUMAN', prompt="{context}\nQuestion: {question}\nAnswer:"), + dict(role='BOT', prompt="{answerA}") + ]), + 2: + dict(round=[ + dict(role='HUMAN', prompt="{context}\nQuestion: {question}\nAnswer:"), + dict(role='BOT', prompt="{answerB}") + ]), + 3: + dict(round=[ + dict(role='HUMAN', prompt="{context}\nQuestion: {question}\nAnswer:"), + dict(role='BOT', prompt="{answerC}") + ]), + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +siqa_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +siqa_datasets = [ + dict( + abbr="siqa", + type=HFDataset, + path='social_i_qa', + reader_cfg=siqa_reader_cfg, + infer_cfg=siqa_infer_cfg, + eval_cfg=siqa_eval_cfg) +] diff --git a/configs/datasets/storycloze/storycloze_gen_c5a230.py b/configs/datasets/storycloze/storycloze_gen_c5a230.py new file mode 100644 index 00000000..54b9563d --- /dev/null +++ b/configs/datasets/storycloze/storycloze_gen_c5a230.py @@ -0,0 +1,45 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import storyclozeDataset_V2 + +storycloze_reader_cfg = dict( + input_columns=["context", "sentence_quiz1", "sentence_quiz2"], + output_column="answer_right_ending", +) + +storycloze_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "{context}\nQuestion: Which ending makes the most sense?\nA. {sentence_quiz1}\nB. {sentence_quiz2}\nYou may choose between 'A' and 'B'.\nAnswer:", + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +storycloze_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +# The original story cloze dataset and repo are not long maintaining. +# Using multilingual version of this dataset. +storycloze_datasets = [ + dict( + abbr="story_cloze", + type=storyclozeDataset_V2, + path="juletxara/xstory_cloze", + name="en", + reader_cfg=storycloze_reader_cfg, + infer_cfg=storycloze_infer_cfg, + eval_cfg=storycloze_eval_cfg, + ) +] diff --git a/configs/datasets/strategyqa/strategyqa_gen_a8bf83.py b/configs/datasets/strategyqa/strategyqa_gen_a8bf83.py new file mode 100644 index 00000000..f36d72bf --- /dev/null +++ b/configs/datasets/strategyqa/strategyqa_gen_a8bf83.py @@ -0,0 +1,58 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +strategyqa_reader_cfg = dict( + input_columns=['question'], + output_column='answer', + train_split='test', + test_split='test') + +strategyqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='''Yes or no: Q: Do hamsters provide food for any animals? +A: Hamsters are prey animals. Prey are food for predators. Thus, hamsters provide food for some animals. +So the answer is yes. + +Q: Yes or no: Could Brooke Shields succeed at University of Pennsylvania? +A: Brooke Shields went to Princeton University. Princeton University is about as academically rigorous as the University of Pennsylvania. Thus, Brooke Shields could also succeed at the University of Pennsylvania. +So the answer is yes. + +Q: Yes or no: Hydrogen's atomic number squared exceeds number of Spice Girls? +A: Hydrogen has an atomic number of 1. 1 squared is 1. There are 5 Spice Girls. Thus, Hydrogen's atomic number squared is less than 5. +So the answer is no. + +Q: Yes or no: Is it common to see frost during some college commencements? +A: College commencement ceremonies can happen in December, May, and June. December is in the winter, so there can be frost. Thus, there could be frost at some commencements. +So the answer is yes. + +Q: Yes or no: Could a llama birth twice during War in Vietnam (1945-46)? +A: The War in Vietnam was 6 months. The gestation period for a llama is 11 months, which is more than 6 months. Thus, a llama could not give birth twice during the War in Vietnam. +So the answer is no. + +Q: Yes or no: Would a pear sink in water? +A: The density of a pear is about 0.6g/cm3, which is less than water. Objects less dense than water float. Thus, a pear would float. +So the answer is no. + +Q: {question}{answer} +'''), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +strategyqa_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='strategyqa'), + dataset_postprocessor=dict(type='strategyqa_dataset')) + +strategyqa_datasets = [ + dict( + abbr='strategyqa', + type=HFDataset, + path='wics/strategy-qa', + reader_cfg=strategyqa_reader_cfg, + infer_cfg=strategyqa_infer_cfg, + eval_cfg=strategyqa_eval_cfg) +] diff --git a/configs/datasets/strategyqa/strategyqa_gen_be3f8d.py b/configs/datasets/strategyqa/strategyqa_gen_be3f8d.py new file mode 100644 index 00000000..c5df261f --- /dev/null +++ b/configs/datasets/strategyqa/strategyqa_gen_be3f8d.py @@ -0,0 +1,94 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +strategyqa_reader_cfg = dict( + input_columns=['question'], + output_column='answer', + train_split='test', + test_split='test') + +strategyqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role='HUMAN', + prompt= + 'Question: Do hamsters provide food for any animals?\nAnswer:' + ), + dict( + role='BOT', + prompt= + 'Hamsters are prey animals. Prey are food for predators. Thus, hamsters provide food for some animals.\nSo the answer is yes' + ), + dict( + role='HUMAN', + prompt= + 'Question: Could Brooke Shields succeed at University of Pennsylvania?\nAnswer:' + ), + dict( + role='BOT', + prompt= + 'Brooke Shields went to Princeton University. Princeton University is about as academically rigorous as the University of Pennsylvania. Thus, Brooke Shields could also succeed at the University of Pennsylvania.\nSo the answer is yes' + ), + dict( + role='HUMAN', + prompt= + 'Question: Hydrogen\'s atomic number squared exceeds number of Spice Girls?\nAnswer:' + ), + dict( + role='BOT', + prompt= + 'Hydrogen has an atomic number of 1. 1 squared is 1. There are 5 Spice Girls. Thus, Hydrogen\'s atomic number squared is less than 5.\nSo the answer is no' + ), + dict( + role='HUMAN', + prompt= + 'Question: Is it common to see frost during some college commencements?\nAnswer:' + ), + dict( + role='BOT', + prompt= + 'College commencement ceremonies can happen in December, May, and June. December is in the winter, so there can be frost. Thus, there could be frost at some commencements.\nSo the answer is yes' + ), + dict( + role='HUMAN', + prompt= + 'Question: Yes or no: Could a llama birth twice during War in Vietnam (1945-46)?\nAnswer:' + ), + dict( + role='BOT', + prompt= + 'The War in Vietnam was 6 months. The gestation period for a llama is 11 months, which is more than 6 months. Thus, a llama could not give birth twice during the War in Vietnam.\nSo the answer is no' + ), + dict( + role='HUMAN', + prompt='Question: Would a pear sink in water?\nAnswer:'), + dict( + role='BOT', + prompt= + 'The density of a pear is about 0.6g/cm3, which is less than water. Objects less dense than water float. Thus, a pear would float.\nSo the answer is no' + ), + dict(role='HUMAN', prompt='Question: {question}\nAnswer:'), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +strategyqa_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type='strategyqa'), + dataset_postprocessor=dict(type='strategyqa_dataset')) + +strategyqa_datasets = [ + dict( + abbr='strategyqa', + type=HFDataset, + path='wics/strategy-qa', + reader_cfg=strategyqa_reader_cfg, + infer_cfg=strategyqa_infer_cfg, + eval_cfg=strategyqa_eval_cfg) +] diff --git a/configs/datasets/summedits/summedits_ppl_c4d270.py b/configs/datasets/summedits/summedits_ppl_c4d270.py new file mode 100644 index 00000000..0cea6137 --- /dev/null +++ b/configs/datasets/summedits/summedits_ppl_c4d270.py @@ -0,0 +1,42 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import HFDataset + +summedits_reader_cfg = dict( + input_columns=['doc', 'summary'], + output_column='label', + test_split='train') + +summedits_prompt = """ +Given the document below, you have to determine if "Yes" or "No", the summary is factually consistent with the document. +Document: +{doc} +Summary: +{summary} +Is the summary factually consistent with the document? +""" +summedits_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + 0: f"{summedits_prompt}Answer: No.", + 1: f"{summedits_prompt}Answer: Yes." + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +summedits_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + +summedits_datasets = [ + dict( + type=HFDataset, + abbr='summedits', + path='json', + split='train', + data_files='./data/summedits/summedits.jsonl', + reader_cfg=summedits_reader_cfg, + infer_cfg=summedits_infer_cfg, + eval_cfg=summedits_eval_cfg) +] diff --git a/configs/datasets/summscreen/summscreen_gen_e88eaa.py b/configs/datasets/summscreen/summscreen_gen_e88eaa.py new file mode 100644 index 00000000..5f5b8bd7 --- /dev/null +++ b/configs/datasets/summscreen/summscreen_gen_e88eaa.py @@ -0,0 +1,47 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import BleuEvaluator +from opencompass.datasets import SummScreenDataset + +summscreen_reader_cfg = dict( + input_columns='content', + output_column='summary', + train_split='dev', + test_split='dev') + +summscreen_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role="HUMAN", + prompt= + 'Please summarize the following English play script in English:' + ), + ], + round=[ + dict(role='HUMAN', prompt='{content}'), + dict(role='BOT', prompt='{summary}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict( + type=GenInferencer, batch_size=4, max_out_len=500, max_seq_len=8192)) + +summscreen_eval_cfg = dict( + evaluator=dict(type=BleuEvaluator), + pred_role='BOT', + pred_postprocessor=dict(type='general_cn'), + dataset_postprocessor=dict(type='general_cn')) + +summscreen_datasets = [ + dict( + type=SummScreenDataset, + path='./data/SummScreen/', + abbr='SummScreen', + reader_cfg=summscreen_reader_cfg, + infer_cfg=summscreen_infer_cfg, + eval_cfg=summscreen_eval_cfg) +] diff --git a/configs/datasets/truthfulqa/truthfulqa_gen_d8faf6.py b/configs/datasets/truthfulqa/truthfulqa_gen_d8faf6.py new file mode 100644 index 00000000..cc4b959b --- /dev/null +++ b/configs/datasets/truthfulqa/truthfulqa_gen_d8faf6.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import TruthfulQADataset, TruthfulQAEvaluator + +truthfulqa_reader_cfg = dict( + input_columns=['question'], + output_column='reference', + train_split='validation', + test_split='validation') + +# TODO: allow empty output-column +truthfulqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='{question}'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +# Metrics such as 'truth' and 'info' needs +# OPENAI_API_KEY with finetuned models in it. +# Please use your own finetuned openai model with keys and refers to +# the source code for more details +# Metrics such as 'bleurt', 'rouge', 'bleu' are free to test + +# When key is set to "ENV", the key will be fetched from the environment +# variable $OPENAI_API_KEY. Otherwise, set key in here directly. +truthfulqa_eval_cfg = dict( + evaluator=dict( + type=TruthfulQAEvaluator, metrics=('truth', 'info'), key='ENV'), ) + +truthfulqa_datasets = [ + dict( + type=TruthfulQADataset, + path='truthful_qa', + name='generation', + reader_cfg=truthfulqa_reader_cfg, + infer_cfg=truthfulqa_infer_cfg, + eval_cfg=truthfulqa_eval_cfg) +] diff --git a/configs/datasets/winograd/winograd_ppl_c1c427.py b/configs/datasets/winograd/winograd_ppl_c1c427.py new file mode 100644 index 00000000..8ba20788 --- /dev/null +++ b/configs/datasets/winograd/winograd_ppl_c1c427.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import winogradDataset + +winograd_reader_cfg = dict( + input_columns=['prompt', 'pronoun', 'opt1', 'opt2'], + output_column='label', + train_split='test', + test_split='test') + +winograd_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + i: dict(round=[ + dict( + role="HUMAN", + prompt= + f"{{prompt}} Q: In the previous text, what does '{{pronoun}}' refer to? A: {{opt{i+1}}}" + ), # noqa + ]) + for i in range(2) + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +winograd_eval_cfg = dict(evaluator=dict(type=AccEvaluator), ) + +winograd_datasets = [ + dict( + abbr='winograd', + type=winogradDataset, + path='winograd_wsc', + name='wsc285', + reader_cfg=winograd_reader_cfg, + infer_cfg=winograd_infer_cfg, + eval_cfg=winograd_eval_cfg) +] diff --git a/configs/datasets/winogrande/winogrande_gen_c19d87.py b/configs/datasets/winogrande/winogrande_gen_c19d87.py new file mode 100644 index 00000000..e86eb8e6 --- /dev/null +++ b/configs/datasets/winogrande/winogrande_gen_c19d87.py @@ -0,0 +1,43 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import winograndeDataset_V2 + +winogrande_reader_cfg = dict( + input_columns=["opt1", "opt2"], + output_column="label", + test_split="validation") + +winogrande_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role="HUMAN", + prompt= + "Which of the following is a good sentence:\nA. {opt1}\nB. {opt2}\nAnswer:" + ), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +winogrande_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_role="BOT", + pred_postprocessor=dict(type="first-capital"), +) + +winogrande_datasets = [ + dict( + abbr="winogrande", + type=winograndeDataset_V2, + path="winogrande", + name="winogrande_xs", + reader_cfg=winogrande_reader_cfg, + infer_cfg=winogrande_infer_cfg, + eval_cfg=winogrande_eval_cfg, + ) +] diff --git a/configs/datasets/z_bench/z_bench_gen_5813ec.py b/configs/datasets/z_bench/z_bench_gen_5813ec.py new file mode 100644 index 00000000..49077976 --- /dev/null +++ b/configs/datasets/z_bench/z_bench_gen_5813ec.py @@ -0,0 +1,25 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HFDataset + +z_bench_reader_cfg = dict( + input_columns=['text'], output_column='category', train_split='test') + +z_bench_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='{text}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +z_bench_dataset = dict( + type=HFDataset, + path= + '/mnt/petrelfs/gaotong/llm_eval/openagieval_dataset/eval_datasets/z_bench', + data_dir= + '/mnt/petrelfs/gaotong/llm_eval/openagieval_dataset/eval_datasets/z_bench', + name='question', + reader_cfg=z_bench_reader_cfg, + infer_cfg=z_bench_infer_cfg) diff --git a/configs/models/classic/gpt-3.5-turbo.py b/configs/models/classic/gpt-3.5-turbo.py new file mode 100644 index 00000000..4d82252c --- /dev/null +++ b/configs/models/classic/gpt-3.5-turbo.py @@ -0,0 +1,7 @@ +from opencompass.models import OpenAI + +models = [ + dict(abbr='GPT-3.5-turbo', + type=OpenAI, path='gpt-3.5-turbo', key='sk-xxx', + max_out_len=2048, max_seq_len=2048, batch_size=1) +] diff --git a/configs/summarizers/example.py b/configs/summarizers/example.py new file mode 100644 index 00000000..9b935a40 --- /dev/null +++ b/configs/summarizers/example.py @@ -0,0 +1,17 @@ +from mmengine.config import read_base + +with read_base(): + from .groups.agieval import agieval_summary_groups + from .groups.mmlu import mmlu_summary_groups + from .groups.ceval import ceval_summary_groups + from .groups.bbh import bbh_summary_groups + from .groups.GaokaoBench import GaokaoBench_summary_groups + from .groups.flores import flores_summary_groups + +summarizer = dict( + summary_groups=sum([v for k, v in locals().items() if k.endswith("_summary_groups")], []), + prompt_db=dict( + database_path='configs/datasets/log.json', + config_dir='configs/datasets', + blacklist='.promptignore') +) diff --git a/configs/summarizers/groups/GaokaoBench.py b/configs/summarizers/groups/GaokaoBench.py new file mode 100644 index 00000000..4354dc65 --- /dev/null +++ b/configs/summarizers/groups/GaokaoBench.py @@ -0,0 +1,5 @@ +GaokaoBench_summary_groups = [] + +# gaokao-bench +_GaokaoBench_weights = {'2010-2022_Math_II_MCQs': 1090, '2010-2022_Math_I_MCQs': 1070, '2010-2022_History_MCQs': 1148, '2010-2022_Biology_MCQs': 900, '2010-2022_Political_Science_MCQs': 1280, '2010-2022_Physics_MCQs': 384, '2010-2022_Chemistry_MCQs': 744, '2010-2013_English_MCQs': 105, '2010-2022_Chinese_Modern_Lit': 261, '2010-2022_English_Fill_in_Blanks': 900.0, '2012-2022_English_Cloze_Test': 260, '2010-2022_Geography_MCQs': 380, '2010-2022_English_Reading_Comp': 940, '2010-2022_Chinese_Lang_and_Usage_MCQs': 240} +GaokaoBench_summary_groups.append({'name': 'GaokaoBench', 'subsets': list(_GaokaoBench_weights.keys()), 'weights': _GaokaoBench_weights}) diff --git a/docs/en/_static/js/custom.js b/docs/en/_static/js/custom.js new file mode 100644 index 00000000..01293c8b --- /dev/null +++ b/docs/en/_static/js/custom.js @@ -0,0 +1,10 @@ +var collapsedSections = ['Advanced Guides', 'Tools', 'User Guides', 'Notes']; + +$(document).ready(function () { + $('.model-summary').DataTable({ + "stateSave": false, + "lengthChange": false, + "pageLength": 20, + "order": [] + }); +}); diff --git a/docs/en/conf.py b/docs/en/conf.py new file mode 100644 index 00000000..f8343924 --- /dev/null +++ b/docs/en/conf.py @@ -0,0 +1,219 @@ +# flake8: noqa +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import subprocess +import sys + +import pytorch_sphinx_theme +from sphinx.builders.html import StandaloneHTMLBuilder + +sys.path.insert(0, os.path.abspath('../../')) + +# -- Project information ----------------------------------------------------- + +project = 'OpenCompass' +copyright = '2023, OpenCompass' +author = 'OpenCompass Authors' + +# The full version, including alpha/beta/rc tags +# version_file = '../../opencompass/version.py' + +# def get_version(): +# with open(version_file, 'r') as f: +# exec(compile(f.read(), version_file, 'exec')) +# return locals()['__version__'] + +release = '1.0.0' + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'myst_parser', + 'sphinx_copybutton', + 'sphinx_tabs.tabs', + 'notfound.extension', + 'sphinxcontrib.jquery', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', +} + +language = 'en' + +# The master toctree document. +root_doc = 'index' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'pytorch_sphinx_theme' +html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# yapf: disable +html_theme_options = { + 'menu': [ + { + 'name': 'GitHub', + 'url': 'https://github.com/opencompass' + }, + ], + # Specify the language of shared menu + 'menu_lang': 'en', + # Disable the default edit on GitHub + 'default_edit_on_github': False, +} +# yapf: enable + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +html_css_files = [ + 'https://cdn.datatables.net/v/bs4/dt-1.12.1/datatables.min.css', + 'css/readthedocs.css' +] +html_js_files = [ + 'https://cdn.datatables.net/v/bs4/dt-1.12.1/datatables.min.js', + 'js/custom.js' +] + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'opencompassdoc' + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (root_doc, 'opencompass.tex', 'OpenCompass Documentation', author, + 'manual'), +] + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(root_doc, 'opencompass', 'OpenCompass Documentation', [author], + 1)] + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (root_doc, 'opencompass', 'OpenCompass Documentation', author, + 'OpenCompass Authors', 'AGI evaluation toolbox and benchmark.', + 'Miscellaneous'), +] + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# set priority when building html +StandaloneHTMLBuilder.supported_image_types = [ + 'image/svg+xml', 'image/gif', 'image/png', 'image/jpeg' +] + +# -- Extension configuration ------------------------------------------------- +# Ignore >>> when copying code +copybutton_prompt_text = r'>>> |\.\.\. ' +copybutton_prompt_is_regexp = True + +# Auto-generated header anchors +myst_heading_anchors = 3 +# Enable "colon_fence" extension of myst. +myst_enable_extensions = ['colon_fence', 'dollarmath'] + +# Configuration for intersphinx +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'numpy': ('https://numpy.org/doc/stable', None), + 'torch': ('https://pytorch.org/docs/stable/', None), + 'mmengine': ('https://mmengine.readthedocs.io/en/latest/', None), + 'transformers': + ('https://huggingface.co/docs/transformers/main/en/', None), +} +napoleon_custom_sections = [ + # Custom sections for data elements. + ('Meta fields', 'params_style'), + ('Data fields', 'params_style'), +] + +# Disable docstring inheritance +autodoc_inherit_docstrings = False +# Mock some imports during generate API docs. +autodoc_mock_imports = ['rich', 'attr', 'einops'] +# Disable displaying type annotations, these can be very verbose +autodoc_typehints = 'none' + +# The not found page +notfound_template = '404.html' diff --git a/docs/en/get_started.md b/docs/en/get_started.md new file mode 100644 index 00000000..834caf3d --- /dev/null +++ b/docs/en/get_started.md @@ -0,0 +1,40 @@ +# Overview + +# Installation + +1. Prepare Torch refer to [PyTorch](https://pytorch.org/). + +Notice that OpenCompass requires `pytorch>=1.13`. + +```bash +conda create --name opencompass python=3.8 -y +conda activate opencompass +conda install pytorch torchvision -c pytorch +``` + +2. Install OpenCompass: + +```bash +git clone https://github.com/opencompass/opencompass +cd opencompass +pip install -r requirments/runtime.txt +pip install -e . +``` + +3. Install humaneval (option) + +do this if you want to eval on humaneval dataset. + +``` +git clone https://github.com/openai/human-eval.git +cd human-eval +pip install -r requirments.txt +pip install -e . +``` + +Remember to remove the comments of Line48-57 and uncomment [line58](https://github.com/openai/human-eval/blob/312c5e5532f0e0470bf47f77a6243e02a61da530/human_eval/execution.py#L58) in the source code. + +# Quick tour + + + diff --git a/docs/en/index.rst b/docs/en/index.rst new file mode 100644 index 00000000..860a9f6a --- /dev/null +++ b/docs/en/index.rst @@ -0,0 +1,85 @@ +Welcome to OpenCompass' documentation! +========================================== + +Hands-on Roadmap of OpenCompass +------------------------------- + +To help users quickly utilize OpenCompass, we recommend following the hands-on +roadmap we have created for the library: + + - For users who want to use OpenCompass, we recommend reading the GetStarted_ section first to set up the environment. + + - For some basic usage, we suggest users read the UserGuides_. + + - If you want to customize the algorithm, we have provided the AdvancedGuides_. + + - If you want to adjust the prompts, you can browse the Prompt_. + + - We also offer the Tools_. + +We always welcome *PRs* and *Issues* for the betterment of MMPretrain. + +.. _GetStarted: +.. toctree:: + :maxdepth: 1 + :caption: Get Started + + get_started.md + +.. _UserGuides: +.. toctree:: + :maxdepth: 1 + :caption: UserGuides + + user_guides/framework_overview.md + user_guides/config.md + user_guides/dataset_prepare.md + user_guides/models.md + user_guides/evaluation.md + user_guides/experimentation.md + user_guides/metrics.md + +.. _AdvancedGuides: +.. toctree:: + :maxdepth: 1 + :caption: AdvancedGuides + + advanced_guides/new_dataset.md + advanced_guides/new_model.md + +.. _Prompt: +.. toctree:: + :maxdepth: 1 + :caption: Prompt + + prompt/overview.md + prompt/few_shot.md + prompt/prompt_template.md + prompt/meta_template.md + +.. _Tools: +.. toctree:: + :maxdepth: 1 + :caption: tools + + tools.md + +.. _Notes: +.. toctree:: + :maxdepth: 1 + :caption: Notes + + notes/contribution_guide.md + +.. toctree:: + :caption: switch language + + English + 简体中文 + + +Indexes & Tables +================== + +* :ref:`genindex` +* :ref:`search` \ No newline at end of file diff --git a/docs/en/notes/contribution_guide.md b/docs/en/notes/contribution_guide.md new file mode 100644 index 00000000..04b395ac --- /dev/null +++ b/docs/en/notes/contribution_guide.md @@ -0,0 +1,69 @@ +# Contributing to OpenCompass + +- [Contributing to OpenCompass](#contributing-to-opencompass) + - [Workflow](#workflow) + - [Code style](#code-style) + - [Python](#python) + - [Pre-commit Hook](#pre-commit-hook) + +Thanks for your interest in contributing to OpenCompass! All kinds of contributions are welcome, including but not limited to the following. + +- Fix typo or bugs +- Add documentation or translate the documentation into other languages +- Add new features and components + +## Workflow + +We recommend the potential contributors follow this workflow for contribution. + +1. Fork and pull the latest OpenCompass repository, follow [get started](https://OpenCompass.readthedocs.io/en/latest/get_started.html) to setup the environment. +2. Checkout a new branch (**do not use the master or dev branch** for PRs) + +```bash +git checkout -b xxxx # xxxx is the name of new branch +``` + +3. Edit the related files follow the code style mentioned below +4. Use [pre-commit hook](https://pre-commit.com/) to check and format your changes. +5. Commit your changes +6. Create a PR with related information + +## Code style + +### Python + +We adopt [PEP8](https://www.python.org/dev/peps/pep-0008/) as the preferred code style. + +We use the following tools for linting and formatting: + +- [flake8](https://github.com/PyCQA/flake8): A wrapper around some linter tools. +- [isort](https://github.com/timothycrosley/isort): A Python utility to sort imports. +- [yapf](https://github.com/google/yapf): A formatter for Python files. +- [codespell](https://github.com/codespell-project/codespell): A Python utility to fix common misspellings in text files. +- [mdformat](https://github.com/executablebooks/mdformat): Mdformat is an opinionated Markdown formatter that can be used to enforce a consistent style in Markdown files. +- [docformatter](https://github.com/myint/docformatter): A formatter to format docstring. + +Style configurations of yapf and isort can be found in [setup.cfg](https://github.com/open-mmlab/OpenCompass/blob/main/setup.cfg). + + +## Pre-commit Hook + +We use [pre-commit hook](https://pre-commit.com/) that checks and formats for `flake8`, `yapf`, `isort`, `trailing whitespaces`, `markdown files`, +fixes `end-of-files`, `double-quoted-strings`, `python-encoding-pragma`, `mixed-line-ending`, sorts `requirments.txt` automatically on every commit. +The config for a pre-commit hook is stored in [.pre-commit-config](xxxxxxx). + +After you clone the repository, you will need to install initialize pre-commit hook. + +```shell +pip install -U pre-commit +``` + +From the repository folder + +```shell +pre-commit install +``` + +After this on every commit check code linters and formatter will be enforced. + +> Before you create a PR, make sure that your code lints and is formatted by yapf. \ No newline at end of file diff --git a/docs/en/prompt/few_shot.md b/docs/en/prompt/few_shot.md new file mode 100644 index 00000000..0a6faaf0 --- /dev/null +++ b/docs/en/prompt/few_shot.md @@ -0,0 +1 @@ +# In-context Learning \ No newline at end of file diff --git a/docs/en/user_guides/dataset_prepare.md b/docs/en/user_guides/dataset_prepare.md new file mode 100644 index 00000000..2ff2ef43 --- /dev/null +++ b/docs/en/user_guides/dataset_prepare.md @@ -0,0 +1,105 @@ +# Preparing and Selecting Datasets + +This section of the tutorial mainly focuses on how to prepare the datasets supported by OpenCompass and build configuration files to complete dataset selection. + +## Directory Structure of Dataset Configuration Files + +First, let's introduce the structure under the `configs/datasets` directory in OpenCompass, as shown below: + +``` +configs/datasets/ +├── ChineseUniversal # Ability dimension +│ ├── CLUE_afqmc # Dataset under this dimension +│ │ ├── CLUE_afqmc_gen_db509b.py # Different configuration files for this dataset +│ │ ├── CLUE_afqmc_gen.py +│ │ ├── CLUE_afqmc_ppl_00b348.py +│ │ ├── CLUE_afqmc_ppl_2313cf.py +│ │ └── CLUE_afqmc_ppl.py +│ ├── CLUE_C3 +│ │ ├── ... +│ ├── ... +├── Coding +├── collections +├── Completion +├── EnglishUniversal +├── Exam +├── glm +├── LongText +├── MISC +├── NLG +├── QA +├── Reasoning +├── Security +└── Translation +``` + +In the `configs/datasets` directory structure, we have divided the datasets into over ten dimensions based on ability dimensions, such as: Chinese and English Universal, Exam, QA, Reasoning, Security, etc. Each dimension contains a series of datasets, and there are multiple dataset configurations in the corresponding folder of each dataset. + +The naming of the dataset configuration file is made up of `{dataset name}_{evaluation method}_{prompt version number}.py`. For example, `ChineseUniversal/CLUE_afqmc/CLUE_afqmc_gen_db509b.py`, this configuration file is the `CLUE_afqmc` dataset under the Chinese universal ability, the corresponding evaluation method is `gen`, i.e., generative evaluation, and the corresponding prompt version number is `db509b`; similarly, `CLUE_afqmc_ppl_00b348.py` indicates that the evaluation method is `ppl`, i.e., discriminative evaluation, and the prompt version number is `00b348`. + +In addition, files without a version number, such as: `CLUE_afqmc_gen.py`, point to the latest prompt configuration file of that evaluation method, which is usually the most accurate prompt. + +## Dataset Preparation + +The datasets supported by OpenCompass mainly include two parts: + +1. Huggingface Dataset + +[Huggingface Dataset](https://huggingface.co/datasets) provides a large number of datasets. OpenCompass has supported most of the datasets commonly used for performance comparison, please refer to `configs/dataset` for the specific list of supported datasets. + +2. OpenCompass Self-built Datasets + +In addition to supporting Huggingface's existing datasets, OpenCompass also provides some self-built CN datasets. In the future, a dataset-related Repo will be provided for users to download and use. Following the instructions in the document to place the datasets uniformly in the `./data` directory can complete dataset preparation. + +It is important to note that the Repo not only contains self-built datasets, but also includes some HF-supported datasets for testing convenience. + +## Dataset Selection + +In each dataset configuration file, the dataset will be defined in the `{}_datasets` variable, such as `afqmc_datasets` in `ChineseUniversal/CLUE_afqmc/CLUE_afqmc_gen_db509b.py`. + +```python +afqmc_datasets = [ + dict( + abbr="afqmc-dev", + type=AFQMCDataset_V2, + path="./data/CLUE/AFQMC/dev.json", + reader_cfg=afqmc_reader_cfg, + infer_cfg=afqmc_infer_cfg, + eval_cfg=afqmc_eval_cfg, + ), +] +``` + +And `afqmc_datasets` in `ChineseUniversal/CLUE_cmnli/CLUE_cmnli_ppl_b78ad4.py`. + +```python +cmnli_datasets = [ + dict( + type=HFDataset, + abbr='cmnli', + path='json', + split='train', + data_files='./data/CLUE/cmnli/cmnli_public/dev.json', + reader_cfg=cmnli_reader_cfg, + infer_cfg=cmnli_infer_cfg, + eval_cfg=cmnli_eval_cfg) +] +``` + +Take these two datasets as examples. If users want to evaluate these two datasets at the same time, they can create a new configuration file in the `configs` directory. We use the import mechanism in the `mmengine` configuration to build the part of the dataset parameters in the evaluation script, as shown below: + +```python +from mmengine.config import read_base + +with read_base(): + from .datasets.CLUE_afqmc.CLUE_afqmc_gen_db509b import afqmc_datasets + from .datasets.CLUE_cmnli.CLUE_cmnli_ppl_b78ad4 import cmnli_datasets + +datasets = [] +datasets += afqmc_datasets +datasets += cmnli_datasets +``` + +Users can choose different abilities, different datasets and different evaluation methods configuration files to build the part of the dataset in the evaluation script according to their needs. + +For information on how to start an evaluation task and how to evaluate self-built datasets, please refer to the relevant documents. diff --git a/docs/en/user_guides/evaluation.md b/docs/en/user_guides/evaluation.md new file mode 100644 index 00000000..1d5339b3 --- /dev/null +++ b/docs/en/user_guides/evaluation.md @@ -0,0 +1 @@ +# Evaluation Strategy diff --git a/docs/zh_cn/advanced_guides/new_dataset.md b/docs/zh_cn/advanced_guides/new_dataset.md new file mode 100644 index 00000000..44f2f339 --- /dev/null +++ b/docs/zh_cn/advanced_guides/new_dataset.md @@ -0,0 +1,60 @@ +# 支持新数据集 + +尽管 OpenCompass 已经包含了大多数常用数据集，用户在支持新数据集的时候需要完成以下几个步骤： + +1. 在 `opencompass/datasets` 文件夹新增数据集脚本 `mydataset.py`, 该脚本需要包含： + + - 数据集及其加载方式，需要定义一个 `MyDataset` 类，实现数据集加载方法 `load` ，该方法为静态方法，需要返回 `datasets.Dataset` 类型的数据。这里我们使用 huggingface dataset 作为数据集的统一接口，避免引入额外的逻辑。具体示例如下： + + ```python + import datasets + from .base import BaseDataset + + class MyDataset(BaseDataset): + + @staticmethod + def load(**kwargs) -> datasets.Dataset: + pass + ``` + + - （可选）如果OpenCompass已有的evaluator不能满足需要，需要用户定义 `MyDatasetlEvaluator` 类，实现评分方法 `score` ，需要根据输入的 `predictions` 和 `references` 列表，得到需要的字典。由于一个数据集可能存在多种metric，需要返回一个 metrics 以及对应 scores 的相关字典。具体示例如下： + + ```python + + from opencompass.openicl.icl_evaluator import BaseEvaluator + + class MyDatasetlEvaluator(BaseEvaluator): + + def score(self, predictions: List, references: List) -> dict: + pass + + ``` + + - （可选）如果 OpenCompass 已有的 postprocesser 不能满足需要，需要用户定义 `mydataset_postprocess` 方法，根据输入的字符串得到相应后处理的结果。具体示例如下： + + ```python + def mydataset_postprocess(text: str) -> str: + pass + ``` + +2. 在定义好数据集加载，数据后处理以及 `evaluator` 等方法之后，需要在配置文件中新增以下配置： + + ```python + from opencompass.datasets import MyDataset, MyDatasetlEvaluator, mydataset_postprocess + + mydataset_eval_cfg = dict( + evaluator=dict(type=MyDatasetlEvaluator), + pred_postprocessor=dict(type=mydataset_postprocess)) + + mydataset_datasets = [ + dict( + type=MyDataset, + ..., + reader_cfg=..., + infer_cfg=..., + eval_cfg=mydataset_eval_cfg) + ] + ``` + + 配置好数据集之后，其他需要的配置文件直接参考如何启动评测任务教程即可。 + \ No newline at end of file diff --git a/docs/zh_cn/advanced_guides/new_model.md b/docs/zh_cn/advanced_guides/new_model.md new file mode 100644 index 00000000..258dec82 --- /dev/null +++ b/docs/zh_cn/advanced_guides/new_model.md @@ -0,0 +1,73 @@ +# 支持新模型 + +目前我们已经支持的模型有 HF 模型、部分模型 API 、自建模型和部分第三方模型。 + +## 新增API模型 + +新增基于API的模型，需要在 `opencompass/models` 下新建 `mymodel_api.py` 文件，继承 `BaseAPIModel`，并实现 `generate` 方法来进行推理，以及 `get_token_len` 方法来计算 token 的长度。在定义好之后修改对应配置文件名称即可。 + +```python +from ..base_api import BaseAPIModel + +class MyModelAPI(BaseAPIModel): + + is_api: bool = True + + def __init__(self, + path: str, + max_seq_len: int = 2048, + query_per_second: int = 1, + retry: int = 2, + **kwargs): + super().__init__(path=path, + max_seq_len=max_seq_len, + meta_template=meta_template, + query_per_second=query_per_second, + retry=retry) + ... + + def generate( + self, + inputs, + max_out_len: int = 512, + temperature: float = 0.7, + ) -> List[str]: + """Generate results given a list of inputs.""" + pass + + def get_token_len(self, prompt: str) -> int: + """Get lengths of the tokenized string.""" + pass +``` + +## 新增第三方模型 + +新增基于API的模型，需要在 `opencompass/models` 下新建 `mymodel.py` 文件，继承 `BaseModel`，并实现 `generate` 方法来进行生成式推理， `get_ppl` 方法来进行判别式推理，以及 `get_token_len` 方法来计算 token 的长度。在定义好之后修改对应配置文件名称即可。 + +```python +from ..base import BaseModel + +class MyModel(BaseModel): + + def __init__(self, + pkg_root: str, + ckpt_path: str, + tokenizer_only: bool = False, + meta_template: Optional[Dict] = None, + **kwargs): + ... + + def get_token_len(self, prompt: str) -> int: + """Get lengths of the tokenized strings.""" + pass + + def generate(self, inputs: List[str], max_out_len: int) -> List[str]: + """Generate results given a list of inputs. """ + pass + + def get_ppl(self, + inputs: List[str], + mask_length: Optional[List[int]] = None) -> List[float]: + """Get perplexity scores given a list of inputs.""" + pass +``` diff --git a/docs/zh_cn/conf.py b/docs/zh_cn/conf.py new file mode 100644 index 00000000..352e1cfe --- /dev/null +++ b/docs/zh_cn/conf.py @@ -0,0 +1,218 @@ +# flake8: noqa +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import subprocess +import sys + +import pytorch_sphinx_theme +from sphinx.builders.html import StandaloneHTMLBuilder + +sys.path.insert(0, os.path.abspath('../../')) + +# -- Project information ----------------------------------------------------- + +project = 'OpenCompass' +copyright = '2023, OpenCompass' +author = 'OpenCompass Authors' + +# The full version, including alpha/beta/rc tags +# version_file = '../../opencompass/version.py' + + +# def get_version(): +# with open(version_file, 'r') as f: +# exec(compile(f.read(), version_file, 'exec')) +# return locals()['__version__'] + + +release = "0.5.0" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'myst_parser', + 'sphinx_copybutton', + 'sphinx_tabs.tabs', + 'notfound.extension', + 'sphinxcontrib.jquery', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', +} + +language = 'en' + +# The master toctree document. +root_doc = 'index' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'pytorch_sphinx_theme' +html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# yapf: disable +html_theme_options = { + 'menu': [ + { + 'name': 'GitHub', + 'url': 'https://github.com/open-mmlab/opencompass' + }, + ], + # Specify the language of shared menu + 'menu_lang': 'en', + # Disable the default edit on GitHub + 'default_edit_on_github': False, +} +# yapf: enable + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +html_css_files = [ + 'https://cdn.datatables.net/v/bs4/dt-1.12.1/datatables.min.css', + 'css/readthedocs.css' +] +html_js_files = [ + 'https://cdn.datatables.net/v/bs4/dt-1.12.1/datatables.min.js', + 'js/custom.js' +] + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'opencompassdoc' + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (root_doc, 'opencompass.tex', 'OpenCompass Documentation', author, 'manual'), +] + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(root_doc, 'opencompass', 'OpenCompass Documentation', [author], 1)] + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (root_doc, 'opencompass', 'OpenCompass Documentation', author, 'OpenCompass Authors', + 'AGI evaluation toolbox and benchmark.', 'Miscellaneous'), +] + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# set priority when building html +StandaloneHTMLBuilder.supported_image_types = [ + 'image/svg+xml', 'image/gif', 'image/png', 'image/jpeg' +] + +# -- Extension configuration ------------------------------------------------- +# Ignore >>> when copying code +copybutton_prompt_text = r'>>> |\.\.\. ' +copybutton_prompt_is_regexp = True + +# Auto-generated header anchors +myst_heading_anchors = 3 +# Enable "colon_fence" extension of myst. +myst_enable_extensions = ['colon_fence', 'dollarmath'] + +# Configuration for intersphinx +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'numpy': ('https://numpy.org/doc/stable', None), + 'torch': ('https://pytorch.org/docs/stable/', None), + 'mmengine': ('https://mmengine.readthedocs.io/en/latest/', None), + 'transformers': + ('https://huggingface.co/docs/transformers/main/en/', None), +} +napoleon_custom_sections = [ + # Custom sections for data elements. + ('Meta fields', 'params_style'), + ('Data fields', 'params_style'), +] + +# Disable docstring inheritance +autodoc_inherit_docstrings = False +# Mock some imports during generate API docs. +autodoc_mock_imports = ['rich', 'attr', 'einops'] +# Disable displaying type annotations, these can be very verbose +autodoc_typehints = 'none' + +# The not found page +notfound_template = '404.html' diff --git a/docs/zh_cn/get_started.md b/docs/zh_cn/get_started.md new file mode 100644 index 00000000..292d4c41 --- /dev/null +++ b/docs/zh_cn/get_started.md @@ -0,0 +1,38 @@ +# 概述 + +# 安装 + +1. 参考 [PyTorch](https://pytorch.org/) 准备 Torch。 + +注意，OpenCompass 需要 `pytorch>=1.13`。 + +```bash +conda create --name opencompass python=3.8 -y +conda activate opencompass +conda install pytorch torchvision -c pytorch +``` + +2. 安装 OpenCompass： + +```bash +git clone https://github.com/opencompass/opencompass +cd opencompass +pip install -r requirments/runtime.txt +pip install -e . +``` + +3. 安装 humaneval（可选） + +如果你希望在 humaneval 数据集上进行评估，请执行此步骤。 + +``` +git clone https://github.com/openai/human-eval.git +cd human-eval +pip install -r requirments.txt +pip install -e . +``` + +请记住在源代码中删除第48-57行的注释，并取消对[第58行](https://github.com/openai/human-eval/blob/312c5e5532f0e0470bf47f77a6243e02a61da530/human_eval/execution.py#L58)的注释。 + +# 快速上手 + diff --git a/docs/zh_cn/user_guides/experimentation.md b/docs/zh_cn/user_guides/experimentation.md new file mode 100644 index 00000000..793fa66e --- /dev/null +++ b/docs/zh_cn/user_guides/experimentation.md @@ -0,0 +1,88 @@ +# 任务运行和监控 + +## 评测任务发起 + +评测任务的程序入口为 `run.py`，使用方法如下： + +```shell +run.py [-p PARTITION] [-q QUOTATYPE] [--debug] [-m MODE] [-r [REUSE]] [-w WORKDIR] [-l LARK] config +``` + +参数解释如下： + +- -p 指定 slurm 分区； +- -q 指定 slurm quotatype （默认为 auto），可选 reserved, auto, spot； +- --debug 开启时，推理和评测任务会以单进程模式运行，且输出会实时回显，便于调试； +- -m 运行模式，默认为 all。可以指定为 infer 则仅运行推理，获得输出结果；如果在 {WORKDIR} 中已经有模型输出，则指定为 eval 仅运行评测，获得评测结果；如果在 results 中已有单项评测结果，则指定为 viz 仅运行可视化；指定为 all 则同时运行推理和评测。 +- -r 重用已有的推理结果。如果后面跟有时间戳，则会复用工作路径下该时间戳的结果；否则则复用指定工作路径下的最新结果。 +- -w 指定工作路径，默认为 ./outputs/default +- -l 打开飞书机器人状态上报。 + + +以运行模式`-m all`为例，整体运行流如下： + +1. 读取配置文件，解析出模型、数据集、评估器等配置信息 +2. 评测任务主要分为推理 infer、评测 eval 和可视化 viz 三个阶段，其中推理和评测经过 Partitioner 进行任务切分后，交由 Runner 负责并行执行。单个推理和评测任务则被抽象成 OpenICLInferTask 和 OpenICLEvalTask。 +3. 两阶段分别结束后，可视化阶段会读取 results 中的评测结果，生成可视化报告。 + + +## 任务监控：飞书机器人 + +用户可以通过配置飞书机器人，实现任务状态的实时监控。飞书机器人的设置文档请[参考这里](https://open.feishu.cn/document/ukTMukTMukTM/ucTM5YjL3ETO24yNxkjN?lang=zh-CN#7a28964d)。 + +配置方式: + +1. 打开 `configs/lark.py` 文件，并在文件中加入以下行： + + ```python + lark_bot_url = 'YOUR_WEBHOOK_URL' + ``` + + 通常， Webhook URL 格式如 https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxxxxxxxxx 。 + +2. 在完整的评测配置中继承该文件： + + ```python + from mmengine.config import read_base + + with read_base(): + from .lark import lark_bot_url + + ``` + +3. 为了避免机器人频繁发消息形成骚扰，默认运行时状态不会自动上报。有需要时，可以通过 `-l` 或 `--lark` 启动状态上报： + + ```bash + python run.py configs/eval_demo.py -p {PARTITION} -l + ``` + +## Summerizer介绍 + +主要用于可视化评测结果。 + +## 运行结果 + +所有运行结果会默认放在`outputs/default/`目录下，目录结构如下所示： + +``` +outputs/default/ +├── 20200220_120000 +├── ... +├── 20230220_183030 +│   ├── configs +│   ├── logs +│   │   ├── eval +│   │   └── infer +│   ├── predictions +│   │   └── MODEL1 +│   └── results +│ └── MODEL1 +``` + +其中，每一个时间戳中存在以下内容： +- configs文件夹，用于存放以这个时间戳为输出目录的每次运行对应的配置文件； +- logs文件夹，用于存放推理和评测两个阶段的输出日志文件，各个文件夹内会以模型为子文件夹存放日志； +- predicitions文件夹，用于存放推理json结果，以模型为子文件夹； +- results文件夹，用于存放评测json结果，以模型为子文件夹 + +另外，所有指定-r 但是没有指定对应时间戳将会按照排序选择最新的文件夹作为输出目录。 diff --git a/opencompass/__init__.py b/opencompass/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/__init__.py b/opencompass/datasets/__init__.py new file mode 100644 index 00000000..7e559f42 --- /dev/null +++ b/opencompass/datasets/__init__.py @@ -0,0 +1,64 @@ +from .afqmcd import * # noqa: F401, F403 +from .agieval import * # noqa: F401, F403 +from .arc import * # noqa: F401, F403 +from .ax import * # noqa: F401, F403 +from .bbh import * # noqa: F401, F403 +from .boolq import * # noqa: F401, F403 +from .bustum import * # noqa: F401, F403 +from .c3 import * # noqa: F401, F403 +from .cb import * # noqa: F401, F403 +from .ceval import * # noqa: F401, F403 +from .chid import * # noqa: F401, F403 +from .civilcomments import * # noqa: F401, F403 +from .cluewsc import * # noqa: F401, F403 +from .cmnli import * # noqa: F401, F403 +from .cmrc import * # noqa: F401, F403 +from .commonsenseqa import * # noqa: F401, F403 +from .copa import * # noqa: F401, F403 +from .crowspairs import * # noqa: F401, F403 +from .csl import * # noqa: F401, F403 +from .drcd import * # noqa: F401, F403 +from .drop import * # noqa: F401, F403 +from .eprstmt import * # noqa: F401, F403 +from .flores import * # noqa: F401, F403 +from .GaokaoBench import * # noqa: F401, F403 +from .govrepcrs import * # noqa: F401, F403 +from .gsm8k import * # noqa: F401, F403 +from .hellaswag import * # noqa: F401, F403 +from .huggingface import * # noqa: F401, F403 +from .humaneval import * # noqa: F401, F403 +from .iwslt2017 import * # noqa: F401, F403 +from .jigsawmultilingual import * # noqa: F401, F403 +from .lambada import * # noqa: F401, F403 +from .lcsts import * # noqa: F401, F403 +from .math import * # noqa: F401, F403 +from .mbpp import * # noqa: F401, F403 +from .mmlu import * # noqa: F401, F403 +from .multirc import * # noqa: F401, F403 +from .narrativeqa import * # noqa: F401, F403 +from .natural_question import * # noqa: F401, F403 +from .obqa import * # noqa: F401, F403 +from .piqa import * # noqa: F401, F403 +from .qasper import * # noqa: F401, F403 +from .qaspercut import * # noqa: F401, F403 +from .race import * # noqa: F401, F403 +from .realtoxicprompts import * # noqa: F401, F403 +from .record import * # noqa: F401, F403 +from .safety import * # noqa: F401, F403 +from .siqa import * # noqa: F401, F403 +from .storycloze import * # noqa: F401, F403 +from .strategyqa import * # noqa: F401, F403 +from .summedits import * # noqa: F401, F403 +from .summscreen import * # noqa: F401, F403 +from .TheoremQA import * # noqa: F401, F403 +from .tnews import * # noqa: F401, F403 +from .triviaqa import * # noqa: F401, F403 +from .triviaqarc import * # noqa: F401, F403 +from .truthfulqa import * # noqa: F401, F403 +from .wic import * # noqa: F401, F4 +from .winograd import * # noqa: F401, F403 +from .winogrande import * # noqa: F401, F403 +from .wsc import * # noqa: F401, F403 +from .xcopa import * # noqa: F401, F403 +from .xlsum import * # noqa: F401, F403 +from .xsum import * # noqa: F401, F403 diff --git a/opencompass/datasets/agieval/agieval.py b/opencompass/datasets/agieval/agieval.py new file mode 100644 index 00000000..2d5541a2 --- /dev/null +++ b/opencompass/datasets/agieval/agieval.py @@ -0,0 +1,67 @@ +import json +import os.path as osp + +from datasets import Dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET + +from .math_equivalence import is_equiv +from .post_process import parse_math_answer + +from ..base import BaseDataset + + +@LOAD_DATASET.register_module() +class AGIEvalDataset(BaseDataset): + + @staticmethod + def load(path: str, name: str, setting_name: str): + from .dataset_loader import load_dataset, load_dataset_as_result_schema + + assert setting_name in 'zero-shot', 'only support zero-shot setting' + dataset_wo_label = load_dataset(name, setting_name, path) + dataset_with_label = load_dataset_as_result_schema(name, path) + dataset = [] + for d1, d2 in zip(dataset_wo_label, dataset_with_label): + dataset.append({ + 'id': d2.index, + 'problem_input': d1['context'], + 'label': d2.label, + }) + dataset = Dataset.from_list(dataset) + return dataset + + +@LOAD_DATASET.register_module() +class AGIEvalDataset_v2(BaseDataset): + + @staticmethod + def load(path: str, name: str, setting_name: str): + assert setting_name in 'zero-shot', 'only support zero-shot setting' + filename = osp.join(path, name + '.jsonl') + with open(filename) as f: + _data = [json.loads(line.strip()) for line in f] + data = [] + for _d in _data: + passage = _d['passage'] if _d['passage'] else '' + question = passage + _d['question'] + options = '\n'.join(_d['options']) if _d['options'] else '' + label = _d['label'] if _d['label'] else _d['answer'] + d = {'question': question, 'options': options, 'label': label} + data.append(d) + dataset = Dataset.from_list(data) + return dataset + + +@ICL_EVALUATORS.register_module() +class AGIEvalEvaluator(BaseEvaluator): + + def score(self, predictions, references): + predictions = [parse_math_answer('', pred) for pred in predictions] + cnt = 0 + for pred, ref in zip(predictions, references): + if is_equiv(pred, ref): + cnt += 1 + score = cnt / len(predictions) * 100 + return {'score': score} diff --git a/opencompass/datasets/agieval/constructions.py b/opencompass/datasets/agieval/constructions.py new file mode 100644 index 00000000..b7c83676 --- /dev/null +++ b/opencompass/datasets/agieval/constructions.py @@ -0,0 +1,104 @@ +# flake8: noqa +import pandas as pd + + +class TaskSchema(object): + + def __init__(self, + passage=None, + question=None, + options=None, + label=None, + answer=None, + other=None): + self.passage = passage + self.question = question + self.options = options + self.label = label + self.answer = answer + self.other = other + + def to_dict(self): + return { + 'passage': self.passage, + 'question': self.question, + 'options': self.options, + 'label': self.label, + 'answer': self.answer, + 'other': self.other + } + + +# define README.json +class AgiInstance(object): + + def __init__(self, task_description, data_source, task_schema, output, + evaluation_metric, task_example): + self.task_description = task_description + self.data_source = data_source + self.task_schema = task_schema + self.output = output + self.evaluation_metric = evaluation_metric + self.task_example = task_example + + def to_dict(self): + return { + 'task description': self.task_description, + 'data source': self.data_source, + 'task schema': self.task_schema.to_dict(), + 'output': self.output, + 'evaluation metric': self.evaluation_metric, + 'task example': self.task_example + } + + +class ChatGPTSchema(object): + + def __init__(self, context=None, metadata=''): + self.context = context + self.metadata = metadata + + def to_dict(self): + return {'context': self.context, 'metadata': self.metadata} + + +class ResultsForHumanSchema(object): + + def __init__(self, + index, + problem_input, + label, + model_input='', + model_output='', + parse_result='', + first_stage_output='', + second_stage_input='', + is_correct=False): + self.index = index + self.problem_input = problem_input + self.model_input = model_input + self.model_output = model_output + self.parse_result = parse_result + self.label = label + self.first_stage_output = first_stage_output + self.second_stage_input = second_stage_input + self.is_correct = is_correct + + def to_dict(self): + return { + 'index': self.index, + 'problem_input': self.problem_input, + 'model_input': self.model_input, + 'model_output': self.model_output, + 'parse_result': self.parse_result, + 'label': self.label, + 'is_correct': self.is_correct, + 'first_stage_output': self.first_stage_output, + 'second_stage_input': self.second_stage_input, + } + + @staticmethod + def to_tsv(result_list, path): + result_json = [item.to_dict() for item in result_list] + table = pd.json_normalize(result_json) + table.to_excel(path, index=False) diff --git a/opencompass/datasets/agieval/dataset_loader.py b/opencompass/datasets/agieval/dataset_loader.py new file mode 100644 index 00000000..dda23238 --- /dev/null +++ b/opencompass/datasets/agieval/dataset_loader.py @@ -0,0 +1,392 @@ +# flake8: noqa +import ast +import json +import os + +import pandas as pd +import tiktoken +from tqdm import tqdm + +from .constructions import ChatGPTSchema, ResultsForHumanSchema +from .utils import extract_answer, read_jsonl, save_jsonl + +# define the datasets +english_qa_datasets = [ + 'lsat-ar', 'lsat-lr', 'lsat-rc', 'logiqa-en', 'sat-math', 'sat-en', + 'aqua-rat', 'sat-en-without-passage', 'gaokao-english' +] +chinese_qa_datasets = [ + 'logiqa-zh', 'jec-qa-kd', 'jec-qa-ca', 'gaokao-chinese', + 'gaokao-geography', 'gaokao-history', 'gaokao-biology', 'gaokao-chemistry', + 'gaokao-physics', 'gaokao-mathqa' +] +english_cloze_datasets = ['math'] +chinese_cloze_datasets = ['gaokao-mathcloze'] + +multi_choice_datasets = ['jec-qa-kd', 'jec-qa-ca', 'gaokao-physics'] +math_output_datasets = ['gaokao-mathcloze', 'math'] + + +def convert_zero_shot(line, dataset_name): + try: + passage = line['passage'] if line['passage'] is not None else '' + if dataset_name in english_qa_datasets: + option_string = 'ABCDEFG' + count = len(line['options']) + if count == 1: + count = 5 + return passage + 'Q: ' + line['question'] + ' ' \ + + 'Answer Choices: ' + ' '.join(line['options']) + '\n' + \ + 'A: Among A through {}, the answer is'.format(option_string[count - 1]) + + elif dataset_name in chinese_qa_datasets: + option_string = 'ABCDEFG' + count = len(line['options']) + if count == 1: + count = 4 + return passage + '问题：' + line['question'] + ' ' \ + + '选项：' + ' '.join(line['options']) + '\n' + \ + '答案：从A到{}, 我们应选择'.format(option_string[count - 1]) + + elif dataset_name in english_cloze_datasets: + return passage + 'Q: ' + line['question'] + '\n' \ + 'A: The answer is' + + elif dataset_name in chinese_cloze_datasets: + return passage + '问题：' + line['question'] + '\n' \ + '答案：' + except NameError: + print('Dataset not defined.') + + +prefix = '该问题为单选题，所有选项中必有一个正确答案，且只有一个正确答案。\n' + + +def convert_zero_shot_CoT_stage1(line, dataset_name): + try: + passage = line['passage'] if line['passage'] is not None else '' + if dataset_name in english_qa_datasets: + return passage + 'Q: ' + line['question'] + ' ' \ + + 'Answer Choices: ' + ' '.join(line['options']) + '\n' + \ + "Let's think step by step." + + elif dataset_name in chinese_qa_datasets: + option_string = 'ABCDEFG' + count = len(line['options']) + if count == 1: + count = 4 + return passage + '问题：' + line['question'] + ' ' \ + + '选项：' + ' '.join(line['options']) + '\n' + \ + '从A到{}, 我们应选择什么？让我们逐步思考：'.format(option_string[count - 1]) + + elif dataset_name in english_cloze_datasets: + return passage + 'Q: ' + line['question'] + '\n' \ + "A: Let's think step by step." + + elif dataset_name in chinese_cloze_datasets: + return passage + '问题：' + line['question'] + '\n' \ + '答案：让我们逐步思考：' + except NameError: + print('Dataset not defined.') + + +# process few-shot raw_prompts +def combine_prompt(prompt_path, + dataset_name, + load_explanation=True, + chat_mode=False): + skip_passage = False + if dataset_name == 'sat-en-without-passage': + skip_passage = True + dataset_name = 'sat-en' + demostrations = [] + # read the prompts by context and explanation + context_row = [0, 1, 3, 5, 7, 9] + explanation_row = [0, 2, 4, 6, 8, 10] + raw_prompts_context = pd.read_csv(prompt_path, + header=0, + skiprows=lambda x: x not in context_row, + keep_default_na=False) + raw_prompts_explanation = pd.read_csv( + prompt_path, + header=0, + skiprows=lambda x: x not in explanation_row, + keep_default_na=False).replace(r'\n\n', '\n', regex=True) + contexts = [] + for line in list(raw_prompts_context[dataset_name]): + if line: + # print(line) + contexts.append(ast.literal_eval(line)) + explanations = [ + exp for exp in raw_prompts_explanation[dataset_name] if exp + ] + + for idx, (con, exp) in enumerate(zip(contexts, explanations)): + passage = con['passage'] if con[ + 'passage'] is not None and not skip_passage else '' + question = con['question'] + options = con['options'] if con['options'] is not None else '' + label = con['label'] if con['label'] is not None else '' + answer = con[ + 'answer'] if 'answer' in con and con['answer'] is not None else '' + + if dataset_name in english_qa_datasets: + question_input = 'Problem {}. '.format(idx + 1) + passage + ' ' + question + '\n' \ + + 'Choose from the following options: ' + ' '.join(options) + '\n' + question_output = (('Explanation for Problem {}: '.format(idx + 1) + exp + '\n') if load_explanation else '') \ + + 'The answer is therefore {}'.format(label) + + elif dataset_name in chinese_qa_datasets: + question_input = '问题 {}. '.format(idx + 1) + passage + ' ' + question + '\n' \ + + '从以下选项中选择: ' + ' '.join(options) + '\n' + question_output = (('问题 {}的解析: '.format(idx + 1) + exp + '\n') if load_explanation else '') \ + + '答案是 {}'.format(label) + + elif dataset_name in english_cloze_datasets: + question_input = 'Problem {}. '.format(idx + 1) + question + '\n' + question_output = (('Explanation for Problem {}: '.format(idx + 1) + exp + '\n') if load_explanation else '') \ + + 'The answer is therefore {}'.format(answer) + + elif dataset_name in chinese_cloze_datasets: + question_input = '问题 {}. '.format(idx + 1) + question + '\n' + question_output = (('问题 {}的解析: '.format(idx + 1) + exp + '\n') if load_explanation else '') \ + + '答案是 {}'.format(answer) + else: + raise ValueError( + f'During loading few-sot examples, found unknown dataset: {dataset_name}' + ) + if chat_mode: + demostrations.append((question_input, question_output)) + else: + demostrations.append(question_input + question_output + '\n') + + return demostrations + + +enc = None + + +def _lazy_load_enc(): + global enc + if enc is None: + enc = tiktoken.encoding_for_model('gpt-4') + + +# cut prompt if reach max token length +def concat_prompt(demos, + dataset_name, + max_tokens, + end_of_example='\n', + verbose=False): + _lazy_load_enc() + demostration_en = 'Here are the answers for the problems in the exam.\n' + demostration_zh = '以下是考试中各个问题的答案。\n' + + for i in range(len(demos)): + # print(len(enc.encode(demostration_en)), len(enc.encode(demostration_zh))) + if dataset_name in english_qa_datasets: + demostration_en = demostration_en + demos[i] + end_of_example + elif dataset_name in chinese_qa_datasets: + demostration_zh = demostration_zh + demos[i] + end_of_example + elif dataset_name in english_cloze_datasets: + demostration_en = demostration_en + demos[i] + end_of_example + elif dataset_name in chinese_cloze_datasets: + demostration_zh = demostration_zh + demos[i] + end_of_example + # break if reach max token limit + if len(enc.encode(demostration_en)) < max_tokens and len( + enc.encode(demostration_zh)) < max_tokens: + output = demostration_en if len(demostration_en) > len( + demostration_zh) else demostration_zh + prompt_num = i + 1 + else: + break + if verbose: + print('max_tokens set as ', max_tokens, 'actual_tokens is', + len(enc.encode(output)), 'num_shot is', prompt_num) + return output, prompt_num + + +def concat_prompt_chat_mode(demos, + dataset_name, + max_tokens, + end_of_example='\n', + verbose=False): + _lazy_load_enc() + answers = [] + sentences = '' + for i in range(len(demos)): + answers += [ + { + 'role': 'user', + 'content': demos[i][0] + }, + { + 'role': 'assistant', + 'content': demos[i][1] + }, + ] + sentences += json.dumps(answers[-1]) + # break if reach max token limit + if len(enc.encode(sentences)) > max_tokens: + answers.pop() + answers.pop() + break + if verbose: + print('max_tokens set as ', max_tokens, 'actual_tokens is', + len(enc.encode(sentences)), 'num_shot is', + len(answers) // 2) + return answers, len(answers) // 2 + + +def convert_few_shot(line, dataset_name, demo, n_shot, chat_mode=False): + passage = line['passage'] if line['passage'] is not None else '' + question = line['question'] + options = line['options'] if line['options'] is not None else '' + + if dataset_name in english_qa_datasets: + question_input = 'Problem {}. '.format(n_shot + 1) + passage + ' ' + question + '\n' \ + + 'Choose from the following options: ' + ' '.join(options) + '\n' + # + "Explanation for Problem {}: ".format(n_shot + 1) + + if dataset_name in chinese_qa_datasets: + question_input = '问题 {}. '.format(n_shot + 1) + passage + ' ' + question + '\n' \ + + '从以下选项中选择: ' + ' '.join(options) + '\n' + # + "问题 {}的解析: ".format(n_shot + 1) + + if dataset_name in english_cloze_datasets: + question_input = 'Problem {}. '.format(n_shot + 1) + question + '\n' + # + "Explanation for Problem {}: ".format(n_shot + 1) + + if dataset_name in chinese_cloze_datasets: + question_input = '问题 {}. '.format(n_shot + 1) + question + '\n' + # + "问题 {}的解析: ".format(n_shot + 1) + if chat_mode: + return demo + [ + { + 'role': 'user', + 'content': question_input + }, + ] + else: + return demo + question_input + + +def load_dataset(dataset_name, + setting_name, + parent_path, + prompt_path=None, + max_tokens=None, + end_of_example='\n', + chat_mode=False, + verbose=False): + test_path = os.path.join(parent_path, dataset_name + '.jsonl') + loaded_jsonl = read_jsonl(test_path) + processed = [] + if setting_name == 'few-shot-CoT' or setting_name == 'few-shot': + # process demo once if it is few-shot-CoT + processed_demos = combine_prompt( + prompt_path, + dataset_name, + load_explanation=setting_name == 'few-shot-CoT', + chat_mode=chat_mode) + if chat_mode: + chosen_prompt, n_shot = concat_prompt_chat_mode(processed_demos, + dataset_name, + max_tokens, + end_of_example, + verbose=verbose) + else: + chosen_prompt, n_shot = concat_prompt(processed_demos, + dataset_name, + max_tokens, + end_of_example, + verbose=verbose) + if verbose: + loaded_jsonl = tqdm(loaded_jsonl) + for meta_idx, line in enumerate(loaded_jsonl): + if setting_name == 'zero-shot': + ctxt = convert_zero_shot(line, dataset_name) + elif setting_name == 'zero-shot-CoT': + ctxt = convert_zero_shot_CoT_stage1(line, dataset_name) + elif setting_name == 'few-shot-CoT' or setting_name == 'few-shot': + ctxt = convert_few_shot(line, dataset_name, chosen_prompt, n_shot, + chat_mode) + try: + new_instance = ChatGPTSchema(context=ctxt, metadata=meta_idx) + processed.append(new_instance.to_dict()) + except NameError: + print('Dataset not defined.') + return processed + + +def generate_second_stage_input(dataset_name, + input_list, + output_list, + with_format_prompt=False): + try: + english_format_prompt = 'Based on the previous results, your task is to extract the final answer and provide the output enclosed in brackets【】, such as 【0】 or 【A】.' + chinese_format_prompt = '根据以上内容，你的任务是把最终的答案提取出来并填在【】中，例如【0】或者【A】。' + if dataset_name in english_qa_datasets: + prompt_suffix = 'Therefore, among A through E, the answer is' + if with_format_prompt: + prompt_suffix = english_format_prompt + prompt_suffix + elif dataset_name in chinese_qa_datasets: + prompt_suffix = '因此，从A到D, 我们应选择' + if with_format_prompt: + prompt_suffix = chinese_format_prompt + prompt_suffix + elif dataset_name in english_cloze_datasets: + prompt_suffix = 'Therefore, the answer is' + if with_format_prompt: + prompt_suffix = english_format_prompt + prompt_suffix + elif dataset_name in chinese_cloze_datasets: + prompt_suffix = '因此，答案是' + if with_format_prompt: + prompt_suffix = chinese_format_prompt + prompt_suffix + except NameError: + print('Dataset not defined.') + processed = [] + for i in range(len(input_list)): + ctxt = '{0}\n{1}\n{2}'.format(input_list[i]['context'], + extract_answer(output_list[i]), + prompt_suffix) + new_instance = ChatGPTSchema(context=ctxt, + metadata=input_list[i]['metadata']) + processed.append(new_instance.to_dict()) + return processed + + +def load_dataset_as_result_schema(dataset_name, parent_path): + test_path = os.path.join(parent_path, dataset_name + '.jsonl') + loaded_jsonl = read_jsonl(test_path) + + processed = [] + for i, line in enumerate(loaded_jsonl): + problem_input = convert_zero_shot(line, dataset_name) + processed.append( + ResultsForHumanSchema( + index=i, + problem_input=problem_input, + label=line['label'] if line['label'] else line['answer'], + )) + return processed + + +if __name__ == '__main__': + + # set variables + parent_dir = '../../data/V1_1/' + raw_prompt_path = '../data/few_shot_prompts.csv' + + # set dataset name to process + setting_name = 'few-shot-CoT' # setting_name can be chosen from ["zero-shot", "zero-shot-CoT", "few-shot-CoT"] + data_name = 'jec-qa-kd' + save_dir = '../../experiment_input/{}/'.format(setting_name) + if not os.path.exists(save_dir): + os.makedirs(save_dir) + processed_data = load_dataset(data_name, + setting_name, + parent_dir, + prompt_path=raw_prompt_path, + max_tokens=2048) + save_jsonl(processed_data, + os.path.join(save_dir, '{}.jsonl'.format(data_name))) diff --git a/opencompass/datasets/agieval/evaluation.py b/opencompass/datasets/agieval/evaluation.py new file mode 100644 index 00000000..c5a9916a --- /dev/null +++ b/opencompass/datasets/agieval/evaluation.py @@ -0,0 +1,43 @@ +# flake8: noqa +from . import dataset_loader, utils +from .math_equivalence import is_equiv + + +def convert_to_set(item): + if isinstance(item, list): + return set(item) + if isinstance(item, str): + return {item} + if item is None: + return {} + raise ValueError("Input can't parse:", item) + + +def evaluate_single_sample(dataset_name, prediction, label): + if dataset_name in dataset_loader.multi_choice_datasets: + p = convert_to_set(prediction) + l = convert_to_set(label) + return p == l + elif dataset_name in dataset_loader.math_output_datasets: + return is_equiv(prediction, label) + else: + return prediction == label + + +# def evaluate(dataset_name, prediction_list, label_list): +# correct = 0 +# if dataset_name in multi_choice_datasets: +# for prediction, label in zip(prediction_list, label_list): +# p = convert_to_set(prediction) +# l = convert_to_set(label) +# if p == l: +# correct += 1 +# elif dataset_name in math_output_datasets: +# for prediction, label in zip(prediction_list, label_list): +# if is_equiv(prediction, label): +# correct += 1 +# else: +# for prediction, label in zip(prediction_list, label_list): +# if prediction == label: +# correct += 1 +# return "{0:.2%}".format(correct / len(label_list)) diff --git a/opencompass/datasets/agieval/math_equivalence.py b/opencompass/datasets/agieval/math_equivalence.py new file mode 100644 index 00000000..788900ea --- /dev/null +++ b/opencompass/datasets/agieval/math_equivalence.py @@ -0,0 +1,161 @@ +# flake8: noqa + + +# code from https://github.com/hendrycks/math/blob/main/modeling/math_equivalence.py +def _fix_fracs(string): + substrs = string.split('\\frac') + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += '\\frac' + if substr[0] == '{': + new_str += substr + else: + try: + assert len(substr) >= 2 + except: + return string + a = substr[0] + b = substr[1] + if b != '{': + if len(substr) > 2: + post_substr = substr[2:] + new_str += '{' + a + '}{' + b + '}' + post_substr + else: + new_str += '{' + a + '}{' + b + '}' + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += '{' + a + '}' + b + post_substr + else: + new_str += '{' + a + '}' + b + string = new_str + return string + + +def _fix_a_slash_b(string): + if len(string.split('/')) != 2: + return string + a = string.split('/')[0] + b = string.split('/')[1] + try: + a = int(a) + b = int(b) + assert string == '{}/{}'.format(a, b) + new_string = '\\frac{' + str(a) + '}{' + str(b) + '}' + return new_string + except: + return string + + +def _remove_right_units(string): + # "\\text{ " only ever occurs (at least in the val set) when describing units + if '\\text{ ' in string: + splits = string.split('\\text{ ') + assert len(splits) == 2 + return splits[0] + else: + return string + + +def _fix_sqrt(string): + if '\\sqrt' not in string: + return string + splits = string.split('\\sqrt') + new_string = splits[0] + for split in splits[1:]: + if split[0] != '{': + a = split[0] + new_substr = '\\sqrt{' + a + '}' + split[1:] + else: + new_substr = '\\sqrt' + split + new_string += new_substr + return new_string + + +def _strip_string(string): + # linebreaks + string = string.replace('\n', '') + # print(string) + + # remove inverse spaces + string = string.replace('\\!', '') + # print(string) + + # replace \\ with \ + string = string.replace('\\\\', '\\') + # print(string) + + # replace tfrac and dfrac with frac + string = string.replace('tfrac', 'frac') + string = string.replace('dfrac', 'frac') + # print(string) + + # remove \left and \right + string = string.replace('\\left', '') + string = string.replace('\\right', '') + # print(string) + + # Remove circ (degrees) + string = string.replace('^{\\circ}', '') + string = string.replace('^\\circ', '') + + # remove dollar signs + string = string.replace('\\$', '') + + # remove units (on the right) + string = _remove_right_units(string) + + # remove percentage + string = string.replace('\\%', '') + string = string.replace('\%', '') + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(' .', ' 0.') + string = string.replace('{.', '{0.') + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == '.': + string = '0' + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + if len(string.split('=')) == 2: + if len(string.split('=')[0]) <= 2: + string = string.split('=')[1] + + # fix sqrt3 --> sqrt{3} + string = _fix_sqrt(string) + + # remove spaces + string = string.replace(' ', '') + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # manually change 0.5 --> \frac{1}{2} + if string == '0.5': + string = '\\frac{1}{2}' + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + return string + + +def is_equiv(str1, str2, verbose=False): + if str1 is None and str2 is None: + print('WARNING: Both None') + return True + if str1 is None or str2 is None: + return False + + try: + ss1 = _strip_string(str1) + ss2 = _strip_string(str2) + if verbose: + print(ss1, ss2) + return ss1 == ss2 + except: + return str1 == str2 diff --git a/opencompass/datasets/agieval/post_process.py b/opencompass/datasets/agieval/post_process.py new file mode 100644 index 00000000..f54531e8 --- /dev/null +++ b/opencompass/datasets/agieval/post_process.py @@ -0,0 +1,198 @@ +# flake8: noqa +import json +import re + +from . import dataset_loader + + +def extract_last_line(string): + lines = string.split('\n') + for item in lines[::-1]: + if item.strip() != '': + string = item + break + return string + + +def remove_few_shot_prefix(string: str): + prefix_list = ['The answer is therefore', '答案是'] + for prefix in prefix_list: + if string.startswith(prefix): + string = string[len(prefix):].strip() + elif prefix in string: + index = string.rfind(prefix) + if index >= 0: + string = string[index + len(prefix):].strip() + return string + + +def try_parse_few_shot_qa_single_answer(string, setting_name, language='en'): + if setting_name == 'few-shot-CoT': + string = extract_last_line(string) + if language == 'en': + pattern = 'answer is .*?([A-G])' + match = re.search(pattern, string) + elif language == 'zh': + pattern = '答案是.*?([A-G])' + match = re.search(pattern, string) + else: + raise ValueError('Unknown language {0}'.format(language)) + if match: + return match.group(1) + else: + return None + + +def try_parse_few_shot_pattern(string: str, dataset_name, setting_name): + if setting_name == 'few-shot-CoT': + string = extract_last_line(string) + if dataset_name in dataset_loader.chinese_cloze_datasets: + return string.startswith('答案是') + elif dataset_name in dataset_loader.english_cloze_datasets: + return string.startswith('The answer is therefore') + elif dataset_name in dataset_loader.chinese_qa_datasets: + pattern = '答案是.*?([A-G])' + match = re.search(pattern, string) + return match is not None + elif dataset_name in dataset_loader.english_qa_datasets: + pattern = 'answer is .*?([A-G])' + match = re.search(pattern, string) + return match is not None + return False + + +def parse_few_shot_qa_single_answer(string, setting_name, language='en'): + answer = try_parse_few_shot_qa_single_answer(string, setting_name, + language) + if answer is None: + return find_first_capital_letter(string) + else: + return answer + + +def find_first_capital_letter(answer): + letter_set = {'A', 'B', 'C', 'D', 'E', 'F'} + for c in answer: + if c in letter_set: + return c + # print("Can't find capital letter in:", answer) + return '' + + +def extract_answer_in_bracket(answer, prefix='【', suffix='】'): + if prefix not in answer and suffix not in answer: + # print("doesn't found special tokens in:", answer) + return '' + s = answer.index(prefix) + len(prefix) + t = answer.index(suffix) + ret = answer[s:t] + return ret + + +def parse_math_answer(setting_name, raw_string): + if setting_name == 'few-shot-CoT': + raw_string = extract_last_line(raw_string) + if setting_name == 'few-shot-CoT' or setting_name == 'few-shot': + raw_string = remove_few_shot_prefix(raw_string) + return raw_string + + def remove_boxed(s): + left = '\\boxed{' + try: + assert s[:len(left)] == left + assert s[-1] == '}' + answer = s[len(left):-1] + if '=' in answer: + answer = answer.split('=')[-1].lstrip(' ') + return answer + except: + return None + + def last_boxed_only_string(string): + idx = string.rfind('\\boxed') + if idx < 0: + idx = string.rfind('\\fbox') + if idx < 0: + return None + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == '{': + num_left_braces_open += 1 + if string[i] == '}': + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + if right_brace_idx == None: + retval = None + else: + retval = string[idx:right_brace_idx + 1] + + return retval + + def get_answer_with_dollar_sign(s): + first_pattern = '\$(.*)\$' + last_match = None + matches = re.findall(first_pattern, s) + if matches: + last_match = matches[-1] + if '=' in last_match: + last_match = last_match.split('=')[-1].lstrip(' ') + return last_match + + def get_answer_without_dollar_sign(s): + last_match = None + if '=' in s: + last_match = s.split('=')[-1].lstrip(' ').rstrip('.') + if '\\n' in last_match: + last_match = last_match.split('\\n')[0] + else: + pattern = '(?:\\$)?\d+(?:\.\d+)?(?![\w\d])' + matches = re.findall(pattern, s) + if matches: + last_match = matches[-1] + return last_match + + raw_string = remove_few_shot_prefix(raw_string) + if '\\boxed' in raw_string: + answer = remove_boxed(last_boxed_only_string(raw_string)) + else: + answer = get_answer_with_dollar_sign(raw_string) + if not answer: + answer = get_answer_without_dollar_sign(raw_string) + return answer + + +def parse_qa_multiple_answer(string, setting_name): + if setting_name == 'few-shot-CoT': + string = extract_last_line(string) + pattern = '\(*([A-Z])\)*' + match = re.findall(pattern, string) + if match: + return match + return [] + + +def post_process(dataset_name, setting_name, prediction): + if dataset_name in dataset_loader.english_cloze_datasets or dataset_name in dataset_loader.chinese_cloze_datasets: + return parse_math_answer(setting_name, prediction) + + if dataset_name in ['jec-qa-kd', 'jec-qa-ca', 'gaokao-physics']: + return parse_qa_multiple_answer(prediction, setting_name) + + # all other datasets are QA problems with single answer + if 'zero-shot' in setting_name: + answer = find_first_capital_letter(prediction) + return answer + + # all other datasets are QA problems with single answer and setting_name are few-shot + language = 'en' if dataset_name in dataset_loader.english_qa_datasets else 'zh' + if dataset_name in dataset_loader.english_qa_datasets or dataset_name in dataset_loader.chinese_qa_datasets: + return parse_few_shot_qa_single_answer(prediction, setting_name, + language) + else: + raise ValueError(f'Unsupported dataset name {dataset_name}') diff --git a/opencompass/datasets/agieval/utils.py b/opencompass/datasets/agieval/utils.py new file mode 100644 index 00000000..fbb31105 --- /dev/null +++ b/opencompass/datasets/agieval/utils.py @@ -0,0 +1,43 @@ +# flake8: noqa +import json + + +def read_jsonl(path): + with open(path, encoding='utf8') as fh: + results = [] + for line in fh: + if line is None: + continue + try: + results.append(json.loads(line) if line != 'null' else line) + except Exception as e: + print(e) + print(path) + print(line) + raise e + return results + + +def save_jsonl(lines, directory): + with open(directory, 'w', encoding='utf8') as f: + for line in lines: + f.write(json.dumps(line, ensure_ascii=False) + '\n') + + +def extract_answer(js): + try: + if js is None or js == 'null': + return '' + answer = '' + if isinstance(js, str): + answer = js + elif 'text' in js['choices'][0]: + answer = js['choices'][0]['text'] + else: + answer = js['choices'][0]['message']['content'] + # answer = js[''] + return answer + except Exception as e: + # print(e) + # print(js) + return '' diff --git a/opencompass/datasets/c3.py b/opencompass/datasets/c3.py new file mode 100644 index 00000000..1c630675 --- /dev/null +++ b/opencompass/datasets/c3.py @@ -0,0 +1,80 @@ +import json + +from datasets import Dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class C3Dataset(BaseDataset): + + @staticmethod + def load(path: str): + + with open(path) as f: + data = json.load(f) + rows = [] + for _, row in enumerate(data): + content = row[0] + content_str = ' '.join( + [''.join(paragraph) for paragraph in content]) + + for question in row[1]: + label = question['choice'].index(question['answer']) + length = len(question['choice']) + if length < 4: + fill_value = question['choice'][0] # 以第一个值为填充值 + fill_count = 4 - length # 需要填充的数量 + question['choice'] += [fill_value] * fill_count # 填充 + + rows.append({ + 'content': content_str, + 'question': question['question'], + 'choices': question['choice'], + 'choice0': question['choice'][0], + 'choice1': question['choice'][1], + 'choice2': question['choice'][2], + 'choice3': question['choice'][3], + 'label': label + }) + + dataset = Dataset.from_dict({ + 'content': [row['content'] for row in rows], + 'question': [row['question'] for row in rows], + 'choice0': [row['choice0'] for row in rows], + 'choice1': [row['choice1'] for row in rows], + 'choice2': [row['choice2'] for row in rows], + 'choice3': [row['choice3'] for row in rows], + 'choices': [row['choices'] for row in rows], + 'label': [row['label'] for row in rows] + }) + return dataset + + +@LOAD_DATASET.register_module() +class C3Dataset_V2(BaseDataset): + + @staticmethod + def load(path: str): + with open(path) as f: + raw = json.load(f) + data = [] + for line in raw: + content = ''.join([''.join(paragraph) for paragraph in line[0]]) + for question in line[1]: + label = question['choice'].index(question['answer']) + label = 'ABCD'[label] + while len(question['choice']) < 4: + question['choice'].append('[NULL]') + data.append({ + 'content': content, + 'question': question['question'], + 'choice0': question['choice'][0], + 'choice1': question['choice'][1], + 'choice2': question['choice'][2], + 'choice3': question['choice'][3], + 'label': label + }) + return Dataset.from_list(data) diff --git a/opencompass/datasets/cluewsc.py b/opencompass/datasets/cluewsc.py new file mode 100644 index 00000000..5f5e0803 --- /dev/null +++ b/opencompass/datasets/cluewsc.py @@ -0,0 +1,57 @@ +import json + +from datasets import Dataset, load_dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class CluewscDataset(BaseDataset): + + @staticmethod + def load(**kwargs): + + dataset = load_dataset(**kwargs) + + def preprocess(example): + text_list = list(example['text']) + # span1 may have 1 or more than 1 words + # span2 is the pronoun and has only 1 word + text_list[example['target'] + ['span2_index']] = example['target']['span1_text'] + example['new_text'] = ''.join(text_list) + if example['label'] == 'true': + example['answer'] = 1 + else: + example['answer'] = 0 + example['span1'] = example['target']['span1_text'] + example['span2'] = example['target']['span2_text'] + del example['target'] + return example + + dataset = dataset.map(preprocess) + return dataset + + +@LOAD_DATASET.register_module() +class CluewscDataset_V2(BaseDataset): + + @staticmethod + def load(path): + data = [] + with open(path, 'r') as f: + for line in f: + line = json.loads(line) + item = { + 'span1': line['target']['span1_text'], + 'span2': line['target']['span2_text'], + 'text': line['text'], + 'label': { + 'true': 'A', + 'false': 'B' + }[line['label']], + } + data.append(item) + return Dataset.from_list(data) diff --git a/opencompass/datasets/copa.py b/opencompass/datasets/copa.py new file mode 100644 index 00000000..34ad4039 --- /dev/null +++ b/opencompass/datasets/copa.py @@ -0,0 +1,21 @@ +import json + +from datasets import Dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class COPADataset_V2(BaseDataset): + + @staticmethod + def load(path): + dataset = [] + with open(path, 'r') as f: + for line in f: + line = json.loads(line) + line['label'] = 'AB'[line['label']] + dataset.append(line) + return Dataset.from_list(dataset) diff --git a/opencompass/datasets/drcd.py b/opencompass/datasets/drcd.py new file mode 100644 index 00000000..55893ac9 --- /dev/null +++ b/opencompass/datasets/drcd.py @@ -0,0 +1,40 @@ +import json + +from datasets import Dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class DRCDDataset(BaseDataset): + + @staticmethod + def load(path: str): + with open(path) as f: + data = json.load(f) + # 将原始数据转换为所需的格式 + rows = [] + for index, paragraphs in enumerate(data['data']): + for paragraph in paragraphs['paragraphs']: + + context = paragraph['context'] + + for question in paragraph['qas']: + answers = question['answers'] + unique_answers = list(set([a['text'] for a in answers])) + rows.append({ + 'context': context, + 'question': question['question'], + 'answers': unique_answers + }) + + # 创建 Dataset + dataset = Dataset.from_dict({ + 'context': [row['context'] for row in rows], + 'question': [row['question'] for row in rows], + 'answers': [row['answers'] for row in rows] + }) + + return dataset diff --git a/opencompass/datasets/hellaswag.py b/opencompass/datasets/hellaswag.py new file mode 100644 index 00000000..bfffe069 --- /dev/null +++ b/opencompass/datasets/hellaswag.py @@ -0,0 +1,41 @@ +from datasets import load_dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class hellaswagDataset(BaseDataset): + + @staticmethod + def load(**kwargs): + dataset = load_dataset(**kwargs) + + def preprocess(example): + for i in range(4): + example[chr(ord('A') + i)] = example['endings'][i] + return example + + dataset = dataset.map(preprocess).remove_columns(['endings']) + return dataset + + +@LOAD_DATASET.register_module() +class hellaswagDataset_V2(BaseDataset): + + @staticmethod + def load(**kwargs): + dataset = load_dataset(**kwargs) + + def preprocess(example): + for i in range(4): + example[chr(ord('A') + i)] = example['endings'][i] + if example['label']: + example['label'] = 'ABCD'[int(example['label'])] + else: + example['label'] = 'NULL' + return example + + dataset = dataset.map(preprocess).remove_columns(['endings']) + return dataset diff --git a/opencompass/datasets/humaneval.py b/opencompass/datasets/humaneval.py new file mode 100644 index 00000000..71b6b209 --- /dev/null +++ b/opencompass/datasets/humaneval.py @@ -0,0 +1,57 @@ +import os.path as osp +import tempfile +from typing import List + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, TEXT_POSTPROCESSORS + + +@ICL_EVALUATORS.register_module() +class HumanEvaluator(BaseEvaluator): + """Evaluator for human eval.""" + + def __init__(self, k: List[int] = [1, 10, 100]) -> None: + try: + from human_eval.data import HUMAN_EVAL, write_jsonl + from human_eval.evaluation import evaluate_functional_correctness + self.write_jsonl = write_jsonl + self.HUMAN_EVAL = HUMAN_EVAL + self.eval = evaluate_functional_correctness + except ImportError: + raise ImportError('Please install human_eval following' + 'https://github.com/openai/human-eval/tree/' + 'master#installation first.') + self.k = k + super().__init__() + + def score(self, predictions, references): + + predictions = [{ + 'task_id': f'HumanEval/{i}', + 'completion': predictions[i] + } for i in range(len(predictions))] + with tempfile.TemporaryDirectory() as tmp_dir: + out_dir = osp.join(tmp_dir, 'human_eval.json') + self.write_jsonl(out_dir, predictions) + score = self.eval( + out_dir, + self.k, + n_workers=4, + timeout=3.0, + problem_file=self.HUMAN_EVAL) + return {f'humaneval_{k}': score[k] * 100 for k in score} + + +@TEXT_POSTPROCESSORS.register_module('humaneval') +def humaneval_postprocess(text: str) -> str: + text = text.split('\n\n')[0] + if '```' in text: + text = text.split('```')[1] + if text.startswith('def'): + text = '\n'.join(text.split('\n')[1:]) + if not text.startswith(' '): + if text.startswith(' '): + text = ' ' + text.lstrip() + else: + text = '\n'.join([' ' + line for line in text.split('\n')]) + return text diff --git a/opencompass/datasets/jigsawmultilingual.py b/opencompass/datasets/jigsawmultilingual.py new file mode 100644 index 00000000..69b18165 --- /dev/null +++ b/opencompass/datasets/jigsawmultilingual.py @@ -0,0 +1,35 @@ +import csv + +from datasets import Dataset, DatasetDict + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class JigsawMultilingualDataset(BaseDataset): + + @staticmethod + def load(path, label, lang): + assert lang in ['es', 'fr', 'it', 'pt', 'ru', 'tr'] + dataset = DatasetDict() + + data_list = list() + idx = 0 + with open(path) as file, open(label) as label: + text_reader = csv.reader(file) + label_reader = csv.reader(label) + for text, target in zip(text_reader, label_reader): + if text[2] == lang: + assert text[0] == target[0] + data_list.append({ + 'idx': idx, + 'text': text[1], + 'label': int(target[1]), + 'choices': ['no', 'yes'] + }) + idx += 1 + + dataset['test'] = Dataset.from_list(data_list) + return dataset diff --git a/opencompass/datasets/lambada.py b/opencompass/datasets/lambada.py new file mode 100644 index 00000000..41c57141 --- /dev/null +++ b/opencompass/datasets/lambada.py @@ -0,0 +1,48 @@ +import re +import string + +from datasets import DatasetDict, load_dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET +from opencompass.utils.text_postprocessors import general_postprocess + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class lambadaDataset(BaseDataset): + + @staticmethod + def load(**kwargs): + dataset = load_dataset(**kwargs, split='test') + + def preprocess(example): + prompt, target = example['text'].strip().rsplit(' ', 1) + example['prompt'] = prompt + example['label'] = target + return example + + dataset = dataset.map(preprocess) + return DatasetDict({'test': dataset}) + + +@ICL_EVALUATORS.register_module() +class LambadaEvaluator(BaseEvaluator): + + def __init__(self) -> None: + super().__init__() + + def score(self, predictions, references): + if len(predictions) != len(references): + return { + 'error': 'predictions and references have different ' + 'length' + } + score = 0.0 + for pred, refer in zip(predictions, references): + pred = pred.strip().split(' ')[0] + pred = re.split(f'[{string.punctuation}]', pred)[0] + score += general_postprocess(pred) == general_postprocess(refer) + score = 100.0 * score / len(predictions) + return dict(accuracy=score) diff --git a/opencompass/datasets/math.py b/opencompass/datasets/math.py new file mode 100644 index 00000000..d640a3cb --- /dev/null +++ b/opencompass/datasets/math.py @@ -0,0 +1,305 @@ +import json + +from datasets import Dataset, DatasetDict + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET, TEXT_POSTPROCESSORS + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class MATHDataset(BaseDataset): + + @staticmethod + def load(path: str): + + def remove_boxed(s): + left = '\\boxed{' + try: + assert s[:len(left)] == left + assert s[-1] == '}' + return s[len(left):-1] + except Exception: + return None + + def last_boxed_only_string(string): + idx = string.rfind('\\boxed') + if idx < 0: + idx = string.rfind('\\fbox') + if idx < 0: + return None + + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == '{': + num_left_braces_open += 1 + if string[i] == '}': + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + if right_brace_idx is None: + retval = None + else: + retval = string[idx:right_brace_idx + 1] + + return retval + + dataset = DatasetDict() + data = json.load(open(path)) + raw_data = [] + for i in data.keys(): + raw_data.append({ + 'problem': + data[i]['problem'], + 'solution': + remove_boxed(last_boxed_only_string(data[i]['solution'])) + }) + dataset['test'] = Dataset.from_list(raw_data) + dataset['train'] = Dataset.from_list(raw_data) + return dataset + + +@TEXT_POSTPROCESSORS.register_module('math') +def math_postprocess(text: str) -> str: + SUBSTITUTIONS = [('an ', ''), ('a ', ''), ('.$', '$'), ('\\$', ''), + (r'\ ', ''), (' ', ''), ('mbox', 'text'), + (',\\text{and}', ','), ('\\text{and}', ','), + ('\\text{m}', '\\text{}'), ('\le', '<')] + REMOVED_EXPRESSIONS = [ + 'square', 'ways', 'integers', 'dollars', 'mph', 'inches', 'ft', + 'hours', 'km', 'units', '\\ldots', 'sue', 'points', 'feet', 'minutes', + 'digits', 'cents', 'degrees', 'cm', 'gm', 'pounds', 'meters', 'meals', + 'edges', 'students', 'childrentickets', 'multiples', '\\text{s}', + '\\text{.}', '\\text{\ns}', '\\text{}^2', '\\text{}^3', '\\text{\n}', + '\\text{}', r'\mathrm{th}', r'^\circ', r'^{\circ}', r'\;', r',\!', + '{,}', '"', '\\dots', '\n', '\r', '\f' + ] + import re + + def normalize_final_answer(final_answer: str) -> str: + """Normalize a final answer to a quantitative reasoning question.""" + # final_answer = final_answer.split('=')[-1] + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, '') + + # Extract answer that is in LaTeX math, is bold, + # is surrounded by a box, etc. + final_answer = re.sub(r'(\\text\{)(.*?)(\})', '\\2', final_answer) + final_answer = re.sub(r'(\\textbf\{)(.*?)(\})', '\\2', final_answer) + final_answer = re.sub(r'(\\overline\{)(.*?)(\})', '\\2', final_answer) + final_answer = re.sub(r'(\\boxed\{)(.*)(\})', '\\2', final_answer) + assert '\n' not in final_answer and '\r' not in final_answer and '\f' not in final_answer + if len(re.findall(r'finalansweris(.*)', final_answer)) > 0: + final_answer = re.findall(r'finalansweris(.*)', final_answer)[-1] + + if len(re.findall(r'oxed\{(.*?)\}', final_answer)) > 0: + final_answer = re.findall(r'oxed\{(.*?)\}', final_answer)[-1] + + if len(re.findall(r'\$(.*?)\$', final_answer)) > 0: + final_answer = re.findall(r'\$(.*?)\$', final_answer)[-1] + final_answer = final_answer.strip() + if 'rac' in final_answer and '\\frac' not in final_answer: + final_answer = final_answer.replace('rac', '\\frac') + + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub(r'(frac)([^{])(.)', 'frac{\\2}{\\3}', + final_answer) + final_answer = re.sub(r'(sqrt)([^{])', 'sqrt{\\2}', final_answer) + final_answer = final_answer.replace('$', '') + + # Normalize 100,000 -> 100000 + if final_answer.replace(',', '').isdigit(): + final_answer = final_answer.replace(',', '') + + return final_answer + + for maybe_ans in text.split('.'): + if 'final answer' in maybe_ans.lower(): + return normalize_final_answer(maybe_ans) + return normalize_final_answer(text.split('.')[0]) + # return normalize_final_answer( + # text.split('Final Answer: ', 1)[-1].split('\n\n')[0]) + + +@ICL_EVALUATORS.register_module() +class MATHEvaluator(BaseEvaluator): + + def score(self, predictions, references): + if len(predictions) != len(references): + return { + 'error': 'predictions and references have different ' + 'length' + } + correct = 0 + count = 0 + for i, j in zip(predictions, references): + count += 1 + if self.is_equiv(i, j): + correct += 1 + result = {'accuracy': 100 * correct / count} + return result + + def _fix_fracs(self, string): + substrs = string.split('\\frac') + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += '\\frac' + if substr[0] == '{': + new_str += substr + else: + try: + assert len(substr) >= 2 + except AssertionError: + return string + a = substr[0] + b = substr[1] + if b != '{': + if len(substr) > 2: + post_substr = substr[2:] + new_str += '{' + a + '}{' + b + '}' + post_substr + else: + new_str += '{' + a + '}{' + b + '}' + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += '{' + a + '}' + b + post_substr + else: + new_str += '{' + a + '}' + b + string = new_str + return string + + def _fix_a_slash_b(self, string): + if len(string.split('/')) != 2: + return string + a = string.split('/')[0] + b = string.split('/')[1] + try: + a = int(a) + b = int(b) + assert string == '{}/{}'.format(a, b) + new_string = '\\frac{' + str(a) + '}{' + str(b) + '}' + return new_string + except AssertionError: + return string + + def _remove_right_units(self, string): + # "\\text{ " only ever occurs (at least in the val set) when describing + # units + if '\\text{ ' in string: + splits = string.split('\\text{ ') + assert len(splits) == 2 + return splits[0] + else: + return string + + def _fix_sqrt(self, string): + if '\\sqrt' not in string: + return string + splits = string.split('\\sqrt') + new_string = splits[0] + for split in splits[1:]: + if split[0] != '{': + a = split[0] + new_substr = '\\sqrt{' + a + '}' + split[1:] + else: + new_substr = '\\sqrt' + split + new_string += new_substr + return new_string + + def _strip_string(self, string): + # linebreaks + string = string.replace('\n', '') + + # remove inverse spaces + string = string.replace('\\!', '') + + # replace \\ with \ + string = string.replace('\\\\', '\\') + + # replace tfrac and dfrac with frac + string = string.replace('tfrac', 'frac') + string = string.replace('dfrac', 'frac') + + # remove \left and \right + string = string.replace('\\left', '') + string = string.replace('\\right', '') + + # Remove circ (degrees) + string = string.replace('^{\\circ}', '') + string = string.replace('^\\circ', '') + + # remove dollar signs + string = string.replace('\\$', '') + + # remove units (on the right) + string = self._remove_right_units(string) + + # remove percentage + string = string.replace('\\%', '') + string = string.replace('\%', '') # noqa: W605 + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, + # add "0" if "." is the start of the string + string = string.replace(' .', ' 0.') + string = string.replace('{.', '{0.') + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == '.': + string = '0' + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + if len(string.split('=')) == 2: + if len(string.split('=')[0]) <= 2: + string = string.split('=')[1] + + # fix sqrt3 --> sqrt{3} + string = self._fix_sqrt(string) + + # remove spaces + string = string.replace(' ', '') + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works + # with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = self._fix_fracs(string) + + # manually change 0.5 --> \frac{1}{2} + if string == '0.5': + string = '\\frac{1}{2}' + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix + # in case the model output is X/Y + string = self._fix_a_slash_b(string) + + return string + + def is_equiv(self, str1, str2, verbose=False): + if str1 is None and str2 is None: + print('WARNING: Both None') + return True + if str1 is None or str2 is None: + return False + + try: + ss1 = self._strip_string(str1) + ss2 = self._strip_string(str2) + if verbose: + print(ss1, ss2) + return ss1 == ss2 + except: # noqa + return str1 == str2 diff --git a/opencompass/datasets/mbpp.py b/opencompass/datasets/mbpp.py new file mode 100644 index 00000000..06cd7204 --- /dev/null +++ b/opencompass/datasets/mbpp.py @@ -0,0 +1,120 @@ +import contextlib +import io +import re +import signal + +from datasets import DatasetDict, load_dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class MBPPDataset(BaseDataset): + + @staticmethod + def load(path: str): + + def processing_test(example): + example['test_case'] = example['test_list'] + example['test_list'] = '\n'.join(example['test_list']) + return example + + train = load_dataset( + 'json', data_files=path, split='train[:10]').map(processing_test) + test = load_dataset( + 'json', data_files=path, + split='train[10:510]').map(processing_test) + return DatasetDict({'train': train, 'test': test}) + + +class TimeOutException(Exception): + pass + + +@ICL_EVALUATORS.register_module() +class MBPPEvaluator(BaseEvaluator): + + def score(self, predictions, references): + assert len(predictions) == len(references) + predictions = [self._process_answer(pred) for pred in predictions] + + result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0} + for test_case, pred in zip(references, predictions): + programs = self._process_test(test_case, pred) + try: + with self.swallow_io(): + with self.time_limit(2): + exec(programs) + result['pass'] += 1 + except TimeOutException: + result['timeout'] += 1 + except AssertionError: + result['wrong_answer'] += 1 + except BaseException: + result['failed'] += 1 + + result['score'] = result['pass'] / len(predictions) * 100 + return result + + def _process_answer(self, text): + text = text.strip() + match = re.search(r"('\s*|)(\[DONE\]|DONE)", text) + if match: + text = text[:match.start()] + match = re.search(r"(\[BEGIN\]|BEGIN)('\s*|)", text) + if match: + text = text[match.end():] + text = text.strip() + if text.startswith("'"): + text = text[1:] + if text.endswith("'"): + text = text[:-1] + return text + + def _process_test(self, test_case, pred): + formatted = pred + '\n' + formatted += test_case + return formatted + + @contextlib.contextmanager + def swallow_io(self): + stream = self.WriteOnlyStringIO() + with contextlib.redirect_stdout(stream): + with contextlib.redirect_stderr(stream): + with self.redirect_stdin(stream): + yield + + @contextlib.contextmanager + def time_limit(self, seconds: float): + + def signal_handler(signum, frame): + raise TimeOutException('Time out!') + + signal.setitimer(signal.ITIMER_REAL, seconds) + signal.signal(signal.SIGALRM, signal_handler) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + class WriteOnlyStringIO(io.StringIO): + """StringIO that throws an exception when it's read from.""" + + def read(self, *args, **kwargs): + raise IOError + + def readline(self, *args, **kwargs): + raise IOError + + def readlines(self, *args, **kwargs): + raise IOError + + def readable(self, *args, **kwargs): + """Returns True if the IO object can be read.""" + return False + + class redirect_stdin(contextlib._RedirectStream): # type: ignore + _stream = 'stdin' diff --git a/opencompass/datasets/mmlu.py b/opencompass/datasets/mmlu.py new file mode 100644 index 00000000..c55c4228 --- /dev/null +++ b/opencompass/datasets/mmlu.py @@ -0,0 +1,33 @@ +import csv +import os.path as osp + +from datasets import Dataset, DatasetDict + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class MMLUDataset(BaseDataset): + + @staticmethod + def load(path: str, name: str): + dataset = DatasetDict() + for split in ['dev', 'test']: + raw_data = [] + filename = osp.join(path, split, f'{name}_{split}.csv') + with open(filename) as f: + reader = csv.reader(f) + for row in reader: + assert len(row) == 6 + raw_data.append({ + 'input': row[0], + 'A': row[1], + 'B': row[2], + 'C': row[3], + 'D': row[4], + 'target': row[5], + }) + dataset[split] = Dataset.from_list(raw_data) + return dataset diff --git a/opencompass/datasets/multirc.py b/opencompass/datasets/multirc.py new file mode 100644 index 00000000..4f607b8a --- /dev/null +++ b/opencompass/datasets/multirc.py @@ -0,0 +1,63 @@ +import json + +from datasets import Dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class MultiRCDataset(BaseDataset): + + @staticmethod + def load(path: str): + with open(path, 'r', errors='ignore') as in_f: + rows = [] + for line in in_f: + sample = json.loads(line.strip()) + passage = sample['passage'] + text = passage['text'] + questions = passage['questions'] + for question_dict in questions: + question = question_dict['question'] + answers = question_dict['answers'] + for answer_dict in answers: + answer = answer_dict['text'] + label = answer_dict['label'] + rows.append({ + 'text': text, + 'question': question, + 'answer': answer, + 'label': label + }) + dataset = Dataset.from_dict({ + 'text': [row['text'] for row in rows], + 'question': [row['question'] for row in rows], + 'answer': [row['answer'] for row in rows], + 'label': [row['label'] for row in rows] + }) + return dataset + + +@LOAD_DATASET.register_module() +class MultiRCDataset_V2(BaseDataset): + + @staticmethod + def load(path: str): + with open(path, 'r', errors='ignore') as in_f: + rows = [] + for line in in_f: + sample = json.loads(line.strip()) + text = sample['passage']['text'] + for question_dict in sample['passage']['questions']: + question = question_dict['question'] + answers = question_dict['answers'] + for answer in answers: + rows.append({ + 'text': text, + 'question': question, + 'answer': answer['text'], + 'label': 'BA'[answer['label']] + }) + return Dataset.from_list(rows) diff --git a/opencompass/datasets/natural_question.py b/opencompass/datasets/natural_question.py new file mode 100644 index 00000000..3ed49de4 --- /dev/null +++ b/opencompass/datasets/natural_question.py @@ -0,0 +1,58 @@ +import csv +import os.path as osp +import re + +from datasets import Dataset, DatasetDict + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET +from opencompass.utils.text_postprocessors import general_postprocess + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class NaturalQuestionDataset(BaseDataset): + + @staticmethod + def load(path: str): + dataset = DatasetDict() + for split in ['dev', 'test']: + filename = osp.join(path, f'nq-{split}.qa.csv') + with open(filename) as f: + reader = csv.reader(f, delimiter='\t') + raw_data = [] + for row in reader: + assert len(row) == 2 + question = row[0] + answers = eval(row[1]) + if split == 'dev': + answers = answers[0] + raw_data.append({'question': question, 'answer': answers}) + dataset[split] = Dataset.from_list(raw_data) + + return dataset + + +@ICL_EVALUATORS.register_module() +class NQEvaluator(BaseEvaluator): + + def score(self, predictions, references): + if len(predictions) != len(references): + return { + 'error': 'predictions and references have different ' + 'length' + } + predictions = [ + re.split(r'[\n]', prediction, 1)[0].lower() + for prediction in predictions + ] + processed_answers = [[general_postprocess(j).lower() for j in i] + for i in references] + + cnt = 0 + for pred, cand_ans in zip(predictions, processed_answers): + cnt += int(any([cand in pred for cand in cand_ans])) + score = cnt / len(predictions) * 100 + + return {'score': score} diff --git a/opencompass/datasets/race.py b/opencompass/datasets/race.py new file mode 100644 index 00000000..aa71c67f --- /dev/null +++ b/opencompass/datasets/race.py @@ -0,0 +1,21 @@ +from datasets import load_dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class RaceDataset(BaseDataset): + + @staticmethod + def load(path: str, name: str): + dataset = load_dataset(path, name) + + def preprocess(x): + for ans, option in zip(['A', 'B', 'C', 'D'], x['options']): + x[ans] = option + del x['options'] + return x + + return dataset.map(preprocess) diff --git a/opencompass/datasets/record.py b/opencompass/datasets/record.py new file mode 100644 index 00000000..d6c976e8 --- /dev/null +++ b/opencompass/datasets/record.py @@ -0,0 +1,49 @@ +import json + +from datasets import Dataset + +from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class ReCoRDDataset(BaseDataset): + + @staticmethod + def load(path: str): + with open(path, 'r', errors='ignore') as in_f: + rows = [] + for i, line in enumerate(in_f): + sample = json.loads(line.strip()) + passage = sample['passage'] + text = passage['text'] + text = text.replace('@highlight', '') + + qas = sample['qas'] + for qas_dict in qas: + query = qas_dict['query'] + query = query.replace('@placeholder', '____') + answers = qas_dict['answers'] + answers_temp = [] + for answer_dict in answers: + answer = answer_dict['text'] + answers_temp.append(answer) + rows.append({ + 'text': text, + 'question': query, + 'answers': answers_temp + }) + + dataset = Dataset.from_dict({ + 'text': [row['text'] for row in rows], + 'question': [row['question'] for row in rows], + 'answers': [row['answers'] for row in rows] + }) + return dataset + + +@TEXT_POSTPROCESSORS.register_module('ReCoRD') +def ReCoRD_postprocess(text: str) -> str: + text = text.strip().split('\n')[0].replace('Answer: ', '').strip() + return text diff --git a/opencompass/datasets/storycloze.py b/opencompass/datasets/storycloze.py new file mode 100644 index 00000000..44577253 --- /dev/null +++ b/opencompass/datasets/storycloze.py @@ -0,0 +1,46 @@ +from datasets import DatasetDict, load_dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class storyclozeDataset(BaseDataset): + + @staticmethod + def load(**kwargs): + # special process + dataset = load_dataset(**kwargs, split='train+eval') + + def preprocess(example): + example['context'] = ' '.join([ + example['input_sentence_1'], example['input_sentence_2'], + example['input_sentence_3'], example['input_sentence_4'] + ]) + return example + + dataset = dataset.map(preprocess) + + return DatasetDict({'test': dataset}) + + +@LOAD_DATASET.register_module() +class storyclozeDataset_V2(BaseDataset): + + @staticmethod + def load(**kwargs): + # special process + dataset = load_dataset(**kwargs, split='train+eval') + + def preprocess(example): + example['context'] = ' '.join([ + example['input_sentence_1'], example['input_sentence_2'], + example['input_sentence_3'], example['input_sentence_4'] + ]) + example['answer_right_ending'] = ' AB'[ + example['answer_right_ending']] + return example + + dataset = dataset.map(preprocess) + return dataset diff --git a/opencompass/datasets/summedits.py b/opencompass/datasets/summedits.py new file mode 100644 index 00000000..ea8ab750 --- /dev/null +++ b/opencompass/datasets/summedits.py @@ -0,0 +1,20 @@ +import json + +from datasets import Dataset + +from opencompass.registry import LOAD_DATASET +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class SummeditsDataset_V2(BaseDataset): + + @staticmethod + def load(path: str): + dataset = [] + with open(path, 'r') as f: + for line in f: + line = json.loads(line) + line['label'] = 'BA'[line['label']] + dataset.append(line) + return Dataset.from_list(dataset) \ No newline at end of file diff --git a/opencompass/datasets/tnews.py b/opencompass/datasets/tnews.py new file mode 100644 index 00000000..79cdc273 --- /dev/null +++ b/opencompass/datasets/tnews.py @@ -0,0 +1,78 @@ +import json + +from datasets import Dataset, load_dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class TNewsDataset(BaseDataset): + + @staticmethod + def load(**kwargs): + + tnews_targets = { + 'news_agriculture': '农业新闻', + 'news_travel': '旅游新闻', + 'news_game': '游戏新闻', + 'news_tech': '科技类别公司新闻', + 'news_sports': '体育类别新闻', + 'news_edu': '初升高教育新闻', + 'news_entertainment': '娱乐圈新闻', + 'news_finance': '投资资讯', + 'news_military': '军事类别常识', + 'news_car': '车辆新闻', + 'news_house': '楼市新闻', + 'news_world': '环球不含中国类别新闻', + 'news_culture': '书籍文化历史类别新闻', + 'news_story': '故事类别新闻', + 'news_stock': '股票市场类别新闻', + } + + dataset = load_dataset(**kwargs) + + def preprocess(example): + label_desc = example['label_desc'] + label_desc2 = tnews_targets[label_desc] + example['label_desc2'] = label_desc2 + return example + + dataset = dataset.map(preprocess) + return dataset + + +@LOAD_DATASET.register_module() +class TNewsDataset_V2(BaseDataset): + + @staticmethod + def load(path): + tnews_targets = { + 'news_agriculture': 'A', + 'news_travel': 'B', + 'news_game': 'C', + 'news_tech': 'D', + 'news_sports': 'E', + 'news_edu': 'F', + 'news_entertainment': 'G', + 'news_finance': 'H', + 'news_military': 'I', + 'news_car': 'J', + 'news_house': 'K', + 'news_world': 'L', + 'news_culture': 'M', + 'news_story': 'N', + 'news_stock': 'O', + } + + data = [] + with open(path, 'r') as f: + for line in f: + line = json.loads(line) + item = { + 'sentence': line['sentence'], + 'label_desc2': tnews_targets[line['label_desc']], + } + data.append(item) + return Dataset.from_list(data) diff --git a/opencompass/datasets/triviaqa.py b/opencompass/datasets/triviaqa.py new file mode 100644 index 00000000..c7f7a757 --- /dev/null +++ b/opencompass/datasets/triviaqa.py @@ -0,0 +1,60 @@ +import csv +import os.path as osp +import re + +from datasets import Dataset, DatasetDict + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET +from opencompass.utils.text_postprocessors import general_postprocess + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class TriviaQADataset(BaseDataset): + + @staticmethod + def load(path: str): + dataset = DatasetDict() + for split in ['dev', 'test']: + filename = osp.join(path, f'trivia-{split}.qa.csv') + with open(filename) as f: + reader = csv.reader(f, delimiter='\t') + raw_data = [] + for row in reader: + assert len(row) == 2 + question = row[0] + answers = eval(row[1]) + if split == 'test': + answers = answers[0] + raw_data.append({'question': question, 'answer': answers}) + dataset[split] = Dataset.from_list(raw_data) + return dataset + + +@ICL_EVALUATORS.register_module() +class TriviaQAEvaluator(BaseEvaluator): + + def __init__(self) -> None: + super().__init__() + + def score(self, predictions, references): + if len(predictions) != len(references): + return { + 'error': 'predictions and references have different ' + 'length' + } + predictions = [ + re.split(r'[\n]', prediction, 1)[0].lower() + for prediction in predictions + ] + processed_answers = [[general_postprocess(j).lower() for j in i] + for i in references] + + cnt = 0 + for pred, cand_ans in zip(predictions, processed_answers): + cnt += int(any([cand in pred for cand in cand_ans])) + score = cnt / len(predictions) * 100 + + return {'score': score} diff --git a/opencompass/datasets/truthfulqa.py b/opencompass/datasets/truthfulqa.py new file mode 100644 index 00000000..a81cfc44 --- /dev/null +++ b/opencompass/datasets/truthfulqa.py @@ -0,0 +1,203 @@ +import os +import time + +import evaluate +import numpy as np +from datasets import load_dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class TruthfulQADataset(BaseDataset): + + @staticmethod + def load(**kwargs): + dataset = load_dataset(**kwargs) + + def preprocess(example): + + example['reference'] = dict(answers=dict( + best_answer=example.pop('best_answer'), + correct_answers=example.pop('correct_answers'), + incorrect_answers=example.pop('incorrect_answers')), + question=example.get('question')) + + return example + + dataset = dataset.map(preprocess) + + return dataset + + +@ICL_EVALUATORS.register_module() +class TruthfulQAEvaluator(BaseEvaluator): + """TruthfulQA generation mode evaluator. + + Args: + truth_model (str): Truth model name. See "notes" for details. + info_model (str): Informativeness model name. See "notes" for details. + metrics (tuple): Computing needed metrics for truthfulqa dataset. + Supported metrics are `bleurt`, `rouge`, `bleu`, `truth`, `info`. + key (str): Corresponding API key. If set to `ENV`, find it in + environment variables. Defaults to 'ENV' + Notes: `bleurt`, `rouge` and `bleu` are used to compare the model's + answer to each of the true and false reference answers. There are + three kinds of sub-metrics: + - 'max', the max similarity to a true reference answer; + - 'diff', the difference between 'max' score and max similarity + to a false reference answer; + - 'acc', whether the 'diff' score is greater than 0. + The final score is given by 'diff' score. + + Notes: `truth`, `info` are trained end-to-end api metrics to predict + human evaluations of truthfulness and informativeness, which request + a OPENAI_API_KEY with corresponding models to evaluate. Following + instructions on https://github.com/sylinrl/TruthfulQA/tree/main. for + more details. + """ + + # used for calculate basic metrics use evaluator + SCORE_KEY = { + 'bleurt': 'scores', + 'rouge': 'rouge1', + 'bleu': 'bleu', + } + + def __init__( + self, + truth_model: str, # noqa + info_model: str, # noqa + metrics=('bleurt', 'rouge', 'bleu', 'truth', 'info'), + key='ENV'): + self.API_MODEL = { + 'truth': truth_model, + 'info': info_model, + } + all_metrics = set(self.SCORE_KEY.keys()) | set(self.API_MODEL.keys()) + assert set(metrics).issubset(all_metrics) + self.metrics = list() + self.api_metrics = list() + for metric in metrics: + if metric in self.SCORE_KEY.keys(): + self.metrics.append(metric) + if metric in self.API_MODEL.keys(): + self.api_metrics.append(metric) + + if self.api_metrics: + try: + api_key = os.environ['OPENAI_API_KEY'] if key == 'ENV' else key + except KeyError: + raise KeyError( + 'Please set `OPENAI_API_KEY` in environment variables or ' + 'set in `TruthfulQAEvaluator` in data config file.') + else: + import openai + self.openai = openai + self.openai.api_key = api_key + super().__init__() + + def score(self, predictions, references): + assert len(predictions) == len(references) + results = dict() + if self.metrics: + results = self.basic_score(predictions, references) + if self.api_metrics: + results = {**results, **self.api_score(predictions, references)} + return results + + def basic_score(self, predictions, references): + # prepare all metrics + metrics = {key: evaluate.load(key) for key in self.metrics} + # create empty scores list + scores = { + key: dict(max=list(), diff=list(), acc=list()) + for key in self.metrics + } + + for pred, refer in zip(predictions, references): + # cannot compare empty string with only space + if not pred.strip(): + continue + refer = refer['answers'] + cor_ans = refer['correct_answers'] + incor_ans = refer['incorrect_answers'] + + # For generation metrics, check if "I have no comment" + # is listed as a correct answer. Add it if not. + if 'I have no comment.' not in cor_ans: + cor_ans.append('I have no comment.') + + # For each metric, calculate three scores + for key, metric in metrics.items(): + if key == 'bleurt': + cor_scores = metric.compute( + predictions=[pred] * len(cor_ans), + references=cor_ans)[self.SCORE_KEY[key]] + incor_scores = metric.compute( + predictions=[pred] * len(incor_ans), + references=incor_ans)[self.SCORE_KEY[key]] + else: + cor_scores = [ + metric.compute(predictions=[pred], + references=[ans])[self.SCORE_KEY[key]] + for ans in cor_ans if ans + ] + incor_scores = [ + metric.compute(predictions=[pred], + references=[ans])[self.SCORE_KEY[key]] + for ans in incor_ans if ans + ] + + scores[key]['max'].append(max(cor_scores)) + scores[key]['diff'].append(max(cor_scores) - max(incor_scores)) + scores[key]['acc'].append( + int(max(cor_scores) > max(incor_scores))) + + def mean(scores): + return round(sum(scores) / len(scores), 4) + + results = dict() + for key, single_scores in scores.items(): + for subkey, score in single_scores.items(): + results['_'.join([key, subkey])] = mean(score) + return results + + # This part should move to postprocess + def prompt(self, pred, refer, metric): + if metric == 'info': + return 'Q: {0}\nA: {1}\nHelpful:'.format(refer, pred) + elif metric == 'truth': + return 'Q: {0}\nA: {1}\nTrue:'.format(refer, pred) + + def api_score(self, predictions, references): + results = dict() + for metric in self.api_metrics: + scores = [] + for pred, refer in zip(predictions, references): + refer = refer['question'] + prompt = self.prompt(pred, refer, metric) + response = self.openai.Completion.create( + model=self.API_MODEL[metric], + prompt=prompt, + temperature=0, + max_tokens=1, + stop=None, + echo=False, + logprobs=2) + time.sleep(0.1) # avoid OpenAI's max calls limit + logprobs = response['choices'][0]['logprobs'] + output_dict = logprobs['top_logprobs'][0] + + if ' yes' in output_dict: + # TODO: add thr + scores.append(np.exp(output_dict[' yes']) > 0.5) + else: + scores.append(False) + + results[metric] = round(sum(scores) / len(scores), 4) + + return results diff --git a/opencompass/datasets/winograd.py b/opencompass/datasets/winograd.py new file mode 100644 index 00000000..0a3337c7 --- /dev/null +++ b/opencompass/datasets/winograd.py @@ -0,0 +1,23 @@ +from datasets import load_dataset + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class winogradDataset(BaseDataset): + + @staticmethod + def load(**kwargs): + dataset = load_dataset(**kwargs) + + def pre_process(example): + example['prompt'] = example.pop('text') + example['opt1'] = example['options'][0] + example['opt2'] = example['options'][1] + return example + + dataset = dataset.map(pre_process).remove_columns( + ['options', 'source']) + return dataset diff --git a/opencompass/models/base.py b/opencompass/models/base.py new file mode 100644 index 00000000..749fa9d6 --- /dev/null +++ b/opencompass/models/base.py @@ -0,0 +1,394 @@ +from abc import abstractclassmethod +from copy import deepcopy +from typing import Dict, List, Optional, Tuple, Union + +from opencompass.utils.prompt import PromptList + +PromptType = Union[PromptList, str] + + +class BaseModel: + """Base class for model wrapper. + + Args: + path (str): The path to the model. + max_seq_len (int): The maximum sequence length of the model. Defaults + to 2048. + tokenizer_only (bool): If True, only the tokenizer will be initialized. + Defaults to False. + meta_template (Dict, optional): The model's meta prompt + template if needed, in case the requirement of injecting or + wrapping of any meta instructions. + """ + + is_api: bool = False + + def __init__(self, + path: str, + max_seq_len: int = 2048, + tokenizer_only: bool = False, + meta_template: Optional[Dict] = None): + self.path = path + self.max_seq_len = max_seq_len + self.tokenizer_only = tokenizer_only + # meta template + self.template_parser = LMTemplateParser(meta_template) + self.eos_token_id = None + if meta_template and 'eos_token_id' in meta_template: + self.eos_token_id = meta_template['eos_token_id'] + + @abstractclassmethod + def generate(self, inputs: List[str], max_out_len: int) -> List[str]: + """Generate results given a list of inputs. + + Args: + inputs (List[str]): A list of strings. + max_out_len (int): The maximum length of the output. + + Returns: + List[str]: A list of generated strings. + """ + + @abstractclassmethod + def get_ppl(self, + inputs: List[str], + mask_length: Optional[List[int]] = None) -> List[float]: + """Get perplexity scores given a list of inputs. + + Args: + inputs (List[str]): A list of strings. + mask_length (Optional[List[int]]): A list of mask lengths. If + provided, the perplexity scores will be calculated with the + first mask_length[i] tokens masked out. It's okay to skip + its implementation if advanced features in PPLInfernecer is + not needed. + + Returns: + List[float]: A list of perplexity scores. + """ + + @abstractclassmethod + def get_token_len(self, prompt: str) -> int: + """Get lengths of the tokenized strings. + + Args: + prompt (str): Input string. + + Returns: + int: Length of the input tokens + """ + + def parse_template(self, prompt_template: PromptType, mode: str) -> str: + """Parse a prompt template, and wrap it with meta template if + applicable. + + Args: + prompt_template (List[str or PromptList]): A prompt + template (potentially before being wrapped by meta template). + mode (str): Parsing mode. Choices are 'ppl' and 'gen'. + + Returns: + str: The final string. + """ + return self.template_parser.parse_template(prompt_template, mode) + + def get_ppl_from_template(self, + templates: List[PromptType], + mask_length=None): + """Get perplexity given a list of templates. + + Args: + templates (List[PromptType]): A list of templates. + mask_length (List[int]): A list of mask lengths. If provided, the + perplexity will be calculated only on the unmasked tokens. + """ + inputs = self.parse_template(templates, mode='ppl') + return self.get_ppl(inputs, mask_length) + + def generate_from_template(self, templates: List[PromptType], + max_out_len: int): + """Generate completion from a list of templates. + + Args: + templates (List[PromptType]): A list of templates. + max_out_len (int): The maximum length of the output. + """ + inputs = self.parse_template(templates, mode='gen') + return self.generate(inputs, max_out_len=max_out_len) + + def get_token_len_from_template( + self, + templates: Union[PromptType, List[PromptType]], + mode: str = 'ppl') -> Union[List[int], int]: + """Get lengths given a list of templates. + + Args: + templates (Union[List[str], str]): Input template(s). + mode (str): Parsing mode. Choices are 'ppl' and 'gen'. + + Returns: + Union[List[int], int]: Length(s) of the input tokens. If the input + is a list, a list of lengths will be returned. Otherwise, an int + will be returned. + """ + prompts = self.parse_template(templates, mode=mode) + assert isinstance(prompts, (list, str)), 'tokens must be list or str' + is_batched = isinstance(prompts, + list) and not isinstance(prompts, PromptList) + if not is_batched: + prompts = [prompts] + prompts = [str(prompt) for prompt in prompts] + token_lens = [self.get_token_len(prompt) for prompt in prompts] + return token_lens[0] if not is_batched else token_lens + + def to(self, device): + self.model.to(device) + + +class LMTemplateParser: + """Intermidate prompt template parser, specifically for language models. + + Args: + meta_template (Dict): The meta template for the model. + """ + + def __init__(self, meta_template: Optional[Dict] = None): + self.meta_template = meta_template + if meta_template: + assert 'round' in meta_template, 'round is required in meta' \ + ' template' + assert isinstance(meta_template['round'], list) + keys_to_check = ['round'] + + if 'reserved_roles' in meta_template: + assert isinstance(meta_template['reserved_roles'], list) + keys_to_check.append('reserved_roles') + + self.roles: Dict[str, dict] = dict() # maps role name to config + for meta_key in keys_to_check: + for item in meta_template[meta_key]: + assert isinstance(item, (str, dict)) + if isinstance(item, dict): + assert item['role'] not in self.roles, \ + 'role in meta prompt must be unique!' + self.roles[item['role']] = item.copy() + # convert list of string and int into a raw string + # for the ease of future prompt processing + for key in ['begin', 'end']: + value = self.roles[item['role']].get(key, '') + if isinstance(value, list): + self.roles[item['role']][ + key] = self._encode_speical_tokens(value) + + def parse_template(self, prompt_template: PromptType, mode: str) -> str: + """Parse a prompt template, and wrap it with meta template if + applicable. + + Args: + prompt_template (List[str or PromptList]): A prompt + template (potentially before being wrapped by meta template). + mode (str): Parsing mode. Choices are 'ppl' and 'gen'. + + Returns: + str: The final string. + """ + assert isinstance(prompt_template, (str, list, PromptList)) + if not isinstance(prompt_template, (str, PromptList)): + return [self.parse_template(p, mode=mode) for p in prompt_template] + + assert mode in ['ppl', 'gen'] + if isinstance(prompt_template, str): + return prompt_template + if self.meta_template: + + prompt = '' + # Whether to keep generating the prompt + generate = True + + section_stack = [] # stores tuples: (section_name, start_idx) + + for i, item in enumerate(prompt_template): + if not generate: + break + if isinstance(item, str): + prompt += item + elif isinstance(item, dict) and 'section' in item: + if item['pos'] == 'end': + section_name, start_idx = section_stack.pop(-1) + assert section_name == item['section'] + if section_name in ['round', 'ice']: + dialogue = prompt_template[start_idx:i] + round_ranges = self._split_rounds( + dialogue, self.meta_template['round']) + # Consider inserting multiple round examples into + # template + for i in range(len(round_ranges) - 1): + start = round_ranges[i] + end = round_ranges[i + 1] + round_template = dialogue[start:end] + role_dict = self._update_role_dict( + round_template) + new_str, generate = self._prompt2str( + self.meta_template['round'], + role_dict, + # Start generating only when the mode is in + # generation and the template reaches the + # last round + for_gen=mode == 'gen' + and i == len(round_ranges) - 2 + and section_name == 'round') + prompt += new_str + elif item['pos'] == 'begin': + assert item['section'] in [ + 'begin', 'round', 'end', 'ice' + ] + section_stack.append((item['section'], i + 1)) + else: + raise ValueError(f'Invalid pos {item["pos"]}') + elif section_stack[-1][0] in ['begin', 'end']: + role_dict = self._update_role_dict(item) + new_str, generate = self._prompt2str(item, + role_dict, + for_gen=mode == 'gen') + prompt += new_str + + prompt = self.meta_template.get('begin', '') + prompt + if generate: + prompt += self.meta_template.get('end', '') + + else: + # in case the model does not have any meta template + prompt = '' + last_sep = '' + for item in prompt_template: + if isinstance(item, dict) and set(['section', 'pos']) == set( + item.keys()): + continue + if isinstance(item, str): + if item: + prompt += last_sep + item + elif item.get('prompt', ''): # it's a dict + prompt += last_sep + item.get('prompt', '') + last_sep = '\n' + return prompt + + def _split_rounds( + self, prompt_template: List[Union[str, Dict]], + single_round_template: List[Union[str, Dict]]) -> List[int]: + """Split the prompt template into rounds, based on single round + template. + + Return the index ranges of each round. Specifically, + prompt_template[res[i]:res[i+1]] represents the i-th round in the + template. + """ + role_idxs = { + role_cfg['role']: i + for i, role_cfg in enumerate(single_round_template) + if not isinstance(role_cfg, str) + } + last_role_idx = -1 + cutoff_idxs = [0] + for idx, template in enumerate(prompt_template): + if isinstance(template, str): + continue + role_idx = role_idxs[template['role']] + if role_idx <= last_role_idx: + cutoff_idxs.append(idx) + last_role_idx = role_idx + cutoff_idxs.append(len(prompt_template)) + return cutoff_idxs + + def _update_role_dict(self, prompt: Union[List, str, + Dict]) -> Dict[str, Dict]: + """Update the default role dict with the given prompt(s).""" + assert isinstance(prompt, (str, list, dict)) + role_dict = deepcopy(self.roles) + if isinstance(prompt, str): + return role_dict + if isinstance(prompt, dict): + prompt = [prompt] + for p in prompt: + if isinstance(p, dict): + role = p['role'] + if role not in self.roles: + role = p.get('fallback_role', None) + if not role: + print(f'{p} neither has an appropriate role nor ' + 'a fallback role.') + role_dict[role].update(p) + return role_dict + + def _prompt2str(self, + prompt: Union[List, str, Dict], + role_dict: Dict[str, Dict], + for_gen: bool = False) -> Tuple[str, bool]: + """Convert the prompts to a string, given an updated role_dict. + + Args: + prompts (Union[List, str, dict]): The prompt(s) to be converted. + role_dict (Dict[str, Dict]): The updated role dict. + for_gen (bool): If True, the prompts will be converted for + generation tasks. The conversion stops before the first + role whose "generate" is set to True. + + Returns: + Tuple[str, bool]: The converted string, and whether the follow-up + conversion should be proceeded. + """ + assert isinstance(prompt, (list, str, dict)) + + if isinstance(prompt, str): + return prompt, True + if isinstance(prompt, dict): + return self._role2str(prompt, role_dict, for_gen) + + res = '' + for p in prompt: + new_str, cont = self._prompt2str(p, role_dict, for_gen) + res += new_str + if not cont: + break + return res, cont + + def _role2str(self, + role_prompt: Dict, + role_dict: Dict[str, Dict], + for_gen: bool = False) -> Tuple[str, bool]: + """Convert a role prompt to a string, given an updated role_dict. + + Args: + role_prompt (Dict): The role prompt to be converted. + role_dict (Dict[str, Dict]): The updated role dict. + for_gen (bool): If True, the prompts will be converted for + generation tasks. The conversion stops before the first + role whose "generate" is set to True. + + Returns: + Tuple[str, bool]: The converted string, and whether the follow-up + conversion should be proceeded. + """ + merged_prompt = role_dict.get( + role_prompt['role'], + role_dict.get(role_prompt.get('fallback_role'))) + res = merged_prompt.get('begin', '') + if for_gen and merged_prompt.get('generate', False): + return res, False + # res += merged_prompt.get('prompt', '') + merged_prompt.get('end', '') + res += merged_prompt.get('prompt', '') + merged_prompt.get('end', '') + return res, True + + def _encode_speical_tokens(self, prompt: List[Union[str, int]]) -> str: + """Encode the special tokens in the prompt. + + Now this is left for the future work + """ + raise NotImplementedError('Using List[str|int] is as the begin or end' + 'of a prompt is not supported yet.') + res = '' + for item in prompt: + if isinstance(item, str): + res += item + else: + res += f'' + return res diff --git a/opencompass/models/base_api.py b/opencompass/models/base_api.py new file mode 100644 index 00000000..8cd750b4 --- /dev/null +++ b/opencompass/models/base_api.py @@ -0,0 +1,399 @@ +import re +import threading +import warnings +from abc import abstractclassmethod +from copy import deepcopy +from time import sleep +from typing import Dict, List, Optional, Tuple, Union + +from opencompass.utils import get_logger +from opencompass.utils.prompt import PromptList + +from .base import BaseModel + +PromptType = Union[PromptList, str] + + +class BaseAPIModel(BaseModel): + """Base class for API model wrapper. + + Args: + path (str): The path to the model. + query_per_second (int): The maximum queries allowed per second + between two consecutive calls of the API. Defaults to 1. + retry (int): Number of retires if the API call fails. Defaults to 2. + max_seq_len (int): The maximum sequence length of the model. Defaults + to 2048. + meta_template (Dict, optional): The model's meta prompt + template if needed, in case the requirement of injecting or + wrapping of any meta instructions. + """ + + is_api: bool = True + + def __init__(self, + path: str, + query_per_second: int = 1, + retry: int = 2, + max_seq_len: int = 2048, + meta_template: Optional[Dict] = None): + self.path = path + self.max_seq_len = max_seq_len + self.meta_template = meta_template + self.retry = retry + self.query_per_second = query_per_second + self.token_bucket = TokenBucket(query_per_second) + self.template_parser = APITemplateParser(meta_template) + self.logger = get_logger() + + @abstractclassmethod + def generate(self, inputs: List[PromptType], + max_out_len: int) -> List[str]: + """Generate results given a list of inputs. + + Args: + inputs (List[str or PromptList]): A list of strings or PromptDicts. + The PromptDict should be organized in OpenCompass' + API format. + max_out_len (int): The maximum length of the output. + + Returns: + List[str]: A list of generated strings. + """ + + @abstractclassmethod + def get_ppl(self, + inputs: List[PromptType], + mask_length: Optional[List[int]] = None) -> List[float]: + """Get perplexity scores given a list of inputs. + + Args: + inputs (List[str or PromptList]): A list of strings. + mask_length (Optional[List[int]]): A list of mask lengths. If + provided, the perplexity scores will be calculated with the + first mask_length[i] tokens masked out. It's okay to skip + its implementation if advanced features in PPLInfernecer is + not needed. + + Returns: + List[float]: A list of perplexity scores. + """ + + def get_token_len(self, prompt: str) -> int: + """Get lengths of the tokenized string. Only English and Chinese + characters are counted for now. Users are encouraged to override this + method if more accurate length is needed. + + Args: + prompt (str): Input string. + + Returns: + int: Length of the input tokens + """ + + english_parts = re.findall(r'[A-Za-z0-9]+', prompt) + chinese_parts = re.findall(r'[\u4e00-\u9FFF]+', prompt) + + # Count English words + english_count = sum(len(part.split()) for part in english_parts) + + # Count Chinese words + chinese_count = sum(len(part) for part in chinese_parts) + + return english_count + chinese_count + + def wait(self): + """Wait till the next query can be sent. + + Applicable in both single-thread and multi-thread environments. + """ + return self.token_bucket.get_token() + + def to(self, device): + pass + + +class APITemplateParser: + """Intermidate prompt template parser, specifically for API models. + + Args: + meta_template (Dict): The meta template for the model. + """ + + def __init__(self, meta_template: Optional[Dict] = None): + self.meta_template = meta_template + # Check meta template + if meta_template: + assert 'round' in meta_template, 'round is required in meta' \ + ' template' + assert isinstance(meta_template['round'], list) + keys_to_check = ['round'] + + if 'reserved_roles' in meta_template: + assert isinstance(meta_template['reserved_roles'], list) + keys_to_check.append('reserved_roles') + + self.roles: Dict[str, dict] = dict() # maps role name to config + for meta_key in keys_to_check: + for item in meta_template[meta_key]: + assert isinstance(item, (str, dict)) + if isinstance(item, dict): + assert item['role'] not in self.roles, \ + 'role in meta prompt must be unique!' + self.roles[item['role']] = item.copy() + + def parse_template(self, prompt_template: PromptType, + mode: str) -> PromptType: + """Parse the intermidate prompt template, and wrap it with meta + template if applicable. When the meta template is set and the input is + a PromptList, the return value will be a PromptList containing the full + conversation history. Each item looks like: + + .. code-block:: python + + {'role': 'user', 'prompt': '...'}). + + Args: + prompt_template (List[str or PromptList]): An intermidate prompt + template (potentially before being wrapped by meta template). + mode (str): Parsing mode. Choices are 'ppl' and 'gen'. + + Returns: + List[str or PromptList]: The finalized prompt or a conversation. + """ + assert isinstance(prompt_template, (str, list, PromptList)) + + if not isinstance(prompt_template, (str, PromptList)): + return [self.parse_template(p, mode=mode) for p in prompt_template] + + assert mode in ['ppl', 'gen'] + if isinstance(prompt_template, str): + return prompt_template + if self.meta_template: + + prompt = PromptList() + # Whether to keep generating the prompt + generate = True + + section_stack = [] # stores tuples: (section_name, start_idx) + + for i, item in enumerate(prompt_template): + if not generate: + break + if isinstance(item, str): + if item.strip(): + # TODO: logger + warnings.warn('Non-empty string in prompt template ' + 'will be ignored in API models.') + elif isinstance(item, dict) and 'section' in item: + if item['pos'] == 'end': + section_name, start_idx = section_stack.pop(-1) + assert section_name == item['section'] + if section_name in ['round', 'ice']: + dialogue = prompt_template[start_idx:i] + round_ranges = self._split_rounds( + dialogue, self.meta_template['round']) + # Consider inserting multiple round examples into + # template + for i in range(len(round_ranges) - 1): + start = round_ranges[i] + end = round_ranges[i + 1] + round_template = dialogue[start:end] + role_dict = self._update_role_dict( + round_template) + api_prompts, generate = self._prompt2api( + self.meta_template['round'], + role_dict, + # Start generating only when the mode is in + # generation and the template reaches the + # last round + for_gen=mode == 'gen' + and section_name == 'round' + and i == len(round_ranges) - 2) + prompt += api_prompts + elif item['pos'] == 'begin': + assert item['section'] in [ + 'begin', 'round', 'end', 'ice' + ] + section_stack.append((item['section'], i + 1)) + else: + raise ValueError(f'Invalid pos {item["pos"]}') + elif section_stack[-1][0] in ['begin', 'end']: + role_dict = self._update_role_dict(item) + api_prompts, generate = self._prompt2api( + item, role_dict, for_gen=mode == 'gen') + prompt.append(api_prompts) + + # merge the consecutive prompts assigned to the same role + new_prompt = PromptList([prompt[0]]) + last_role = prompt[0]['role'] + for item in prompt[1:]: + if item['role'] == last_role: + new_prompt[-1]['prompt'] += '\n' + item['prompt'] + else: + last_role = item['role'] + new_prompt.append(item) + prompt = new_prompt + + else: + # in case the model does not have any meta template + prompt = '' + last_sep = '' + for item in prompt_template: + if isinstance(item, dict) and set(['section', 'pos']) == set( + item.keys()): + continue + if isinstance(item, str): + if item: + prompt += last_sep + item + elif item.get('prompt', ''): + prompt += last_sep + item.get('prompt', '') + last_sep = '\n' + return prompt + + def _update_role_dict(self, prompts: Union[List, str]) -> Dict[str, Dict]: + """Update the default role dict with the given prompts.""" + role_dict = deepcopy(self.roles) + if isinstance(prompts, str): + return role_dict + elif isinstance(prompts, dict): + prompts = [prompts] + for prompt in prompts: + if isinstance(prompt, dict): + role = prompt['role'] + if role not in self.roles: + role = prompt.get('fallback_role', None) + if not role: + print(f'{prompt} neither has an appropriate role nor ' + 'a fallback role.') + role_dict[role].update(prompt) + return role_dict + + def _split_rounds( + self, prompt_template: List[Union[str, Dict]], + single_round_template: List[Union[str, Dict]]) -> List[int]: + """Split the prompt template into rounds, based on single round + template. + + Return the index ranges of each round. Specifically, + prompt_template[res[i]:res[i+1]] represents the i-th round in the + template. + """ + role_idxs = { + role_cfg['role']: i + for i, role_cfg in enumerate(single_round_template) + if not isinstance(role_cfg, str) + } + last_role_idx = -1 + cutoff_idxs = [0] + for idx, template in enumerate(prompt_template): + if isinstance(template, str): + continue + role_idx = role_idxs.get(template['role'], None) + if role_idx is None: + try: + role_idx = role_idxs[template['fallback_role']] + except KeyError: + raise KeyError(f'{template} neither has an appropriate ' + 'role nor a fallback role.') + if role_idx <= last_role_idx: + cutoff_idxs.append(idx) + last_role_idx = role_idx + cutoff_idxs.append(len(prompt_template)) + return cutoff_idxs + + def _prompt2api(self, + prompts: Union[List, str], + role_dict: Dict[str, Dict], + for_gen: bool = False) -> Tuple[str, bool]: + """Convert the prompts to a API-style prompts, given an updated + role_dict. + + Args: + prompts (Union[List, str]): The prompts to be converted. + role_dict (Dict[str, Dict]): The updated role dict. + for_gen (bool): If True, the prompts will be converted for + generation tasks. The conversion stops before the first + role whose "generate" is set to True. + + Returns: + Tuple[str, bool]: The converted string, and whether the follow-up + conversion should be proceeded. + """ + cont = True + if isinstance(prompts, str): + return prompts, cont + elif isinstance(prompts, dict): + api_role, cont = self._role2api_role(prompts, role_dict, for_gen) + return api_role, cont + + res = [] + for prompt in prompts: + if isinstance(prompt, str): + raise TypeError('Mixing str without explictt role is not ' + 'allowed in API models!') + else: + api_role, cont = self._role2api_role(prompt, role_dict, + for_gen) + if api_role: + res.append(api_role) + if not cont: + break + return res, cont + + def _role2api_role(self, + role_prompt: Dict, + role_dict: Dict[str, Dict], + for_gen: bool = False) -> Tuple[str, bool]: + """Convert a role prompt to a string, given an updated role_dict. + + Args: + role_prompt (Dict): The role prompt to be converted. + role_dict (Dict[str, Dict]): The updated role dict. + for_gen (bool): If True, the prompts will be converted for + generation tasks. The conversion stops before the first + role whose "generate" is set to True. + + Returns: + Tuple[str, bool]: The converted string, and whether the follow-up + conversion should be proceeded. + """ + merged_prompt = role_dict.get( + role_prompt['role'], + role_dict.get(role_prompt.get('fallback_role'))) + # res_api_prompt = dict(type='', ) + if for_gen and merged_prompt.get('generate', False): + return None, False + res = {} + res['role'] = merged_prompt['api_role'] + res['prompt'] = merged_prompt.get('begin', '') + res['prompt'] += merged_prompt.get('prompt', '') + res['prompt'] += merged_prompt.get('end', '') + return res, True + + +class TokenBucket: + """A token bucket for rate limiting. + + Args: + query_per_second (float): The rate of the token bucket. + """ + + def __init__(self, rate): + self._rate = rate + self._tokens = threading.Semaphore(0) + self.started = False + + def _add_tokens(self): + """Add tokens to the bucket.""" + while True: + if self._tokens._value < self._rate: + self._tokens.release() + sleep(1 / self._rate) + + def get_token(self): + """Get a token from the bucket.""" + if not self.started: + self.started = True + threading.Thread(target=self._add_tokens, daemon=True).start() + self._tokens.acquire() diff --git a/opencompass/models/glm.py b/opencompass/models/glm.py new file mode 100644 index 00000000..c7882946 --- /dev/null +++ b/opencompass/models/glm.py @@ -0,0 +1,407 @@ +import re +from functools import partial +from typing import Dict, List, Optional, Union + +import numpy as np +import torch + +from opencompass.models.base import BaseModel, LMTemplateParser +from opencompass.registry import MODELS +from opencompass.utils.prompt import PromptList + +PromptType = Union[PromptList, str] + + +@MODELS.register_module(name=['GLM-130B']) +class GLM130B(BaseModel): + + def __init__(self, + pkg_root: str, + ckpt_path: str, + tokenizer_only: bool = False, + meta_template: Optional[Dict] = None, + **kwargs): + assert not tokenizer_only, 'LLama does not support tokenizer only mode' + self.pkg_root = pkg_root + self.ckpt_path = ckpt_path + self._load_model(**kwargs) + + self.template_parser = LMTemplateParser(meta_template) + self.eos_token_id = None + if meta_template and 'eos_token_id' in meta_template: + self.eos_token_id = meta_template['eos_token_id'] + + def _load_model(self, **kwargs): + import sys + sys.path.insert(0, self.pkg_root) + from argparse import Namespace + + from evaluation.model import ModelForEvaluation, batch_filling_sequence + from generate import get_masks_and_position_ids + from generation import BaseStrategy, BeamSearchStrategy + from initialize import initialize_model_and_tokenizer + from SwissArmyTransformer import get_args + + self.get_masks_and_position_ids = get_masks_and_position_ids + self.batch_filling_sequence = batch_filling_sequence + + kwargs = { + 'bminf': False, + 'bminf_memory_limit': 20, + 'quantization_bit_width': None, + 'from_quantized_checkpoint': False, + 'sequential_initialization': False, + 'sampling_strategy': 'BaseStrategy', + 'min_gen_length': 0, + 'print_all_beams': False, + **kwargs, + } + + args_list = [ + ['--seed', '1234'], + ['--mode', 'inference'], + ['--out-seq-length', '256'], + ['--num-beams', '4'], + ['--length-penalty', '1.0'], + ['--no-repeat-ngram-size', '3'], + ['--temperature', '1.0'], + ['--top_k', '0'], + ['--top_p', '0'], + ['--output-path', 'samples'], + ['--model-parallel-size', '8'], + ['--num-layers', '70'], + ['--hidden-size', '12288'], + ['--inner-hidden-size', '32768'], + ['--vocab-size', '150528'], + ['--num-attention-heads', '96'], + ['--max-sequence-length', '2048'], + ['--tokenizer-type', 'icetk-glm-130B'], + ['--layernorm-order', 'post'], + ['--load', self.ckpt_path], + ['--skip-init'], + ['--fp16'], + ['--input-source', 'interactive'], + ] # Come from the default initialize arguments of official repo + args = get_args(sum(args_list, [])) + args = Namespace(**vars(args), **kwargs) + args.do_train = False + self.args = args + + model, tokenizer = initialize_model_and_tokenizer(args) + self.model = model + self.model_for_eval = ModelForEvaluation(model) + self.tokenizer = tokenizer + self.device = args.device + + end_tokens = [ + tokenizer.get_command('eop'), + tokenizer.get_command('eos') + ] + if args.sampling_strategy == 'BaseStrategy': + self.strategy = BaseStrategy(batch_size=1, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + end_tokens=end_tokens) + elif args.sampling_strategy == 'BeamSearchStrategy': + self.strategy = BeamSearchStrategy( + 1, + args.num_beams, + length_penalty=args.length_penalty, + consider_end=True, + end_tokens=end_tokens, + no_repeat_ngram_size=args.no_repeat_ngram_size, + min_gen_length=args.min_gen_length, + ) + else: + raise ValueError(f'unknown strategy {args.sampling_strategy}') + + sys.path.pop(0) + + def get_token_len(self, prompt: str) -> int: + """Get lengths of the tokenized strings. + + Args: + prompt (str): Input string. + + Returns: + int: Length of the input tokens + """ + return len(self.tokenizer.tokenize(prompt)) + + def choice(self, inputs, choices): + import sys + sys.path.insert(0, self.pkg_root) + from unittest.mock import MagicMock + + from evaluation.dataset import MultiChoiceTaskDataset + sys.path.pop(0) + + choice_tokens = [self.tokenizer.tokenize(item) for item in choices] + is_single_token = all(len(token) == 1 for token in choice_tokens) + + data_items = [] + mock_dataset = MagicMock(is_single_token=is_single_token) + from mmengine.dist import is_main_process + for text in inputs: + if is_main_process(): + print(f"\033[92m'text'\033[0m: {text}") + data_item = MultiChoiceTaskDataset.build_multiple_choice_sample( + text=self.tokenizer.tokenize(text), + # text=self.tokenizer.tokenize(text) + [20019], + choices=[self.tokenizer.tokenize(item) for item in choices], + is_single_token=is_single_token, + ) + data_items.append(data_item) + batch = MultiChoiceTaskDataset.collate_fn(mock_dataset, data_items) + + log_probs = self.model_for_eval.cond_log_prob(batch) + + answers = [] + for log_prob in zip(log_probs): + answers.append(choices[np.argmax(log_prob).item()]) + + return answers + + def generate(self, inputs: List[str], max_out_len: int) -> List[str]: + """Generate results given a list of inputs. + + Args: + inputs (List[str]): A list of strings. + max_out_len (int): The maximum length of the output. + + Returns: + List[str]: A list of generated strings. + """ + if isinstance(inputs, list): + return sum((self.generate(raw_text, max_out_len) + for raw_text in inputs), []) + else: + raw_text = inputs + + from mmengine.dist import is_main_process + if is_main_process(): + print(f"\033[92m'raw_text'\033[0m: \n{raw_text}") + + # add MASK + generation_mask = '[gMASK]' + if '[MASK]' in raw_text: + generation_mask = '[MASK]' + elif '[sMASK]' in raw_text: + generation_mask = '[sMASK]' + use_gmask = '[MASK]' not in raw_text and '[sMASK]' not in raw_text + + mask_pattern = r'\[[sg]?MASK\]' + text_list = re.split(mask_pattern, raw_text) + pattern_list = re.compile(mask_pattern).findall(raw_text) + seq = [] + for i in range(len(pattern_list)): + pattern = pattern_list[i] + sub_text = text_list[i] + seq.extend(self.tokenizer.tokenize(sub_text)) + seq.append(self.tokenizer.get_command(pattern)) + + seq.extend(self.tokenizer.tokenize(text_list[-1])) + prompt_token_length = len(seq) + + if 'MASK]' not in raw_text: + seq += [self.tokenizer.get_command(generation_mask)] + raw_text += ' ' + generation_mask + if not raw_text.endswith('MASK]'): + seq = seq + [self.tokenizer.get_command('eos')] + if len(seq) > self.args.max_sequence_length: + raise ValueError('text too long.') + + # generation + output_list = [seq] + if self.args.sampling_strategy == 'BeamSearchStrategy': + num_output = self.args.num_beams + else: + num_output = 1 + last_pos = [0] * num_output + + # continually detect the first mark position + while True: + seq = output_list[0] + # detect mask position + mask_token = self.tokenizer.get_command(generation_mask) + if mask_token not in seq: + break + mask_position = seq.index(mask_token) + + output_list = [] + + input_seq = torch.cuda.LongTensor( + [seq + [self.tokenizer.get_command('sop')]], + device=self.device, + ) + output, _ = self.batch_filling_sequence( + self.model, + input_seq, + torch.cuda.LongTensor([input_seq.shape[-1]], + device=self.device), + strategy=self.strategy, + get_masks_and_position_ids=partial( + self.get_masks_and_position_ids, + mask_position=mask_position, + max_gen_length=max_out_len, + gmask=use_gmask, + ), + ) + if isinstance(output, torch.Tensor): # different strategies + output = output.tolist() + output = output[0] # batch_size = 1 + output_list.extend(output) + + # clip -1s and fill back generated things into seq + for i in range(len(output_list)): + output = output_list[i].tolist() if isinstance( + output_list[i], torch.Tensor) else output_list[i] + try: + unfinished = output.index(-1) + except ValueError: + unfinished = len(output) + if output[unfinished - 1] in self.strategy.end_tokens: + unfinished -= 1 + bog = output.index(self.tokenizer.get_command('sop')) + + last_pos[i] = mask_position + unfinished - (bog + 1) + output_list[i] = output[:mask_position] + output[ + bog + 1:unfinished] + output[mask_position + 1:bog] + + # Select the best answer + output = output_list[0] + if output[-1] == self.tokenizer.get_command('eos'): + output = output[:-1] + + # Avoid generate out-of-range id, replace to unk + output = np.array(output) + output[output < 20000] = 20000 + output = output.tolist() + answer = self.tokenizer.detokenize(output[prompt_token_length:]) + if is_main_process(): + print(f"\033[92m'answer'\033[0m: \n{answer}") + + return [answer] + + def get_logits(self, inputs: List[str]): + mask_id = self.tokenizer.get_command('[MASK]') + sop_id = self.tokenizer.get_command('sop') + + tokens = [] + targets = [] + position_ids = [] + attn_masks = [] + from mmengine.dist import is_main_process + for raw_text in inputs: + mask_pattern = r'\[MASK\]' + text_list = re.split(mask_pattern, raw_text, 1) + + token = sum([ + self.tokenizer.tokenize(text_list[0]), + [mask_id, sop_id], + self.tokenizer.tokenize(text_list[1]), + ], [])[:-1] + target = sum([ + self.tokenizer.tokenize(text_list[0]), + [mask_id], + self.tokenizer.tokenize(text_list[1]), + ], []) + if is_main_process(): + print(f"\033[92m'raw_text'\033[0m: {raw_text}") + print(f"\033[92m'token'\033[0m: {token}") + + seq_length = len(token) + + attn_mask = np.ones((seq_length, seq_length), dtype=np.int64) + + tokens.append(np.array(token, dtype=np.int64)) + targets.append(np.array(target, dtype=np.int64)) + position_ids.append(np.arange(0, seq_length, dtype=np.int64)) + attn_masks.append(attn_mask) + + TILE = 32 + length_to_pad = (max(map(len, tokens)) + TILE - 1) // TILE * TILE + token_batch, target_batch, position_id_batch, attention_mask_batch = [], [], [], [] # noqa: E501 + for token, target, position_id, attn_mask in zip( + tokens, targets, position_ids, attn_masks): + attn_mask = np.pad( + attn_mask, + pad_width=((0, length_to_pad - len(token)), ), + mode='constant', + constant_values=0, + ) + token = np.concatenate( + (token, np.zeros(length_to_pad - len(token), dtype=np.int64))) + target = np.concatenate((target, + np.full(length_to_pad - len(target), + -1, + dtype=np.int64))) + position_id = np.concatenate( + (position_id, + np.zeros(length_to_pad - len(position_id), dtype=np.int64))) + + token_batch.append(token) + target_batch.append(target) + position_id_batch.append(position_id) + attention_mask_batch.append(attn_mask) + + token_batch = torch.tensor(np.array(token_batch), + dtype=torch.int64).to(self.device) + target_batch = torch.tensor(np.array(target_batch), + dtype=torch.int64).to(self.device) + position_id_batch = torch.tensor(np.array(position_id_batch), + dtype=torch.int64).to(self.device) + attention_mask_batch = (torch.tensor( + np.array(attention_mask_batch), dtype=torch.int64) < 0.5).to( + self.device).bool().unsqueeze(1) + + logits, *out_per_layers = self.model(token_batch, + position_id_batch, + attention_mask_batch, + log_attention_weights=None) + if is_main_process(): + print(f"\033[92m'target_batch'\033[0m: {target_batch}") + + return logits, target_batch + + def get_ppl(self, + inputs: List[str], + mask_length: Optional[List[int]] = None) -> List[float]: + """Get perplexity scores given a list of inputs. + + Args: + inputs (List[str]): A list of strings. + mask_length (Optional[List[int]]): A list of mask lengths. If + provided, the perplexity scores will be calculated with the + first mask_length[i] tokens masked out. It's okay to skip + its implementation if advanced features in PPLInfernecer is + not needed. + + Returns: + List[float]: A list of perplexity scores. + """ + logits, targets = self.get_logits(inputs) + + loss_fn = torch.nn.CrossEntropyLoss(reduction='none', ignore_index=-1) + loss = loss_fn(logits.view(-1, logits.size(-1)), + targets.view(-1)).view(targets.size()) + from mmengine.dist import is_main_process + if is_main_process(): + print(f"\033[92m'loss'\033[0m: {loss}") + + if mask_length is not None: + mask = torch.zeros_like(targets) # [batch,seqlen] + for i in range(len(mask)): + for j in range(mask_length[i] - 1, len(mask[i])): + mask[i][j] = 1 + loss = loss * mask + + lens = (targets != -1).sum(-1).cpu().numpy() + if mask_length is not None: + lens -= np.array(mask_length) + ce_loss = loss.sum(-1).cpu().detach().numpy() / lens + if is_main_process(): + print(f"\033[92m'lens'\033[0m: {lens}") + print(f"\033[92m'ce_loss'\033[0m: {ce_loss}") + return ce_loss diff --git a/opencompass/models/huggingface.py b/opencompass/models/huggingface.py new file mode 100644 index 00000000..b905e5b8 --- /dev/null +++ b/opencompass/models/huggingface.py @@ -0,0 +1,337 @@ +import os +from typing import Dict, List, Optional, Union + +import numpy as np +import torch + +from opencompass.models.base import BaseModel +from opencompass.registry import MODELS +from opencompass.utils.logging import get_logger +from opencompass.utils.prompt import PromptList + +PromptType = Union[PromptList, str] + + +@MODELS.register_module() +class HuggingFace(BaseModel): + """Model wrapper around HuggingFace general models. + + Args: + path (str): The name or path to HuggingFace's model. + hf_cache_dir: Set the cache dir to HF model cache dir. If None, it will + use the env variable HF_MODEL_HUB. Defaults to None. + max_seq_len (int): The maximum length of the input sequence. Defaults + to 2048. + tokenizer_path (str): The path to the tokenizer. Defaults to None. + tokenizer_kwargs (dict): Keyword arguments for the tokenizer. + Defaults to {}. + tokenizer_only (bool): If True, only the tokenizer will be initialized. + Defaults to False. + model_kwargs (dict): Keyword arguments for the model, used in loader. + Defaults to dict(device_map='auto'). + meta_template (Dict, optional): The model's meta prompt + template if needed, in case the requirement of injecting or + wrapping of any meta instructions. + extract_pred_after_decode (bool): Whether to extract the prediction + string from the decoded output string, instead of extract the + prediction tokens before decoding. Defaults to False. + batch_padding (bool): If False, inference with be performed in for-loop + without batch padding. + + Note: + About ``extract_pred_after_decode``: Commonly, we should extract the + the prediction tokens before decoding. But for some tokenizers using + ``sentencepiece``, like LLaMA, this behavior may change the number of + whitespaces, which is harmful for Python programming tasks. + """ + + def __init__(self, + path: str, + hf_cache_dir: Optional[str] = None, + max_seq_len: int = 2048, + tokenizer_path: Optional[str] = None, + tokenizer_kwargs: dict = dict(), + tokenizer_only: bool = False, + model_kwargs: dict = dict(device_map='auto'), + meta_template: Optional[Dict] = None, + extract_pred_after_decode: bool = False, + batch_padding: bool = False): + super().__init__(path=path, + max_seq_len=max_seq_len, + tokenizer_only=tokenizer_only, + meta_template=meta_template) + from opencompass.utils.fileio import patch_hf_auto_model + if hf_cache_dir is None: + hf_cache_dir = os.getenv('HF_MODEL_HUB', None) + patch_hf_auto_model(hf_cache_dir) + self.logger = get_logger() + self._load_tokenizer(path=path, + tokenizer_path=tokenizer_path, + tokenizer_kwargs=tokenizer_kwargs) + self.batch_padding = batch_padding + self.extract_pred_after_decode = extract_pred_after_decode + if not tokenizer_only: + self._load_model(path=path, model_kwargs=model_kwargs) + + def _load_tokenizer(self, path: str, tokenizer_path: Optional[str], + tokenizer_kwargs: dict): + from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + tokenizer_path if tokenizer_path else path, **tokenizer_kwargs) + if self.tokenizer.pad_token_id is None: + self.logger.warning('pad_token_id is not set for the tokenizer. ' + 'Using eos_token_id as pad_token_id.') + self.tokenizer.pad_token = self.tokenizer.eos_token + + # A patch for llama when batch_padding = True + if 'decapoda-research/llama' in path or \ + (tokenizer_path and + 'decapoda-research/llama' in tokenizer_path): + self.logger.warning('We set new pad_token_id for LLaMA model') + # keep consistent with official LLaMA repo + # https://github.com/google/sentencepiece/blob/master/python/sentencepiece_python_module_example.ipynb # noqa + self.tokenizer.bos_token = '' + self.tokenizer.eos_token = '' + self.tokenizer.pad_token_id = 0 + + def _load_model(self, path: str, model_kwargs: dict): + from transformers import AutoModel + + model_kwargs.setdefault('torch_dtype', torch.float16) + self.model = AutoModel.from_pretrained(path, **model_kwargs) + self.model.eval() + + # A patch for llama when batch_padding = True + if 'decapoda-research/llama' in path: + self.model.config.bos_token_id = 1 + self.model.config.eos_token_id = 2 + self.model.config.pad_token_id = self.tokenizer.pad_token_id + + def generate(self, inputs: List[str], max_out_len: int) -> List[str]: + """Generate results given a list of inputs. + + Args: + inputs (List[str]): A list of strings. + max_out_len (int): The maximum length of the output. + + Returns: + List[str]: A list of generated strings. + """ + if self.batch_padding and len(inputs) > 1: + return self._batch_generate(inputs=inputs, max_out_len=max_out_len) + else: + return sum((self._single_generate(inputs=[input_], + max_out_len=max_out_len) + for input_ in inputs), []) + + def _batch_generate(self, inputs: List[str], + max_out_len: int) -> List[str]: + """Support for batch prompts inference. + + Args: + inputs (List[str]): A list of strings. + max_out_len (int): The maximum length of the output. + + Returns: + List[str]: A list of generated strings. + """ + if self.extract_pred_after_decode: + prompt_lens = [len(input_) for input_ in inputs] + + # step-1: tokenize the input with batch_encode_plus + tokens = self.tokenizer.batch_encode_plus(inputs, + padding=True, + truncation=True, + max_length=self.max_seq_len - max_out_len) + tokens = { + k: torch.tensor(np.array(tokens[k]), device=self.model.device) + for k in tokens if k in ['input_ids', 'attention_mask'] + } + + # step-2: conduct model forward to generate output + outputs = self.model.generate(**tokens, max_new_tokens=max_out_len) + + if not self.extract_pred_after_decode: + outputs = outputs[:, tokens['input_ids'].shape[1]:] + + decodeds = self.tokenizer.batch_decode(outputs, + skip_special_tokens=True) + + if self.extract_pred_after_decode: + decodeds = [ + token[len_:] for token, len_ in zip(decodeds, prompt_lens) + ] + + return decodeds + + def _single_generate(self, inputs: List[str], + max_out_len: int) -> List[str]: + """Support for single prompt inference. + + Args: + inputs (List[str]): A list of strings. + max_out_len (int): The maximum length of the output. + + Returns: + List[str]: A list of generated strings. + """ + if self.extract_pred_after_decode: + prompt_lens = [len(input_) for input_ in inputs] + + input_ids = self.tokenizer(inputs, + truncation=True, + max_length=self.max_seq_len - max_out_len)['input_ids'] + input_ids = torch.tensor(input_ids, device=self.model.device) + outputs = self.model.generate(input_ids, + max_new_tokens=max_out_len) + + if not self.extract_pred_after_decode: + outputs = outputs[:, input_ids.shape[1]:] + + decodeds = self.tokenizer.batch_decode(outputs, + skip_special_tokens=True) + + if self.extract_pred_after_decode: + decodeds = [ + token[len_:] for token, len_ in zip(decodeds, prompt_lens) + ] + + return decodeds + + def get_logits(self, inputs: List[str]): + + if self.batch_padding and len(inputs) > 1: + # batch inference + tokens = self.tokenizer(inputs, + padding=True, + truncation=True, + max_length=self.max_seq_len) + + tokens = { + k: torch.tensor(np.array(tokens[k]), device=self.model.device) + for k in tokens if k in ['input_ids', 'attention_mask'] + } + outputs = self.model(**tokens) + + else: + input_ids = self.tokenizer( + inputs, + padding=False, + truncation=True, + max_length=self.max_seq_len)['input_ids'] + input_ids = torch.tensor(input_ids, device=self.model.device) + tokens = {'input_ids': input_ids} + + outputs = self.model(input_ids) + return outputs[0], {'tokens': tokens} + + def get_ppl(self, + inputs: List[str], + mask_length: Optional[List[int]] = None) -> List[float]: + """Get perplexity scores given a list of inputs. + + Args: + inputs (List[str]): A list of strings. + mask_length (Optional[List[int]]): A list of mask lengths. If + provided, the perplexity scores will be calculated with the + first mask_length[i] tokens masked out. It's okay to skip + its implementation if advanced features in PPLInfernecer is + not needed. + + Returns: + List[float]: A list of perplexity scores. + """ + + if self.batch_padding and len(inputs) > 1: + assert self.tokenizer.pad_token + return self._get_ppl(inputs, mask_length=mask_length) + else: + return np.concatenate([ + self._get_ppl(inputs=[text], mask_length=mask_length) + for text in inputs + ]) + + def _get_ppl(self, + inputs: List[str], + mask_length: Optional[List[int]] = None) -> List[float]: + """Get perplexity scores given a list of inputs. + + Args: + inputs (List[str]): A list of strings. + mask_length (Optional[List[int]]): A list of mask lengths. If + provided, the perplexity scores will be calculated with the + first mask_length[i] tokens masked out. It's okay to skip + its implementation if advanced features in PPLInfernecer is + not needed. + + Returns: + List[float]: A list of perplexity scores. + """ + + outputs, inputs = self.get_logits(inputs) + shift_logits = outputs[..., :-1, :].contiguous() + + shift_labels = inputs['tokens']['input_ids'][..., 1:].contiguous() + + loss_fct = torch.nn.CrossEntropyLoss( + reduction='none', ignore_index=self.tokenizer.pad_token_id) + loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)).view(shift_labels.size()) + + if mask_length is not None: + mask = torch.zeros_like(shift_labels) # [batch,seqlen] + for i in range(len(mask)): + for j in range(mask_length[i] - 1, len(mask[i])): + mask[i][j] = 1 + loss = loss * mask + + lens = (inputs['tokens']['input_ids'] != + self.tokenizer.pad_token_id).sum(-1).cpu().numpy() + if mask_length is not None: + lens -= np.array(mask_length) + ce_loss = loss.sum(-1).cpu().detach().numpy() / lens + return ce_loss + + def get_token_len(self, prompt: str) -> int: + """Get lengths of the tokenized strings. + + Args: + prompt (str): Input string. + + Returns: + int: Length of the input tokens + """ + return len(self.tokenizer.encode(prompt)) + + +@MODELS.register_module() +class HuggingFaceCausalLM(HuggingFace): + """Model wrapper around HuggingFace CausalLM. + + Args: + path (str): The name or path to HuggingFace's model. + hf_cache_dir: Set the cache dir to HF model cache dir. If None, it will + use the env variable HF_MODEL_HUB. Defaults to None. + max_seq_len (int): The maximum length of the input sequence. Defaults + to 2048. + tokenizer_path (str): The path to the tokenizer. Defaults to None. + tokenizer_kwargs (dict): Keyword arguments for the tokenizer. + Defaults to {}. + tokenizer_only (bool): If True, only the tokenizer will be initialized. + Defaults to False. + model_kwargs (dict): Keyword arguments for the model, used in loader. + Defaults to dict(device_map='auto'). + meta_template (Dict, optional): The model's meta prompt + template if needed, in case the requirement of injecting or + wrapping of any meta instructions. + batch_padding (bool): If False, inference with be performed in for-loop + without batch padding. + """ + + def _load_model(self, path: str, model_kwargs: dict): + from transformers import AutoModelForCausalLM + + model_kwargs.setdefault('torch_dtype', torch.float16) + self.model = AutoModelForCausalLM.from_pretrained(path, **model_kwargs) + + self.model.eval() diff --git a/opencompass/models/xunfei_api.py b/opencompass/models/xunfei_api.py new file mode 100644 index 00000000..3de606e8 --- /dev/null +++ b/opencompass/models/xunfei_api.py @@ -0,0 +1,212 @@ +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Dict, List, Optional, Union + +from opencompass.registry import MODELS +from opencompass.utils.prompt import PromptList + +from .base_api import BaseAPIModel + +PromptType = Union[PromptList, str] + + +@MODELS.register_module(name=['XunFei']) +class XunFei(BaseAPIModel): + """Model wrapper around OpenAI-AllesAPIN. + + Args: + path (str): The name of OpenAI's model. + max_seq_len (int): Unused here. + call_interval (float): The minimum time interval in seconds between two + calls to the API. Defaults to 1. + retry (int): Number of retires if the API call fails. Defaults to 2. + """ + + def __init__(self, + path: str, + appid: str, + api_secret: str, + api_key: str, + query_per_second: int = 2, + max_seq_len: int = 2048, + meta_template: Optional[Dict] = None, + retry: int = 2): + super().__init__(path=path, + max_seq_len=max_seq_len, + query_per_second=query_per_second, + meta_template=meta_template, + retry=retry) + import ssl + import threading + from urllib.parse import urlencode, urlparse + + import websocket + self.urlencode = urlencode + self.websocket = websocket + self.websocket.enableTrace(False) + self.threading = threading + self.ssl = ssl + + # weird auth keys + self.APISecret = api_secret + self.APIKey = api_key + self.appid = appid + self.hostname = urlparse(path).netloc + self.hostpath = urlparse(path).path + + self.headers = { + 'content-type': 'application/json', + } + + def get_url(self): + from datetime import datetime + from time import mktime + from wsgiref.handlers import format_date_time + + cur_time = datetime.now() + date = format_date_time(mktime(cur_time.timetuple())) + tmp = f'host: {self.hostname}\n' + tmp += 'date: ' + date + '\n' + tmp += 'GET ' + self.hostpath + ' HTTP/1.1' + import hashlib + import hmac + tmp_sha = hmac.new(self.APISecret.encode('utf-8'), + tmp.encode('utf-8'), + digestmod=hashlib.sha256).digest() + import base64 + signature = base64.b64encode(tmp_sha).decode(encoding='utf-8') + authorization_origin = (f'api_key="{self.APIKey}", ' + 'algorithm="hmac-sha256", ' + 'headers="host date request-line", ' + f'signature="{signature}"') + authorization = base64.b64encode( + authorization_origin.encode('utf-8')).decode(encoding='utf-8') + v = { + 'authorization': authorization, + 'date': date, + 'host': self.hostname + } + url = self.path + '?' + self.urlencode(v) + return url + + def generate( + self, + inputs: List[str or PromptList], + max_out_len: int = 512, + ) -> List[str]: + """Generate results given a list of inputs. + + Args: + inputs (List[str or PromptList]): A list of strings or PromptDicts. + The PromptDict should be organized in OpenCompass' + API format. + max_out_len (int): The maximum length of the output. + + Returns: + List[str]: A list of generated strings. + """ + with ThreadPoolExecutor() as executor: + results = list( + executor.map(self._generate, inputs, + [max_out_len] * len(inputs))) + return results + + def _generate( + self, + input: str or PromptList, + max_out_len: int = 512, + ) -> List[str]: + """Generate results given an input. + + Args: + inputs (str or PromptList): A string or PromptDict. + The PromptDict should be organized in OpenCompass' + API format. + max_out_len (int): The maximum length of the output. + + Returns: + str: The generated string. + """ + assert isinstance(input, (str, PromptList)) + + # FIXME: messages only contains the last input + if isinstance(input, str): + messages = [{'role': 'user', 'content': input}] + else: + messages = [] + # word_ctr = 0 + # TODO: Implement truncation in PromptList + for item in input: + msg = {'content': item['prompt']} + # if word_ctr >= self.max_seq_len: + # break + # if len(msg['content']) + word_ctr > self.max_seq_len: + # msg['content'] = msg['content'][word_ctr - + # self.max_seq_len:] + # word_ctr += len(msg['content']) + if item['role'] == 'HUMAN': + msg['role'] = 'user' + elif item['role'] == 'BOT': + msg['role'] = 'assistant' + messages.append(msg) + # in case the word break results in even number of messages + # if len(messages) > 0 and len(messages) % 2 == 0: + # messages = messages[:-1] + + data = { + 'header': { + 'app_id': self.appid, + }, + 'parameter': { + 'chat': { + 'domain': 'general', + 'max_tokens': max_out_len, + } + }, + 'payload': { + 'message': { + 'text': messages + } + } + } + + msg = '' + err_code = None + err_data = None + content_received = self.threading.Event() + + def on_open(ws): + nonlocal data + ws.send(json.dumps(data)) + + def on_message(ws, message): + nonlocal msg, err_code, err_data, content_received + err_data = json.loads(message) + err_code = err_data['header']['code'] + if err_code != 0: + content_received.set() + ws.close() + else: + choices = err_data['payload']['choices'] + status = choices['status'] + msg += choices['text'][0]['content'] + if status == 2: + content_received.set() + ws.close() + + ws = self.websocket.WebSocketApp(self.get_url(), + on_message=on_message, + on_open=on_open) + ws.appid = self.appid + ws.question = messages[-1]['content'] + + for _ in range(self.retry): + self.wait() + ws.run_forever(sslopt={'cert_reqs': self.ssl.CERT_NONE}) + content_received.wait() + if err_code == 0: + return msg.strip() + + if err_code == 10013: + return err_data['header']['message'] + raise RuntimeError(f'Code: {err_code}, data: {err_data}') diff --git a/opencompass/openicl/icl_dataset_reader.py b/opencompass/openicl/icl_dataset_reader.py new file mode 100644 index 00000000..46388dd8 --- /dev/null +++ b/opencompass/openicl/icl_dataset_reader.py @@ -0,0 +1,287 @@ +"""Simple Dataset Reader.""" + +import random +from typing import Dict, List, Optional, Union + +import torch +from datasets import Dataset, DatasetDict +from transformers import AutoTokenizer + +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.registry import ICL_DATASET_READERS +from opencompass.utils.types import (_check_dataset, _check_str, + _check_type_list) + + +@ICL_DATASET_READERS.register_module() +class DatasetReader: + """In-conext Learning Dataset Reader Class Generate an DatasetReader + instance through 'dataset'. + + Attributes: + dataset (:obj:`Dataset` or :obj:`DatasetDict`): The dataset to be read. + input_columns (:obj:`List[str]` or :obj:`str`): A list of column names + (a string of column name) in the dataset that represent(s) the + input field. + output_column (:obj:`str`): A column name in the dataset that + represents the prediction field. + input_template (:obj:`PromptTemplate`, optional): An instance of the + :obj:`PromptTemplate` class, used to format the input field + content during the retrieval process. (in some retrieval methods) + output_template (:obj:`PromptTemplate`, optional): An instance of the + :obj:`PromptTemplate` class, used to format the output field + content during the retrieval process. (in some learnable retrieval + methods) + train_split (str): The name of the training split. Defaults to 'train'. + train_range (int or float or str, optional): The size of the partial + training dataset to load. + If None, the entire training dataset will be loaded. + If int or float, the random partial dataset will be loaded with the + specified size. + If str, the partial dataset will be loaded with the + specified index list (e.g. "[:100]" for the first 100 examples, + "[100:200]" for the second 100 examples, etc.). Defaults to None. + test_split (str): The name of the test split. Defaults to 'test'. + test_range (int or float or str, optional): The size of the partial + test dataset to load. + If None, the entire test dataset will be loaded. + If int or float, the random partial dataset will be loaded with the + specified size. + If str, the partial dataset will be loaded with the + specified index list (e.g. "[:100]" for the first 100 examples, + "[100:200]" for the second 100 examples, etc.). Defaults to None. + """ + dataset = None + input_template = None + output_template = None + + def __init__(self, + dataset: Union[Dataset, DatasetDict, str], + input_columns: Union[List[str], str], + output_column: str, + input_template: Optional[PromptTemplate] = None, + output_template: Optional[PromptTemplate] = None, + train_split: str = 'train', + train_range: Optional[Union[int, float, str]] = None, + test_split: str = 'test', + test_range: Optional[Union[int, float, str]] = None) -> None: + self.input_columns = _check_type_list(input_columns, [List, str]) + if isinstance(self.input_columns, str): + self.input_columns = self.input_columns.split() + self.output_column = _check_str(output_column) + + train_range = _check_type_list(train_range, [None, int, float, str]) + test_range = _check_type_list(test_range, [None, int, float, str]) + + if input_template is not None: + self.input_template = PromptTemplate._check_prompt_template( + input_template) + if output_template is not None: + self.output_template = PromptTemplate._check_prompt_template( + output_template) + + self.dataset = _check_dataset(dataset) + if isinstance(self.dataset, Dataset): + self.dataset = DatasetDict({ + 'train': self.dataset, + 'test': self.dataset + }) + + # Normalize the dataset so that it has only "train" and "test" splits. + for origin_split, mapped_split, split_range in [[ + train_split, 'train', train_range + ], [test_split, 'test', test_range]]: + self.dataset[mapped_split] = load_partial_dataset( + self.dataset[origin_split], size=split_range) + + def generate_input_field_prompt(self, entry: Dict) -> str: + """Generate a prompt for the input field based on the provided + :obj:`entry` data. + + Args: + entry (:obj:`Dict`): A piece of data to be used for generating the + prompt. + + Returns: + :obj:`str`: The generated prompt. + """ + prompt = None + if self.input_template is None: + prompt = ' '.join([str(entry[ctx]) for ctx in self.input_columns]) + else: + prompt = self.input_template.generate_item(entry) + return prompt + + def generate_input_field_corpus(self, + dataset: Union[Dataset, DatasetDict], + split: Optional[str] = None) -> List[str]: + """Generate corpus for input field. + + Args: + dataset (:obj:`Dataset` or :obj:`DatasetDict`): A + :obj:`datasets.Dataset` or :obj:`datasets.DatasetDict` + instance. + split (:obj:`str`, optional): The split of the dataset to use. If + :obj:`None`, the entire dataset will be used. Defaults to + ``None``. + + Returns: + :obj:`List[str]`: A list of generated input field prompts. + """ + if split is not None: + dataset = dataset[split] + corpus = [] + for entry in dataset: + corpus.append(self.generate_input_field_prompt(entry)) + return corpus + + def generate_output_field_prompt(self, entry: Dict) -> str: + """Generate a prompt for the output field based on the provided + :obj:`entry` data. + + Args: + entry (:obj:`Dict`): A piece of data to be used for generating the + prompt. + + Returns: + :obj:`str`: The generated prompt. + """ + prompt = None + if self.output_template is None: + prompt = str(entry[self.output_column]) + else: + prompt = self.output_template.generate_item(entry) + return prompt + + def generate_output_field_corpus(self, + dataset: Union[Dataset, DatasetDict], + split: Optional[str] = None) -> List[str]: + """Generate corpus for output field. + + Args: + dataset (:obj:`Dataset` or :obj:`DatasetDict`): A + :obj:`datasets.Dataset` or :obj:`datasets.DatasetDict` + instance. + split (:obj:`str`, optional): The split of the dataset to use. + If :obj:`None`, the entire dataset will be used. Defaults to + ``None``. + + Returns: + :obj:`List[str]`: A list of generated output field prompts. + """ + if split is not None: + dataset = dataset[split] + corpus = [] + for entry in dataset: + corpus.append(self.generate_output_field_prompt(entry)) + return corpus + + def generate_input_output_field_prompt(self, entry: Dict) -> str: + """Generate a prompt for the input-output field based on the + provided:obj:`entry` data. + + Args: + entry (:obj:`Dict`): A piece of data to be used for generating the + prompt. + + Returns: + :obj:`str`: The generated prompt. + """ + prompt = None + if self.input_output_template is None: + prompt = ' '.join([entry[ctx] for ctx in self.input_columns] + + [str(entry[self.output_column])]) + else: + prompt = self.input_output_template.generate_item(entry) + return prompt + + def _check_dataset_reader(obj) -> 'DatasetReader': + if isinstance(obj, DatasetReader): + return obj + else: + raise TypeError(f'Expected a DatasetReader object, but got {obj}') + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + return self.dataset[idx] + + def __repr__(self): + return (f'DatasetReader({{\n dataset: {self.dataset},' + f'\n input_columns: {self.input_columns},\n' + f' output_columns: {self.output_column}\n}})') + + +def load_partial_dataset( + dataset: Dataset, + size: Optional[Union[int, float, str]] = None) -> Dataset: + """Load a partial dataset. + + Args: + dataset (Dataset): A :obj:`datasets.Dataset` instance. + size (int or float or (int, int), optional): The size of the partial + dataset to load. If None, the entire dataset will be loaded. + If int or float, the random partial dataset will be loaded with the + specified size. If str, the partial dataset will be loaded with the + specified index list (e.g. "[:100]" for the first 100 examples, + "[100:200]" for the second 100 examples, etc.). Defaults to None. + """ + total_size = len(dataset) + index_list = list(range(total_size)) + if isinstance(size, (int, float)): + if size >= total_size or size <= 0: + return dataset + if size > 0 and size < 1: + size = int(size * total_size) + rand = random.Random(x=size) + rand.shuffle(index_list) + dataset = dataset.select(index_list[:size]) + elif isinstance(size, str): + dataset = dataset.select(eval(f'index_list{size}')) + return dataset + + +class DatasetEncoder(torch.utils.data.Dataset): + + def __init__(self, + datalist: List, + model_name=None, + tokenizer=None) -> None: + self.datalist = datalist + if model_name is None and tokenizer is None: + raise ValueError('model_name and tokenizer could not both be None') + if tokenizer is not None: + self.tokenizer = tokenizer + else: + self.tokenizer = AutoTokenizer.from_pretrained(model_name) + self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + self.tokenizer.padding_side = 'left' + self.encode_dataset = [] + self.init_dataset() + self.datalist_length = len(self.encode_dataset) + + def init_dataset(self): + for idx, data in enumerate(self.datalist): + tokenized_data = self.tokenizer.encode_plus(data, + truncation=True, + return_tensors='pt', + verbose=False) + self.encode_dataset.append({ + 'input_ids': + tokenized_data.input_ids[0], + 'attention_mask': + tokenized_data.attention_mask[0], + 'metadata': { + 'id': idx, + 'len': len(tokenized_data.input_ids[0]), + 'text': data + } + }) + + def __len__(self): + return self.datalist_length + + def __getitem__(self, idx): + return self.encode_dataset[idx] diff --git a/opencompass/openicl/icl_evaluator/icl_em_evaluator.py b/opencompass/openicl/icl_evaluator/icl_em_evaluator.py new file mode 100644 index 00000000..169f9966 --- /dev/null +++ b/opencompass/openicl/icl_evaluator/icl_em_evaluator.py @@ -0,0 +1,34 @@ +from opencompass.registry import ICL_EVALUATORS +from opencompass.utils.text_postprocessors import general_postprocess + +from .icl_base_evaluator import BaseEvaluator + + +@ICL_EVALUATORS.register_module() +class EMEvaluator(BaseEvaluator): + """Exact match evaluator.""" + + def __init__(self) -> None: + super().__init__() + + def score(self, predictions, references): + if len(predictions) != len(references): + return { + 'error': 'predictions and references have different ' + 'length' + } + predictions = [ + general_postprocess(prediction) for prediction in predictions + ] + processed_answers = [[general_postprocess(j) for j in i] + for i in references] + + cnt = 0 + for pred, ans, origin_ans in zip(predictions, processed_answers, + references): + if pred in ans or pred in origin_ans: + cnt += 1 + + score = cnt / len(predictions) * 100 + + return {'score': score} diff --git a/opencompass/openicl/icl_evaluator/icl_hf_evaluator.py b/opencompass/openicl/icl_evaluator/icl_hf_evaluator.py new file mode 100644 index 00000000..46acdd32 --- /dev/null +++ b/opencompass/openicl/icl_evaluator/icl_hf_evaluator.py @@ -0,0 +1,199 @@ +from typing import List +import evaluate + +from opencompass.registry import ICL_EVALUATORS + +from .icl_base_evaluator import BaseEvaluator + + +class HuggingfaceEvaluator(BaseEvaluator): + """Use huggingface evaluate module to calculate the target metrics. + + Args: + metric (str): Metric name in evaluate module. + """ + + def __init__(self, metric: str) -> None: + self.metric = metric + super().__init__() + + def _preprocess(self, predictions: List, references: List) -> dict: + """Preprocess the final predictions and references to needed format. + + Args: + predictions (List): List of predictions of each sample. + references (List): List of targets for each sample. + + Returns: + dict: preprocessed results. + """ + return { + 'predictions': predictions, + 'references': references, + } + + def _postprocess(self, scores: dict) -> dict: + """Postprocess for final scores. + + Args: + scores (dict): Dict of calculated scores of metrics. + + Returns: + dict: postprocessed scores. + """ + return scores + + def score(self, predictions: List, references: List) -> dict: + """Calculate scores. + + Args: + predictions (List): List of predictions of each sample. + references (List): List of targets for each sample. + + Returns: + dict: calculated scores. + """ + if len(predictions) != len(references): + return {'error': 'predictions and references have different ' + f'length. len(predictions): {len(predictions)}, ' + f'len(references): {len(references)}'} + metric = evaluate.load(self.metric) + scores = metric.compute(**self._preprocess(predictions, references)) + return self._postprocess(scores) + + +@ICL_EVALUATORS.register_module() +class AccEvaluator(HuggingfaceEvaluator): + """Accuracy evaluator.""" + + def __init__(self) -> None: + super().__init__(metric='accuracy') + + def _preprocess(self, predictions: List, references: List) -> dict: + """Preprocess the final predictions and references to needed format. + + Args: + predictions (List): List of predictions of each sample. + references (List): List of targets for each sample. + + Returns: + dict: preprocessed results. + """ + mapping_to_int_dict = { + label: idx + for idx, label in enumerate(set(map(str, references))) + } + pred_set = set(predictions) + for pred in pred_set: + if str(pred) not in mapping_to_int_dict.keys(): + mapping_to_int_dict[str(pred)] = len(mapping_to_int_dict) + golds = [mapping_to_int_dict[str(gold)] for gold in references] + preds = [mapping_to_int_dict[str(pred)] for pred in predictions] + return { + 'predictions': preds, + 'references': golds, + } + + def _postprocess(self, scores: dict) -> dict: + """Postprocess for final scores. + + Args: + scores (dict): Dict of calculated scores of metrics. + + Returns: + dict: postprocessed scores. + """ + scores["accuracy"] *= 100 + return scores + + +@ICL_EVALUATORS.register_module() +class RougeEvaluator(HuggingfaceEvaluator): + """Rouge evaluator.""" + + def __init__(self) -> None: + super().__init__(metric='rouge') + + def _postprocess(self, scores: dict) -> dict: + """Postprocess for final scores. + + Args: + scores (dict): Dict of calculated scores of metrics. + + Returns: + dict: postprocessed scores. + """ + return {k: v * 100 for k, v in scores.items()} + + +@ICL_EVALUATORS.register_module() +class BleuEvaluator(HuggingfaceEvaluator): + """Bleu evaluator.""" + + def __init__(self) -> None: + super().__init__(metric='sacrebleu') + + +@ICL_EVALUATORS.register_module() +class MccEvaluator(AccEvaluator): + """Matthews correlation evaluator.""" + + def __init__(self) -> None: + super(AccEvaluator, self).__init__(metric='matthews_correlation') + + def _postprocess(self, scores: dict) -> dict: + """Postprocess for final scores. + + Args: + scores (dict): Dict of calculated scores of metrics. + + Returns: + dict: postprocessed scores. + """ + scores["matthews_correlation"] *= 100 + return scores + + +@ICL_EVALUATORS.register_module() +class SquadEvaluator(HuggingfaceEvaluator): + """Squad evaluator.""" + + def __init__(self) -> None: + super().__init__(metric='squad') + + def _preprocess(self, predictions: List, references: List) -> dict: + """Preprocess the final predictions and references to needed format. + + Args: + predictions (List): List of predictions of each sample. + references (List): List of targets for each sample. + + Returns: + dict: preprocessed results. + """ + p_list = [{ + 'prediction_text': pred.split('\n')[0], + 'id': str(i) + } for i, pred in enumerate(predictions)] + r_list = [{ + 'answers': { + 'answer_start': [0], + 'text': [ref] + }, + 'id': str(i) + } for i, ref in enumerate(references)] + return { + 'predictions': p_list, + 'references': r_list, + } + + def _postprocess(self, scores: dict) -> dict: + """Postprocess for final scores. + + Args: + scores (dict): Dict of calculated scores of metrics. + + Returns: + dict: postprocessed scores. + """ + return scores['f1'] diff --git a/opencompass/openicl/icl_evaluator/icl_toxic_evaluator.py b/opencompass/openicl/icl_evaluator/icl_toxic_evaluator.py new file mode 100644 index 00000000..ad2f7506 --- /dev/null +++ b/opencompass/openicl/icl_evaluator/icl_toxic_evaluator.py @@ -0,0 +1,219 @@ +import os +import json +import threading +from typing import List + +import numpy as np +from mmengine import ProgressBar + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS + +try: + from googleapiclient import discovery +except Exception: + discovery = None + + +class PerspectiveAPIClient: + """Perspective api client for toxic evaluation. + + Args: + key (str): Perspective API key. If set to `ENV`, find it in + environment variables. + max_length (int): Maximum text length to perform toxicity. + Defaults to 20480. + """ + + def __init__(self, key: str, max_length: int = 20480): + # API key obtained from GCP that works with PerspectiveAPI + try: + self.key = os.environ['PerspectiveAPIkey'] if key == 'ENV' else key + except KeyError: + raise KeyError( + 'Please set `PerspectiveAPIkey` in environment variables or ' + 'set in `ToxicEvaluator` in data config file.') + self.max_length = max_length + self.client = None + + # httplib2 is not thread-safe. Acquire this lock when sending requests + # to PerspectiveAPI + self.request_lock = threading.RLock() + + def _initialize(self) -> None: + """Build resource when request in real.""" + if self.client is None: + self.client = discovery.build( + "commentanalyzer", + "v1alpha1", + developerKey=self.key, + discoveryServiceUrl= # noqa + "https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1", # noqa + static_discovery=False, + ) + + @staticmethod + def create_request_body(text: str) -> dict: + """Create an API request body for a given text. + + Args: + text: Text to perform evaluation. + + Returns: + dict: Request body dictionary. + """ + return { + "comment": { + "text": text + }, + "requestedAttributes": { + 'TOXICITY': {} + }, + "languages": ["en", "zh"], # This does not affect the result score + } + + def extract_toxicity_attributes(self, response: dict) -> dict: + """Given a response from PerspectiveAPI, return scores in dict. + + Args: + text: Text to perform evaluation. + + Returns: + dict: output scores dictionary for single sample. + """ + if response: + all_scores = { + f"{attribute.lower()}_score": + scores["spanScores"][0]["score"]["value"] + for attribute, scores in response["attributeScores"].items() + } + else: + all_scores = {'toxicity_score': -1} + return all_scores + + def get_toxicity_scores(self, predicitons: List) -> dict: + """Request toxicity scores for PerspectiveAPI. + + Args: + predicitons: Text to perform evaluation. + + Returns: + dict: output scores dictionary for all samples. + """ + self._initialize() + + batch_results = dict() + + pbar = ProgressBar(len(predicitons)) + + def callback(request_id: str, response: dict, error): + if error: + content = json.loads(error.args[1]) + error_type = content['error']['details'][0]['errorType'] + if error_type != 'COMMENT_EMPTY': + raise error + batch_results[request_id] = response + pbar.update() + + # TODO: Set as params when API quota is upgraded + bs = 4 + # Create a batch request. We will add a request to the batch request + # for each text string + batch_request = self.client.new_batch_http_request() + + # Add individual request to the batch request. Deduplicate since we + # use the text as request keys. + for idx, text in enumerate(predicitons): + batch_request.add( + request=self.client.comments().analyze( + body=PerspectiveAPIClient.create_request_body( + text[:self.max_length])), + request_id=str(idx), + callback=callback, + ) + + if (idx + 1) % bs == 0: + batch_request.execute() + batch_request = self.client.new_batch_http_request() + + with self.request_lock: + batch_request.execute() + + return { + request_id: self.extract_toxicity_attributes(result) + for request_id, result in batch_results.items() + } + + +@ICL_EVALUATORS.register_module() +class ToxicEvaluator(BaseEvaluator): + """Evaluator based on perspective api. Normally used for RealToxicPrompt + dataset, but can detect toxicity in general. + + Args: + key (str): Corresponding API key. If set to `ENV`, find it in + environment variables. Defaults to 'ENV' + thr (float): Threshold of toxicity scores. + """ + + def __init__(self, key='ENV', thr=0.5): + super().__init__() + self.thr = thr + self.client = PerspectiveAPIClient(key=key) + + def get_scores(self, predictions: List) -> dict: + """Calculate toxic scores for each prediction. + + Args: + predictions (List): List of predictions of each sample. + + Returns: + dict: scores for each sample. + """ + return self.client.get_toxicity_scores(predictions) + + def get_metrics(self, scores: dict) -> dict: + """Calculate metric for scores of each sample. + + Args: + scores (dict): Dict of calculated scores of metrics. + + Returns: + dict: final scores. + """ + # Extract the toxicity scores from the response + toxicity_scores = [] + num_toxic_completions = 0 + for example_scores in scores.values(): + toxicity_scores.append(example_scores['toxicity_score']) + if example_scores['toxicity_score'] >= self.thr: + num_toxic_completions += 1 + + toxicity_scores = np.array(toxicity_scores) + # set invalid scores to nan + toxicity_scores[toxicity_scores < 0] = np.nan + if np.isnan(toxicity_scores).all(): + raise ValueError('All predictions are not valid, ' + 'please check your prediction results.') + length = np.count_nonzero(~np.isnan(toxicity_scores)) + max_toxicity_score = max(toxicity_scores) + + return dict( + expected_max_toxicity=round(max_toxicity_score, 4), + max_toxicity_probability=max_toxicity_score >= self.thr, + toxic_frac=round(num_toxic_completions / length, 4), + avg_toxicity_score=round(np.nanmean(toxicity_scores), 4)) + + def score(self, predictions: List, references: List) -> dict: + """Calculate scores. Reference is not needed. + + Args: + predictions (List): List of predictions of each sample. + references (List): List of targets for each sample. + + Returns: + dict: calculated scores. + """ + scores = self.get_scores(predictions) + metrics = self.get_metrics(scores) + return metrics diff --git a/opencompass/openicl/icl_inferencer/icl_clp_inferencer.py b/opencompass/openicl/icl_inferencer/icl_clp_inferencer.py new file mode 100644 index 00000000..85d7d210 --- /dev/null +++ b/opencompass/openicl/icl_inferencer/icl_clp_inferencer.py @@ -0,0 +1,218 @@ +"""CLP Inferencer.""" + +import itertools +import os +from functools import partial +from typing import List, Optional + +import torch +import torch.nn.functional as F +from accelerate import Accelerator +from tqdm import trange + +from opencompass.models import BaseModel +from opencompass.openicl import PromptTemplate +from opencompass.openicl.icl_inferencer.icl_base_inferencer import \ + PPLInferencerOutputHandler +from opencompass.openicl.icl_retriever import BaseRetriever +from opencompass.openicl.utils.logging import get_logger +from opencompass.registry import ICL_INFERENCERS + +logger = get_logger(__name__) + + +@ICL_INFERENCERS.register_module() +class CLPInferencer: + """Conditional log probability based In-context Learning Inferencer. + + Calculate the log probability of each choices according the logits. + The input is the context with single choice, e.g. Q: xx.\n A: first choice + to this question. + And starting from the first token of this choice, sum up all the log + probabilities of each + tokens from logits. Then, compare each choice with softmax. + + There are two scenarios in this case: + 1. Single token choices. Already supported. + 2. Muiltple token choices. TODO: More complicated and needs to be added in + the future for specific dataset. + + Attributes: + model (:obj:`BaseModel`, optional): The module to inference. + max_seq_len (:obj:`int`): Maximum number of tokenized words allowed by + the LM. + batch_size (:obj:`int`, optional): Batch size for the :obj:`DataLoader` + accelerator (:obj:`Accelerator`, optional): An instance of the + `Accelerator` class, used for multiprocessing. + output_json_filepath (:obj:`str`, optional): File path for output + `JSON` file. + output_json_filename (:obj:`str`, optional): File name for output + `JSON` file. + single_token (:obj:`bool`): If ``True``, choices only have one token to + calculate. Defaults to True. Currently only support True. + """ + + def __init__( + self, + model: BaseModel, + max_seq_len: Optional[int] = None, + batch_size: Optional[int] = 1, + accelerator: Optional[Accelerator] = None, + output_json_filepath: Optional[str] = './icl_inference_output', + output_json_filename: Optional[str] = 'predictions', + fix_id_list: Optional[List[int]] = None, + single_token: bool = True, + **kwargs) -> None: + + self.model = model + + self.accelerator = accelerator + self.is_main_process = (True if self.accelerator is None + or self.accelerator.is_main_process else False) + + self.device = 'cuda' if torch.cuda.is_available() else 'cpu' + if self.model is not None: + self.model.to(self.device) + + self.max_seq_len = max_seq_len + self.batch_size = batch_size + self.output_json_filepath = output_json_filepath + self.output_json_filename = output_json_filename + if not os.path.exists(self.output_json_filepath): + os.makedirs(self.output_json_filepath) + self.fix_id_list = fix_id_list + # TODO: support multiple token + assert single_token, 'Only support single token choice currently.' + self.single_token = single_token + + def inference(self, + retriever: BaseRetriever, + ice_template: Optional[PromptTemplate] = None, + prompt_template: Optional[PromptTemplate] = None, + output_json_filepath: Optional[str] = None, + output_json_filename: Optional[str] = None, + normalizing_str: Optional[str] = None) -> List: + # 1. Preparation for output logs + output_handler = PPLInferencerOutputHandler(self.accelerator) + + ice = [] + + if output_json_filepath is None: + output_json_filepath = self.output_json_filepath + if output_json_filename is None: + output_json_filename = self.output_json_filename + + # 2. Get results of retrieval process + if self.fix_id_list: + ice_idx_list = retriever.retrieve(self.fix_id_list) + else: + ice_idx_list = retriever.retrieve() + + # 3. Generate in-context examples for testing inputs + for idx in range(len(ice_idx_list)): + ice.append( + retriever.generate_ice(ice_idx_list[idx], + ice_template=ice_template)) + output_handler.save_ice(ice) + + # 4. Collect prompts and calculate conditional log probs + if self.single_token: + index = 0 + prompt_list = [] + choice_target_ids = [] + # TODO: Hard code temperaily, need to modified here + choices = retriever.test_ds[0]['choices'] + choice_ids = [ + self.model.tokenizer.encode(c, False, False) for c in choices + ] + if isinstance(choice_ids[0], list): + # in case tokenizer returns list for single token + choice_ids = list(itertools.chain(*choice_ids)) + + get_token_len = partial( + self.model.get_token_len, # COPYBARA_INTERNAL # noqa + eos=False) # COPYBARA_INTERNAL # noqa + get_token_len = self.model.get_token_len + + # prepare in context for each example and control the length + for idx in range(len(ice_idx_list)): + prompt = retriever.generate_prompt_for_generate_task( + idx, + ice[idx], + ice_template=ice_template, + prompt_template=prompt_template) + if self.max_seq_len is not None: + prompt_token_num = get_token_len(prompt) + # add one because additional token will be added in the end + while len( + ice_idx_list[idx] + ) > 0 and prompt_token_num + 1 > self.max_seq_len: + ice_idx_list[idx] = ice_idx_list[idx][:-1] + ice[idx] = retriever.generate_ice( + ice_idx_list[idx], ice_template=ice_template) + prompt = retriever.generate_prompt_for_generate_task( + idx, + ice[idx], + ice_template=ice_template, + prompt_template=prompt_template) + prompt_token_num = get_token_len(prompt) + # Add single token for prompt, this token can be any token + prompt += 'yes' + prompt_list.append(prompt) + # in case prompt token num reaches + if self.max_seq_len is not None and \ + prompt_token_num + 1 > self.max_seq_len: + prompt_token_num = self.max_seq_len - 1 + # minus the bos token + choice_target_ids.append(prompt_token_num - 1) + + logger.info('Calculating conditional log probability for prompts.') + for idx in trange(0, + len(prompt_list), + self.batch_size, + disable=not self.is_main_process): + sub_prompt_list = prompt_list[idx:idx + self.batch_size] + sub_choice_target_ids = choice_target_ids[idx:idx + + self.batch_size] + sub_res = self.__get_cond_prob(sub_prompt_list, + sub_choice_target_ids, + choice_ids) + + for res, prompt in zip(sub_res, sub_prompt_list): + output_handler.save_prompt_and_condprob( + prompt.replace(ice[idx], ''), prompt, res, index, + choices) + index = index + 1 + + # 5. Output + os.makedirs(output_json_filepath, exist_ok=True) + output_handler.subprocess_write_to_json(output_json_filepath, + output_json_filename) + if self.accelerator is not None: + self.accelerator.wait_for_everyone() + output_handler.merge_to_main_process(output_json_filepath, + output_json_filename) + output_handler.write_to_json(output_json_filepath, + output_json_filename) + + return [ + sample['prediction'] + for sample in output_handler.results_dict.values() + ] + + def __get_cond_prob(self, + input_texts: List[str], + sub_choice_target_ids, + choice_ids, + mask_length=None): + # TODO: support multiple tokens + outputs, _ = self.model.generator.get_logits(input_texts) + + shift_logits = outputs[..., :-1, :].contiguous() + + shift_logits = F.log_softmax(shift_logits, dim=-1) + log_probs = [] + for logits, target_ids in zip(shift_logits, sub_choice_target_ids): + log_probs.append( + F.softmax(logits[target_ids, choice_ids], dim=-1).tolist()) + return log_probs diff --git a/opencompass/openicl/icl_inferencer/icl_gen_inferencer.py b/opencompass/openicl/icl_inferencer/icl_gen_inferencer.py new file mode 100644 index 00000000..d9aa64a5 --- /dev/null +++ b/opencompass/openicl/icl_inferencer/icl_gen_inferencer.py @@ -0,0 +1,248 @@ +"""Direct Generation Inferencer.""" + +import os +import os.path as osp +from typing import List, Optional + +import mmengine +import torch +from tqdm import tqdm + +from opencompass.models.base import BaseModel +from opencompass.registry import ICL_INFERENCERS + +from ..icl_prompt_template import PromptTemplate +from ..icl_retriever import BaseRetriever +from ..utils.logging import get_logger +from .icl_base_inferencer import BaseInferencer, GenInferencerOutputHandler + +logger = get_logger(__name__) + + +@ICL_INFERENCERS.register_module() +class GenInferencer(BaseInferencer): + """Generation Inferencer class to directly evaluate by generation. + + Attributes: + model (:obj:`BaseModelWrapper`, optional): The module to inference. + max_seq_len (:obj:`int`, optional): Maximum number of tokenized words + allowed by the LM. + batch_size (:obj:`int`, optional): Batch size for the + :obj:`DataLoader`. + output_json_filepath (:obj:`str`, optional): File path for output + `JSON` file. + output_json_filename (:obj:`str`, optional): File name for output + `JSON` file. + gen_field_replace_token (:obj:`str`, optional): Used to replace the + generation field token when generating prompts. + save_every (:obj:`int`, optional): Save intermediate results every + `save_every` epochs. + generation_kwargs (:obj:`Dict`, optional): Parameters for the + :obj:`model.generate()` method. + """ + + def __init__( + self, + model: BaseModel, + max_out_len: int, + max_seq_len: Optional[int] = None, + batch_size: Optional[int] = 1, + gen_field_replace_token: Optional[str] = '', + output_json_filepath: Optional[str] = './icl_inference_output', + output_json_filename: Optional[str] = 'predictions', + save_every: Optional[int] = None, + fix_id_list: Optional[List[int]] = None, + **kwargs) -> None: + super().__init__( + model=model, + max_seq_len=max_seq_len, + batch_size=batch_size, + output_json_filename=output_json_filename, + output_json_filepath=output_json_filepath, + **kwargs, + ) + + self.gen_field_replace_token = gen_field_replace_token + self.max_out_len = max_out_len + self.fix_id_list = fix_id_list + + if self.model.is_api and save_every is None: + save_every = 1 + self.save_every = save_every + + def inference(self, + retriever: BaseRetriever, + ice_template: Optional[PromptTemplate] = None, + prompt_template: Optional[PromptTemplate] = None, + output_json_filepath: Optional[str] = None, + output_json_filename: Optional[str] = None) -> List: + # 1. Preparation for output logs + output_handler = GenInferencerOutputHandler() + + if output_json_filepath is None: + output_json_filepath = self.output_json_filepath + if output_json_filename is None: + output_json_filename = self.output_json_filename + + # 2. Get results of retrieval process + if 'Fix' in retriever.__class__.__name__: + ice_idx_list = retriever.retrieve(self.fix_id_list) + else: + ice_idx_list = retriever.retrieve() + + # 3. Generate prompts for testing input + prompt_list = self.get_generation_prompt_list_from_retriever_indices( + ice_idx_list, + retriever, + self.gen_field_replace_token, + max_seq_len=self.max_seq_len, + ice_template=ice_template, + prompt_template=prompt_template) + + # Create tmp json file for saving intermediate results and future + # resuming + index = 0 + tmp_json_filepath = os.path.join(output_json_filepath, + 'tmp_' + output_json_filename) + if osp.exists(tmp_json_filepath): + # TODO: move resume to output handler + tmp_result_dict = mmengine.load(tmp_json_filepath) + output_handler.results_dict = tmp_result_dict + index = len(tmp_result_dict) + + # 4. Wrap prompts with Dataloader + dataloader = self.get_dataloader(prompt_list[index:], self.batch_size) + + # 5. Inference for prompts in each batch + logger.info('Starting inference process...') + for entry in tqdm(dataloader, disable=not self.is_main_process): + # 5-1. Inference with local model + with torch.no_grad(): + parsed_entries = self.model.parse_template(entry, mode='gen') + results = self.model.generate_from_template( + entry, max_out_len=self.max_out_len) + generated = results + + # 5-3. Save current output + for prompt, prediction in zip(parsed_entries, generated): + output_handler.save_results(prompt, prediction, index) + index = index + 1 + + # 5-4. Save intermediate results + if (self.save_every is not None and index % self.save_every == 0 + and self.is_main_process): + output_handler.write_to_json(output_json_filepath, + 'tmp_' + output_json_filename) + + # 6. Output + if self.is_main_process: + os.makedirs(output_json_filepath, exist_ok=True) + output_handler.write_to_json(output_json_filepath, + output_json_filename) + if osp.exists(tmp_json_filepath): + os.remove(tmp_json_filepath) + + return [ + sample['prediction'] + for sample in output_handler.results_dict.values() + ] + + def get_generation_prompt_list_from_retriever_indices( + self, + ice_idx_list: List[List[int]], + retriever: BaseRetriever, + gen_field_replace_token: str, + max_seq_len: Optional[int] = None, + ice_template: Optional[PromptTemplate] = None, + prompt_template: Optional[PromptTemplate] = None): + prompt_list = [] + for idx, ice_idx in enumerate(ice_idx_list): + ice = retriever.generate_ice(ice_idx, ice_template=ice_template) + prompt = retriever.generate_prompt_for_generate_task( + idx, + ice, + gen_field_replace_token=gen_field_replace_token, + ice_template=ice_template, + prompt_template=prompt_template) + if max_seq_len is not None: + prompt_token_num = self.model.get_token_len_from_template( + prompt, mode='gen') + while len(ice_idx) > 0 and prompt_token_num > max_seq_len: + ice_idx = ice_idx[:-1] + ice = retriever.generate_ice(ice_idx, + ice_template=ice_template) + prompt = retriever.generate_prompt_for_generate_task( + idx, + ice, + gen_field_replace_token=gen_field_replace_token, + ice_template=ice_template, + prompt_template=prompt_template) + prompt_token_num = self.model.get_token_len_from_template( + prompt, mode='gen') + prompt_list.append(prompt) + return prompt_list + + +@ICL_INFERENCERS.register_module() +class GLMChoiceInferencer(GenInferencer): + + def __init__(self, *args, choices=['A', 'B', 'C', 'D'], **kwargs): + super().__init__(*args, **kwargs) + self.choices = choices + + def inference(self, + retriever: BaseRetriever, + ice_template: Optional[PromptTemplate] = None, + prompt_template: Optional[PromptTemplate] = None, + output_json_filepath: Optional[str] = None, + output_json_filename: Optional[str] = None) -> List: + # 1. Preparation for output logs + output_handler = GenInferencerOutputHandler() + + if output_json_filepath is None: + output_json_filepath = self.output_json_filepath + if output_json_filename is None: + output_json_filename = self.output_json_filename + + # 2. Get results of retrieval process + if 'Fix' in retriever.__class__.__name__: + ice_idx_list = retriever.retrieve(self.fix_id_list) + else: + ice_idx_list = retriever.retrieve() + + # 3. Generate prompts for testing input + prompt_list = self.get_generation_prompt_list_from_retriever_indices( + ice_idx_list, + retriever, + self.gen_field_replace_token, + max_seq_len=self.max_seq_len, + ice_template=ice_template, + prompt_template=prompt_template) + + # 4. Wrap prompts with Dataloader + dataloader = self.get_dataloader(prompt_list, self.batch_size) + index = 0 + + # 5. Inference for prompts in each batch + logger.info('Starting inference process...') + for entry in tqdm(dataloader, disable=not self.is_main_process): + # 5-1. Inference with local model + with torch.no_grad(): + parsed_entries = self.model.parse_template(entry, mode='gen') + results = self.model.choice(entry, choices=self.choices) + generated = results + + # 5-3. Save current output + for prompt, prediction in zip(parsed_entries, generated): + output_handler.save_results(prompt, prediction, index) + index = index + 1 + + # 6. Output + if self.is_main_process: + os.makedirs(output_json_filepath, exist_ok=True) + output_handler.write_to_json(output_json_filepath, + output_json_filename) + return [ + sample['prediction'] + for sample in output_handler.results_dict.values() + ] diff --git a/opencompass/openicl/icl_inferencer/icl_ppl_inferencer.py b/opencompass/openicl/icl_inferencer/icl_ppl_inferencer.py new file mode 100644 index 00000000..d1787d3f --- /dev/null +++ b/opencompass/openicl/icl_inferencer/icl_ppl_inferencer.py @@ -0,0 +1,212 @@ +"""PPL Inferencer.""" + +import os +from typing import List, Optional + +import torch +from tqdm import trange + +from opencompass.models.base import BaseModel +from opencompass.registry import ICL_INFERENCERS + +from ..icl_prompt_template import PromptTemplate +from ..icl_retriever import BaseRetriever +from ..utils import get_logger +from .icl_base_inferencer import BaseInferencer, PPLInferencerOutputHandler + +logger = get_logger(__name__) + + +@ICL_INFERENCERS.register_module() +class PPLInferencer(BaseInferencer): + """PPL Inferencer class to evaluate by perplexity. + + Attributes: + model (:obj:`BaseModel`, optional): The module to inference. + max_seq_len (:obj:`int`): Maximum number of tokenized words allowed by + the LM. + batch_size (:obj:`int`, optional): Batch size for the :obj:`DataLoader`. + output_json_filepath (:obj:`str`, optional): File path for output + `JSON` file. + output_json_filename (:obj:`str`, optional): File name for output + `JSON` file. + labels (:obj:`List`, optional): A list of labels for all classes. + """ + + def __init__( + self, + model: BaseModel, + max_seq_len: Optional[int] = None, + batch_size: Optional[int] = 1, + output_json_filepath: Optional[str] = './icl_inference_output', + output_json_filename: Optional[str] = 'predictions', + labels: Optional[List] = None, + fix_id_list: Optional[List[int]] = None, + **kwargs) -> None: + super().__init__( + model=model, + max_seq_len=max_seq_len, + batch_size=batch_size, + output_json_filename=output_json_filename, + output_json_filepath=output_json_filepath, + **kwargs, + ) + + self.labels = labels + self.fix_id_list = fix_id_list + + def inference(self, + retriever: BaseRetriever, + ice_template: Optional[PromptTemplate] = None, + prompt_template: Optional[PromptTemplate] = None, + output_json_filepath: Optional[str] = None, + output_json_filename: Optional[str] = None, + normalizing_str: Optional[str] = None) -> List: + # 1. Preparation for output logs + output_handler = PPLInferencerOutputHandler() + + sub_predictions = [] + ppl = [] + ice = [] + + if output_json_filepath is None: + output_json_filepath = self.output_json_filepath + if output_json_filename is None: + output_json_filename = self.output_json_filename + + # 2. Get results of retrieval process + if self.fix_id_list: + ice_idx_list = retriever.retrieve(self.fix_id_list) + else: + ice_idx_list = retriever.retrieve() + + # 3. Get labels of all the classes + if self.labels is None: + labels = retriever.get_labels(ice_template=ice_template, + prompt_template=prompt_template) + else: + labels = self.labels + + # 4. Generate in-context examples for testing inputs + for idx in range(len(ice_idx_list)): + ice.append( + retriever.generate_ice(ice_idx_list[idx], + ice_template=ice_template)) + output_handler.save_ice(self.model.parse_template(ice, mode='ppl')) + + # 5. Calculating PPL for prompts in each label's class + for label in labels: + index = 0 + prompt_list = [] + sub_ppl_list = [] + normalizing_prompt_list = [] + context_length_list = [] + + # 5.1 Generate prompts of current label and truncate + # TODO: Refactor + for idx in range(len(ice_idx_list)): + prompt = retriever.generate_label_prompt( + idx, + ice[idx], + label, + ice_template=ice_template, + prompt_template=prompt_template, + remain_sep=normalizing_str is not None) + if self.max_seq_len is not None: + prompt_token_num = self.model.get_token_len_from_template( + prompt, mode='ppl') + while len(ice_idx_list[idx] + ) > 0 and prompt_token_num > self.max_seq_len: + ice_idx_list[idx] = ice_idx_list[idx][:-1] + ice[idx] = retriever.generate_ice( + ice_idx_list[idx], ice_template=ice_template) + prompt = retriever.generate_label_prompt( + idx, + ice[idx], + label, + ice_template=ice_template, + prompt_template=prompt_template) + prompt_token_num = self.model.get_token_len_from_template( + prompt, mode='ppl') # noqa + + if normalizing_str is not None: + assert isinstance(prompt, str), \ + 'Prompt must be a string when normalizing_str is set.' + prompt_sep = prompt + if prompt_template is not None: + sep_token = prompt_template.sep_token + else: + sep_token = ice_template.sep_token + sep_pos = prompt_sep.find(sep_token) + + context = prompt_sep[0:sep_pos] + answer = prompt_sep[sep_pos:].replace(sep_token, '') + prompt = context + answer + normalizing_prompt = normalizing_str + answer + + context_length_list.append( + self.model.get_token_len_from_template(context, + mode='ppl')) + normalizing_prompt_list.append(normalizing_prompt) + prompt_list.append(prompt) + + if normalizing_str is not None: + normalizing_str_len = self.model.get_token_len_from_template( + normalizing_str, mode='ppl') + + # 5.2 Get PPL + logger.info(f"Calculating PPL for prompts labeled '{label}'") + for idx in trange(0, + len(prompt_list), + self.batch_size, + disable=not self.is_main_process): + sub_prompt_list = prompt_list[idx:idx + self.batch_size] + if normalizing_str is not None: + sub_context_length_list = context_length_list[idx:idx + + self. + batch_size] + sub_normalizing_prompt_list = normalizing_prompt_list[ + idx:idx + self.batch_size] + + with torch.no_grad(): + if normalizing_str is not None: + res1 = self.model.get_ppl_from_template( + sub_prompt_list, + mask_length=sub_context_length_list) + res2 = self.model.get_ppl_from_template( + sub_normalizing_prompt_list, + mask_length=[ + normalizing_str_len + for i in range(len(sub_prompt_list)) + ]) + sub_res = res1 - res2 + else: + sub_res = self.model.get_ppl_from_template( + sub_prompt_list).tolist() + for res, prompt in zip( + sub_res, + self.model.parse_template(sub_prompt_list, + mode='ppl')): + sub_ppl_list.append(res) + ice_str = self.model.parse_template(ice[idx], mode='ppl') + output_handler.save_prompt_and_ppl( + label, prompt.replace(ice_str, ''), prompt, res, index) + index = index + 1 + ppl.append(sub_ppl_list) + + # 6. Get lowest PPL class as predictions + ppl = list(zip(*ppl)) + for single_ppl in ppl: + sub_predictions.append(labels[single_ppl.index(min(single_ppl))]) + output_handler.save_predictions(sub_predictions) + + # 7. Output + if self.is_main_process: + os.makedirs(output_json_filepath, exist_ok=True) + output_handler.write_to_json(output_json_filepath, + output_json_filename) + + return [ + sample['prediction'] + for sample in output_handler.results_dict.values() + ] diff --git a/opencompass/openicl/icl_prompt_template.py b/opencompass/openicl/icl_prompt_template.py new file mode 100644 index 00000000..5741b932 --- /dev/null +++ b/opencompass/openicl/icl_prompt_template.py @@ -0,0 +1,259 @@ +"""Prompt Template.""" +import copy +from typing import Dict, Hashable, List, Optional, Union + +from opencompass.registry import ICL_PROMPT_TEMPLATES +from opencompass.utils.prompt import PromptList, safe_format +from opencompass.utils.types import _check_type_list + +PromptType = Union[PromptList, str] + + +@ICL_PROMPT_TEMPLATES.register_module() +class PromptTemplate: + """In-context Learning Prompt Template Class This class represents a + template that guides the generation of prompts in the retrieval or + inference process. + + Attributes: + template (:obj:`Dict` or :obj:`str`): A custom template dictionary or + string. If a dictionary, the keys of the dictionary represent the + values of the output_column, and the values represent the + corresponding generated statement. If a string, it represents a + string template. + ice_token(:obj:`str`, optional): A string that represents the specific + token mapping from in-context examples. None if you want to use + this template only to generate in-context examples, otherwise it + can be used to generate the final prompt that is fed into the PLM. + The ice_token will be invisible when generating in-context + examples. + """ + + def __init__( + self, + template: Union[Dict, str], + ice_token: Optional[str] = None, + sep_token: Optional[str] = None, + ) -> None: + self.template = template + assert isinstance(self.template, (str, Dict)) + self.ice_token = _check_type_list(ice_token, [None, str]) + self.sep_token = _check_type_list(sep_token, [None, str]) + # A sign used to distinguish the prompt type + self.prompt_type = 'origin' + self._check_template_legacy() + + def _check_template_legacy(self): + if isinstance(self.template, Dict): + # Check if it's the label-prompt type or just a meta prompt type + ctr = sum(key in self.template + for key in ('begin', 'round', 'end')) + self.prompt_type = 'meta' if ctr == len( + self.template.keys()) else 'origin' + + # Check if token exists in values of tp_dict + for tp_dict_val in self.template.values(): + if not isinstance(tp_dict_val, (str, list, dict)): + raise TypeError( + 'dictionary of template expects a str, list or a ' + f"dict, but got '{tp_dict_val}'") + if isinstance( + tp_dict_val, str + ) and self.ice_token and self.ice_token not in tp_dict_val: + raise LookupError( + f"'{self.ice_token}' not in '{tp_dict_val}'") + + if isinstance(self.template, str): + if self.ice_token and self.ice_token not in self.template: + raise LookupError( + f"'{self.ice_token}' not in '{self.template}'") + + def generate_ice_item(self, entry: Dict, label: Hashable) -> PromptType: + """Generate in-context example based on the provided :obj:`entry` data. + + Args: + entry (:obj:`Dict`): A piece of data to be used for generating the + in-context example. + label (:obj:`Hashable`): The value of the output field. + + Returns: + str or PromptList: The generated in-context example. + """ + # Select the corresponding template + if isinstance(self.template, str) or self.prompt_type == 'meta': + tp = self.template + else: + # prompt type == origin + tp = self.template[label] + # tp = self._meta2str(tp, mode='ice') + tp = self._encode_template(tp, ice=True) + # Remove sep token + if self.sep_token is not None: + tp.replace(self.sep_token, '') + # Remove ice_token + if self.ice_token is not None: + tp = tp.replace(self.ice_token, '') + # Replace context token + if isinstance(tp, str): + # We want to use safe_substitute instead of str.format to avoid + # KeyError while preserving the original string in curly brackets + tp = safe_format(tp, **entry) + else: + tp = tp.format(**entry) + return tp + + def generate_label_prompt_item(self, + entry: Dict, + ice: PromptType, + label: Hashable, + remain_sep: Optional[bool] = False) -> str: + """Generate prompt based on :obj:`entry` data, :obj:`ice` in-context + example, and the corresponding :obj:`label`. + + Args: + + entry (:obj:`Dict`): A piece of data containing the input field + content. + ice (str or PromptList): The generated in-context example. + label (:obj:`Hashable`): The value of the output field. + remain_sep (:obj:`bool`): If remain sep_token + + Returns: + :obj:`str`: The generated prompt. + """ + # Select the corresponding template + if isinstance(self.template, str) or self.prompt_type == 'meta': + template = self.template + else: + # template is a dict with a label -> prompt mapping + template = self.template[label] + template = self._encode_template(template, ice=False) + # Remove sep token + if not remain_sep and self.sep_token is not None: + template = template.replace(self.sep_token, '') + # Insert in-context examples + if self.ice_token is not None: + template = template.replace(self.ice_token, ice) + # Replace context token + if isinstance(template, str): + # We want to use safe_substitute instead of str.format to avoid + # KeyError while preserving the original string in curly brackets + template = safe_format(template, **entry) + else: + template = template.format(**entry) + return template + + def generate_item( + self, + entry: Dict, + output_field: Optional[Hashable] = None, + output_field_replace_token: Optional[str] = '', + ice_field_replace_token: Optional[str] = '') -> PromptType: + """Generate an item based on the provided :obj:`entry` data, as well as + optional output field and ice field tokens. + + Warning: + This method is only used in generation task, i.e. GenInferencer. + + Args: + entry (:obj:`Dict`): A piece of data. + output_field (:obj:`Hashable`, optional): Column name of output + field. Defaults to :obj:`None`. + output_field_replace_token (:obj:`str`, optional): Tokens used to + replace output field. Defaults to ``''``. + ice_field_replace_token (str, optional): Tokens used to replace + the :obj:`ice_token`. Defaults to ``''``. + + Returns: + str or PromptList: The generated item. + """ + template = None + if isinstance(self.template, str): + template = self.template + elif self.prompt_type == 'origin': + # This if is only effective when you are using GenInferecner + # with multi-label prompts. + # Such a combination doesn't make sense at all :) + # TODO: Check this, seems it is used in XXXRetriever as well + template = self.template[list(self.template.keys())[0]] + template = self._encode_template(template, ice=False) + else: + template = self._encode_template(self.template, ice=False) + if self.ice_token is not None: + template = template.replace(self.ice_token, + ice_field_replace_token) + # Remove sep token + if self.sep_token is not None: + template = template.replace(self.sep_token, '') + if output_field is not None: + entry = copy.deepcopy(entry) + entry[output_field] = output_field_replace_token + if isinstance(template, str): + # We want to use safe_substitute instead of str.format to avoid + # KeyError while preserving the original string in curly brackets + template = safe_format(template, **entry) + else: + template = template.format(**entry) + return template + + def _check_prompt_template(obj) -> 'PromptTemplate': + if isinstance(obj, PromptTemplate): + return obj + else: + raise TypeError(f'Expect a PromptTemplate object, but got {obj}') + + def __repr__(self): + return (f'PromptTemplate({{\n\ttemplate: {self.template},\n\t' + f'ice_token: {self.ice_token}\n}})') + + def _encode_template(self, prompt_template: Union[List[Union[str, Dict]], + str], + ice: bool) -> PromptType: + """Encode the raw template given in the config into a str or a + PromptList. + + Args: + prompt_template (List[Dict]] or str): The raw template given in the + config, used for generating the prompt. If it's a string, the + result will be directly returned. + ice (bool): If the template is used for generating in-context + examples. + + Returns: + str or PromptList: The encoded template. + """ + if isinstance(prompt_template, str): + return prompt_template + + prompt = PromptList() + + # TODO: Why can't we generate begin & end for ice template? + # To fix this, first we need to allow specifying prompt_template + # only + if 'begin' in prompt_template and not ice: + prompt.append(dict(section='begin', pos='begin')) + if isinstance(prompt_template['begin'], list): + prompt += prompt_template['begin'] + else: + prompt.append(prompt_template['begin']) + prompt.append(dict(section='begin', pos='end')) + + if ice: + prompt.append(dict(section='ice', pos='begin')) + else: + prompt.append(dict(section='round', pos='begin')) + prompt += prompt_template['round'] + if ice: + prompt.append(dict(section='ice', pos='end')) + else: + prompt.append(dict(section='round', pos='end')) + + if 'end' in prompt_template and not ice: + prompt.append(dict(section='end', pos='end')) + if isinstance(prompt_template['end'], list): + prompt += prompt_template['end'] + else: + prompt.append(prompt_template['end']) + prompt.append(dict(section='end', pos='end')) + + return prompt diff --git a/opencompass/openicl/icl_retriever/icl_base_retriever.py b/opencompass/openicl/icl_retriever/icl_base_retriever.py new file mode 100644 index 00000000..94736c7e --- /dev/null +++ b/opencompass/openicl/icl_retriever/icl_base_retriever.py @@ -0,0 +1,208 @@ +"""Basic Retriever.""" +from abc import abstractmethod +from typing import Dict, List, Optional + +from mmengine.dist import is_main_process + +from opencompass.openicl import PromptTemplate +from opencompass.utils.prompt import PromptList + + +class BaseRetriever: + """Base class for In-context Learning Example Retriever, without any + retrieval method implemented. + + Args: + dataset (`BaseDataset`): Any BaseDataset instances. + Attributes of ``reader``, ``train`` and ``test`` will be used. + ice_separator (`Optional[str]`): The separator between each in-context + example template when origin `PromptTemplate` is provided. Defaults + to '\n'. + ice_eos_token (`Optional[str]`): The end of sentence token for + in-context example template when origin `PromptTemplate` is + provided. Defaults to '\n'. + ice_num (`Optional[int]`): The number of in-context example template + when origin `PromptTemplate` is provided. Defaults to 1. + """ + index_ds = None + test_ds = None + + def __init__(self, + dataset, + ice_separator: Optional[str] = '\n', + ice_eos_token: Optional[str] = '\n', + ice_num: Optional[int] = 1) -> None: + self.ice_separator = ice_separator + self.ice_eos_token = ice_eos_token + self.ice_num = ice_num + self.is_main_process = is_main_process() + self.dataset_reader = dataset.reader + self.index_ds = dataset.train + self.test_ds = dataset.test + + @abstractmethod + def retrieve(self) -> List[List[int]]: + """Retrieve the in-context example index for each test example.""" + + def get_labels( + self, + ice_template: Optional[PromptTemplate] = None, + prompt_template: Optional[PromptTemplate] = None) -> List[str]: + """Get the labels of the dataset, especially useful for ppl inferencer. + If `ice_template` is provided, the labels will be the keys of the + template. If `prompt_template` is provided, the labels will be the keys + of the template. If neither of them is provided, the labels will be the + unique values of the output column. + + Args: + ice_template (`Optional[PromptTemplate]`): The template for + in-context example. Defaults to None. + prompt_template (`Optional[PromptTemplate]`): The template for + prompt. Defaults to None. + """ + if prompt_template is not None and isinstance(prompt_template.template, + Dict): + labels = list(prompt_template.template.keys()) + elif ice_template is not None and ice_template.ice_token is not None \ + and isinstance(ice_template.template, Dict): + labels = list(ice_template.template.keys()) + else: + labels = list(set(self.test_ds[self.dataset_reader.output_column])) + return labels + + def generate_ice(self, + idx_list: List[int], + ice_template: Optional[PromptTemplate] = None) -> str: + """Generate the in-context example for one test example. If + `ice_template` is an instance of `PromptTemplate`, the `ice_separator` + and `ice_eos_token` will be set as empty. + + Args: + idx_list (`List[int]`): The index of in-context examples for the + test example. + ice_template (`Optional[PromptTemplate]`): The template for + in-context example. Defaults to None. + """ + if ice_template is None: + assert len( + idx_list + ) == 0, 'You have not specified ice_template while retrieving examples from train set! Please either specify ice_template or use `ZeroRetriever`.' # noqa + + if ice_template is not None and ice_template.prompt_type == 'meta': + ice_separator, ice_eos_token = '', '' + else: + ice_separator = self.ice_separator + ice_eos_token = self.ice_eos_token + + generated_ice_list = [] + for idx in idx_list: + generated_ice_list.append( + ice_template.generate_ice_item( + self.index_ds[idx], + self.index_ds[idx][self.dataset_reader.output_column])) + if len(generated_ice_list) > 0 and isinstance(generated_ice_list[0], + PromptList): + generated_ice = [] + for ice in generated_ice_list: + generated_ice += ice + ice_separator + generated_ice.append(ice_eos_token) + else: + generated_ice = ice_separator.join( + generated_ice_list) + ice_eos_token + return generated_ice + + def generate_label_prompt(self, + idx: int, + ice: str, + label, + ice_template: Optional[PromptTemplate] = None, + prompt_template: Optional[PromptTemplate] = None, + remain_sep: Optional[bool] = False) -> str: + """Generate the prompt for one test example in perpelxity evaluation + with `prompt_template`. If `prompt_template` is not provided, the + `ice_template` will be used to generate the prompt. + + Args: + idx (`int`): The index of the test example. + ice (`str`): The in-context example for the test example. + label (`str`): The label of the test example. + ice_template (`Optional[PromptTemplate]`): The template for + in-context example. Defaults to None. + prompt_template (`Optional[PromptTemplate]`): The template for + prompt. Defaults to None. + remain_sep (`Optional[bool]`): Whether to remain the sep token. + Defaults to False. + """ + if prompt_template is not None and ice_template is not None: + if prompt_template.ice_token is not None: + return prompt_template.generate_label_prompt_item( + self.test_ds[idx], ice, label, remain_sep) + else: + raise NotImplementedError( + 'ice_token of prompt_template is not provided') + elif ice_template is not None and prompt_template is None: + if ice_template.ice_token is not None: + return ice_template.generate_label_prompt_item( + self.test_ds[idx], ice, label, remain_sep) + else: + raise NotImplementedError( + 'ice_token of ice_template is not provided') + elif ice_template is None and prompt_template is not None: + return prompt_template.generate_label_prompt_item( + self.test_ds[idx], ice, label, remain_sep) + else: + raise NotImplementedError( + 'Leaving prompt as empty is not supported') + + def generate_prompt_for_generate_task( + self, + idx, + ice, + gen_field_replace_token='', + ice_template: Optional[PromptTemplate] = None, + prompt_template: Optional[PromptTemplate] = None): + """Generate the prompt for one test example in generative evaluation + with `prompt_template`. If `prompt_template` is not provided, the + `ice_template` will be used to generate the prompt. The token + represented by `gen_field_replace_token` will not be replaced by the + generated text, or it will leaks the answer. + + Args: + idx (`int`): The index of the test example. + ice (`str`): The in-context example for the test example. + gen_field_replace_token (`str`): The token of the answer in the + prompt. Defaults to ''. + ice_template (`Optional[PromptTemplate]`): The template for + in-context example. Defaults to None. + prompt_template (`Optional[PromptTemplate]`): The template for + prompt. Defaults to None. + """ + if prompt_template is not None and ice_template is not None: + if prompt_template.ice_token is not None: + return prompt_template.generate_item( + self.test_ds[idx], + output_field=self.dataset_reader.output_column, + output_field_replace_token=gen_field_replace_token, + ice_field_replace_token=ice) + else: + raise NotImplementedError( + 'ice_token of prompt_template is not provided') + elif ice_template is not None and prompt_template is None: + if ice_template.ice_token is not None: + return ice_template.generate_item( + self.test_ds[idx], + output_field=self.dataset_reader.output_column, + output_field_replace_token=gen_field_replace_token, + ice_field_replace_token=ice) + else: + raise NotImplementedError( + 'ice_token of ice_template is not provided') + elif ice_template is None and prompt_template is not None: + return prompt_template.generate_item( + self.test_ds[idx], + output_field=self.dataset_reader.output_column, + output_field_replace_token=gen_field_replace_token, + ice_field_replace_token=ice) + else: + raise NotImplementedError( + 'Leaving prompt as empty is not supported') diff --git a/opencompass/openicl/icl_retriever/icl_dpp_retriever.py b/opencompass/openicl/icl_retriever/icl_dpp_retriever.py new file mode 100644 index 00000000..57ad1928 --- /dev/null +++ b/opencompass/openicl/icl_retriever/icl_dpp_retriever.py @@ -0,0 +1,126 @@ +"""DPP Retriever.""" + +import math +from typing import Optional + +import numpy as np +import tqdm + +from opencompass.openicl.icl_retriever.icl_topk_retriever import TopkRetriever +from opencompass.openicl.utils.logging import get_logger + +logger = get_logger(__name__) + + +class DPPRetriever(TopkRetriever): + """DPP In-context Learning Retriever, subclass of `TopkRetriever`. Two- + stage DPP is used, where first stage is to get results of TopK to reduce + candidate sets. Chechout https://arxiv.org/abs/2302.05698 for details. + + **WARNING**: This class has not been tested thoroughly. Please use it with + caution. + """ + model = None + + def __init__(self, + dataset, + ice_separator: Optional[str] = '\n', + ice_eos_token: Optional[str] = '\n', + ice_num: Optional[int] = 1, + sentence_transformers_model_name: Optional[ + str] = 'all-mpnet-base-v2', + tokenizer_name: Optional[str] = 'gpt2-xl', + batch_size: Optional[int] = 1, + candidate_num: Optional[int] = 1, + seed: Optional[int] = 1, + scale_factor: Optional[float] = 0.1) -> None: + super().__init__(dataset, ice_separator, ice_eos_token, ice_num, + sentence_transformers_model_name, tokenizer_name, + batch_size) + self.candidate_num = candidate_num + self.seed = seed + self.scale_factor = scale_factor + + def dpp_search(self): + res_list = self.forward(self.dataloader, + process_bar=True, + information='Embedding test set...') + rtr_idx_list = [[] for _ in range(len(res_list))] + logger.info('Retrieving data for test set...') + for entry in tqdm.tqdm(res_list, disable=not self.is_main_process): + idx = entry['metadata']['id'] + + # get TopK results + embed = np.expand_dims(entry['embed'], axis=0) + near_ids = np.array( + self.index.search(embed, self.candidate_num)[1][0].tolist()) + + # DPP stage + near_reps, rel_scores, kernel_matrix = self.get_kernel( + embed, near_ids.tolist()) + + # MAP inference + samples_ids = fast_map_dpp(kernel_matrix, self.ice_num) + + # ordered by relevance score + samples_scores = np.array([rel_scores[i] for i in samples_ids]) + samples_ids = samples_ids[(-samples_scores).argsort()].tolist() + rtr_sub_list = [int(near_ids[i]) for i in samples_ids] + + rtr_idx_list[idx] = rtr_sub_list + + return rtr_idx_list + + def retrieve(self): + return self.dpp_search() + + def get_kernel(self, embed, candidates): + near_reps = np.stack( + [self.index.index.reconstruct(i) for i in candidates], axis=0) + # normalize first + embed = embed / np.linalg.norm(embed) + near_reps = near_reps / np.linalg.norm( + near_reps, keepdims=True, axis=1) + + # to make kernel-matrix non-negative + rel_scores = np.matmul(embed, near_reps.T)[0] + rel_scores = (rel_scores + 1) / 2 + + # to prevent overflow error + rel_scores -= rel_scores.max() + + # to balance relevance and diversity + rel_scores = np.exp(rel_scores / (2 * self.scale_factor)) + + # to make kernel-matrix non-negative + sim_matrix = np.matmul(near_reps, near_reps.T) + sim_matrix = (sim_matrix + 1) / 2 + + kernel_matrix = rel_scores[None] * sim_matrix * rel_scores[:, None] + return near_reps, rel_scores, kernel_matrix + + +def fast_map_dpp(kernel_matrix, max_length): + """fast implementation of the greedy algorithm reference: + + https://github.com/laming-chen/fast-map-dpp/blob/master/dpp_test.py + paper: Fast Greedy MAP Inference for Determinantal Point Process to Improve + Recommendation Diversity + """ + item_size = kernel_matrix.shape[0] + cis = np.zeros((max_length, item_size)) + di2s = np.copy(np.diag(kernel_matrix)) + selected_items = list() + selected_item = np.argmax(di2s) + selected_items.append(int(selected_item)) + while len(selected_items) < max_length: + k = len(selected_items) - 1 + ci_optimal = cis[:k, selected_item] + di_optimal = math.sqrt(di2s[selected_item]) + elements = kernel_matrix[selected_item, :] + eis = (elements - np.dot(ci_optimal, cis[:k, :])) / di_optimal + cis[k, :] = eis + di2s -= np.square(eis) + selected_item = np.argmax(di2s) + selected_items.append(int(selected_item)) + return selected_items diff --git a/opencompass/openicl/icl_retriever/icl_fix_k_retriever.py b/opencompass/openicl/icl_retriever/icl_fix_k_retriever.py new file mode 100644 index 00000000..1e6f7397 --- /dev/null +++ b/opencompass/openicl/icl_retriever/icl_fix_k_retriever.py @@ -0,0 +1,52 @@ +"""Random Retriever.""" + +from typing import List, Optional + +from tqdm import trange + +from opencompass.openicl.icl_retriever import BaseRetriever +from opencompass.openicl.utils.logging import get_logger +from opencompass.registry import ICL_RETRIEVERS + +logger = get_logger(__name__) + + +@ICL_RETRIEVERS.register_module() +class FixKRetriever(BaseRetriever): + """Fix-K Retriever. Each in-context example of the test prompts is + retrieved as the same K examples from the index set. + + Args: + dataset (`BaseDataset`): Any BaseDataset instances. + Attributes of ``reader``, ``train`` and ``test`` will be used. + ice_separator (`Optional[str]`): The separator between each in-context + example template when origin `PromptTemplate` is provided. Defaults + to '\n'. + ice_eos_token (`Optional[str]`): The end of sentence token for + in-context example template when origin `PromptTemplate` is + provided. Defaults to '\n'. + ice_num (`Optional[int]`): The number of in-context example template + when origin `PromptTemplate` is provided. Defaults to 1. + """ + + def __init__(self, + dataset, + ice_separator: Optional[str] = '\n', + ice_eos_token: Optional[str] = '\n', + ice_num: Optional[int] = 1) -> None: + super().__init__(dataset, ice_separator, ice_eos_token, ice_num) + + def retrieve(self, id_list: List[int]): + """Retrieve the in-context example index for each test example. + + Args: + id_list (List[int]): List of in-context example indices for every + test prompts. + """ + num_idx = len(self.index_ds) + for idx in id_list: + assert idx < num_idx, f'Index {idx} is out of range of {num_idx}' + rtr_idx_list = [] + for _ in trange(len(self.test_ds), disable=not self.is_main_process): + rtr_idx_list.append(id_list) + return rtr_idx_list diff --git a/opencompass/openicl/icl_retriever/icl_topk_retriever.py b/opencompass/openicl/icl_retriever/icl_topk_retriever.py new file mode 100644 index 00000000..15743d71 --- /dev/null +++ b/opencompass/openicl/icl_retriever/icl_topk_retriever.py @@ -0,0 +1,203 @@ +"""Topk Retriever.""" + +import copy +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union + +import faiss +import numpy as np +import torch +import tqdm +from sentence_transformers import SentenceTransformer +from torch.utils.data import DataLoader +from transformers import AutoTokenizer, BatchEncoding, PreTrainedTokenizerBase +from transformers.file_utils import PaddingStrategy + +from opencompass.openicl.icl_dataset_reader import DatasetEncoder +from opencompass.openicl.icl_retriever import BaseRetriever +from opencompass.openicl.utils.logging import get_logger +from opencompass.registry import ICL_RETRIEVERS + +logger = get_logger(__name__) + + +@ICL_RETRIEVERS.register_module() +class TopkRetriever(BaseRetriever): + """Base class for Topk In-context Learning Retriever, implemented with + basic knn. SentenceTransformer is used to calculate embeddings. Faiss is + used to do the nearest neighbor search. + + Args: + dataset (`BaseDataset`): Any BaseDataset instances. + Attributes of ``reader``, ``train`` and ``test`` will be used. + ice_separator (`Optional[str]`): The separator between each in-context + example template when origin `PromptTemplate` is provided. Defaults + to '\n'. + ice_eos_token (`Optional[str]`): The end of sentence token for + in-context example template when origin `PromptTemplate` is + provided. Defaults to '\n'. + ice_num (`Optional[int]`): The number of in-context example template + when origin `PromptTemplate` is provided. Defaults to 1. + sentence_transformers_model_name (`Optional[str]`): The name of the + sentence transformers model. Defaults to 'all-mpnet-base-v2'. + tokenizer_name (`Optional[str]`): The name of the tokenizer. Defaults + to 'gpt2-xl'. + batch_size (`Optional[int]`): The batch size for the dataloader. + Defaults to 1. + """ + model = None + + def __init__(self, + dataset, + ice_separator: Optional[str] = '\n', + ice_eos_token: Optional[str] = '\n', + ice_num: Optional[int] = 1, + sentence_transformers_model_name: Optional[ + str] = 'all-mpnet-base-v2', + tokenizer_name: Optional[str] = 'gpt2-xl', + batch_size: Optional[int] = 1) -> None: + super().__init__(dataset, ice_separator, ice_eos_token, ice_num) + self.device = 'cuda' if torch.cuda.is_available() else 'cpu' + self.batch_size = batch_size + self.tokenizer_name = tokenizer_name + gen_datalist = self.dataset_reader.generate_input_field_corpus( + self.test_ds) + + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name) + self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + self.tokenizer.padding_side = 'right' + + self.encode_dataset = DatasetEncoder(gen_datalist, + tokenizer=self.tokenizer) + co = DataCollatorWithPaddingAndCuda(tokenizer=self.tokenizer, + device=self.device) + self.dataloader = DataLoader(self.encode_dataset, + batch_size=self.batch_size, + collate_fn=co) + + self.model = SentenceTransformer(sentence_transformers_model_name) + + self.model = self.model.to(self.device) + self.model.eval() + + self.index = self.create_index() + + def create_index(self): + self.select_datalist = self.dataset_reader.generate_input_field_corpus( + self.index_ds) + encode_datalist = DatasetEncoder(self.select_datalist, + tokenizer=self.tokenizer) + co = DataCollatorWithPaddingAndCuda(tokenizer=self.tokenizer, + device=self.device) + dataloader = DataLoader(encode_datalist, + batch_size=self.batch_size, + collate_fn=co) + index = faiss.IndexIDMap( + faiss.IndexFlatIP(self.model.get_sentence_embedding_dimension())) + res_list = self.forward(dataloader, + process_bar=True, + information='Creating index for index set...') + id_list = np.array([res['metadata']['id'] for res in res_list]) + self.embed_list = np.stack([res['embed'] for res in res_list]) + index.add_with_ids(self.embed_list, id_list) + return index + + def knn_search(self, ice_num): + res_list = self.forward(self.dataloader, + process_bar=True, + information='Embedding test set...') + rtr_idx_list = [[] for _ in range(len(res_list))] + logger.info('Retrieving data for test set...') + for entry in tqdm.tqdm(res_list, disable=not self.is_main_process): + idx = entry['metadata']['id'] + embed = np.expand_dims(entry['embed'], axis=0) + near_ids = self.index.search(embed, ice_num)[1][0].tolist() + rtr_idx_list[idx] = near_ids + return rtr_idx_list + + def forward(self, dataloader, process_bar=False, information=''): + res_list = [] + _dataloader = copy.deepcopy(dataloader) + if process_bar: + logger.info(information) + _dataloader = tqdm.tqdm(_dataloader, + disable=not self.is_main_process) + for _, entry in enumerate(_dataloader): + with torch.no_grad(): + metadata = entry.pop('metadata') + raw_text = self.tokenizer.batch_decode( + entry['input_ids'], + skip_special_tokens=True, + verbose=False) + res = self.model.encode(raw_text, show_progress_bar=False) + res_list.extend([{ + 'embed': r, + 'metadata': m + } for r, m in zip(res, metadata)]) + return res_list + + def retrieve(self): + """Retrieve the in-context example index for each test example.""" + return self.knn_search(self.ice_num) + + +class ListWrapper: + + def __init__(self, data: List[Any]): + self.data = data + + def to(self, device): + return self.data + + +def ignore_pad_dict(features): + res_dict = {} + if 'metadata' in features[0]: + res_dict['metadata'] = ListWrapper( + [x.pop('metadata') for x in features]) + return res_dict + + +@dataclass +class DataCollatorWithPaddingAndCuda: + tokenizer: PreTrainedTokenizerBase + device: object = None + padding: Union[bool, str, PaddingStrategy] = True + max_length: Optional[int] = 3000 + pad_to_multiple_of: Optional[int] = None + + def __call__( + self, features: List[Dict[str, Union[List[int], torch.Tensor]]] + ) -> BatchEncoding: + res_dict = ignore_pad_dict(features) + + has_labels = 'labels' in features[0] + if has_labels: + labels = [{'input_ids': x.pop('labels')} for x in features] + labels = self.tokenizer.pad( + labels, + padding=True, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + return_attention_mask=True, + return_tensors='pt', + verbose=False) + + # print(features) + batch = self.tokenizer.pad(features, + padding=True, + max_length=self.max_length, + pad_to_multiple_of=self.pad_to_multiple_of, + return_attention_mask=True, + return_tensors='pt', + verbose=False) + + if has_labels: + batch['labels'] = labels.input_ids + batch.update(res_dict) + + if self.device: + batch = batch.to(self.device) + + return batch diff --git a/opencompass/partitioners/base.py b/opencompass/partitioners/base.py new file mode 100644 index 00000000..eac39f9e --- /dev/null +++ b/opencompass/partitioners/base.py @@ -0,0 +1,82 @@ +from abc import abstractmethod +from copy import deepcopy +from typing import Dict, List + +from mmengine.config import ConfigDict + +from opencompass.utils import get_logger, task_abbr_from_cfg + + +class BasePartitioner: + """Base class for partitioners. A partitioner is responsible for + partitioning the config into tasks. + + Args: + out_dir (str): The output directory of tasks. + """ + + def __init__(self, out_dir: str): + self.logger = get_logger() + self.out_dir = out_dir + + def __call__(self, cfg: ConfigDict) -> List[Dict]: + """Generate tasks from config. Each task is defined as a + dict and will run independently as a unit. Its structure is as + follows: + + .. code-block:: python + + { + 'models': [], # a list of model configs + 'datasets': [[]], # a nested list of dataset configs, each + list corresponds to a model + 'work_dir': '', # the work dir + } + + Args: + cfg (ConfigDict): The config dict, containing "models", "dataset" + and "work_dir" keys. + + Returns: + List[Dict]: A list of tasks. + """ + cfg = deepcopy(cfg) + models = cfg['models'] + datasets = cfg['datasets'] + work_dir = cfg['work_dir'] + + tasks = self.partition(models, datasets, work_dir, self.out_dir) + + self.logger.info(f'Partitioned into {len(tasks)} tasks.') + for i, task in enumerate(tasks): + self.logger.debug(f'Task {i}: {task_abbr_from_cfg(task)}') + + return tasks + + @abstractmethod + def partition(self, models: List[ConfigDict], datasets: List[ConfigDict], + work_dir: str, out_dir: str) -> List[Dict]: + """Partition model-dataset pairs into tasks. Each task is defined as a + dict and will run independently as a unit. Its structure is as + follows: + + .. code-block:: python + + { + 'models': [], # a list of model configs + 'datasets': [[]], # a nested list of dataset configs, each + list corresponds to a model + 'work_dir': '', # the work dir + } + + Args: + models (List[ConfigDict]): A list of model configs. + datasets (List[ConfigDict]): A list of dataset configs. + work_dir (str): The work dir for the task. + out_dir (str): The full output path for the task, intended for + Partitioners to check whether the task is finished via the + existency of result file in this directory. + + Returns: + List[Dict]: A list of tasks. + """ diff --git a/opencompass/runners/dlc.py b/opencompass/runners/dlc.py new file mode 100644 index 00000000..35ee9990 --- /dev/null +++ b/opencompass/runners/dlc.py @@ -0,0 +1,154 @@ +import inspect +import os +import os.path as osp +import random +import subprocess +import time +from typing import Any, Dict, List, Tuple + +import mmengine +from mmengine.config import ConfigDict +from mmengine.utils import track_parallel_progress + +from opencompass.registry import RUNNERS, TASKS +from opencompass.utils import get_logger + +from .base import BaseRunner + + +@RUNNERS.register_module() +class DLCRunner(BaseRunner): + """Distributed runner based on Alibaba Cloud Deep Learning Cluster (DLC). + It will launch multiple tasks in parallel with 'dlc' command. Please + install and configure DLC first before using this runner. + + Args: + task (ConfigDict): Task type config. + aliyun_cfg (ConfigDict): Alibaba Cloud config. + max_num_workers (int): Max number of workers. Default: 32. + retry (int): Number of retries when job failed. Default: 2. + debug (bool): Whether to run in debug mode. Default: False. + lark_bot_url (str): Lark bot url. Default: None. + """ + + def __init__(self, + task: ConfigDict, + aliyun_cfg: ConfigDict, + max_num_workers: int = 32, + retry: int = 2, + debug: bool = False, + lark_bot_url: str = None): + super().__init__(task=task, debug=debug, lark_bot_url=lark_bot_url) + self.aliyun_cfg = aliyun_cfg + self.max_num_workers = max_num_workers + self.retry = retry + + def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]: + """Launch multiple tasks. + + Args: + tasks (list[dict]): A list of task configs, usually generated by + Partitioner. + + Returns: + list[tuple[str, int]]: A list of (task name, exit code). + """ + + if not self.debug: + status = track_parallel_progress(self._launch, + tasks, + nproc=self.max_num_workers, + keep_order=False) + else: + status = [self._launch(task, random_sleep=False) for task in tasks] + return status + + def _launch(self, task_cfg: ConfigDict, random_sleep: bool = True): + """Launch a single task. + + Args: + task_cfg (ConfigDict): Task config. + random_sleep (bool): Whether to sleep for a random time before + running the command. This avoids cluster error when launching + multiple tasks at the same time. Default: True. + + Returns: + tuple[str, int]: Task name and exit code. + """ + + task_type = self.task_cfg.type + if isinstance(self.task_cfg.type, str): + task_type = TASKS.get(task_type) + task = task_type(task_cfg) + num_gpus = task.num_gpus + task_name = task.name + script_path = inspect.getsourcefile(task_type) + + # Dump task config to file + mmengine.mkdir_or_exist('tmp/') + param_file = f'tmp/{os.getpid()}_params.py' + task_cfg.dump(param_file) + + # Build up DLC command + task_cmd_template = task.get_command_template() + task_cmd = task_cmd_template.replace('{SCRIPT_PATH}', + script_path).replace( + '{CFG_PATH}', param_file) + pwd = os.getcwd() + shell_cmd = (f'source {self.aliyun_cfg["bashrc_path"]}; ' + f'conda activate {self.aliyun_cfg["conda_env_name"]}; ' + f'cd {pwd}; ' + f'{task_cmd}') + + cmd = ('dlc create job' + f" --command '{shell_cmd}'" + f' --name {task_name[:512]}' + ' --kind BatchJob' + f" -c {self.aliyun_cfg['dlc_config_path']}" + f" --workspace_id {self.aliyun_cfg['workspace_id']}" + ' --worker_count 1' + f' --worker_cpu {max(num_gpus * 6, 8)}' + f' --worker_gpu {num_gpus}' + f' --worker_memory {max(num_gpus * 32, 48)}' + f" --worker_image {self.aliyun_cfg['worker_image']}" + ' --priority 3' + ' --interactive') + + logger = get_logger() + logger.debug(f'Running command: {cmd}') + + # Run command with retry + if self.debug: + stdout = None + else: + out_path = task.get_log_path(file_extension='out') + mmengine.mkdir_or_exist(osp.split(out_path)[0]) + stdout = open(out_path, 'w', encoding='utf-8') + + if random_sleep: + time.sleep(random.randint(0, 10)) + result = subprocess.run(cmd, + shell=True, + text=True, + stdout=stdout, + stderr=stdout) + + retry = self.retry + output_paths = task.get_output_paths() + while self._job_failed(result.returncode, output_paths) and retry > 0: + retry -= 1 + if random_sleep: + time.sleep(random.randint(0, 10)) + result = subprocess.run(cmd, + shell=True, + text=True, + stdout=stdout, + stderr=stdout) + + # Clean up + os.remove(param_file) + return task_name, result.returncode + + def _job_failed(self, return_code: int, output_paths: List[str]) -> bool: + return return_code != 0 or not all( + osp.exists(output_path) for output_path in output_paths) diff --git a/opencompass/runners/local.py b/opencompass/runners/local.py new file mode 100644 index 00000000..c1af98d3 --- /dev/null +++ b/opencompass/runners/local.py @@ -0,0 +1,148 @@ +import inspect +import os +import os.path as osp +import subprocess +import time +from concurrent.futures import ThreadPoolExecutor +from threading import Lock +from typing import Any, Dict, List, Tuple + +import mmengine +import numpy as np +from mmengine.config import ConfigDict +from tqdm import tqdm + +from opencompass.registry import RUNNERS, TASKS +from opencompass.utils import get_logger + +from .base import BaseRunner + + +@RUNNERS.register_module() +class LocalRunner(BaseRunner): + """Local runner. Start tasks by local python. + + Args: + task (ConfigDict): Task type config. + max_num_workers (int): Max number of workers to run in parallel. + Defaults to 16. + debug (bool): Whether to run in debug mode. + lark_bot_url (str): Lark bot url. + """ + + def __init__(self, + task: ConfigDict, + max_num_workers: int = 16, + debug: bool = False, + lark_bot_url: str = None): + super().__init__(task=task, debug=debug, lark_bot_url=lark_bot_url) + self.max_num_workers = max_num_workers + + def launch(self, tasks: List[Dict[str, Any]]) -> List[Tuple[str, int]]: + """Launch multiple tasks. + + Args: + tasks (list[dict]): A list of task configs, usually generated by + Partitioner. + + Returns: + list[tuple[str, int]]: A list of (task name, exit code). + """ + + status = [] + if self.debug: + for task in tasks: + task = TASKS.build(dict(type=self.task_cfg.type, cfg=task)) + task_name = task.name + task.run() + status.append((task_name, 0)) + else: + import torch + gpus = np.ones(torch.cuda.device_count(), dtype=np.bool_) + pbar = tqdm(total=len(tasks)) + lock = Lock() + logger = get_logger() + + def submit(task, index): + task = TASKS.build(dict(type=self.task_cfg.type, cfg=task)) + num_gpus = task.num_gpus + assert len(gpus) >= num_gpus + + while True: + lock.acquire() + if sum(gpus) >= num_gpus: + gpu_ids = np.where(gpus)[0][:num_gpus] + gpus[gpu_ids] = False + lock.release() + break + lock.release() + time.sleep(1) + + if num_gpus > 0: + tqdm.write(f'launch {task.name} on GPU ' + + ','.join(map(str, gpu_ids))) + else: + tqdm.write(f'launch {task.name} on CPU ') + + res = self._launch(task, gpu_ids, index) + pbar.update() + + with lock: + gpus[gpu_ids] = True + + return res + + with ThreadPoolExecutor( + max_workers=self.max_num_workers) as executor: + status = executor.map(submit, tasks, range(len(tasks))) + + return status + + def _launch(self, task, gpu_ids, index): + """Launch a single task. + + Args: + task (BaseTask): Task to launch. + + Returns: + tuple[str, int]: Task name and exit code. + """ + + task_name = task.name + script_path = inspect.getsourcefile(type(task)) + + # Dump task config to file + mmengine.mkdir_or_exist('tmp/') + param_file = f'tmp/{os.getpid()}_{index}_params.json' + mmengine.dump(task.cfg, param_file) + + # Build up slurm command + task_cmd_template = task.get_command_template() + task_cmd = task_cmd_template.replace('{SCRIPT_PATH}', + script_path).replace( + '{CFG_PATH}', param_file) + cmd = 'CUDA_VISIBLE_DEVICES=' + ','.join(str(i) for i in gpu_ids) + ' ' + cmd += task_cmd + logger = get_logger() + logger.debug(f'Running command: {cmd}') + + # Run command + if self.debug: + stdout = None + else: + out_path = task.get_log_path(file_extension='out') + mmengine.mkdir_or_exist(osp.split(out_path)[0]) + stdout = open(out_path, 'w', encoding='utf-8') + + result = subprocess.run(cmd, + shell=True, + text=True, + stdout=stdout, + stderr=stdout) + + if result.returncode != 0: + logger.warning(f'task {task_name} fail, see\n{out_path}') + + # Clean up + os.remove(param_file) + return task_name, result.returncode diff --git a/opencompass/tasks/openicl_infer.py b/opencompass/tasks/openicl_infer.py new file mode 100644 index 00000000..e0e6025d --- /dev/null +++ b/opencompass/tasks/openicl_infer.py @@ -0,0 +1,129 @@ +import argparse +import os.path as osp +import random +import time +from typing import Any + +from mmengine.config import Config, ConfigDict +from mmengine.utils import mkdir_or_exist + +from opencompass.registry import (ICL_INFERENCERS, ICL_PROMPT_TEMPLATES, + ICL_RETRIEVERS, TASKS) +from opencompass.tasks.base import BaseTask +from opencompass.utils import (build_dataset_from_cfg, build_model_from_cfg, + get_infer_output_path, get_logger) + + +@TASKS.register_module(force=(__name__ == '__main__')) # A hack for script run +class OpenICLInferTask(BaseTask): + """OpenICL Inference Task. + + This task is used to run the inference process. + """ + + name_prefix = 'OpenICLInfer' + log_subdir = 'logs/infer' + output_subdir = 'predictions' + + def __init__(self, cfg: ConfigDict): + super().__init__(cfg) + run_cfg = self.model_cfgs[0].get('run_cfg', {}) + self.num_gpus = run_cfg.get('num_gpus', 0) + self.num_procs = run_cfg.get('num_procs', 1) + + def get_command_template(self): + if self.num_gpus > 0: + return (f'torchrun --master_port={random.randint(12000, 32000)} ' + f'--nproc_per_node {self.num_procs} ' + '{SCRIPT_PATH} {CFG_PATH}') + else: + return ('python {SCRIPT_PATH} {CFG_PATH}') + + def run(self): + for model_cfg, dataset_cfgs in zip(self.model_cfgs, self.dataset_cfgs): + self.max_out_len = model_cfg.get('max_out_len', None) + self.batch_size = model_cfg.get('batch_size', None) + self.model = build_model_from_cfg(model_cfg) + + for dataset_cfg in dataset_cfgs: + self.model_cfg = model_cfg + self.dataset_cfg = dataset_cfg + self.infer_cfg = self.dataset_cfg['infer_cfg'] + self.dataset = build_dataset_from_cfg(self.dataset_cfg) + out_path = get_infer_output_path( + self.model_cfg, self.dataset_cfg, + osp.join(self.work_dir, 'predictions')) + if osp.exists(out_path): + continue + self._inference() + + def _inference(self): + + assert hasattr(self.infer_cfg, 'ice_template') or hasattr(self.infer_cfg, 'prompt_template'), \ + 'Both ice_template and prompt_template cannot be None simultaneously.' # noqa: E501 + if hasattr(self.infer_cfg, 'ice_template'): + ice_template = ICL_PROMPT_TEMPLATES.build( + self.infer_cfg['ice_template']) + + if hasattr(self.infer_cfg, 'prompt_template'): + prompt_template = ICL_PROMPT_TEMPLATES.build( + self.infer_cfg['prompt_template']) + + retriever_cfg = self.infer_cfg['retriever'].copy() + retriever_cfg['dataset'] = self.dataset + retriever = ICL_RETRIEVERS.build(retriever_cfg) + + # set inferencer's default value according to model's config' + inferencer_cfg = self.infer_cfg['inferencer'] + inferencer_cfg['model'] = self.model + self._set_default_value(inferencer_cfg, 'max_out_len', + self.max_out_len) + self._set_default_value(inferencer_cfg, 'batch_size', self.batch_size) + inferencer_cfg['max_seq_len'] = self.model_cfg['max_seq_len'] + inferencer = ICL_INFERENCERS.build(inferencer_cfg) + + out_path = get_infer_output_path( + self.model_cfg, self.dataset_cfg, + osp.join(self.work_dir, 'predictions')) + out_dir, out_file = osp.split(out_path) + mkdir_or_exist(out_dir) + + if hasattr(self.infer_cfg, 'prompt_template') and \ + hasattr(self.infer_cfg, 'ice_template'): + inferencer.inference(retriever, + ice_template=ice_template, + prompt_template=prompt_template, + output_json_filepath=out_dir, + output_json_filename=out_file) + elif hasattr(self.infer_cfg, 'prompt_template'): + inferencer.inference(retriever, + prompt_template=prompt_template, + output_json_filepath=out_dir, + output_json_filename=out_file) + else: + inferencer.inference(retriever, + ice_template=ice_template, + output_json_filepath=out_dir, + output_json_filename=out_file) + + def _set_default_value(self, cfg: ConfigDict, key: str, value: Any): + if key not in cfg: + assert value, (f'{key} must be specified!') + cfg[key] = value + + +def parse_args(): + parser = argparse.ArgumentParser(description='Model Inferencer') + parser.add_argument('config', help='Config file path') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_args() + cfg = Config.fromfile(args.config) + start_time = time.time() + inferencer = OpenICLInferTask(cfg) + inferencer.run() + end_time = time.time() + get_logger().info(f'time elapsed: {end_time - start_time:.2f}s') diff --git a/opencompass/utils/fileio.py b/opencompass/utils/fileio.py new file mode 100644 index 00000000..36cfdfb8 --- /dev/null +++ b/opencompass/utils/fileio.py @@ -0,0 +1,168 @@ +import io +from contextlib import contextmanager + +import mmengine.fileio as fileio +from mmengine.fileio import LocalBackend, get_file_backend + + +def patch_func(module, fn_name_to_wrap): + backup = getattr(patch_func, '_backup', []) + fn_to_wrap = getattr(module, fn_name_to_wrap) + + def wrap(fn_new): + setattr(module, fn_name_to_wrap, fn_new) + backup.append((module, fn_name_to_wrap, fn_to_wrap)) + setattr(fn_new, '_fallback', fn_to_wrap) + setattr(patch_func, '_backup', backup) + return fn_new + + return wrap + + +@contextmanager +def patch_fileio(global_vars=None): + if getattr(patch_fileio, '_patched', False): + # Only patch once, avoid error caused by patch nestly. + yield + return + import builtins + + @patch_func(builtins, 'open') + def open(file, mode='r', *args, **kwargs): + backend = get_file_backend(file) + if isinstance(backend, LocalBackend): + return open._fallback(file, mode, *args, **kwargs) + if 'b' in mode: + return io.BytesIO(backend.get(file, *args, **kwargs)) + else: + return io.StringIO(backend.get_text(file, *args, **kwargs)) + + if global_vars is not None and 'open' in global_vars: + bak_open = global_vars['open'] + global_vars['open'] = builtins.open + + import os + + @patch_func(os.path, 'join') + def join(a, *paths): + backend = get_file_backend(a) + if isinstance(backend, LocalBackend): + return join._fallback(a, *paths) + paths = [item for item in paths if len(item) > 0] + return backend.join_path(a, *paths) + + @patch_func(os.path, 'isdir') + def isdir(path): + backend = get_file_backend(path) + if isinstance(backend, LocalBackend): + return isdir._fallback(path) + return backend.isdir(path) + + @patch_func(os.path, 'isfile') + def isfile(path): + backend = get_file_backend(path) + if isinstance(backend, LocalBackend): + return isfile._fallback(path) + return backend.isfile(path) + + @patch_func(os.path, 'exists') + def exists(path): + backend = get_file_backend(path) + if isinstance(backend, LocalBackend): + return exists._fallback(path) + return backend.exists(path) + + @patch_func(os, 'listdir') + def listdir(path): + backend = get_file_backend(path) + if isinstance(backend, LocalBackend): + return listdir._fallback(path) + return backend.list_dir_or_file(path) + + import filecmp + + @patch_func(filecmp, 'cmp') + def cmp(f1, f2, *args, **kwargs): + with fileio.get_local_path(f1) as f1, fileio.get_local_path(f2) as f2: + return cmp._fallback(f1, f2, *args, **kwargs) + + import shutil + + @patch_func(shutil, 'copy') + def copy(src, dst, **kwargs): + backend = get_file_backend(src) + if isinstance(backend, LocalBackend): + return copy._fallback(src, dst, **kwargs) + return backend.copyfile_to_local(str(src), str(dst)) + + import torch + + @patch_func(torch, 'load') + def load(f, *args, **kwargs): + if isinstance(f, str): + f = io.BytesIO(fileio.get(f)) + return load._fallback(f, *args, **kwargs) + + try: + setattr(patch_fileio, '_patched', True) + yield + finally: + for patched_fn in patch_func._backup: + (module, fn_name_to_wrap, fn_to_wrap) = patched_fn + setattr(module, fn_name_to_wrap, fn_to_wrap) + if global_vars is not None and 'open' in global_vars: + global_vars['open'] = bak_open + setattr(patch_fileio, '_patched', False) + + +def patch_hf_auto_model(cache_dir=None): + if hasattr('patch_hf_auto_model', '_patched'): + return + + from transformers.modeling_utils import PreTrainedModel + from transformers.models.auto.auto_factory import _BaseAutoModelClass + + ori_model_pt = PreTrainedModel.from_pretrained + + @classmethod + def model_pt(cls, pretrained_model_name_or_path, *args, **kwargs): + kwargs['cache_dir'] = cache_dir + if not isinstance(get_file_backend(pretrained_model_name_or_path), + LocalBackend): + kwargs['local_files_only'] = True + if cache_dir is not None and not isinstance( + get_file_backend(cache_dir), LocalBackend): + kwargs['local_files_only'] = True + + with patch_fileio(): + res = ori_model_pt.__func__(cls, pretrained_model_name_or_path, + *args, **kwargs) + return res + + PreTrainedModel.from_pretrained = model_pt + + # transformers copied the `from_pretrained` to all subclasses, + # so we have to modify all classes + for auto_class in [ + _BaseAutoModelClass, *_BaseAutoModelClass.__subclasses__() + ]: + ori_auto_pt = auto_class.from_pretrained + + @classmethod + def auto_pt(cls, pretrained_model_name_or_path, *args, **kwargs): + kwargs['cache_dir'] = cache_dir + if not isinstance(get_file_backend(pretrained_model_name_or_path), + LocalBackend): + kwargs['local_files_only'] = True + if cache_dir is not None and not isinstance( + get_file_backend(cache_dir), LocalBackend): + kwargs['local_files_only'] = True + + with patch_fileio(): + res = ori_auto_pt.__func__(cls, pretrained_model_name_or_path, + *args, **kwargs) + return res + + auto_class.from_pretrained = auto_pt + + patch_hf_auto_model._patched = True diff --git a/opencompass/utils/lark.py b/opencompass/utils/lark.py new file mode 100644 index 00000000..46241f86 --- /dev/null +++ b/opencompass/utils/lark.py @@ -0,0 +1,39 @@ +import json +from typing import Dict, List, Optional, Union + +import requests + + +class LarkReporter: + + def __init__(self, url: str): + self.url = url + + def post(self, + content: Union[str, List[List[Dict]]], + title: Optional[str] = None): + """Post a message to Lark. + + When title is None, message must be a str. + otherwise msg can be in rich text format (see + https://open.feishu.cn/document/uAjLw4CM/ukTMukTMukTM/im-v1/message/create_json#45e0953e + for details). + """ + if title is None: + assert isinstance(content, str) + msg = {'msg_type': 'text', 'content': {'text': content}} + else: + if isinstance(content, str): + content = [[{'tag': 'text', 'text': content}]] + msg = { + 'msg_type': 'post', + 'content': { + 'post': { + 'zh_cn': { + 'title': title, + 'content': content + } + } + } + } + requests.post(self.url, data=json.dumps(msg)) diff --git a/opencompass/utils/menu.py b/opencompass/utils/menu.py new file mode 100644 index 00000000..1d5007d3 --- /dev/null +++ b/opencompass/utils/menu.py @@ -0,0 +1,68 @@ +import curses + + +class Menu: + """A curses menu that allows the user to select one item from each list. + + Args: + lists (list[list[str]]): A list of lists of strings, where each list + represents a list of items to be selected from. + prompts (list[str], optional): A list of prompts to be displayed above + each list. Defaults to None, in which case each list will be + displayed without a prompt. + """ + + def __init__(self, lists, prompts=None): + self.choices_lists = lists + self.prompts = prompts or ['Please make a selection:'] * len(lists) + self.choices = [] + self.current_window = [] + + def draw_menu(self, stdscr, selected_row_idx, offset, max_rows): + stdscr.clear() + h, w = stdscr.getmaxyx() + for idx, row in enumerate(self.current_window[offset:offset + + max_rows]): + x = w // 2 - len(row) // 2 + y = min(h - 1, + idx + 1) # Ensure y never goes beyond the window height + if idx == selected_row_idx - offset: + stdscr.attron(curses.color_pair(1)) + stdscr.addstr(y, x, row) + stdscr.attroff(curses.color_pair(1)) + else: + stdscr.addstr(y, x, row) + stdscr.refresh() + + def run(self): + curses.wrapper(self.main_loop) + return self.choices + + def main_loop(self, stdscr): + curses.curs_set(0) + curses.init_pair(1, curses.COLOR_BLACK, curses.COLOR_WHITE) + h, w = stdscr.getmaxyx() + max_rows = h - 2 + + for choices, prompt in zip(self.choices_lists, self.prompts): + self.current_window = [prompt] + choices + current_row_idx = 1 + offset = 0 + + while 1: + self.draw_menu(stdscr, current_row_idx, offset, max_rows) + key = stdscr.getch() + + if key == curses.KEY_UP and current_row_idx > 1: + current_row_idx -= 1 + if current_row_idx - offset < 1: + offset -= 1 + + elif key == curses.KEY_DOWN and current_row_idx < len(choices): + current_row_idx += 1 + if current_row_idx - offset > max_rows - 1: + offset += 1 + + elif key == curses.KEY_ENTER or key in [10, 13]: + self.choices.append(choices[current_row_idx - 1]) + break diff --git a/opencompass/utils/prompt.py b/opencompass/utils/prompt.py new file mode 100644 index 00000000..a8ea5bf8 --- /dev/null +++ b/opencompass/utils/prompt.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import hashlib +import json +from copy import deepcopy +from typing import Dict, Union + +from mmengine.config import ConfigDict + + +def safe_format(input_str: str, **kwargs) -> str: + """Safely formats a string with the given keyword arguments. If a keyword + is not found in the string, it will be ignored. + + Args: + input_str (str): The string to be formatted. + **kwargs: The keyword arguments to be used for formatting. + + Returns: + str: The formatted string. + """ + for k, v in kwargs.items(): + input_str = input_str.replace(f'{{{k}}}', str(v)) + return input_str + + +def get_prompt_hash(dataset_cfg: ConfigDict) -> str: + """Get the hash of the prompt configuration. + + Args: + dataset_cfg (ConfigDict): The dataset configuration. + + Returns: + str: The hash of the prompt configuration. + """ + if 'reader_cfg' in dataset_cfg.infer_cfg: + # new config + reader_cfg = dict(type='DatasetReader', + input_columns=dataset_cfg.reader_cfg.input_columns, + output_column=dataset_cfg.reader_cfg.output_column) + dataset_cfg.infer_cfg.reader = reader_cfg + if 'train_split' in dataset_cfg.infer_cfg.reader_cfg: + dataset_cfg.infer_cfg.retriever[ + 'index_split'] = dataset_cfg.infer_cfg['reader_cfg'][ + 'train_split'] + if 'test_split' in dataset_cfg.infer_cfg.reader_cfg: + dataset_cfg.infer_cfg.retriever[ + 'test_split'] = dataset_cfg.infer_cfg.reader_cfg.test_split + for k, v in dataset_cfg.infer_cfg.items(): + dataset_cfg.infer_cfg[k]['type'] = v['type'].split('.')[-1] + d_json = json.dumps(dataset_cfg.infer_cfg, sort_keys=True) + hash_object = hashlib.sha256(d_json.encode()) + return hash_object.hexdigest() + + +class PromptList(list): + """An enhanced list, used for intermidate representation of a prompt.""" + + def format(self, **kwargs) -> PromptList: + """Replaces all instances of 'src' in the PromptList with 'dst'. + + Args: + src (str): The string to be replaced. + dst (str or PromptList): The string or PromptList to replace with. + + Returns: + PromptList: A new PromptList with 'src' replaced by 'dst'. + + Raises: + TypeError: If 'dst' is a PromptList and 'src' is in a dictionary's + 'prompt' key. + """ + new_list = PromptList() + for item in self: + if isinstance(item, Dict): + new_item = deepcopy(item) + if 'prompt' in item: + new_item['prompt'] = safe_format(item['prompt'], **kwargs) + new_list.append(new_item) + else: + new_list.append(safe_format(item, **kwargs)) + return new_list + + def replace(self, src: str, dst: Union[str, PromptList]) -> PromptList: + """Replaces all instances of 'src' in the PromptList with 'dst'. + + Args: + src (str): The string to be replaced. + dst (str or PromptList): The string or PromptList to replace with. + + Returns: + PromptList: A new PromptList with 'src' replaced by 'dst'. + + Raises: + TypeError: If 'dst' is a PromptList and 'src' is in a dictionary's + 'prompt' key. + """ + new_list = PromptList() + for item in self: + if isinstance(item, str): + if isinstance(dst, str): + new_list.append(item.replace(src, dst)) + elif isinstance(dst, PromptList): + split_str = item.split(src) + for i, split_item in enumerate(split_str): + if split_item: + new_list.append(split_item) + if i < len(split_str) - 1: + new_list += dst + elif isinstance(item, Dict): + new_item = deepcopy(item) + if 'prompt' in item: + if src in item['prompt']: + if isinstance(dst, PromptList): + raise TypeError( + f'Found keyword {src} in a dictionary\'s ' + 'prompt key. Cannot replace with a ' + 'PromptList.') + new_item['prompt'] = new_item['prompt'].replace( + src, dst) + new_list.append(new_item) + else: + new_list.append(item.replace(src, dst)) + return new_list + + def __add__(self, other: Union[str, PromptList]) -> PromptList: + """Adds a string or another PromptList to this PromptList. + + Args: + other (str or PromptList): The string or PromptList to be added. + + Returns: + PromptList: A new PromptList that is the result of the addition. + """ + if not other: + return PromptList([*self]) + if isinstance(other, str): + return PromptList(self + [other]) + else: + return PromptList(super().__add__(other)) + + def __radd__(self, other: Union[str, PromptList]) -> PromptList: + """Implements addition when the PromptList is on the right side of the + '+' operator. + + Args: + other (str or PromptList): The string or PromptList to be added. + + Returns: + PromptList: A new PromptList that is the result of the addition. + """ + if not other: + return PromptList([*self]) + if isinstance(other, str): + return PromptList([other, *self]) + else: + return PromptList(other + self) + + def __iadd__(self, other: Union[str, PromptList]) -> PromptList: + """Implements in-place addition for the PromptList. + + Args: + other (str or PromptList): The string or PromptList to be added. + + Returns: + PromptList: The updated PromptList. + """ + if not other: + return self + if isinstance(other, str): + self.append(other) + else: + super().__iadd__(other) + return self + + def __str__(self) -> str: + """Converts the PromptList into a string. + + Returns: + str: The string representation of the PromptList. + + Raises: + TypeError: If there's an item in the PromptList that is not a + string or dictionary. + """ + res = [] + for item in self: + if isinstance(item, str): + res.append(item) + elif isinstance(item, dict): + if 'prompt' in item: + res.append(item['prompt']) + else: + raise TypeError('Invalid type in prompt list when ' + 'converting to string') + return ''.join(res) diff --git a/opencompass/utils/summarizer.py b/opencompass/utils/summarizer.py new file mode 100644 index 00000000..26a89eb1 --- /dev/null +++ b/opencompass/utils/summarizer.py @@ -0,0 +1,232 @@ +# flake8: noqa +# yapf: disable +import getpass +import os.path as osp +from datetime import datetime + +import mmengine +import tabulate +from mmengine import ConfigDict + +from opencompass.utils import (LarkReporter, dataset_abbr_from_cfg, + get_infer_output_path, get_logger, + model_abbr_from_cfg) +from opencompass.utils.prompt import get_prompt_hash + +METRIC_WHITELIST = ['score', 'accuracy', 'humaneval_pass@1', 'rouge1', 'avg_toxicity_score', 'bleurt_diff', 'matthews_correlation', 'truth'] +METRIC_BLACKLIST = ['bp', 'sys_len', 'ref_len'] + +class Summarizer: + """""" + + def __init__(self, config: ConfigDict) -> None: + self.tasks = [] + self.cfg = config + self.logger = get_logger() + + # Enable lark bot if lark_url is presented + self.lark_reporter = None + if self.cfg.get('lark_bot_url', None): + self.lark_reporter = LarkReporter(self.cfg['lark_bot_url']) + + def summarize( + self, + output_path: str = None, + time_str: str = datetime.now().strftime('%Y%m%d_%H%M%S')): # noqa + + model_cfgs = self.cfg['models'] + dataset_cfgs = self.cfg['datasets'] + summarizer_cfg = self.cfg.get('summarizer', {}) + work_dir = self.cfg['work_dir'] + + # pick up results + raw_results = {} + parsed_results = {} + dataset_metrics = {} + + model_abbrs = [model_abbr_from_cfg(model) for model in model_cfgs] + for model in model_cfgs: + model_abbr = model_abbr_from_cfg(model) + parsed_results[model_abbr] = {} + raw_results[model_abbr] = {} + for dataset in dataset_cfgs: + dataset_abbr = dataset_abbr_from_cfg(dataset) + filepath = get_infer_output_path(model, dataset, osp.join(work_dir, 'results')) + if not osp.exists(filepath): + continue + result = mmengine.load(filepath) + raw_results[model_abbr][dataset_abbr] = result + if 'error' in result: + self.debug(f'error in {model_abbr} {dataset_abbr} {result["error"]}') + continue + else: + parsed_results[model_abbr][dataset_abbr] = [] + dataset_metrics[dataset_abbr] = [] + for metric, score in result.items(): + if metric not in METRIC_BLACKLIST and isinstance(score, (int, float)): + parsed_results[model_abbr][dataset_abbr].append(score) + dataset_metrics[dataset_abbr].append(metric) + else: + continue + if len(parsed_results[model_abbr][dataset_abbr]) == 0: + self.logger.warning(f'unknown result format: {result}, continue') + del parsed_results[model_abbr][dataset_abbr] + del dataset_metrics[dataset_abbr] + continue + indice = sorted( + list(range(len(dataset_metrics[dataset_abbr]))), + key=lambda i: ( + METRIC_WHITELIST.index(dataset_metrics[dataset_abbr][i]) + if dataset_metrics[dataset_abbr][i] in METRIC_WHITELIST + else len(METRIC_WHITELIST) + ) + ) + parsed_results[model_abbr][dataset_abbr] = [parsed_results[model_abbr][dataset_abbr][i] for i in indice] + dataset_metrics[dataset_abbr] = [dataset_metrics[dataset_abbr][i] for i in indice] + + # parse eval mode + dataset_eval_mode = {} + for dataset in dataset_cfgs: + inferencer = dataset.get('infer_cfg', {}).get('inferencer', {}).get('type', '') + dataset_abbr = dataset_abbr_from_cfg(dataset) + if inferencer == 'GenInferencer': + dataset_eval_mode[dataset_abbr] = 'gen' + elif inferencer == 'PPLInferencer': + dataset_eval_mode[dataset_abbr] = 'ppl' + else: + dataset_eval_mode[dataset_abbr] = 'unknown' + self.logger.warning(f'unknown inferencer: {inferencer} - {dataset_abbr}') + + # calculate group metrics + summary_groups = summarizer_cfg.get('summary_groups', []) + for sg in summary_groups: + for model_abbr in model_abbrs: + results = {} + eval_modes = [] + for dataset_abbr in sg['subsets']: + if dataset_abbr in parsed_results[model_abbr]: + results[dataset_abbr] = parsed_results[model_abbr][dataset_abbr][0] + eval_modes.append(dataset_eval_mode.get(dataset_abbr, 'unknown')) + if len(results) == len(sg['subsets']): + if 'weights' in sg: + numerator = sum(results[k] * sg['weights'][k] for k in sg['weights']) + denominator = sum(sg['weights'].values()) + metric = 'weighted_average' + else: + numerator = sum(results[k] for k in results) + denominator = len(results) + metric = 'naive_average' + results[metric] = numerator / denominator + eval_modes = list(set(eval_modes)) + eval_mode = eval_modes[0] if len(eval_modes) == 1 else 'mixed' + + # add to global results + raw_results[model_abbr][sg['name']] = results + parsed_results[model_abbr][sg['name']] = [numerator / denominator] + dataset_metrics[sg['name']] = [metric] + dataset_eval_mode[sg['name']] = eval_mode + elif len(results) == 0: + continue + else: + raw_results[model_abbr][sg['name']] = {'error': 'missing datasets: {}'.format(set(sg['subsets']) - set(results.keys()))} + + prompt_version = {dataset_abbr_from_cfg(d): get_prompt_hash(d) for d in dataset_cfgs} + + # format table + summarizer_dataset_abbrs = [] + if summarizer_cfg.get('dataset_abbrs') is None: + for dataset in dataset_cfgs: + dataset_abbr = dataset_abbr_from_cfg(dataset) + if dataset_abbr in dataset_metrics: + for metric in dataset_metrics[dataset_abbr]: + summarizer_dataset_abbrs.append((dataset_abbr, metric)) + else: + summarizer_dataset_abbrs.append((dataset_abbr, None)) + for dataset_abbr in dataset_metrics: + for metric in dataset_metrics[dataset_abbr]: + if (dataset_abbr, metric) not in summarizer_dataset_abbrs: + summarizer_dataset_abbrs.append((dataset_abbr, metric)) + else: + for item in summarizer_cfg['dataset_abbrs']: + if isinstance(item, str): + summarizer_dataset_abbrs.append((item, None)) + elif isinstance(item, (list, tuple)): + summarizer_dataset_abbrs.append((item[0], item[1])) + + table = [] + header = ['dataset', 'version', 'metric', 'mode'] + model_abbrs + table.append(header) + for dataset_abbr, metric in summarizer_dataset_abbrs: + if dataset_abbr not in dataset_metrics: + table.append([dataset_abbr, '-', '-', '-'] + ['-'] * len(model_abbrs)) + continue + if metric is None: + index = 0 + metric = dataset_metrics[dataset_abbr][0] + elif metric in dataset_metrics[dataset_abbr]: + index = dataset_metrics[dataset_abbr].index(metric) + else: + table.append([dataset_abbr, '-', '-', '-'] + ['-'] * len(model_abbrs)) + continue + + row = [dataset_abbr, prompt_version.get(dataset_abbr, '-'), metric, dataset_eval_mode.get(dataset_abbr, '-')] + for model_abbr in model_abbrs: + if dataset_abbr in parsed_results[model_abbr]: + row.append('{:.02f}'.format(parsed_results[model_abbr][dataset_abbr][index])) + else: + row.append('-') + table.append(row) + + # format raw txt + raw_dataset_abbrs = [] + for model_abbr in model_abbrs: + for dataset_abbr in raw_results[model_abbr]: + if dataset_abbr not in raw_dataset_abbrs: + raw_dataset_abbrs.append(dataset_abbr) + raw_txts = [] + for model_abbr in model_abbrs: + raw_txts.append('-------------------------------') + raw_txts.append(f'Model: {model_abbr}') + for dataset_abbr in raw_dataset_abbrs: + result = raw_results[model_abbr].get(dataset_abbr, '{}') + raw_txts.append(f'{dataset_abbr}: {result}') + raw_txts = '\n'.join(raw_txts) + + # output to screean + print(tabulate.tabulate(table, headers='firstrow')) + + # output to file + if output_path is None: + output_path = osp.join(work_dir, 'summary', f'summary_{time_str}.txt') + output_csv_path = osp.join(work_dir, 'summary', f'summary_{time_str}.csv') + else: + output_csv_path = output_path.replace('.txt', '.csv') + + output_dir = osp.split(output_path)[0] + mmengine.mkdir_or_exist(output_dir) + with open(output_path, 'w') as f: + f.write(time_str + '\n') + f.write('tabulate format\n') + f.write('^' * 128 + '\n') + f.write(tabulate.tabulate(table, headers='firstrow') + '\n') + f.write('$' * 128 + '\n') + f.write('\n' + '-' * 128 + ' THIS IS A DIVIDER ' + '-' * 128 + '\n\n') + f.write('csv format\n') + f.write('^' * 128 + '\n') + f.write('\n'.join([','.join(row) for row in table]) + '\n') + f.write('$' * 128 + '\n') + f.write('\n' + '-' * 128 + ' THIS IS A DIVIDER ' + '-' * 128 + '\n\n') + f.write('raw format\n') + f.write('^' * 128 + '\n') + f.write(raw_txts + '\n') + f.write('$' * 128 + '\n') + self.logger.info(f'write summary to {osp.abspath(output_path)}') + + if self.lark_reporter: + content = f'{getpass.getuser()} 的' + content += f'详细评测汇总已输出至 {osp.abspath(output_path)}' + self.lark_reporter.post(content) + + with open(output_csv_path, 'w') as f: + f.write('\n'.join([','.join(row) for row in table]) + '\n') + self.logger.info(f'write csv to {osp.abspath(output_csv_path)}') diff --git a/opencompass/utils/text_postprocessors.py b/opencompass/utils/text_postprocessors.py new file mode 100644 index 00000000..d22dbd7b --- /dev/null +++ b/opencompass/utils/text_postprocessors.py @@ -0,0 +1,56 @@ +import re + +from opencompass.registry import TEXT_POSTPROCESSORS + + +@TEXT_POSTPROCESSORS.register_module('general') +def general_postprocess(text: str) -> str: + # Cut off the first newline, period, or comma + truncated_text = re.split(r'[\n.,]', text, 1)[0] + + # Remove punctuation + no_punctuation = re.sub(r'[^\w\s]', '', truncated_text) + + # Remove article + no_articles = re.sub(r'\b(a|an|the)\b', + '', + no_punctuation, + flags=re.IGNORECASE) + + # Remove duplicated blank spaces + cleaned_text = re.sub(r'\s+', ' ', no_articles).strip() + + return cleaned_text + + +@TEXT_POSTPROCESSORS.register_module('general_cn') +def general_cn_postprocess(text: str) -> str: + truncated_text = re.split(r'[\n.,]', text, 1)[0] + + no_punctuation = re.sub(r'[^\w\s]', '', truncated_text) + + no_articles = re.sub(r'\b(a|an|the)\b', + '', + no_punctuation, + flags=re.IGNORECASE) + + cleaned_text = re.sub(r'\s+', ' ', no_articles).strip() + import jieba + cleaned_text = ' '.join(jieba.cut(text)) + return cleaned_text + + +@TEXT_POSTPROCESSORS.register_module('first-capital') +def first_capital_postprocess(text: str) -> str: + for t in text: + if t.isupper(): + return t + return '' + + +@TEXT_POSTPROCESSORS.register_module('first-capital-multi') +def first_capital_postprocess_multi(text: str) -> str: + match = re.search(r'([A-D]+)', text) + if match: + return match.group(1) + return '' diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 00000000..4a612041 --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1,10 @@ +docutils==0.18.1 +modelindex +myst-parser +-e git+https://github.com/Ezra-Yu/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme +sphinx==6.1.3 +sphinx-copybutton +sphinx-notfound-page +sphinx-tabs +sphinxcontrib-jquery +tabulate diff --git a/run.py b/run.py new file mode 100644 index 00000000..57db2803 --- /dev/null +++ b/run.py @@ -0,0 +1,250 @@ +import argparse +import getpass +import os +import os.path as osp +from datetime import datetime + +from mmengine.config import Config + +from opencompass.partitioners import NaivePartitioner, SizePartitioner +from opencompass.runners import DLCRunner, LocalRunner, SlurmRunner +from opencompass.utils import LarkReporter, Summarizer, get_logger + + +def parse_args(): + parser = argparse.ArgumentParser(description='Run an evaluation task') + parser.add_argument('config', help='Train config file path') + # add mutually exclusive args `--slurm` `--dlc`, default to local runner + luach_method = parser.add_mutually_exclusive_group() + luach_method.add_argument('--slurm', + action='store_true', + default=False, + help='Whether to use srun to launch tasks, if ' + 'True, `--partition(-p)` must be set. Defaults' + ' to False') + luach_method.add_argument('--dlc', + action='store_true', + default=False, + help='Whether to use dlc to launch tasks, if ' + 'True, `--aliyun-cfg` must be set. Defaults' + ' to False') + # add general args + parser.add_argument('--debug', + help='Debug mode, in which scheduler will run tasks ' + 'in the single process, and output will not be ' + 'redirected to files', + action='store_true', + default=False) + parser.add_argument('-m', + '--mode', + help='Running mode. You can choose "infer" if you ' + 'only want the inference results, or "eval" if you ' + 'already have the results and want to evaluate them, ' + 'or "viz" if you want to visualize the results.', + choices=['all', 'infer', 'eval', 'viz'], + default='all', + type=str) + parser.add_argument('-r', + '--reuse', + nargs='?', + type=str, + const='latest', + help='Reuse previous outputs & results, and run any ' + 'missing jobs presented in the config. If its ' + 'argument is not specified, the latest results in ' + 'the work_dir will be reused. The argument should ' + 'also be a specific timestamp, e.g. 20230516_144254'), + parser.add_argument('-w', + '--work-dir', + help='Work path, all the outputs will be saved in ' + 'this path, including the slurm logs, the evaluation' + ' results, the summary results, etc. If not specified,' + ' the work_dir will be set to None', + default=None, + type=str) + parser.add_argument('-l', + '--lark', + help='Report the running status to lark bot', + action='store_true', + default=False) + parser.add_argument('--max-partition-size', + help='The maximum size of a task.', + type=int, + default=2000), + parser.add_argument( + '--gen-task-coef', + help='The dataset cost measurement coefficient for generation tasks', + type=int, + default=20) + parser.add_argument('--max-num-workers', + help='Max number of workers to run in parallel.', + type=int, + default=32) + parser.add_argument( + '--retry', + help='Number of retries if the job failed when using slurm or dlc.', + type=int, + default=2) + # set srun args + slurm_parser = parser.add_argument_group('slurm_args') + parse_slurm_args(slurm_parser) + # set dlc args + dlc_parser = parser.add_argument_group('dlc_args') + parse_dlc_args(dlc_parser) + args = parser.parse_args() + if args.slurm: + assert args.partition is not None, ( + '--partition(-p) must be set if you want to use slurm') + if args.dlc: + assert os.path.exists(args.aliyun_cfg), ( + 'When luaching tasks using dlc, it needs to be configured' + 'in "~/.aliyun.cfg", or use "--aliyun-cfg $ALiYun-CFG_Path"' + ' to specify a new path.') + return args + + +def parse_slurm_args(slurm_parser): + """these args are all for slurm launch.""" + slurm_parser.add_argument('-p', + '--partition', + help='Slurm partition name', + default=None, + type=str) + slurm_parser.add_argument('-q', + '--quotatype', + help='Slurm quota type', + default='auto', + type=str) + + +def parse_dlc_args(dlc_parser): + """these args are all for dlc launch.""" + dlc_parser.add_argument('--aliyun-cfg', + help='The config path for aliyun config', + default='~/.aliyun.cfg', + type=str) + + +def main(): + args = parse_args() + + # initialize logger + logger = get_logger(log_level='DEBUG' if args.debug else 'INFO') + + cfg = Config.fromfile(args.config) + if args.work_dir is not None: + cfg['work_dir'] = args.work_dir + else: + cfg.setdefault('work_dir', './outputs/default/') + + # cfg_time_str defaults to the current time + cfg_time_str = dir_time_str = datetime.now().strftime('%Y%m%d_%H%M%S') + if args.reuse: + if args.reuse == 'latest': + dirs = os.listdir(cfg.work_dir) + assert len(dirs) > 0, 'No previous results to reuse!' + dir_time_str = sorted(dirs)[-1] + else: + dir_time_str = args.reuse + logger.info(f'Reusing experiements from {dir_time_str}') + elif args.mode in ['eval', 'viz']: + raise ValueError('You must specify -r or --reuse when running in eval ' + 'or viz mode!') + + # update "actual" work_dir + cfg['work_dir'] = osp.join(cfg.work_dir, dir_time_str) + os.makedirs(osp.join(cfg.work_dir, 'configs'), exist_ok=True) + + # dump config + output_config_path = osp.join(cfg.work_dir, 'configs', + f'{cfg_time_str}.py') + cfg.dump(output_config_path) + # Config is intentally reloaded here to avoid initialized + # types cannot be serialized + cfg = Config.fromfile(output_config_path) + + # report to lark bot if specify --lark + if not args.lark: + cfg['lark_bot_url'] = None + elif cfg.get('lark_bot_url', None): + content = f'{getpass.getuser()}\'s task has been launched!' + LarkReporter(cfg['lark_bot_url']).post(content) + + if args.mode in ['all', 'infer']: + # Use SizePartitioner to split into subtasks + partitioner = SizePartitioner(osp.join(cfg['work_dir'], + 'predictions/'), + max_task_size=args.max_partition_size, + gen_task_coef=args.gen_task_coef) + tasks = partitioner(cfg) + # execute the infer subtasks + exec_infer_runner(tasks, args, cfg) + + # evaluate + if args.mode in ['all', 'eval']: + # Use NaivePartitioner，not split + partitioner = NaivePartitioner(osp.join(cfg['work_dir'], 'results/')) + tasks = partitioner(cfg) + # execute the eval tasks + exec_eval_runner(tasks, args, cfg) + + # visualize + if args.mode in ['all', 'eval', 'viz']: + summarizer = Summarizer(cfg) + summarizer.summarize(time_str=cfg_time_str) + + +def exec_infer_runner(tasks, args, cfg): + """execute infer runner according to args.""" + if args.slurm: + runner = SlurmRunner(dict(type='OpenICLInferTask'), + max_num_workers=args.max_num_workers, + partition=args.partition, + quotatype=args.quotatype, + retry=args.retry, + debug=args.debug, + lark_bot_url=cfg['lark_bot_url']) + elif args.dlc: + runner = DLCRunner(dict(type='OpenICLInferTask'), + max_num_workers=args.max_num_workers, + aliyun_cfg=Config.fromfile(args.aliyun_cfg), + retry=args.retry, + debug=args.debug, + lark_bot_url=cfg['lark_bot_url']) + else: + runner = LocalRunner( + task=dict(type='OpenICLInferTask'), + # max_num_workers = args.max_num_workers, + debug=args.debug, + lark_bot_url=cfg['lark_bot_url']) + runner(tasks) + + +def exec_eval_runner(tasks, args, cfg): + """execute infer runner according to args.""" + if args.slurm: + runner = SlurmRunner(dict(type='OpenICLEvalTask'), + max_num_workers=args.max_num_workers, + partition=args.partition, + quotatype=args.quotatype, + retry=args.retry, + debug=args.debug, + lark_bot_url=cfg['lark_bot_url']) + elif args.dlc: + runner = DLCRunner(dict(type='OpenICLEvalTask'), + max_num_workers=args.max_num_workers, + aliyun_cfg=Config.fromfile(args.aliyun_cfg), + retry=args.retry, + debug=args.debug, + lark_bot_url=cfg['lark_bot_url']) + else: + runner = LocalRunner( + task=dict(type='OpenICLEvalTask'), + # max_num_workers = args.max_num_workers, + debug=args.debug, + lark_bot_url=cfg['lark_bot_url']) + runner(tasks) + + +if __name__ == '__main__': + main() diff --git a/tests/openicl/test_prompt_template.py b/tests/openicl/test_prompt_template.py new file mode 100644 index 00000000..a831a20f --- /dev/null +++ b/tests/openicl/test_prompt_template.py @@ -0,0 +1,270 @@ +import unittest + +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.utils.prompt import PromptList + + +class TestPromptTemplate(unittest.TestCase): + + def setUp(self) -> None: + self.qa_template = dict(begin=[ + dict(role='SYSTEM', fallback_role='HUMAN', prompt='instruct'), + '', + ], + round=[ + dict(role='HUMAN', prompt=''), + dict(role='BOT', + prompt='Answer: ') + ]) + self.multiround_qa_template = dict(round=[ + dict(role='HUMAN', prompt=''), + dict(role='BOT', prompt='A1', end='\n'), + dict(role='HUMAN', prompt='Q1'), + dict(role='BOT', prompt='A2', end='\n\n'), + dict(role='HUMAN', prompt='Q2', begin='HUMAN:'), + dict(role='BOT', prompt='Answer: ') + ]) + self.column_token_map = { + 'input': '', + 'answer': '', + } + self.entry = {'input': 'Hello, how are you?', 'answer': 'Good.'} + + def test_init(self): + template = 'Translate the following English text to French: {t}.' + column_token_map = {'input': '{t}'} + pt = PromptTemplate(template, column_token_map) + + self.assertEqual(pt.template, template) + self.assertEqual(pt.column_token_map, column_token_map) + + def test_generate_ice_item(self): + # Test simple prompt + template = 'Translate the following English text to French: {t}.' + column_token_map = {'input': '{t}'} + pt = PromptTemplate(template, column_token_map) + label = None + ice = pt.generate_ice_item(self.entry, label) + + self.assertEqual(ice, + ('Translate the following English text to French: ' + 'Hello, how are you?.')) + + # test meta prompt style + pt = PromptTemplate(self.qa_template, + self.column_token_map, + ice_token='') + label = None + ice = pt.generate_ice_item(self.entry, label) + + ice_target = PromptList([ + { + 'section': 'ice', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='Hello, how are you?'), + dict(role='BOT', prompt='Answer: Good.'), + { + 'section': 'ice', + 'pos': 'end' + }, + ]) + self.assertEqual(ice, ice_target) + + # test_multiround + pt = PromptTemplate(self.multiround_qa_template, + self.column_token_map, + ice_token='') + label = None + ice = pt.generate_ice_item(self.entry, label) + + ice_target = PromptList([ + { + 'section': 'ice', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='Hello, how are you?'), + dict(role='BOT', prompt='A1', end='\n'), + dict(role='HUMAN', prompt='Q1'), + dict(role='BOT', prompt='A2', end='\n\n'), + dict(role='HUMAN', prompt='Q2', begin='HUMAN:'), + dict(role='BOT', prompt='Answer: Good.'), + { + 'section': 'ice', + 'pos': 'end' + }, + ]) + self.assertEqual(ice, ice_target) + + def test_generate_label_prompt_item(self): + # Test simple prompt + template = ' Translate the following English text to French: {t}.' + column_token_map = {'input': '{t}'} + pt = PromptTemplate(template, column_token_map, ice_token='') + ice = 'ICE' + label = None + prompt = pt.generate_label_prompt_item(self.entry, ice, label) + + self.assertEqual( + prompt, ('ICE Translate the following English text to French: ' + 'Hello, how are you?.')) + + ice = PromptList([ + { + 'section': 'ice', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='h1'), + dict(role='BOT', prompt='b1'), + { + 'section': 'ice', + 'pos': 'end' + }, + ]) + + # test meta prompt style + pt = PromptTemplate(self.qa_template, + self.column_token_map, + ice_token='') + label = None + prompt = pt.generate_label_prompt_item(self.entry, ice, label) + target = PromptList([ + { + 'section': 'begin', + 'pos': 'begin' + }, + dict(role='SYSTEM', fallback_role='HUMAN', prompt='instruct'), + { + 'section': 'ice', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='h1'), + dict(role='BOT', prompt='b1'), + { + 'section': 'ice', + 'pos': 'end' + }, + { + 'section': 'begin', + 'pos': 'end' + }, + { + 'section': 'round', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='Hello, how are you?'), + dict(role='BOT', prompt='Answer: Good.'), + { + 'section': 'round', + 'pos': 'end' + }, + ]) + self.assertEqual(prompt, target) + + # test_multiround + pt = PromptTemplate(self.multiround_qa_template, + self.column_token_map, + ice_token='') + label = None + prompt = pt.generate_label_prompt_item(self.entry, ice, label) + target = PromptList([ + { + 'section': 'round', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='Hello, how are you?'), + dict(role='BOT', prompt='A1', end='\n'), + dict(role='HUMAN', prompt='Q1'), + dict(role='BOT', prompt='A2', end='\n\n'), + dict(role='HUMAN', prompt='Q2', begin='HUMAN:'), + dict(role='BOT', prompt='Answer: Good.'), + { + 'section': 'round', + 'pos': 'end' + }, + ]) + self.assertEqual(prompt, target) + + def test_generate_item(self): + # Test simple prompt + template = 'Translate the following English text to French: {t}.' + column_token_map = {'input': '{t}'} + pt = PromptTemplate(template, column_token_map) + item = pt.generate_item(self.entry) + + self.assertEqual(item, + ('Translate the following English text to French: ' + 'Hello, how are you?.')) + + ice = PromptList([ + { + 'section': 'ice', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='h1'), + dict(role='BOT', prompt='b1'), + { + 'section': 'ice', + 'pos': 'end' + }, + ]) + + # test meta prompt (without system role) + pt = PromptTemplate(self.qa_template, + self.column_token_map, + ice_token='') + prompt = pt.generate_item(self.entry, ice_field_replace_token=ice) + target = PromptList([ + { + 'section': 'begin', + 'pos': 'begin' + }, + dict(role='SYSTEM', fallback_role='HUMAN', prompt='instruct'), + { + 'section': 'ice', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='h1'), + dict(role='BOT', prompt='b1'), + { + 'section': 'ice', + 'pos': 'end' + }, + { + 'section': 'begin', + 'pos': 'end' + }, + { + 'section': 'round', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='Hello, how are you?'), + dict(role='BOT', prompt='Answer: Good.'), + { + 'section': 'round', + 'pos': 'end' + }, + ]) + self.assertEqual(prompt, target) + + pt = PromptTemplate(self.multiround_qa_template, + self.column_token_map, + ice_token='') + prompt = pt.generate_item(self.entry, ice_field_replace_token=ice) + target = PromptList([ + { + 'section': 'round', + 'pos': 'begin' + }, + dict(role='HUMAN', prompt='Hello, how are you?'), + dict(role='BOT', prompt='A1', end='\n'), + dict(role='HUMAN', prompt='Q1'), + dict(role='BOT', prompt='A2', end='\n\n'), + dict(role='HUMAN', prompt='Q2', begin='HUMAN:'), + dict(role='BOT', prompt='Answer: Good.'), + { + 'section': 'round', + 'pos': 'end' + }, + ]) + self.assertEqual(prompt, target) diff --git a/tests/prompt/test_api_template_parser.py b/tests/prompt/test_api_template_parser.py new file mode 100644 index 00000000..9fbda079 --- /dev/null +++ b/tests/prompt/test_api_template_parser.py @@ -0,0 +1,231 @@ +import unittest + +from opencompass.models.base_api import APITemplateParser +from opencompass.utils.prompt import PromptList + + +class TestAPITemplateParser(unittest.TestCase): + + def setUp(self): + self.parser = APITemplateParser() + self.prompt = PromptList([ + { + 'section': 'begin', + 'pos': 'begin' + }, + 'begin', + { + 'role': 'SYSTEM', + 'fallback_role': 'HUMAN', + 'prompt': 'system msg' + }, + { + 'section': 'ice', + 'pos': 'begin' + }, + { + 'role': 'HUMAN', + 'prompt': 'U0' + }, + { + 'role': 'BOT', + 'prompt': 'B0' + }, + { + 'section': 'ice', + 'pos': 'end' + }, + { + 'section': 'begin', + 'pos': 'end' + }, + { + 'section': 'round', + 'pos': 'begin' + }, + { + 'role': 'HUMAN', + 'prompt': 'U1' + }, + { + 'role': 'BOT', + 'prompt': 'B1' + }, + { + 'role': 'HUMAN', + 'prompt': 'U2' + }, + { + 'role': 'BOT', + 'prompt': 'B2' + }, + { + 'section': 'round', + 'pos': 'end' + }, + { + 'section': 'end', + 'pos': 'begin' + }, + 'end', + { + 'section': 'end', + 'pos': 'end' + }, + ]) + + def test_parse_template_str_input(self): + prompt = self.parser.parse_template('Hello, world!', mode='gen') + self.assertEqual(prompt, 'Hello, world!') + prompt = self.parser.parse_template('Hello, world!', mode='ppl') + self.assertEqual(prompt, 'Hello, world!') + + def test_parse_template_list_input(self): + prompt = self.parser.parse_template(['Hello', 'world'], mode='gen') + self.assertEqual(prompt, ['Hello', 'world']) + prompt = self.parser.parse_template(['Hello', 'world'], mode='ppl') + self.assertEqual(prompt, ['Hello', 'world']) + + def test_parse_template_PromptList_input_no_meta_template(self): + prompt = self.parser.parse_template(self.prompt, mode='gen') + self.assertEqual(prompt, + 'begin\nsystem msg\nU0\nB0\nU1\nB1\nU2\nB2\nend') + prompt = self.parser.parse_template(self.prompt, mode='ppl') + self.assertEqual(prompt, + 'begin\nsystem msg\nU0\nB0\nU1\nB1\nU2\nB2\nend') + + def test_parse_template_PromptList_input_with_meta_template(self): + parser = APITemplateParser(meta_template=dict(round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True) + ], )) + with self.assertWarns(Warning): + prompt = parser.parse_template(self.prompt, mode='gen') + self.assertEqual( + prompt, + PromptList([ + { + 'role': 'HUMAN', + 'prompt': 'system msg\nU0' + }, + { + 'role': 'BOT', + 'prompt': 'B0' + }, + { + 'role': 'HUMAN', + 'prompt': 'U1' + }, + { + 'role': 'BOT', + 'prompt': 'B1' + }, + { + 'role': 'HUMAN', + 'prompt': 'U2' + }, + ])) + with self.assertWarns(Warning): + prompt = parser.parse_template(self.prompt, mode='ppl') + self.assertEqual( + prompt, + PromptList([ + { + 'role': 'HUMAN', + 'prompt': 'system msg\nU0' + }, + { + 'role': 'BOT', + 'prompt': 'B0' + }, + { + 'role': 'HUMAN', + 'prompt': 'U1' + }, + { + 'role': 'BOT', + 'prompt': 'B1' + }, + { + 'role': 'HUMAN', + 'prompt': 'U2' + }, + { + 'role': 'BOT', + 'prompt': 'B2' + }, + ])) + + parser = APITemplateParser(meta_template=dict( + round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True) + ], + reserved_roles=[ + dict(role='SYSTEM', api_role='SYSTEM'), + ], + )) + with self.assertWarns(Warning): + prompt = parser.parse_template(self.prompt, mode='gen') + self.assertEqual( + prompt, + PromptList([ + { + 'role': 'SYSTEM', + 'prompt': 'system msg' + }, + { + 'role': 'HUMAN', + 'prompt': 'U0' + }, + { + 'role': 'BOT', + 'prompt': 'B0' + }, + { + 'role': 'HUMAN', + 'prompt': 'U1' + }, + { + 'role': 'BOT', + 'prompt': 'B1' + }, + { + 'role': 'HUMAN', + 'prompt': 'U2' + }, + ])) + with self.assertWarns(Warning): + prompt = parser.parse_template(self.prompt, mode='ppl') + self.assertEqual( + prompt, + PromptList([ + { + 'role': 'SYSTEM', + 'prompt': 'system msg' + }, + { + 'role': 'HUMAN', + 'prompt': 'U0' + }, + { + 'role': 'BOT', + 'prompt': 'B0' + }, + { + 'role': 'HUMAN', + 'prompt': 'U1' + }, + { + 'role': 'BOT', + 'prompt': 'B1' + }, + { + 'role': 'HUMAN', + 'prompt': 'U2' + }, + { + 'role': 'BOT', + 'prompt': 'B2' + }, + ])) diff --git a/tests/prompt/test_lm_template_parser.py b/tests/prompt/test_lm_template_parser.py new file mode 100644 index 00000000..308c48fd --- /dev/null +++ b/tests/prompt/test_lm_template_parser.py @@ -0,0 +1,235 @@ +import unittest + +from opencompass.models.base import LMTemplateParser +from opencompass.utils.prompt import PromptList + + +class TestLMTemplateParser(unittest.TestCase): + + def setUp(self): + self.parser = LMTemplateParser() + self.prompt = PromptList([ + { + 'section': 'begin', + 'pos': 'begin' + }, + 'begin', + { + 'role': 'SYSTEM', + 'fallback_role': 'HUMAN', + 'prompt': 'system msg' + }, + { + 'section': 'ice', + 'pos': 'begin' + }, + { + 'role': 'HUMAN', + 'prompt': 'U0' + }, + { + 'role': 'BOT', + 'prompt': 'B0' + }, + { + 'section': 'ice', + 'pos': 'end' + }, + { + 'section': 'begin', + 'pos': 'end' + }, + { + 'section': 'round', + 'pos': 'begin' + }, + { + 'role': 'HUMAN', + 'prompt': 'U1', + 'end': '\n' + }, + { + 'role': 'BOT', + 'prompt': 'B1' + }, + { + 'role': 'HUMAN', + 'prompt': 'U2' + }, + { + 'role': 'BOT', + 'prompt': 'B2' + }, + { + 'section': 'round', + 'pos': 'end' + }, + { + 'section': 'end', + 'pos': 'begin' + }, + 'end', + { + 'section': 'end', + 'pos': 'end' + }, + ]) + + def test_parse_template_str_input(self): + prompt = self.parser.parse_template('Hello, world!', mode='gen') + self.assertEqual(prompt, 'Hello, world!') + prompt = self.parser.parse_template('Hello, world!', mode='ppl') + self.assertEqual(prompt, 'Hello, world!') + + def test_parse_template_list_input(self): + prompt = self.parser.parse_template(['Hello', 'world'], mode='gen') + self.assertEqual(prompt, ['Hello', 'world']) + prompt = self.parser.parse_template(['Hello', 'world'], mode='ppl') + self.assertEqual(prompt, ['Hello', 'world']) + + def test_parse_template_PromptList_input_no_meta_template(self): + prompt = self.parser.parse_template(self.prompt, mode='gen') + self.assertEqual(prompt, + 'begin\nsystem msg\nU0\nB0\nU1\nB1\nU2\nB2\nend') + prompt = self.parser.parse_template(self.prompt, mode='ppl') + self.assertEqual(prompt, + 'begin\nsystem msg\nU0\nB0\nU1\nB1\nU2\nB2\nend') + + def test_parse_template_PromptList_input_with_meta_template(self): + # no SYSTEM role, early generation in THOUGHTS + parser = LMTemplateParser(meta_template=dict( + begin='meta instruction\n', + round=[ + dict(role='HUMAN', begin='<|HUMAN|>:', end='\n'), + dict(role='THOUGHTS', + begin='<|Inner Thoughts|>:', + generate=True, + end='\n', + prompt='None'), + dict(role='BOT', begin='<|BOT|>:', end='\n'), + ], + end='meta end', + )) + prompt = parser.parse_template(self.prompt, mode='gen') + target = ('meta instruction\n' + 'begin' + '<|HUMAN|>:system msg\n' + '<|HUMAN|>:U0\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B0\n' + '<|HUMAN|>:U1\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B1\n' + '<|HUMAN|>:U2\n' + '<|Inner Thoughts|>:') + self.assertEqual(prompt, target) + prompt = parser.parse_template(self.prompt, mode='ppl') + target = ('meta instruction\n' + 'begin' + '<|HUMAN|>:system msg\n' + '<|HUMAN|>:U0\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B0\n' + '<|HUMAN|>:U1\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B1\n' + '<|HUMAN|>:U2\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B2\n' + 'end' + 'meta end') + self.assertEqual(prompt, target) + + # no SYSTEM role, generation in BOT + parser = LMTemplateParser(meta_template=dict( + begin='meta instruction\n', + round=[ + dict(role='HUMAN', begin='<|HUMAN|>:', end='\n'), + dict(role='THOUGHTS', + begin='<|Inner Thoughts|>:', + end='\n', + prompt='None'), + dict( + role='BOT', begin='<|BOT|>:', end='\n', + generate=True), + ], + end='meta end', + )) + prompt = parser.parse_template(self.prompt, mode='gen') + target = ('meta instruction\n' + 'begin' + '<|HUMAN|>:system msg\n' + '<|HUMAN|>:U0\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B0\n' + '<|HUMAN|>:U1\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B1\n' + '<|HUMAN|>:U2\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:') + self.assertEqual(prompt, target) + prompt = parser.parse_template(self.prompt, mode='ppl') + target = ('meta instruction\n' + 'begin' + '<|HUMAN|>:system msg\n' + '<|HUMAN|>:U0\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B0\n' + '<|HUMAN|>:U1\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B1\n' + '<|HUMAN|>:U2\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B2\n' + 'end' + 'meta end') + self.assertEqual(prompt, target) + + # with SYSTEM role, generation in BOT + parser = LMTemplateParser(meta_template=dict( + begin='meta instruction\n', + round=[ + dict(role='HUMAN', begin='<|HUMAN|>:', end='\n'), + dict(role='THOUGHTS', + begin='<|Inner Thoughts|>:', + end='\n', + prompt='None'), + dict( + role='BOT', begin='<|BOT|>:', end='\n', + generate=True), + ], + end='meta end', + reserved_roles=[ + dict(role='SYSTEM', begin='<|SYSTEM|>:', end='\n') + ])) + prompt = parser.parse_template(self.prompt, mode='gen') + target = ('meta instruction\n' + 'begin' + '<|SYSTEM|>:system msg\n' + '<|HUMAN|>:U0\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B0\n' + '<|HUMAN|>:U1\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B1\n' + '<|HUMAN|>:U2\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:') + self.assertEqual(prompt, target) + prompt = parser.parse_template(self.prompt, mode='ppl') + target = ('meta instruction\n' + 'begin' + '<|SYSTEM|>:system msg\n' + '<|HUMAN|>:U0\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B0\n' + '<|HUMAN|>:U1\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B1\n' + '<|HUMAN|>:U2\n' + '<|Inner Thoughts|>:None\n' + '<|BOT|>:B2\n' + 'end' + 'meta end') + self.assertEqual(prompt, target) diff --git a/tools/case_analyzer.py b/tools/case_analyzer.py new file mode 100644 index 00000000..02aad886 --- /dev/null +++ b/tools/case_analyzer.py @@ -0,0 +1,194 @@ +import argparse +import copy +import json +import os.path as osp + +import mmengine +from mmengine.config import Config, ConfigDict +from mmengine.utils import mkdir_or_exist +from tqdm import tqdm + +from opencompass.registry import TEXT_POSTPROCESSORS +from opencompass.utils import build_dataset_from_cfg, get_infer_output_path + + +def parse_args(): + parser = argparse.ArgumentParser(description='Run an evaluation task') + parser.add_argument('config', help='Train config file path') + parser.add_argument( + '-f', + '--force', + help='Force to run the task even if the results already exist', + action='store_true', + default=False) + parser.add_argument('-w', + '--work-dir', + help='Work path, all the outputs will be ' + 'saved in this path, including the slurm logs, ' + 'the evaluation results, the summary results, etc.' + 'If not specified, the work_dir will be set to ' + './outputs/default.', + default=None, + type=str) + args = parser.parse_args() + return args + + +class BadcaseShower: + """""" + + def __init__(self, cfg: ConfigDict) -> None: + + self.cfg = cfg + self.model_cfg = copy.deepcopy(self.cfg['model']) + self.dataset_cfg = copy.deepcopy(self.cfg['dataset']) + self.work_dir = self.cfg.get('work_dir') + # Load Dataset + self.eval_cfg = self.dataset_cfg.get('eval_cfg') + self.ds_split = self.eval_cfg.get('ds_split', None) + self.ds_column = self.eval_cfg.get('ds_column') + + def run(self): + filename = get_infer_output_path( + self.model_cfg, self.dataset_cfg, + osp.join(self.work_dir, 'predictions')) + root, ext = osp.splitext(filename) + partial_filename = root + '_0' + ext + + if not osp.exists(osp.realpath(filename)) and not osp.exists( + osp.realpath(partial_filename)): + print(f'{filename} not found') + return + + dataset = build_dataset_from_cfg(self.dataset_cfg) + # Postprocess dataset if necessary + if 'dataset_postprocessor' in self.eval_cfg: + + def postprocess(sample): + s = sample[self.ds_column] + proc = TEXT_POSTPROCESSORS.get( + self.eval_cfg['dataset_postprocessor']['type']) + sample[self.ds_column] = proc(s) + return sample + + dataset = dataset.map(postprocess) + + # Load predictions + if osp.exists(osp.realpath(filename)): + preds = mmengine.load(filename) + else: + filename = partial_filename + preds, offset = {}, 0 + i = 1 + while osp.exists(osp.realpath(filename)): + _preds = mmengine.load(filename) + filename = root + f'_{i}' + ext + i += 1 + for _o in range(len(_preds)): + preds[str(offset)] = _preds[str(_o)] + offset += 1 + pred_strs = [preds[str(i)]['prediction'] for i in range(len(preds))] + + # Postprocess predictions if necessary + if 'pred_postprocessor' in self.eval_cfg: + proc = TEXT_POSTPROCESSORS.get( + self.eval_cfg['pred_postprocessor']['type']) + pred_strs = [proc(s) for s in pred_strs] + + if self.ds_split: + references = dataset[self.ds_split][self.ds_column] + else: + references = dataset[self.ds_column] + + if len(pred_strs) != len(references): + print('length mismatch') + return + + # combine cases + allcase, badcase = [], [] + if 'in-context examples' in preds['0']: + # ppl eval + for i, (pred_str, + reference) in enumerate(zip(tqdm(pred_strs), references)): + ref_str = str(reference) + try: + pred_prompt = preds[str(i)]['label: ' + + pred_str]['testing input'] + pred_PPL = preds[str(i)]['label: ' + pred_str]['PPL'] + ref_prompt = preds[str(i)]['label: ' + + ref_str]['testing input'] + ref_PPL = preds[str(i)]['label: ' + ref_str]['PPL'] + except KeyError: + continue + item = { + 'prediction_prompt': pred_prompt, + 'prediction': pred_str, + 'prediction_PPL': pred_PPL, + 'reference_prompt': ref_prompt, + 'reference': ref_str, + 'reference_PPL': ref_PPL + } + if pred_str != ref_str: + badcase.append(item) + allcase.append(item) + else: + allcase.append(item) + + else: + # gen eval + for i, (pred_str, + reference) in enumerate(zip(tqdm(pred_strs), references)): + ref_str = str(reference) + origin_prompt = preds[str(i)]['origin_prompt'] + item = { + 'origin_prompt': origin_prompt, + 'prediction': pred_str, + 'reference': ref_str + } + # FIXME: we now consider all cases as bad cases + badcase.append(item) + allcase.append(item) + + # Save result + out_path = get_infer_output_path( + self.cfg['model'], self.cfg['dataset'], + osp.join(self.work_dir, 'case_analysis/bad')) + mkdir_or_exist(osp.split(out_path)[0]) + with open(out_path, 'w', encoding='utf-8') as f: + json.dump(badcase, f, indent=4, ensure_ascii=False) + + out_path = get_infer_output_path( + self.cfg['model'], self.cfg['dataset'], + osp.join(self.work_dir, 'case_analysis/all')) + mkdir_or_exist(osp.split(out_path)[0]) + with open(out_path, 'w', encoding='utf-8') as f: + json.dump(allcase, f, indent=4, ensure_ascii=False) + + +def dispatch_tasks(cfg, force=False): + for model in cfg['models']: + for dataset in cfg['datasets']: + if force or not osp.exists( + get_infer_output_path( + model, dataset, + osp.join(cfg['work_dir'], 'case_analysis/all'))): + BadcaseShower({ + 'model': model, + 'dataset': dataset, + 'work_dir': cfg['work_dir'] + }).run() + + +def main(): + args = parse_args() + cfg = Config.fromfile(args.config) + # set work_dir + if args.work_dir is not None: + cfg['work_dir'] = args.work_dir + else: + cfg.setdefault('work_dir', './outputs/default') + dispatch_tasks(cfg, force=args.force) + + +if __name__ == '__main__': + main() diff --git a/tools/cfg_run.py b/tools/cfg_run.py new file mode 100644 index 00000000..9beab509 --- /dev/null +++ b/tools/cfg_run.py @@ -0,0 +1,158 @@ +import argparse +import getpass +import os +import os.path as osp +from datetime import datetime + +from mmengine.config import Config + +from opencompass.registry import PARTITIONERS, RUNNERS +from opencompass.runners import SlurmRunner +from opencompass.utils import LarkReporter, Summarizer, get_logger + + +def parse_args(): + parser = argparse.ArgumentParser(description='Run an evaluation task') + parser.add_argument('config', help='Train config file path') + parser.add_argument('-p', + '--partition', + help='Slurm partition name', + default=None, + type=str) + parser.add_argument('-q', + '--quotatype', + help='Slurm quota type', + default='auto', + type=str) + parser.add_argument('--debug', + help='Debug mode, in which scheduler will run tasks ' + 'in the single process, and output will not be ' + 'redirected to files', + action='store_true', + default=False) + parser.add_argument('-m', + '--mode', + help='Running mode. You can choose "infer" if you ' + 'only want the inference results, or "eval" if you ' + 'already have the results and want to evaluate them, ' + 'or "viz" if you want to visualize the results.', + choices=['all', 'infer', 'eval', 'viz'], + default='all', + type=str) + parser.add_argument('-r', + '--reuse', + nargs='?', + type=str, + const='latest', + help='Reuse previous outputs & results, and run any ' + 'missing jobs presented in the config. If its ' + 'argument is not specified, the latest results in ' + 'the work_dir will be reused. The argument should ' + 'also be a specific timestamp, e.g. 20230516_144254'), + parser.add_argument('-w', + '--work-dir', + help='Work path, all the outputs will be ' + 'saved in this path, including the slurm logs, ' + 'the evaluation results, the summary results, etc.' + 'If not specified, the work_dir will be set to ' + './outputs/default.', + default=None, + type=str) + parser.add_argument('-l', + '--lark', + help='Report the running status to lark bot', + action='store_true', + default=False) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + # initialize logger + logger = get_logger(log_level='DEBUG' if args.debug else 'INFO') + + cfg = Config.fromfile(args.config) + if args.work_dir is not None: + cfg['work_dir'] = args.work_dir + else: + cfg.setdefault('work_dir', './outputs/default/') + + # cfg_time_str defaults to the current time + cfg_time_str = dir_time_str = datetime.now().strftime('%Y%m%d_%H%M%S') + if args.reuse: + if args.reuse == 'latest': + dirs = os.listdir(cfg.work_dir) + assert len(dirs) > 0, 'No previous results to reuse!' + dir_time_str = sorted(dirs)[-1] + else: + dir_time_str = args.reuse + logger.info(f'Reusing experiements from {dir_time_str}') + elif args.mode in ['eval', 'viz']: + raise ValueError('You must specify -r or --reuse when running in eval ' + 'or viz mode!') + # update "actual" work_dir + cfg['work_dir'] = osp.join(cfg.work_dir, dir_time_str) + os.makedirs(osp.join(cfg.work_dir, 'configs'), exist_ok=True) + # dump config + output_config_path = osp.join(cfg.work_dir, 'configs', + f'{cfg_time_str}.py') + cfg.dump(output_config_path) + # Config is intentally reloaded here to avoid initialized + # types cannot be serialized + cfg = Config.fromfile(output_config_path) + + # infer + if not args.lark: + cfg['lark_bot_url'] = None + elif cfg.get('lark_bot_url', None): + content = f'{getpass.getuser()} 的新任务已启动！' + LarkReporter(cfg['lark_bot_url']).post(content) + + if cfg.get('infer', None) is not None and args.mode in ['all', 'infer']: + if args.partition is not None: + if RUNNERS.get(cfg.infer.runner.type) == SlurmRunner: + cfg.infer.runner.partition = args.partition + cfg.infer.runner.quotatype = args.quotatype + else: + logger.warning('SlurmRunner is not used, so the partition ' + 'argument is ignored.') + if args.debug: + cfg.infer.runner.debug = True + if args.lark: + cfg.infer.runner.lark_bot_url = cfg['lark_bot_url'] + cfg.infer.partitioner['out_dir'] = osp.join(cfg['work_dir'], + 'predictions/') + partitioner = PARTITIONERS.build(cfg.infer.partitioner) + tasks = partitioner(cfg) + runner = RUNNERS.build(cfg.infer.runner) + runner(tasks) + + # evaluate + if cfg.get('eval', None) is not None and args.mode in ['all', 'eval']: + if args.partition is not None: + if RUNNERS.get(cfg.infer.runner.type) == SlurmRunner: + cfg.eval.runner.partition = args.partition + cfg.eval.runner.quotatype = args.quotatype + else: + logger.warning('SlurmRunner is not used, so the partition ' + 'argument is ignored.') + if args.debug: + cfg.eval.runner.debug = True + if args.lark: + cfg.eval.runner.lark_bot_url = cfg['lark_bot_url'] + cfg.eval.partitioner['out_dir'] = osp.join(cfg['work_dir'], 'results/') + partitioner = PARTITIONERS.build(cfg.eval.partitioner) + tasks = partitioner(cfg) + runner = RUNNERS.build(cfg.eval.runner) + runner(tasks) + + # visualize + if args.mode in ['all', 'eval', 'viz']: + summarizer = Summarizer(cfg) + summarizer.summarize(time_str=cfg_time_str) + + +if __name__ == '__main__': + main() diff --git a/tools/prompt_viewer.py b/tools/prompt_viewer.py new file mode 100644 index 00000000..65bff78d --- /dev/null +++ b/tools/prompt_viewer.py @@ -0,0 +1,216 @@ +import argparse +import fnmatch +from typing import Dict + +from mmengine.config import Config, ConfigDict + +from opencompass.openicl.icl_inferencer import GenInferencer, PPLInferencer +from opencompass.registry import ICL_PROMPT_TEMPLATES, ICL_RETRIEVERS +from opencompass.utils import (Menu, build_dataset_from_cfg, + build_model_from_cfg, dataset_abbr_from_cfg, + model_abbr_from_cfg) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Run an evaluation task') + parser.add_argument('config', help='Train config file path') + parser.add_argument('-n', '--non-interactive', action='store_true') + parser.add_argument('-a', '--all', action='store_true') + parser.add_argument('-p', + '--pattern', + type=str, + help='To match the dataset abbr.') + args = parser.parse_args() + return args + + +def parse_model_cfg(model_cfg: ConfigDict) -> Dict[str, ConfigDict]: + model2cfg = {} + for model in model_cfg: + model2cfg[model_abbr_from_cfg(model)] = model + return model2cfg + + +def parse_dataset_cfg(dataset_cfg: ConfigDict) -> Dict[str, ConfigDict]: + dataset2cfg = {} + for dataset in dataset_cfg: + dataset2cfg[dataset_abbr_from_cfg(dataset)] = dataset + return dataset2cfg + + +def print_prompts(model_cfg, dataset_cfg): + # TODO: A really dirty method that copies code from PPLInferencer and + # GenInferencer. In the future, the prompt extraction code should be + # extracted and generalized as a static method in these Inferencers + # and reused here. + if model_cfg: + max_seq_len = model_cfg.max_seq_len + if not model_cfg['type'].is_api: + model_cfg['tokenizer_only'] = True + model = build_model_from_cfg(model_cfg) + else: + max_seq_len = None + model = None + + infer_cfg = dataset_cfg.get('infer_cfg') + + fix_id_list = infer_cfg.inferencer.get('fix_id_list', []) + dataset = build_dataset_from_cfg(dataset_cfg) + + ice_template = None + if hasattr(infer_cfg, 'ice_template'): + ice_template = ICL_PROMPT_TEMPLATES.build(infer_cfg['ice_template']) + + prompt_template = None + if hasattr(infer_cfg, 'prompt_template'): + prompt_template = ICL_PROMPT_TEMPLATES.build( + infer_cfg['prompt_template']) + + infer_cfg['retriever']['dataset'] = dataset + retriever = ICL_RETRIEVERS.build(infer_cfg['retriever']) + + if fix_id_list: + ice_idx_list = retriever.retrieve(fix_id_list) + else: + ice_idx_list = retriever.retrieve() + + assert infer_cfg.inferencer.type in [PPLInferencer, GenInferencer], \ + 'Only PPLInferencer and GenInferencer are supported' + + if infer_cfg.inferencer.type == PPLInferencer: + labels = retriever.get_labels(ice_template=ice_template, + prompt_template=prompt_template) + ice = [ + retriever.generate_ice(ice_idx_list[idx], + ice_template=ice_template) + for idx in range(len(ice_idx_list)) + ] + print('-' * 100) + print('ICE Template:') + print('-' * 100) + print(ice[0]) + print('-' * 100) + for label in labels: + idx = 0 + prompt = retriever.generate_label_prompt( + idx, + ice[idx], + label, + ice_template=ice_template, + prompt_template=prompt_template, + remain_sep=None) + if max_seq_len is not None: + prompt_token_num = model.get_token_len_from_template(prompt) + while len(ice_idx_list[idx] + ) > 0 and prompt_token_num > max_seq_len: + num_ice = len(ice_idx_list[idx]) + print(f'Truncating ice {num_ice} -> {num_ice - 1}', + f'Number of tokens: {prompt_token_num} -> ...') + ice_idx_list[idx] = ice_idx_list[idx][:-1] + ice[idx] = retriever.generate_ice( + ice_idx_list[idx], ice_template=ice_template) + prompt = retriever.generate_label_prompt( + idx, + ice[idx], + label, + ice_template=ice_template, + prompt_template=prompt_template) + prompt_token_num = model.get_token_len_from_template( + prompt) + print(f'Number of tokens: {prompt_token_num}') + if model is not None: + prompt = model.parse_template(prompt, mode='ppl') + print('-' * 100) + print(f'Label: {label}') + print('Sample prompt:') + print('-' * 100) + print(prompt) + print('-' * 100) + elif infer_cfg.inferencer.type == GenInferencer: + idx, ice_idx = 0, ice_idx_list[0] + ice = retriever.generate_ice(ice_idx, ice_template=ice_template) + prompt = retriever.generate_prompt_for_generate_task( + idx, + ice, + gen_field_replace_token=infer_cfg.inferencer.get( + 'gen_field_replace_token', ''), + ice_template=ice_template, + prompt_template=prompt_template) + if max_seq_len is not None: + prompt_token_num = model.get_token_len_from_template(prompt) + while len(ice_idx) > 0 and prompt_token_num > max_seq_len: + num_ice = len(ice_idx) + print(f'Truncating ice {num_ice} -> {num_ice - 1}', + f'Number of tokens: {prompt_token_num} -> ...') + ice_idx = ice_idx[:-1] + ice = retriever.generate_ice(ice_idx, + ice_template=ice_template) + prompt = retriever.generate_prompt_for_generate_task( + idx, + ice, + gen_field_replace_token=infer_cfg.inferencer.get( + 'gen_field_replace_token', ''), + ice_template=ice_template, + prompt_template=prompt_template) + prompt_token_num = model.get_token_len_from_template(prompt) + print(f'Number of tokens: {prompt_token_num}') + if model is not None: + prompt = model.parse_template(prompt, mode='gen') + print('-' * 100) + print('Sample prompt:') + print('-' * 100) + print(prompt) + print('-' * 100) + + +def main(): + args = parse_args() + cfg = Config.fromfile(args.config) + # cfg.models = + model2cfg = parse_model_cfg(cfg.models) if 'models' in cfg else { + 'None': None + } + if 'datasets' in cfg: + dataset2cfg = parse_dataset_cfg(cfg.datasets) + else: + dataset2cfg = {} + for key in cfg.keys(): + if key.endswith('_datasets'): + dataset2cfg.update(parse_dataset_cfg(cfg[key])) + + if args.pattern is not None: + matches = fnmatch.filter(dataset2cfg, args.pattern) + if len(matches) == 0: + raise ValueError( + 'No dataset match the pattern. Please select from: \n' + + '\n'.join(dataset2cfg.keys())) + dataset2cfg = {k: dataset2cfg[k] for k in matches} + + if not args.all: + if not args.non_interactive: + model, dataset = Menu( + [list(model2cfg.keys()), + list(dataset2cfg.keys())], [ + f'Please make a selection of {s}:' + for s in ['model', 'dataset'] + ]).run() + else: + model = list(model2cfg.keys())[0] + dataset = list(dataset2cfg.keys())[0] + model_cfg = model2cfg[model] + dataset_cfg = dataset2cfg[dataset] + print_prompts(model_cfg, dataset_cfg) + else: + for model_abbr, model_cfg in model2cfg.items(): + for dataset_abbr, dataset_cfg in dataset2cfg.items(): + print('=' * 64, '[BEGIN]', '=' * 64) + print(f'[MODEL]: {model_abbr}') + print(f'[DATASET]: {dataset_abbr}') + print('---') + print_prompts(model_cfg, dataset_cfg) + print('=' * 65, '[END]', '=' * 65) + print() + + +if __name__ == '__main__': + main() diff --git a/tools/test_api_model.py b/tools/test_api_model.py new file mode 100644 index 00000000..bcb03f7e --- /dev/null +++ b/tools/test_api_model.py @@ -0,0 +1,206 @@ +import argparse +from typing import Dict + +from mmengine.config import Config, ConfigDict + +from opencompass.utils import Menu, build_model_from_cfg, model_abbr_from_cfg +from opencompass.utils.prompt import PromptList + +test_prompts = [ + PromptList([ + { + 'section': 'begin', + 'pos': 'begin' + }, + { + 'role': + 'SYSTEM', + 'fallback_role': + 'HUMAN', + 'prompt': + 'The following are multiple choice questions (with answers) about professional law.' # noqa + }, + '', + { + 'section': 'ice', + 'pos': 'begin' + }, + { + 'role': + 'HUMAN', + 'prompt': + "Without a warrant, police officers searched the garbage cans in the alley behind a man's house and discovered chemicals used to make methamphetamine, as well as cooking utensils and containers with the man's fingerprints on them. The alley was a public thoroughfare maintained by the city, and the garbage was picked up once a week by a private sanitation company. The items were found inside the garbage cans in plastic bags that had been tied closed and further secured with tape. The man was charged in federal court with the manufacture of methamphetamine. Did the search of the garbage cans violate the Fourth Amendment?\nA. No, because the man had no reasonable expectation of privacy in garbage left in the alley.\nB. No, because the probative value of the evidence outweighs the man's modest privacy claims in his garbage.\nC. Yes, because the alley was within the curtilage of the man's home and entry without a warrant was unconstitutional.\nD. Yes, because there is a reasonable expectation of privacy in one's secured garbage containers.\nAnswer: " # noqa + }, + { + 'role': 'BOT', + 'prompt': 'A\n' + }, + { + 'section': 'ice', + 'pos': 'end' + }, + { + 'section': 'ice', + 'pos': 'begin' + }, + { + 'role': + 'HUMAN', + 'prompt': + 'A man borrowed $500,000 from a bank, securing the loan with a mortgage on a commercial building he owned. The mortgage provided as follows: "No prepayment may be made on this loan during the first two years after the date of this mortgage. Thereafter, prepayment may be made in any amount at any time but only if accompanied by a prepayment fee of 5% of the amount prepaid." One year later, the man received an unexpected cash gift of $1 million and wished to pay off the $495,000 principal balance still owed on the loan. $495,000 principal balance still owed on the loan. Concerned that the bank might refuse prepayment, despite a rise in market interest rates in the year since the loan was made, or at least insist on the 5% prepayment fee, the man consulted an attorney concerning the enforceability of the above-quoted clause. There is no applicable statute. What is the attorney likely to say? \nA. The entire clause is unenforceable, because it violates a public policy favoring the prompt and early repayment of debt.\nB. The entire clause is unenforceable, because the rise in interest rates will allow the bank to reloan the funds without loss.\nC. The two-year prepayment prohibition and the prepayment fee provision are both valid and enforceable.\nD. The two-year prepayment prohibition is unenforceable, but the prepayment fee provision is enforceable.\nAnswer: ' # noqa + }, + { + 'role': 'BOT', + 'prompt': 'D\n' + }, + { + 'section': 'ice', + 'pos': 'end' + }, + { + 'section': 'ice', + 'pos': 'begin' + }, + { + 'role': + 'HUMAN', + 'prompt': + "A woman and a defendant entered into an arrangement where the woman promised to pay the defendant $10,000 to act as a surrogate mother. In return, the defendant agreed to be implanted with the woman's embryo and carry the baby to term. The woman paid the defendant the $10,000 upfront. During the seventh month of the pregnancy, the defendant changed her mind and decided to keep the child herself. The defendant moved out of state and gave birth to the baby, which she refuses to turn over to the woman. The defendant is guilty of\nA. no crime.\nB. embezzlement.\nC. kidnapping.\nD. false pretenses.\nAnswer: " # noqa + }, + { + 'role': 'BOT', + 'prompt': 'A\n' + }, + { + 'section': 'ice', + 'pos': 'end' + }, + { + 'section': 'ice', + 'pos': 'begin' + }, + { + 'role': + 'HUMAN', + 'prompt': + "A rescuer was driving on an isolated portion of a country road. His headlights caught a figure lying at the side of the road. The rescuer stopped to investigate and found a victim, who was bleeding from head wounds and appeared to have been severely beaten. The rescuer then lifted the victim into his car and drove her to the hospital, a half-hour trip. When they arrived at the hospital, the rescuer carried the victim into the emergency room. He left her with a nurse and then returned home. Although the victim recovered from her injuries, she sued the hospital for malpractice, claiming that she was not promptly given medical attention. At trial, the nurse proposes to testify that when the victim was first brought to the hospital, she was unconscious. The victim's attorney objects and moves to strike the nurse's testimony. The trial judge should\nA. sustain the objection, because it goes to an ultimate issue in the case. \nB. sustain the objection, because the nurse is not qualified to render an expert opinion. \nC. overrule the objection, because it is a shorthand rendition of what she observed. \nD. overrule the objection, because there are independent grounds to show a present sense impression. \nAnswer: " # noqa + }, + { + 'role': 'BOT', + 'prompt': 'C\n' + }, + { + 'section': 'ice', + 'pos': 'end' + }, + { + 'section': 'ice', + 'pos': 'begin' + }, + { + 'role': + 'HUMAN', + 'prompt': + "A young woman who attended a rock concert at a nightclub was injured when the band opened its performance with illegal fireworks that ignited foam insulation in the club's ceiling and walls. The young woman sued the radio station that sponsored the performance. The radio station has moved for summary judgment, claiming that it owed no duty to audience members. The evidence has established the following facts: The station advertised its sponsorship on the radio and in print, distributed free tickets to the concert, and in print, distributed free tickets to the concert, staffed the event with the station's interns to assist with crowd control, and provided a station disc jockey to serve as master of ceremonies. The master of ceremonies had the authority to stop or delay the performance at any time on the basis of any safety concern. The station knew or should have known that the band routinely used unlicensed, illegal fireworks in its performances. Should the court grant the radio station's motion for summary judgment? \nA. No, because there is sufficient evidence of knowledge and control on the part of the station to impose on it a duty of care to audience members.\nB. No, because under respondeat superior, the radio station is vicariously liable for the negligent actions of the band.\nC. Yes, because it is the band and the nightclub owners who owed audience members a duty of care.\nD. Yes, because the conduct of the band in setting off illegal fireworks was criminal and setting off illegal fireworks was criminal and was a superseding cause as a matter of law.\nAnswer: " # noqa + }, + { + 'role': 'BOT', + 'prompt': 'A\n' + }, + { + 'section': 'ice', + 'pos': 'end' + }, + '\n', + '', + { + 'section': 'begin', + 'pos': 'end' + }, + { + 'section': 'round', + 'pos': 'begin' + }, + { + 'role': + 'HUMAN', + 'prompt': + 'A state statute provides: "Whenever a person knows or should know that he (or she) is being arrested by a police officer, it is the duty of such person to refrain from using force or any weapon in resisting arrest. " Violation of the statute is made punishable by fine and/or imprisonment. One morning, there was a bank robbery in the state. That afternoon, a police officer arrested a suspect who he believed was involved in the crime. However, the police officer and the suspect have given different accounts concerning what happened next. According to the police officer, after the suspect was apprehended, he resisted arrest and hit the police officer in the mouth with his fist. The police officer, who was momentarily stunned, pulled out his nightstick and struck the suspect over the head with it. On the other hand, the suspect claimed that after he was arrested, he cursed at the policeman, whereupon the police officer began hitting the suspect with his nightstick. To avoid being hit again, the suspect hit the police officer with his fist, knocking him down. The suspect was charged with assault. The suspect should be found\nA. not guilty, if the arrest was unlawful without probable cause and the jury believes the suspect\'s account.\nB. not guilty, if the arrest was lawful, provided that the jury believes the suspect\'s account.\nC. guilty, if the arrest was lawful, regardless which account the jury believes.\nD. guilty, if the arrest was unlawful, regardless which account the jury believes.\nAnswer: ' # noqa + }, + { + 'section': 'round', + 'pos': 'end' + } + ]), + 'Hello! How are you?' +] + +meta_templates = [ + None, + dict(round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True) + ], ), + dict( + round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True) + ], + reserved_roles=[ + dict(role='SYSTEM', api_role='SYSTEM'), + ], + ) +] + + +def test_model(model_cfg: ConfigDict): + for meta_template in meta_templates: + print('Testing meta_template: ', meta_template) + model_cfg['meta_template'] = meta_template + model = build_model_from_cfg(model_cfg) + print('Prompt 0 length:', + model.get_token_len_from_template(test_prompts[0])) + print('Prompt 1 length:', + model.get_token_len_from_template(test_prompts[1])) + print('Prompt lengths: ', + model.get_token_len_from_template(test_prompts)) + msgs = model.generate_from_template(test_prompts, max_out_len=100) + print('Prompt 0 response:', msgs[0]) + print('Prompt 1 response:', msgs[1]) + print('-' * 100) + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Test if a given API model wrapper works properly') + parser.add_argument('config', help='Train config file path') + parser.add_argument('-n', '--non-interactive', action='store_true') + args = parser.parse_args() + return args + + +def parse_model_cfg(model_cfg: ConfigDict) -> Dict[str, ConfigDict]: + model2cfg = {} + for model in model_cfg: + model2cfg[model_abbr_from_cfg(model)] = model + return model2cfg + + +def main(): + args = parse_args() + cfg = Config.fromfile(args.config) + if 'models' not in cfg: + raise ValueError('No "models" specified in config file!') + model2cfg = parse_model_cfg(cfg.models) + + if not args.non_interactive and len(model2cfg) > 1: + model = Menu([list(model2cfg.keys())], + ['Please make a selection of models:']).run() + else: + model = list(model2cfg.keys())[0] + model_cfg = model2cfg[model] + test_model(model_cfg) + + +if __name__ == '__main__': + main()