diff --git a/README.md b/README.md index 07dc610a..26a9fd4b 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,7 @@ Just like a compass guides us on our journey, OpenCompass will guide you through 🔥🔥🔥 We are delighted to announce that **the OpenCompass has been recommended by the Meta AI**, click [Get Started](https://ai.meta.com/llama/get-started/#validation) of Llama for more information. > **Attention**
-> We launch the OpenCompass Collaboration project, welcome to support diverse evaluation benchmarks into OpenCompass! -> Clike [Issue](https://github.com/open-compass/opencompass/issues/248) for more information. -> Let's work together to build a more powerful OpenCompass toolkit! +> Breaking Change Notice: In version 0.4.0, we are consolidating all AMOTIC configuration files (previously located in ./configs/datasets, ./configs/models, and ./configs/summarizers) into the opencompass package. Users are advised to update their configuration references to reflect this structural change. ## 🚀 What's New diff --git a/README_zh-CN.md b/README_zh-CN.md index 8ac8cf3a..02f59284 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -53,9 +53,7 @@ 🔥🔥🔥 祝贺 **OpenCompass 作为大模型标准测试工具被Meta AI官方推荐**, 点击 Llama 的 [入门文档](https://ai.meta.com/llama/get-started/#validation) 获取更多信息。 > **注意**
-> 我们正式启动 OpenCompass 共建计划，诚邀社区用户为 OpenCompass 提供更具代表性和可信度的客观评测数据集! -> 点击 [Issue](https://github.com/open-compass/opencompass/issues/248) 获取更多数据集. -> 让我们携手共进，打造功能强大易用的大模型评测平台！ +> 重要通知：从 v0.4.0 版本开始，所有位于 ./configs/datasets、./configs/models 和 ./configs/summarizers 目录下的 AMOTIC 配置文件将迁移至 opencompass 包中。请及时更新您的配置文件路径。 ## 🚀 最新进展 diff --git a/configs/datasets/GaokaoBench/GaokaoBench_gen_5cfe9e.py b/configs/datasets/GaokaoBench/GaokaoBench_gen_5cfe9e.py index 480eeadb..a1c7da8a 100644 --- a/configs/datasets/GaokaoBench/GaokaoBench_gen_5cfe9e.py +++ b/configs/datasets/GaokaoBench/GaokaoBench_gen_5cfe9e.py @@ -284,11 +284,12 @@ for _folder, _prompts in [ }, 'pred_role': 'BOT', } - _base_path = './data/GAOKAO-BENCH/data' + _base_path = 'opencompass/GAOKAO-BENCH' _dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + _p['keyword'], - 'path': _base_path + '/' + _folder + '/' + _p['keyword'] + '.json', + 'path': _base_path, + 'filename': '/' + _folder + '/' + _p['keyword'] + '.json', 'name': _p['keyword'], 'reader_cfg': _reader_cfg, 'infer_cfg': _infer_cfg, diff --git a/configs/datasets/GaokaoBench/GaokaoBench_mixed_9af5ee.py b/configs/datasets/GaokaoBench/GaokaoBench_mixed_9af5ee.py index 637f4f51..15b9f3dd 100644 --- a/configs/datasets/GaokaoBench/GaokaoBench_mixed_9af5ee.py +++ b/configs/datasets/GaokaoBench/GaokaoBench_mixed_9af5ee.py @@ -288,7 +288,8 @@ for _folder, _prompts in [ _dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + _p['keyword'], - 'path': _base_path + '/' + _folder + '/' + _p['keyword'] + '.json', + 'path': _base_path, + 'filename': '/' + _folder + '/' + _p['keyword'] + '.json', 'name': _p['keyword'], 'reader_cfg': _reader_cfg, 'infer_cfg': _infer_cfg, @@ -335,11 +336,12 @@ for _p in _MCQ_prompts: }, 'pred_role': 'BOT', } - _base_path = './data/GAOKAO-BENCH/data' + _base_path = 'opencompass/GAOKAO-BENCH' _dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + _p['keyword'], - 'path': _base_path + '/' + _folder + '/' + _p['keyword'] + '.json', + 'path': _base_path, + 'filename': '/' + _folder + '/' + _p['keyword'] + '.json', 'name': _p['keyword'], 'reader_cfg': _reader_cfg, 'infer_cfg': _infer_cfg, diff --git a/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py b/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py index e3c251aa..e1bbdf7e 100644 --- a/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py +++ b/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py @@ -31,10 +31,12 @@ for folder, prompts in [ 'evaluator': {'type': 'GaokaoBenchEvaluator' + '_' + p['type']}, 'pred_role': 'BOT', } + _base_path = 'opencompass/GAOKAO-BENCH' dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + p['keyword'], - 'path': os.path.join('data', 'GAOKAO-BENCH', 'data', folder, p['keyword'] + '.json'), + 'path': _base_path, + 'filename': '/' + folder + '/' + p['keyword'] + '.json', 'name': p['keyword'], 'reader_cfg': reader_cfg, 'infer_cfg': infer_cfg, diff --git a/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_d21e37.py b/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_d21e37.py index 1f50030b..a561af9a 100644 --- a/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_d21e37.py +++ b/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_d21e37.py @@ -30,10 +30,12 @@ for folder, prompts in [ 'evaluator': {'type': 'GaokaoBenchEvaluator' + '_' + p['type']}, 'pred_role': 'BOT', } + _base_path = 'opencompass/GAOKAO-BENCH' dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + p['keyword'], - 'path': os.path.join('data', 'GAOKAO-BENCH', 'data', folder, p['keyword'] + '.json'), + 'path': _base_path, + 'filename': '/' + folder + '/' + p['keyword'] + '.json', 'name': p['keyword'], 'reader_cfg': reader_cfg, 'infer_cfg': infer_cfg, diff --git a/configs/datasets/nq/nq_open_1shot_gen_01cf41.py b/configs/datasets/nq/nq_open_1shot_gen_01cf41.py index ef15e81e..db67906b 100644 --- a/configs/datasets/nq/nq_open_1shot_gen_01cf41.py +++ b/configs/datasets/nq/nq_open_1shot_gen_01cf41.py @@ -54,7 +54,7 @@ for k in [1]: dict( type=NQOpenDataset, abbr=f'nq_open_{k}shot', - path='./data/nq-open/', + path='opencompass/nq_open', reader_cfg=nq_reader_cfg, infer_cfg=nq_infer_cfg, eval_cfg=nq_eval_cfg) diff --git a/configs/datasets/nq/nq_open_1shot_gen_20a989.py b/configs/datasets/nq/nq_open_1shot_gen_20a989.py index 1df301f6..b1ac35ff 100644 --- a/configs/datasets/nq/nq_open_1shot_gen_20a989.py +++ b/configs/datasets/nq/nq_open_1shot_gen_20a989.py @@ -38,7 +38,7 @@ for k in [1]: dict( type=NQOpenDataset, abbr=f'nq_open_{k}shot', - path='./data/nq-open/', + path='opencompass/nq_open', reader_cfg=nq_reader_cfg, infer_cfg=nq_infer_cfg, eval_cfg=nq_eval_cfg) diff --git a/configs/datasets/nq/nq_open_1shot_gen_2e45e5.py b/configs/datasets/nq/nq_open_1shot_gen_2e45e5.py index d676b949..e877b397 100644 --- a/configs/datasets/nq/nq_open_1shot_gen_2e45e5.py +++ b/configs/datasets/nq/nq_open_1shot_gen_2e45e5.py @@ -54,7 +54,7 @@ for k in [1]: dict( type=NQOpenDataset, abbr=f'nq_open_{k}shot', - path='./data/nq-open/', + path='opencompass/nq_open', reader_cfg=nq_reader_cfg, infer_cfg=nq_infer_cfg, eval_cfg=nq_eval_cfg) diff --git a/configs/datasets/nq/nq_open_gen_e93f8a.py b/configs/datasets/nq/nq_open_gen_e93f8a.py index 224d9776..07d5b2a8 100644 --- a/configs/datasets/nq/nq_open_gen_e93f8a.py +++ b/configs/datasets/nq/nq_open_gen_e93f8a.py @@ -54,7 +54,7 @@ for k in [0, 1, 5, 25]: dict( type=NQOpenDataset, abbr=f'nq_open_{k}shot', - path='./data/nq-open/', + path='opencompass/nq_open', reader_cfg=nq_reader_cfg, infer_cfg=nq_infer_cfg, eval_cfg=nq_eval_cfg) diff --git a/configs/datasets/wikibench/wikibench_few_shot_ppl_c23d79.py b/configs/datasets/wikibench/wikibench_few_shot_ppl_c23d79.py index 0669bd7b..81056a4f 100644 --- a/configs/datasets/wikibench/wikibench_few_shot_ppl_c23d79.py +++ b/configs/datasets/wikibench/wikibench_few_shot_ppl_c23d79.py @@ -60,7 +60,8 @@ for _split in list(wikibench_sets.keys()): wikibench_datasets.append( dict( type=WikiBenchDataset, - path=f'./data/WikiBench/{_name}.jsonl', + path='opencompass/WikiBench', + filename=f'{_name}.jsonl', name='circular_' + _name if do_circular else _name, abbr='wikibench-' + _split + '-' + _name + 'circular' if do_circular else '', diff --git a/configs/datasets/wikibench/wikibench_gen_0978ad.py b/configs/datasets/wikibench/wikibench_gen_0978ad.py index 871133f9..73d35f19 100644 --- a/configs/datasets/wikibench/wikibench_gen_0978ad.py +++ b/configs/datasets/wikibench/wikibench_gen_0978ad.py @@ -43,7 +43,8 @@ for _split in list(wikibench_sets.keys()): wikibench_datasets.append( dict( type=WikiBenchDataset, - path=f'./data/WikiBench/{_name}.jsonl', + path='opencompass/WikiBench', + filename=f'{_name}.jsonl', name='circular_' + _name if do_circular else _name, abbr='wikibench-' + _split + '-' + _name + 'circular' if do_circular else '', reader_cfg=dict( diff --git a/configs/datasets/wikibench/wikibench_gen_f96ece.py b/configs/datasets/wikibench/wikibench_gen_f96ece.py index 5bf9d34e..80f03e5a 100644 --- a/configs/datasets/wikibench/wikibench_gen_f96ece.py +++ b/configs/datasets/wikibench/wikibench_gen_f96ece.py @@ -43,7 +43,8 @@ for _split in list(wikibench_sets.keys()): wikibench_datasets.append( dict( type=WikiBenchDataset, - path=f'./data/WikiBench/{_name}.jsonl', + path='opencompass/WikiBench', + filename=f'{_name}.jsonl', name='circular_' + _name if do_circular else _name, abbr='wikibench-' + _split + '-' + _name + 'circular' if do_circular else '', reader_cfg=dict( diff --git a/opencompass/__init__.py b/opencompass/__init__.py index bfeb9e74..6e384829 100644 --- a/opencompass/__init__.py +++ b/opencompass/__init__.py @@ -1 +1,17 @@ __version__ = '0.3.4' + + +def _warn_about_config_migration(): + import warnings + warnings.warn( + 'Starting from v0.4.0, all AMOTIC configuration files currently ' + 'located in `./configs/datasets`, `./configs/models`, and ' + '`./configs/summarizers` will be migrated to the ' + '`opencompass/configs/` package. Please update your configuration ' + 'file paths accordingly.', + UserWarning, # Changed to UserWarning + stacklevel=2) + + +# Trigger the warning +_warn_about_config_migration() diff --git a/opencompass/configs/datasets/GaokaoBench/GaokaoBench_gen_5cfe9e.py b/opencompass/configs/datasets/GaokaoBench/GaokaoBench_gen_5cfe9e.py index 480eeadb..a1c7da8a 100644 --- a/opencompass/configs/datasets/GaokaoBench/GaokaoBench_gen_5cfe9e.py +++ b/opencompass/configs/datasets/GaokaoBench/GaokaoBench_gen_5cfe9e.py @@ -284,11 +284,12 @@ for _folder, _prompts in [ }, 'pred_role': 'BOT', } - _base_path = './data/GAOKAO-BENCH/data' + _base_path = 'opencompass/GAOKAO-BENCH' _dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + _p['keyword'], - 'path': _base_path + '/' + _folder + '/' + _p['keyword'] + '.json', + 'path': _base_path, + 'filename': '/' + _folder + '/' + _p['keyword'] + '.json', 'name': _p['keyword'], 'reader_cfg': _reader_cfg, 'infer_cfg': _infer_cfg, diff --git a/opencompass/configs/datasets/GaokaoBench/GaokaoBench_mixed_9af5ee.py b/opencompass/configs/datasets/GaokaoBench/GaokaoBench_mixed_9af5ee.py index 637f4f51..15b9f3dd 100644 --- a/opencompass/configs/datasets/GaokaoBench/GaokaoBench_mixed_9af5ee.py +++ b/opencompass/configs/datasets/GaokaoBench/GaokaoBench_mixed_9af5ee.py @@ -288,7 +288,8 @@ for _folder, _prompts in [ _dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + _p['keyword'], - 'path': _base_path + '/' + _folder + '/' + _p['keyword'] + '.json', + 'path': _base_path, + 'filename': '/' + _folder + '/' + _p['keyword'] + '.json', 'name': _p['keyword'], 'reader_cfg': _reader_cfg, 'infer_cfg': _infer_cfg, @@ -335,11 +336,12 @@ for _p in _MCQ_prompts: }, 'pred_role': 'BOT', } - _base_path = './data/GAOKAO-BENCH/data' + _base_path = 'opencompass/GAOKAO-BENCH' _dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + _p['keyword'], - 'path': _base_path + '/' + _folder + '/' + _p['keyword'] + '.json', + 'path': _base_path, + 'filename': '/' + _folder + '/' + _p['keyword'] + '.json', 'name': _p['keyword'], 'reader_cfg': _reader_cfg, 'infer_cfg': _infer_cfg, diff --git a/opencompass/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py b/opencompass/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py index e3c251aa..e1bbdf7e 100644 --- a/opencompass/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py +++ b/opencompass/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_4c31db.py @@ -31,10 +31,12 @@ for folder, prompts in [ 'evaluator': {'type': 'GaokaoBenchEvaluator' + '_' + p['type']}, 'pred_role': 'BOT', } + _base_path = 'opencompass/GAOKAO-BENCH' dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + p['keyword'], - 'path': os.path.join('data', 'GAOKAO-BENCH', 'data', folder, p['keyword'] + '.json'), + 'path': _base_path, + 'filename': '/' + folder + '/' + p['keyword'] + '.json', 'name': p['keyword'], 'reader_cfg': reader_cfg, 'infer_cfg': infer_cfg, diff --git a/opencompass/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_d21e37.py b/opencompass/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_d21e37.py index 1f50030b..a561af9a 100644 --- a/opencompass/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_d21e37.py +++ b/opencompass/configs/datasets/GaokaoBench/GaokaoBench_no_subjective_gen_d21e37.py @@ -30,10 +30,12 @@ for folder, prompts in [ 'evaluator': {'type': 'GaokaoBenchEvaluator' + '_' + p['type']}, 'pred_role': 'BOT', } + _base_path = 'opencompass/GAOKAO-BENCH' dataset = { 'type': GaokaoBenchDataset, 'abbr': 'GaokaoBench_' + p['keyword'], - 'path': os.path.join('data', 'GAOKAO-BENCH', 'data', folder, p['keyword'] + '.json'), + 'path': _base_path, + 'filename': '/' + folder + '/' + p['keyword'] + '.json', 'name': p['keyword'], 'reader_cfg': reader_cfg, 'infer_cfg': infer_cfg, diff --git a/opencompass/configs/datasets/nq/nq_open_1shot_gen_01cf41.py b/opencompass/configs/datasets/nq/nq_open_1shot_gen_01cf41.py index ef15e81e..db67906b 100644 --- a/opencompass/configs/datasets/nq/nq_open_1shot_gen_01cf41.py +++ b/opencompass/configs/datasets/nq/nq_open_1shot_gen_01cf41.py @@ -54,7 +54,7 @@ for k in [1]: dict( type=NQOpenDataset, abbr=f'nq_open_{k}shot', - path='./data/nq-open/', + path='opencompass/nq_open', reader_cfg=nq_reader_cfg, infer_cfg=nq_infer_cfg, eval_cfg=nq_eval_cfg) diff --git a/opencompass/configs/datasets/nq/nq_open_1shot_gen_20a989.py b/opencompass/configs/datasets/nq/nq_open_1shot_gen_20a989.py index 1df301f6..b1ac35ff 100644 --- a/opencompass/configs/datasets/nq/nq_open_1shot_gen_20a989.py +++ b/opencompass/configs/datasets/nq/nq_open_1shot_gen_20a989.py @@ -38,7 +38,7 @@ for k in [1]: dict( type=NQOpenDataset, abbr=f'nq_open_{k}shot', - path='./data/nq-open/', + path='opencompass/nq_open', reader_cfg=nq_reader_cfg, infer_cfg=nq_infer_cfg, eval_cfg=nq_eval_cfg) diff --git a/opencompass/configs/datasets/nq/nq_open_1shot_gen_2e45e5.py b/opencompass/configs/datasets/nq/nq_open_1shot_gen_2e45e5.py index d676b949..e877b397 100644 --- a/opencompass/configs/datasets/nq/nq_open_1shot_gen_2e45e5.py +++ b/opencompass/configs/datasets/nq/nq_open_1shot_gen_2e45e5.py @@ -54,7 +54,7 @@ for k in [1]: dict( type=NQOpenDataset, abbr=f'nq_open_{k}shot', - path='./data/nq-open/', + path='opencompass/nq_open', reader_cfg=nq_reader_cfg, infer_cfg=nq_infer_cfg, eval_cfg=nq_eval_cfg) diff --git a/opencompass/configs/datasets/nq/nq_open_gen_e93f8a.py b/opencompass/configs/datasets/nq/nq_open_gen_e93f8a.py index 224d9776..07d5b2a8 100644 --- a/opencompass/configs/datasets/nq/nq_open_gen_e93f8a.py +++ b/opencompass/configs/datasets/nq/nq_open_gen_e93f8a.py @@ -54,7 +54,7 @@ for k in [0, 1, 5, 25]: dict( type=NQOpenDataset, abbr=f'nq_open_{k}shot', - path='./data/nq-open/', + path='opencompass/nq_open', reader_cfg=nq_reader_cfg, infer_cfg=nq_infer_cfg, eval_cfg=nq_eval_cfg) diff --git a/opencompass/configs/datasets/wikibench/wikibench_few_shot_ppl_c23d79.py b/opencompass/configs/datasets/wikibench/wikibench_few_shot_ppl_c23d79.py index 0669bd7b..81056a4f 100644 --- a/opencompass/configs/datasets/wikibench/wikibench_few_shot_ppl_c23d79.py +++ b/opencompass/configs/datasets/wikibench/wikibench_few_shot_ppl_c23d79.py @@ -60,7 +60,8 @@ for _split in list(wikibench_sets.keys()): wikibench_datasets.append( dict( type=WikiBenchDataset, - path=f'./data/WikiBench/{_name}.jsonl', + path='opencompass/WikiBench', + filename=f'{_name}.jsonl', name='circular_' + _name if do_circular else _name, abbr='wikibench-' + _split + '-' + _name + 'circular' if do_circular else '', diff --git a/opencompass/configs/datasets/wikibench/wikibench_gen_0978ad.py b/opencompass/configs/datasets/wikibench/wikibench_gen_0978ad.py index 871133f9..73d35f19 100644 --- a/opencompass/configs/datasets/wikibench/wikibench_gen_0978ad.py +++ b/opencompass/configs/datasets/wikibench/wikibench_gen_0978ad.py @@ -43,7 +43,8 @@ for _split in list(wikibench_sets.keys()): wikibench_datasets.append( dict( type=WikiBenchDataset, - path=f'./data/WikiBench/{_name}.jsonl', + path='opencompass/WikiBench', + filename=f'{_name}.jsonl', name='circular_' + _name if do_circular else _name, abbr='wikibench-' + _split + '-' + _name + 'circular' if do_circular else '', reader_cfg=dict( diff --git a/opencompass/configs/datasets/wikibench/wikibench_gen_f96ece.py b/opencompass/configs/datasets/wikibench/wikibench_gen_f96ece.py index 5bf9d34e..80f03e5a 100644 --- a/opencompass/configs/datasets/wikibench/wikibench_gen_f96ece.py +++ b/opencompass/configs/datasets/wikibench/wikibench_gen_f96ece.py @@ -43,7 +43,8 @@ for _split in list(wikibench_sets.keys()): wikibench_datasets.append( dict( type=WikiBenchDataset, - path=f'./data/WikiBench/{_name}.jsonl', + path='opencompass/WikiBench', + filename=f'{_name}.jsonl', name='circular_' + _name if do_circular else _name, abbr='wikibench-' + _split + '-' + _name + 'circular' if do_circular else '', reader_cfg=dict( diff --git a/opencompass/datasets/GaokaoBench.py b/opencompass/datasets/GaokaoBench.py index d3cd31a0..c1ae6d10 100644 --- a/opencompass/datasets/GaokaoBench.py +++ b/opencompass/datasets/GaokaoBench.py @@ -15,8 +15,10 @@ from .base import BaseDataset class GaokaoBenchDataset(BaseDataset): @staticmethod - def load(path: str, name: str): - path = get_data_path(path, local_mode=True) + def load(path: str, filename: str, name: str): + path = get_data_path(path) + path = path + filename + if environ.get('DATASET_SOURCE') == 'ModelScope': from modelscope import MsDataset return MsDataset.load(path, subset_name=name, split='test') diff --git a/opencompass/datasets/natural_question.py b/opencompass/datasets/natural_question.py index ab8356cd..e1ca1632 100644 --- a/opencompass/datasets/natural_question.py +++ b/opencompass/datasets/natural_question.py @@ -55,7 +55,7 @@ class NQOpenDataset(BaseDataset): @staticmethod def load(path: str): - path = get_data_path(path, local_mode=True) + path = get_data_path(path) dataset = DatasetDict() for split in ['validation', 'train']: filename = osp.join(path, f'nq-open-{split}.jsonl') diff --git a/opencompass/datasets/wikibench.py b/opencompass/datasets/wikibench.py index 9e0e5743..e2309999 100644 --- a/opencompass/datasets/wikibench.py +++ b/opencompass/datasets/wikibench.py @@ -21,8 +21,9 @@ def get_number(options): class WikiBenchDataset(BaseDataset): @staticmethod - def load(path: str, name: str): - path = get_data_path(path, local_mode=True) + def load(path: str, filename: str, name: str): + path = get_data_path(path) + path = path + filename circular_patterns = ['ABCD', 'BCDA', 'CDAB', 'DABC'] diff --git a/opencompass/utils/datasets_info.py b/opencompass/utils/datasets_info.py index 06555925..c877adf9 100644 --- a/opencompass/utils/datasets_info.py +++ b/opencompass/utils/datasets_info.py @@ -301,6 +301,21 @@ DATASETS_MAPPING = { "ms_id": "", "hf_id": "", "local": "./data/cmo.jsonl", + }, + "opencompass/nq_open": { + "ms_id": "", + "hf_id": "", + "local": "./data/nq-open/", + }, + "opencompass/GAOKAO-BENCH": { + "ms_id": "", + "hf_id": "", + "local": "./data/GAOKAO-BENCH/data", + }, + "opencompass/WikiBench": { + "ms_id": "", + "hf_id": "", + "local": "./data/WikiBench/", } } @@ -472,5 +487,25 @@ DATASETS_URL = { "/cmo": { "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/cmo.zip", "md5": "fad52c81290506a8ca74f46b5400d8fc" + }, + "/nq-open": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/nq-open.zip", + "md5": "a340521e5c9ec591227dcb367f718b25", + }, + "/winogrande": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/winogrande.zip", + "md5": "9e949a75eacc26ed4fd2b9aa870b495b", + }, + "/triviaqa": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/triviaqa.zip", + "md5": "e6a118d744236814926b2ec7ec66c034", + }, + "/GAOKAO-BENCH": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/GAOKAO-BENCH.zip", + "md5": "ba3c71b8b9db96d2a0664b977c4f9784", + }, + "/WikiBench": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/WikiBench.zip", + "md5": "6dac1d1a3133fe1effff185cbf71d928", } }