jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity
46cc7894e1
OpenCompass/opencompass/utils/datasets.py
Xingjun.Wang edab1c07ba
[Feature] Support ModelScope datasets (#1289) 
* add ceval, gsm8k modelscope surpport

* update race, mmlu, arc, cmmlu, commonsenseqa, humaneval and unittest

* update bbh, flores, obqa, siqa, storycloze, summedits, winogrande, xsum datasets

* format file

* format file

* update dataset format

* support ms_dataset

* udpate dataset for modelscope support

* merge myl_dev and update test_ms_dataset

* udpate dataset for modelscope support

* update readme

* update eval_api_zhipu_v2

* remove unused code

* add get_data_path function

* update readme

* remove tydiqa japanese subset

* add ceval, gsm8k modelscope surpport

* update race, mmlu, arc, cmmlu, commonsenseqa, humaneval and unittest

* update bbh, flores, obqa, siqa, storycloze, summedits, winogrande, xsum datasets

* format file

* format file

* update dataset format

* support ms_dataset

* udpate dataset for modelscope support

* merge myl_dev and update test_ms_dataset

* update readme

* udpate dataset for modelscope support

* update eval_api_zhipu_v2

* remove unused code

* add get_data_path function

* remove tydiqa japanese subset

* update util

* remove .DS_Store

* fix md format

* move util into package

* update docs/get_started.md

* restore eval_api_zhipu_v2.py, add environment setting

* Update dataset

* Update

* Update

* Update

* Update

---------

Co-authored-by: Yun lin <yunlin@U-Q9X2K4QV-1904.local>
Co-authored-by: Yunnglin <mao.looper@qq.com>
Co-authored-by: Yun lin <yunlin@laptop.local>
Co-authored-by: Yunnglin <maoyl@smail.nju.edu.cn>
Co-authored-by: zhangsongyang <zhangsongyang@pjlab.org.cn>
2024-07-29 13:48:32 +08:00

302 lines
9.0 KiB
Python
Raw Blame History

import os
DATASETS_MAPPING = {
# ADVGLUE Datasets
'opencompass/advglue-dev': {
'ms_id': None,
'hf_id': None,
'local': './data/adv_glue/dev_ann.json',
},
# AGIEval Datasets
'opencompass/agieval': {
'ms_id': 'opencompass/agieval',
'hf_id': 'opencompass/agieval',
'local': './data/AGIEval/data/v1/',
},
# ARC Datasets(Test)
'opencompass/ai2_arc-test': {
'ms_id': 'opencompass/ai2_arc',
'hf_id': 'opencompass/ai2_arc',
'local': './data/ARC/ARC-c/ARC-Challenge-Test.jsonl',
},
'opencompass/ai2_arc-dev': {
'ms_id': 'opencompass/ai2_arc',
'hf_id': 'opencompass/ai2_arc',
'local': './data/ARC/ARC-c/ARC-Challenge-Dev.jsonl',
},
'opencompass/ai2_arc-easy-dev': {
'ms_id': 'opencompass/ai2_arc',
'hf_id': 'opencompass/ai2_arc',
'local': './data/ARC/ARC-e/ARC-Easy-Dev.jsonl',
},
# BBH
'opencompass/bbh': {
'ms_id': 'opencompass/bbh',
'hf_id': 'opencompass/bbh',
'local': './data/BBH/data',
},
# C-Eval
'opencompass/ceval-exam': {
'ms_id': 'opencompass/ceval-exam',
'hf_id': 'opencompass/ceval-exam',
'local': './data/ceval/formal_ceval',
},
# AFQMC
'opencompass/afqmc-dev': {
'ms_id': 'opencompass/afqmc',
'hf_id': 'opencompass/afqmc',
'local': './data/CLUE/AFQMC/dev.json',
},
# CMNLI
'opencompass/cmnli-dev': {
'ms_id': 'opencompass/cmnli',
'hf_id': 'opencompass/cmnli',
'local': './data/CLUE/cmnli/cmnli_public/dev.json',
},
# OCNLI
'opencompass/OCNLI-dev': {
'ms_id': 'opencompass/OCNLI',
'hf_id': 'opencompass/OCNLI',
'local': './data/CLUE/OCNLI/dev.json',
},
# ChemBench
'opencompass/ChemBench': {
'ms_id': 'opencompass/ChemBench',
'hf_id': 'opencompass/ChemBench',
'local': './data/ChemBench/',
},
# CMMLU
'opencompass/cmmlu': {
'ms_id': 'opencompass/cmmlu',
'hf_id': 'opencompass/cmmlu',
'local': './data/cmmlu/',
},
# CommonsenseQA
'opencompass/commonsense_qa': {
'ms_id': 'opencompass/commonsense_qa',
'hf_id': 'opencompass/commonsense_qa',
'local': './data/commonsenseqa',
},
# CMRC
'opencompass/cmrc_dev': {
'ms_id': 'opencompass/cmrc_dev',
'hf_id': 'opencompass/cmrc_dev',
'local': './data/CLUE/CMRC/dev.json'
},
# DRCD_dev
'opencompass/drcd_dev': {
'ms_id': 'opencompass/drcd_dev',
'hf_id': 'opencompass/drcd_dev',
'local': './data/CLUE/DRCD/dev.json'
},
# clozeTest_maxmin
'opencompass/clozeTest_maxmin': {
'ms_id': None,
'hf_id': None,
'local': './data/clozeTest-maxmin/python/clozeTest.json',
},
# clozeTest_maxmin
'opencompass/clozeTest_maxmin_answers': {
'ms_id': None,
'hf_id': None,
'local': './data/clozeTest-maxmin/python/answers.txt',
},
# Flores
'opencompass/flores': {
'ms_id': 'opencompass/flores',
'hf_id': 'opencompass/flores',
'local': './data/flores_first100',
},
# MBPP
'opencompass/mbpp': {
'ms_id': 'opencompass/mbpp',
'hf_id': 'opencompass/mbpp',
'local': './data/mbpp/mbpp.jsonl',
},
# 'opencompass/mbpp': {
# 'ms_id': 'opencompass/mbpp',
# 'hf_id': 'opencompass/mbpp',
# 'local': './data/mbpp/mbpp.jsonl',
# },
'opencompass/sanitized_mbpp': {
'ms_id': 'opencompass/mbpp',
'hf_id': 'opencompass/mbpp',
'local': './data/mbpp/sanitized-mbpp.jsonl',
},
# GSM
'opencompass/gsm8k': {
'ms_id': 'opencompass/gsm8k',
'hf_id': 'opencompass/gsm8k',
'local': './data/gsm8k/',
},
# HellaSwag
'opencompass/hellaswag': {
'ms_id': 'opencompass/hellaswag',
'hf_id': 'opencompass/hellaswag',
'local': './data/hellaswag/hellaswag.jsonl',
},
# HellaSwagICE
'opencompass/hellaswag_ice': {
'ms_id': 'opencompass/hellaswag',
'hf_id': 'opencompass/hellaswag',
'local': './data/hellaswag/',
},
# HumanEval
'opencompass/humaneval': {
'ms_id': 'opencompass/humaneval',
'hf_id': 'opencompass/humaneval',
'local': './data/humaneval/human-eval-v2-20210705.jsonl',
},
# HumanEvalCN
'opencompass/humaneval_cn': {
'ms_id': 'opencompass/humaneval',
'hf_id': 'opencompass/humaneval',
'local': './data/humaneval_cn/human-eval-cn-v2-20210705.jsonl',
},
# Lambada
'opencompass/lambada': {
'ms_id': 'opencompass/lambada',
'hf_id': 'opencompass/lambada',
'local': './data/lambada/test.jsonl',
},
# LCSTS
'opencompass/LCSTS': {
'ms_id': 'opencompass/LCSTS',
'hf_id': 'opencompass/LCSTS',
'local': './data/LCSTS',
},
# MATH
'opencompass/math': {
'ms_id': 'opencompass/math',
'hf_id': 'opencompass/math',
'local': './data/math/math.json',
},
# MMLU
'opencompass/mmlu': {
'ms_id': 'opencompass/mmlu',
'hf_id': 'opencompass/mmlu',
'local': './data/mmlu/',
},
# NQ
'opencompass/natural_question': {
'ms_id': 'opencompass/natural_question',
'hf_id': 'opencompass/natural_question',
'local': './data/nq/',
},
# OpenBook QA-test
'opencompass/openbookqa_test': {
'ms_id': 'opencompass/openbookqa',
'hf_id': 'opencompass/openbookqa',
'local': './data/openbookqa/Main/test.jsonl',
},
# OpenBook QA-fact
'opencompass/openbookqa_fact': {
'ms_id': 'opencompass/openbookqa',
'hf_id': 'opencompass/openbookqa',
'local': './data/openbookqa/Additional/test_complete.jsonl',
},
# PIQA
'opencompass/piqa': {
'ms_id': 'opencompass/piqa',
'hf_id': 'opencompass/piqa',
'local': './data/piqa',
},
# RACE
'opencompass/race': {
'ms_id': 'opencompass/race',
'hf_id': 'opencompass/race',
'local': './data/race',
},
# SIQA
'opencompass/siqa': {
'ms_id': 'opencompass/siqa',
'hf_id': 'opencompass/siqa',
'local': './data/siqa',
},
# XStoryCloze
'opencompass/xstory_cloze': {
'ms_id': 'opencompass/xstory_cloze',
'hf_id': 'opencompass/xstory_cloze',
'local': './data/xstory_cloze',
},
# StrategyQA
'opencompass/strategy_qa': {
'ms_id': 'opencompass/strategy_qa',
'hf_id': 'opencompass/strategy_qa',
'local': './data/strategyqa/strategyQA_train.json',
},
# SummEdits
'opencompass/summedits': {
'ms_id': 'opencompass/summedits',
'hf_id': 'opencompass/summedits',
'local': './data/summedits/summedits.jsonl',
},
# TriviaQA
'opencompass/trivia_qa': {
'ms_id': 'opencompass/trivia_qa',
'hf_id': 'opencompass/trivia_qa',
'local': './data/triviaqa/',
},
# TydiQA
'opencompass/tydiqa': {
'ms_id': 'opencompass/tydiqa',
'hf_id': 'opencompass/tydiqa',
'local': './data/tydiqa/',
},
# Winogrande
'opencompass/winogrande': {
'ms_id': 'opencompass/winogrande',
'hf_id': 'opencompass/winogrande',
'local': './data/winogrande/',
},
# XSum
'opencompass/xsum': {
'ms_id': 'opencompass/xsum',
'hf_id': 'opencompass/xsum',
'local': './data/Xsum/dev.jsonl',
}
}
def get_data_path(dataset_id: str, local_mode: bool = False):
"""return dataset id when getting data from ModelScope repo, otherwise just
return local path as is.
Args:
dataset_id (str): dataset id or data path
local_mode (bool): whether to use local path or
ModelScope/HuggignFace repo
"""
# update the path with CACHE_DIR
cache_dir = os.environ.get('COMPASS_DATA_CACHE', '')
# For absolute path customized by the users
if dataset_id.startswith('/'):
return dataset_id
# For relative path, with CACHE_DIR
if local_mode:
local_path = os.path.join(cache_dir, dataset_id)
assert os.path.exists(local_path), f'{local_path} does not exist!'
return local_path
dataset_source = os.environ.get('DATASET_SOURCE', None)
if dataset_source == 'ModelScope':
ms_id = DATASETS_MAPPING[dataset_id]['ms_id']
assert ms_id is not None, \
f'{dataset_id} is not supported in ModelScope'
return ms_id
elif dataset_source == 'HF':
# TODO: HuggingFace mode is currently not supported!
hf_id = DATASETS_MAPPING[dataset_id]['hf_id']
assert hf_id is not None, \
f'{dataset_id} is not supported in HuggingFace'
return hf_id
else:
# for the local path
local_path = DATASETS_MAPPING[dataset_id]['local']
local_path = os.path.join(cache_dir, local_path)
assert os.path.exists(local_path), f'{local_path} does not exist!'
return local_path
Reference in New Issue View Git Blame Copy Permalink