mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
46cc7894e1
* [Feature] Support import configs/models/summarizers from whl * Update LCBench configs * Update * Update * Update * Update * update * Update * Update * Update * Update * Update
55 lines
1.9 KiB
Python
55 lines
1.9 KiB
Python
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||
from opencompass.openicl.icl_inferencer import GenInferencer
|
||
from opencompass.datasets import PJExamDataset, PJExamEvaluator
|
||
|
||
PJExam_datasets = []
|
||
for _name in [
|
||
'gk-2022-v1', 'gk-2022-v1-math', 'gk-2023-v1', 'gk-2023-v1-math',
|
||
'gk-2023-v2', 'gk-2023-v2-math', 'zk-2022-v1'
|
||
]:
|
||
_hint = '请你做一道</major>选择题\n请你一步一步思考并将思考过程写在【解析】和<eoe>之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和<eoa>之间。\n例如：【答案】A<eoa>\n完整的题目回答的格式如下：\n【解析】...<eoe>\n【答案】...<eoa>\n请你严格按照上述格式作答。\n题目如下：\n'
|
||
_reader_cfg = {
|
||
'input_columns': ['question'],
|
||
'output_column': 'std_ans',
|
||
},
|
||
_infer_cfg = {
|
||
'ice_template': {
|
||
'type': PromptTemplate,
|
||
'template': {
|
||
'round': [{
|
||
'role': 'HUMAN',
|
||
'prompt': _hint + '{question}',
|
||
}]
|
||
},
|
||
'ice_token': '</E>'
|
||
},
|
||
'retriever': {
|
||
'type': ZeroRetriever
|
||
},
|
||
'inferencer': {
|
||
'type': GenInferencer,
|
||
'max_out_len': 1024,
|
||
}
|
||
}
|
||
_eval_cfg = {
|
||
'evaluator': {
|
||
'type': PJExamEvaluator
|
||
},
|
||
'pred_role': 'BOT',
|
||
'ds_column': 'eval_infos'
|
||
}
|
||
_dataset = {
|
||
'type': PJExamDataset,
|
||
'abbr': 'PJExamDataset-' + _name,
|
||
'path': './data/PJExam',
|
||
'name': _name,
|
||
'reader_cfg': _reader_cfg,
|
||
'infer_cfg': _infer_cfg,
|
||
'eval_cfg': _eval_cfg,
|
||
}
|
||
|
||
PJExam_datasets.append(_dataset)
|
||
|
||
del _name, _hint, _reader_cfg, _infer_cfg, _eval_cfg, _dataset
|