mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
46cc7894e1
* [Feature] Support import configs/models/summarizers from whl * Update LCBench configs * Update * Update * Update * Update * update * Update * Update * Update * Update * Update
56 lines
1.5 KiB
Python
56 lines
1.5 KiB
Python
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
|
from opencompass.openicl.icl_inferencer import PPLOnlyInferencer
|
|
from opencompass.openicl.icl_evaluator import AveragePPLEvaluator
|
|
from opencompass.datasets import JsonlDataset
|
|
|
|
mmlu_datasets = []
|
|
|
|
mmlu_infer_cfg = dict(
|
|
prompt_template=dict(type=PromptTemplate, template='{text}'),
|
|
retriever=dict(type=ZeroRetriever),
|
|
inferencer=dict(type=PPLOnlyInferencer),
|
|
)
|
|
|
|
mmlu_eval_cfg = dict(evaluator=dict(type=AveragePPLEvaluator))
|
|
|
|
mmlu_reader_cfg = dict(
|
|
input_columns=['text'],
|
|
output_column=None,
|
|
)
|
|
|
|
mmlu_datasets.append(
|
|
dict(
|
|
abbr=f'mmlu-test-ppl',
|
|
type=JsonlDataset,
|
|
path='/mnt/petrelfs/zhoufengzhe/repos/cscripts/mock-datas/mmlu_test_content.jsonl',
|
|
reader_cfg=mmlu_reader_cfg,
|
|
infer_cfg=mmlu_infer_cfg,
|
|
eval_cfg=mmlu_eval_cfg
|
|
)
|
|
)
|
|
|
|
mmlu_infer_cfg = dict(
|
|
prompt_template=dict(type=PromptTemplate, template='{rephrase}'),
|
|
retriever=dict(type=ZeroRetriever),
|
|
inferencer=dict(type=PPLOnlyInferencer),
|
|
)
|
|
|
|
mmlu_eval_cfg = dict(evaluator=dict(type=AveragePPLEvaluator))
|
|
|
|
mmlu_reader_cfg = dict(
|
|
input_columns=['rephrase'],
|
|
output_column=None,
|
|
)
|
|
|
|
mmlu_datasets.append(
|
|
dict(
|
|
abbr=f'mmlu-ref-ppl',
|
|
type=JsonlDataset,
|
|
path='/mnt/petrelfs/zhoufengzhe/repos/cscripts/mock-datas/mmlu_test_content.jsonl',
|
|
reader_cfg=mmlu_reader_cfg,
|
|
infer_cfg=mmlu_infer_cfg,
|
|
eval_cfg=mmlu_eval_cfg
|
|
)
|
|
)
|