mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
41 lines
1.5 KiB
Python
41 lines
1.5 KiB
Python
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||
|
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||
|
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||
|
|
from opencompass.datasets.xruler.ruler_qa import xRulerQaDataset
|
||
|
|
from opencompass.datasets.xruler.ruler_qa import xRulerQaEvaluator
|
||
|
|
LANGS = ['ar', 'bn', 'cs']
|
||
|
|
qa_configurations = [
|
||
|
|
{'lang': lang, 'dataset': 'squad', 'path': f'./data/xruler/xquad/xquad_{lang}.json'} for lang in LANGS
|
||
|
|
# {'dataset': 'hotpotqa', 'path': './data/ruler/hotpotqa.json'},
|
||
|
|
]
|
||
|
|
|
||
|
|
qa_datasets = []
|
||
|
|
for index, config in enumerate(qa_configurations):
|
||
|
|
dataset_dict = {
|
||
|
|
'abbr': f'xruler_qa_{config["dataset"]}_{config["lang"]}',
|
||
|
|
'dataset': config['dataset'],
|
||
|
|
'path': config['path'],
|
||
|
|
'lang': config['lang'],
|
||
|
|
'type': xRulerQaDataset,
|
||
|
|
'tokens_to_generate': 50,
|
||
|
|
'max_seq_length': 1024 * 8,
|
||
|
|
'reader_cfg': dict(input_columns=['prompt'], output_column='answer'),
|
||
|
|
'infer_cfg': dict(
|
||
|
|
prompt_template=dict(
|
||
|
|
type=PromptTemplate,
|
||
|
|
template=dict(
|
||
|
|
round=[
|
||
|
|
dict(role='HUMAN', prompt='{prompt}'),
|
||
|
|
dict(role='BOT', prompt='{answer}\n'),
|
||
|
|
]
|
||
|
|
),
|
||
|
|
),
|
||
|
|
retriever=dict(type=ZeroRetriever),
|
||
|
|
inferencer=dict(type=GenInferencer),
|
||
|
|
),
|
||
|
|
'eval_cfg': dict(
|
||
|
|
evaluator=dict(type=xRulerQaEvaluator),
|
||
|
|
),
|
||
|
|
}
|
||
|
|
qa_datasets.append(dataset_dict)
|