from opencompass.openicl.icl_prompt_template import PromptTemplate from opencompass.openicl.icl_retriever import ZeroRetriever from opencompass.openicl.icl_inferencer import GenInferencer from opencompass.openicl.icl_evaluator import AccEvaluator from opencompass.datasets import ARCDataset from opencompass.utils.text_postprocessors import first_capital_postprocess ARC_e_reader_cfg = dict( input_columns=["question", "textA", "textB", "textC", "textD"], output_column="answerKey") ARC_e_infer_cfg = dict( prompt_template=dict( type=PromptTemplate, template=dict( round=[ dict( role="HUMAN", prompt= "Question: {question}\nA. {textA}\nB. {textB}\nC. {textC}\nD. {textD}\nAnswer:" ) ], ), ), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer), ) ARC_e_eval_cfg = dict( evaluator=dict(type=AccEvaluator), pred_role="BOT", pred_postprocessor=dict(type=first_capital_postprocess), ) ARC_e_datasets = [ dict( abbr="ARC-e", type=ARCDataset, path="./data/ARC/ARC-e/ARC-Easy-Dev.jsonl", reader_cfg=ARC_e_reader_cfg, infer_cfg=ARC_e_infer_cfg, eval_cfg=ARC_e_eval_cfg, ) ]