from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import FixKRetriever
from opencompass.openicl.icl_inferencer import LLInferencer
from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import winograndeDataset_V3

winogrande_reader_cfg = dict(
    input_columns=['opt1', 'opt2'],
    output_column='answer',
    train_split="train_xs",
    test_split="dev",
)

question_and_options = "Which of the following is a good sentence:\nA. {opt1}\nB. {opt2}"
winogrande_infer_cfg = dict(
    ice_template=dict(
        type=PromptTemplate,
        template={'A': '{opt1}', 'B': '{opt2}'},
    ),
    prompt_template=dict(
        type=PromptTemplate,
        template={'A': '</E>{opt1}', 'B': '</E>{opt2}'},
        ice_token="</E>",
    ),
    retriever=dict(type=FixKRetriever, fix_id_list=[0, 2, 4, 6, 8]),
    inferencer=dict(type=LLInferencer),
)
winogrande_eval_cfg = dict(evaluator=dict(type=AccEvaluator))

winogrande_datasets = [
    dict(
        abbr='winogrande',
        type=winograndeDataset_V3,
        path='./data/winogrande',
        reader_cfg=winogrande_reader_cfg,
        infer_cfg=winogrande_infer_cfg,
        eval_cfg=winogrande_eval_cfg)
]