2023-07-04 21:34:55 +08:00
from opencompass . openicl . icl_prompt_template import PromptTemplate
from opencompass . openicl . icl_retriever import ZeroRetriever
from opencompass . openicl . icl_inferencer import GenInferencer
from opencompass . openicl . icl_evaluator import AccEvaluator
2023-07-06 12:27:41 +08:00
from opencompass . datasets import TheoremQADataset , TheoremQA_postprocess
2023-07-04 21:34:55 +08:00
TheoremQA_reader_cfg = dict (
input_columns = [ ' Question ' , ' Answer_type ' ] ,
output_column = ' Answer ' ,
train_split = ' test ' )
TheoremQA_prompt1 = """ You are a mathematician, you are supposed to answer the given question. You need to output the answer in your final sentence like " Therefore, the answer is ... " . The answer can only be one of the following forms:
1. a numerical value like 0.1 , no symbol and no unit at all .
2. a list of number like [ 2 , 3 , 4 ] .
3. True / False .
4. an option like ( a ) , ( b ) , ( c ) , ( d )
"""
TheoremQA_prompt2 = ' Question: {Question} \n Let \' s think step by step. '
TheoremQA_infer_cfg = dict (
prompt_template = dict (
type = PromptTemplate ,
template = dict (
begin = [
dict (
role = ' SYSTEM ' ,
fallback_role = ' HUMAN ' ,
prompt = TheoremQA_prompt1 ) ,
] ,
round = [
dict ( role = ' HUMAN ' , prompt = TheoremQA_prompt2 ) ,
] ) ) ,
retriever = dict ( type = ZeroRetriever ) ,
2023-07-06 12:27:41 +08:00
inferencer = dict ( type = GenInferencer , max_out_len = 512 ) )
2023-07-04 21:34:55 +08:00
TheoremQA_eval_cfg = dict (
evaluator = dict ( type = AccEvaluator ) ,
2023-07-06 12:27:41 +08:00
pred_postprocessor = dict ( type = TheoremQA_postprocess ) )
2023-07-04 21:34:55 +08:00
TheoremQA_datasets = [
dict (
abbr = ' TheoremQA ' ,
type = TheoremQADataset ,
path = " ./data/TheoremQA/test.csv " ,
reader_cfg = TheoremQA_reader_cfg ,
infer_cfg = TheoremQA_infer_cfg ,
eval_cfg = TheoremQA_eval_cfg )
]