mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
79 lines
2.6 KiB
Python
79 lines
2.6 KiB
Python
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||
|
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||
|
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||
|
|
from opencompass.datasets import MATHDataset, MATHEvaluator, math_postprocess_v2, GaoKaoMATHEvaluator
|
||
|
|
from opencompass.utils.model_postprocessors import naive_model_postprocess, xfinder_postprocess
|
||
|
|
from opencompass.utils.postprocessors.naive import MATH_NAVIE_PROMPT_TEMPLATE
|
||
|
|
|
||
|
|
# ----------------------------- Eval Parameters -----------------------------
|
||
|
|
## Postprocess function
|
||
|
|
post_func = 're' # 're', 'xfinder_model', 'naive_model'
|
||
|
|
|
||
|
|
## Evalute function
|
||
|
|
eval_func = 'naive_model' # 're', 'naive_model'
|
||
|
|
|
||
|
|
## Model api url
|
||
|
|
xfinder_url = 'http://0.0.0.0:23333/v1' # for 'xFinder-qwen1505' if post_func is 'xfinder_model'
|
||
|
|
naive_model_name = 'Qwen/Qwen2.5-72B-Instruct' # replace with your model name
|
||
|
|
naive_model_url = ['http://22.8.6.22:23333/v1', 'http://22.8.67.84:23333/v1', 'http://22.8.72.81:23333/v1', 'http://22.9.42.143:23333/v1'] # Multi-apis for accerlation
|
||
|
|
|
||
|
|
# ----------------------------- Detailed Config -----------------------------
|
||
|
|
|
||
|
|
math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
|
||
|
|
|
||
|
|
math_infer_cfg = dict(
|
||
|
|
prompt_template=dict(
|
||
|
|
type=PromptTemplate,
|
||
|
|
template=dict(
|
||
|
|
round=[
|
||
|
|
dict(role='HUMAN', prompt='{problem}\nPlease reason step by step, and put your final answer within \\boxed{}.'),
|
||
|
|
]
|
||
|
|
),
|
||
|
|
),
|
||
|
|
retriever=dict(type=ZeroRetriever),
|
||
|
|
inferencer=dict(type=GenInferencer, max_out_len=1024),
|
||
|
|
)
|
||
|
|
|
||
|
|
if post_func == 're':
|
||
|
|
pred_postprocessor = dict(type=math_postprocess_v2)
|
||
|
|
elif post_func == 'xfinder_model':
|
||
|
|
pred_postprocessor = dict(
|
||
|
|
type=xfinder_postprocess,
|
||
|
|
question_type='math',
|
||
|
|
model_name='xFinder-qwen1505',
|
||
|
|
num_processes=128,
|
||
|
|
api_url=xfinder_url,
|
||
|
|
)
|
||
|
|
elif post_func == 'naive_model':
|
||
|
|
pred_postprocessor = dict(
|
||
|
|
type=naive_model_postprocess,
|
||
|
|
custom_instruction=MATH_NAVIE_PROMPT_TEMPLATE,
|
||
|
|
model_name=naive_model_name,
|
||
|
|
num_processes=64,
|
||
|
|
api_url=naive_model_url,
|
||
|
|
)
|
||
|
|
|
||
|
|
if eval_func == 're':
|
||
|
|
evaluator = dict(type=MATHEvaluator, version='v2')
|
||
|
|
elif eval_func == 'naive_model':
|
||
|
|
evaluator = dict(
|
||
|
|
type=GaoKaoMATHEvaluator,
|
||
|
|
model_name=naive_model_name,
|
||
|
|
url=naive_model_url,
|
||
|
|
)
|
||
|
|
|
||
|
|
math_eval_cfg = dict(
|
||
|
|
evaluator=evaluator, pred_postprocessor=pred_postprocessor,
|
||
|
|
)
|
||
|
|
|
||
|
|
math_datasets = [
|
||
|
|
dict(
|
||
|
|
type=MATHDataset,
|
||
|
|
abbr='math',
|
||
|
|
path='opencompass/math',
|
||
|
|
reader_cfg=math_reader_cfg,
|
||
|
|
infer_cfg=math_infer_cfg,
|
||
|
|
eval_cfg=math_eval_cfg,
|
||
|
|
)
|
||
|
|
]
|