mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
a2093a81ef
* add support for matbench * fix dataset path * fix data load * fix * fix lint --------- Co-authored-by: Jucheng Hu <jucheng.hu.20@ucl.ac.uk> Co-authored-by: Myhs-phz <demarcia2014@126.com>
56 lines
2.0 KiB
Python
56 lines
2.0 KiB
Python
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
|
from opencompass.openicl.icl_evaluator import AccEvaluator
|
|
from opencompass.datasets.matbench.matbench import MatbenchDataset, MatbenchEvaluator_regression, MatbenchEvaluator_classification
|
|
|
|
|
|
|
|
matbench_reader_cfg = dict(
|
|
input_columns=['problem'], output_column='answer')
|
|
|
|
|
|
matbench_tasks = ['matbench_steels','matbench_expt_gap', 'matbench_expt_is_metal','matbench_glass']
|
|
|
|
matbench_datasets = []
|
|
|
|
for task in matbench_tasks:
|
|
if task in ['matbench_expt_is_metal','matbench_glass']:
|
|
matbench_infer_cfg = dict(
|
|
prompt_template=dict(
|
|
type=PromptTemplate,
|
|
template=dict(
|
|
round=[dict(role='HUMAN', prompt=f'{{problem}} Please present your answer by yes or no, do not output anything else.')])),
|
|
retriever=dict(type=ZeroRetriever),
|
|
inferencer=dict(type=GenInferencer))
|
|
|
|
matbench_eval_cfg = dict(
|
|
evaluator=dict(type=MatbenchEvaluator_classification),
|
|
pred_role='BOT')
|
|
|
|
elif task in ['matbench_steels','matbench_expt_gap']:
|
|
matbench_infer_cfg = dict(
|
|
prompt_template=dict(
|
|
type=PromptTemplate,
|
|
template=dict(
|
|
round=[dict(role='HUMAN', prompt=f'{{problem}} Please present your answer by one float number, do not output anything else.')])),
|
|
retriever=dict(type=ZeroRetriever),
|
|
inferencer=dict(type=GenInferencer))
|
|
|
|
|
|
matbench_eval_cfg = dict(
|
|
evaluator=dict(type=MatbenchEvaluator_regression),
|
|
pred_role='BOT')
|
|
|
|
|
|
matbench_datasets.append(
|
|
dict(
|
|
type=MatbenchDataset,
|
|
path=f'opencompass/Matbench',
|
|
task=task,
|
|
abbr=task,
|
|
reader_cfg=matbench_reader_cfg,
|
|
infer_cfg=matbench_infer_cfg,
|
|
eval_cfg=matbench_eval_cfg))
|
|
|