mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
b35d991786
* [Feature] Add reasonbench dataset * add configs for supporting generative inference & merge datasets in the same category * modify config filename to prompt version * fix codes to meet pre-commit requirements * lint the code to meet pre-commit requirements * Align Load_data Sourcecode Briefly * fix bugs * reduce code redundancy
141 lines
3.9 KiB
Python
141 lines
3.9 KiB
Python
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
|
from opencompass.openicl.icl_retriever import FixKRetriever
|
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
|
from opencompass.openicl.icl_evaluator import AccEvaluator
|
|
from opencompass.utils.text_postprocessors import first_capital_postprocess
|
|
from opencompass.datasets.reasonbench import ReasonBenchDataset
|
|
|
|
reasonbench_eval_cfg = dict(
|
|
evaluator=dict(type=AccEvaluator),
|
|
pred_postprocessor=dict(type=first_capital_postprocess)
|
|
)
|
|
|
|
reader_cfgs = []
|
|
for i in range(2, 5):
|
|
choices = ["A", "B", "C", "D"][:i]
|
|
|
|
reader_cfgs.append(dict(
|
|
input_columns=["prompt_ppl"],
|
|
output_column="label_ppl")
|
|
)
|
|
|
|
infer_cfg=dict(
|
|
ice_template=dict(
|
|
type=PromptTemplate,
|
|
template=dict(
|
|
begin="</E>",
|
|
round=[
|
|
dict(
|
|
role="HUMAN",
|
|
prompt="</E>{prompt_ppl}"
|
|
),
|
|
dict(role="BOT", prompt="Answer: {label_ppl}"),
|
|
]),
|
|
ice_token="</E>",
|
|
),
|
|
retriever=dict(type=FixKRetriever, fix_id_list=[]),
|
|
inferencer=dict(type=GenInferencer)
|
|
)
|
|
|
|
|
|
CausalReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-causal",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/causal.jsonl",
|
|
reader_cfg=reader_cfgs[0],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
CommonsenseReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-commonsense",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/commonsense.jsonl",
|
|
reader_cfg=reader_cfgs[1],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
AbductiveReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-abductive",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/abductive.jsonl",
|
|
reader_cfg=reader_cfgs[0],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
DeductiveReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-deductive",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/deductive.jsonl",
|
|
reader_cfg=reader_cfgs[1],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
InductiveReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-inductive",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/inductive.jsonl",
|
|
reader_cfg=reader_cfgs[0],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
SymbolicReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-symbolic",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/symbolic.jsonl",
|
|
reader_cfg=reader_cfgs[2],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
CLEVA_CommonsenseReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-cleva_commonsense",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/cleva_commonsense.jsonl",
|
|
reader_cfg=reader_cfgs[1],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
CLEVA_DeductiveReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-cleva_deductive",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/cleva_deductive.jsonl",
|
|
reader_cfg=reader_cfgs[1],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
CLEVA_InductiveReasoningDataset = [
|
|
dict(
|
|
abbr="reasonbench-cleva_inductive",
|
|
type=ReasonBenchDataset,
|
|
path="data/reasonbench/cleva_inductive.jsonl",
|
|
reader_cfg=reader_cfgs[0],
|
|
infer_cfg=infer_cfg,
|
|
eval_cfg=reasonbench_eval_cfg),
|
|
]
|
|
|
|
reasonbench_datasets = \
|
|
CLEVA_CommonsenseReasoningDataset + \
|
|
CLEVA_DeductiveReasoningDataset + \
|
|
CLEVA_InductiveReasoningDataset + \
|
|
CausalReasoningDataset + \
|
|
CommonsenseReasoningDataset + \
|
|
AbductiveReasoningDataset + \
|
|
DeductiveReasoningDataset + \
|
|
InductiveReasoningDataset + \
|
|
SymbolicReasoningDataset
|