diff --git a/opencompass/configs/datasets/PromptCBLUE/PromptCBLUE_gen.py b/opencompass/configs/datasets/PromptCBLUE/PromptCBLUE_gen.py new file mode 100644 index 00000000..b81f5caa --- /dev/null +++ b/opencompass/configs/datasets/PromptCBLUE/PromptCBLUE_gen.py @@ -0,0 +1,64 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.utils.text_postprocessors import first_capital_postprocess +from opencompass.datasets import PromptCBLUEDataset + +# 1. 子数据集名称 +PromptCBLUE_lifescience_sets = [ + 'CHIP-CDN', 'CHIP-CTC', 'KUAKE-QIC', 'IMCS-V2-DAC', + 'CHIP-STS', 'KUAKE-QQR', 'KUAKE-IR', 'KUAKE-QTR' +] + +# 2. Reader 配置 +reader_cfg = dict( + input_columns=['input', 'answer_choices', 'options_str'], + output_column='target', + train_split='dev', +) + +# 3. Prompt 模板:末行固定 ANSWER: $LETTER +_HINT = 'Given the ICD-10 candidate terms below, choose the normalized term(s) matching the original diagnosis.' + +query_template = f"""{_HINT} + +Original diagnosis: {{input}} + +Options: +{{options_str}} + +The last line of your response must be exactly: +ANSWER: $LETTER +""".strip() + +infer_cfg_common = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[dict(role='HUMAN', prompt=query_template)]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +# 4. 评估配置:与 MMLU 同款 +eval_cfg_common = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type=first_capital_postprocess), +) + +# 5. 组装数据集配置 +promptcblue_datasets = [] +for ds_name in PromptCBLUE_lifescience_sets: + promptcblue_datasets.append(dict( + abbr=f'promptcblue_{ds_name.lower().replace("-", "_")}_norm', + type=PromptCBLUEDataset, + path='/fs-computility/ai4sData/shared/lifescience/tangcheng/LifeScience/opencompass_val/datasets/PromptCBLUE', + name=ds_name, + reader_cfg=reader_cfg, + infer_cfg=infer_cfg_common, + eval_cfg=eval_cfg_common, + )) + +# ★ OpenCompass 识别的出口变量 +datasets = promptcblue_datasets diff --git a/opencompass/configs/datasets/PromptCBLUE/PromptCBLUE_llm_judge_gen.py b/opencompass/configs/datasets/PromptCBLUE/PromptCBLUE_llm_judge_gen.py new file mode 100644 index 00000000..22de56a4 --- /dev/null +++ b/opencompass/configs/datasets/PromptCBLUE/PromptCBLUE_llm_judge_gen.py @@ -0,0 +1,102 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.evaluator import GenericLLMEvaluator +from opencompass.datasets import generic_llmjudge_postprocess +from opencompass.datasets import PromptCBLUEDataset + +PromptCBLUE_lifescience_sets = [ + 'CHIP-CDN', 'CHIP-CTC', 'KUAKE-QIC', 'IMCS-V2-DAC', + 'CHIP-STS', 'KUAKE-QQR', 'KUAKE-IR', 'KUAKE-QTR' +] +# Query template (keep original) +QUERY_TEMPLATE = """ +Given a medical diagnosis description and labeled ICD-10 candidate terms below, select the matching normalized term(s). +Original diagnosis: {input} + +Options: +{options_str} + +The last line of your response must be exactly in the format: +ANSWER: +""".strip() + +# Grader template (keep original) +GRADER_TEMPLATE = """ +As an expert evaluator, judge whether the candidate's answer matches the gold standard below. +Return 'A' for CORRECT or 'B' for INCORRECT, with no additional text. + +Original diagnosis: {input} + +Options: +{options_str} + +Gold answer: {target} + +Candidate answer: {prediction} +""".strip() + +# Common reader config +reader_cfg = dict( + input_columns=['input', 'answer_choices', 'options_str'], + output_column='target', + train_split='dev' +) + +# Assemble LLM evaluation datasets +promptcblue_llm_datasets = [] +for name in PromptCBLUE_lifescience_sets: + infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role='HUMAN', prompt=QUERY_TEMPLATE), + ]), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ) + + eval_cfg = dict( + evaluator=dict( + type=GenericLLMEvaluator, + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt='You are an expert judge for medical term normalization tasks.', + ) + ], + round=[ + dict(role='HUMAN', prompt=GRADER_TEMPLATE), + ], + ) + ), + dataset_cfg=dict( + type=PromptCBLUEDataset, + path='/fs-computility/ai4sData/shared/lifescience/tangcheng/LifeScience/opencompass_val/datasets/PromptCBLUE', + name=name, + reader_cfg=reader_cfg, + ), + judge_cfg=dict(), + dict_postprocessor=dict(type=generic_llmjudge_postprocess), + ), + pred_role='BOT', + ) + + promptcblue_llm_datasets.append( + dict( + abbr=f"promptcblue_{name.lower().replace('-', '_')}_norm_llm", + type=PromptCBLUEDataset, + path='/fs-computility/ai4sData/shared/lifescience/tangcheng/LifeScience/opencompass_val/datasets/PromptCBLUE', + name=name, + reader_cfg=reader_cfg, + infer_cfg=infer_cfg, + eval_cfg=eval_cfg, + mode='singlescore', + ) + ) diff --git a/opencompass/configs/datasets/PromptCBLUE/README.md b/opencompass/configs/datasets/PromptCBLUE/README.md new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/PromptCBLUE.py b/opencompass/datasets/PromptCBLUE.py new file mode 100644 index 00000000..0d71916e --- /dev/null +++ b/opencompass/datasets/PromptCBLUE.py @@ -0,0 +1,61 @@ +import json +import os.path as osp + +from datasets import Dataset, DatasetDict + +from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path + +from .base import BaseDataset # 保持与 MMLUDataset 同级的导包风格 + + +@LOAD_DATASET.register_module() +class PromptCBLUEDataset(BaseDataset): + """Loader for PromptCBLUE life-science tasks (CHIP-CDN, CHIP-CTC …). + + - 只读 `dev.json`。 + - 保留指定 `task_dataset` 的所有任务类型(包括 normalization、cls 等)。 + - 若 `target` 不在 `answer_choices`,自动追加;并生成 `options_str` + (形如 “A. 选项1\\nB. 选项2 …”)。 + - 返回 `DatasetDict`,将 dev 复制到 test 以满足评估流程。 + """ + + @staticmethod + def load(path: str, name: str, **kwargs): + path = get_data_path(path) + file_path = osp.join(path, 'dev.json') + if not osp.exists(file_path): + raise FileNotFoundError(f'`dev.json` not found under {path}') + + records = [] + with open(file_path, 'r', encoding='utf-8') as f: + for line in f: + rec = json.loads(line) + if rec.get('task_dataset') != name: + continue # 过滤子数据集 + + choices = rec.get('answer_choices', []).copy() + target = rec.get('target') + if target not in choices: + choices.append(target) + + options_str = '\n'.join(f'{chr(65+i)}. {opt}' + for i, opt in enumerate(choices)) + + records.append({ + 'input': rec['input'], + 'answer_choices': choices, + 'options_str': options_str, + 'target': target, + }) + + # 保证列完整,即使 records 为空 + if records: + ds = Dataset.from_list(records) + else: + ds = Dataset.from_dict({ + k: [] + for k in ['input', 'answer_choices', 'options_str', 'target'] + }) + dataset = DatasetDict(dev=ds, test=ds) # dev 与 test 指向同一份 + return dataset diff --git a/opencompass/datasets/__init__.py b/opencompass/datasets/__init__.py index b00162d1..c034715a 100644 --- a/opencompass/datasets/__init__.py +++ b/opencompass/datasets/__init__.py @@ -115,6 +115,7 @@ from .OlympiadBench import * # noqa: F401, F403 from .OpenFinData import * # noqa: F401, F403 from .physics import * # noqa: F401, F403 from .piqa import * # noqa: F401, F403 +from .PromptCBLUE import PromptCBLUEDataset # noqa: F401, F403 from .py150 import * # noqa: F401, F403 from .qasper import * # noqa: F401, F403 from .qaspercut import * # noqa: F401, F403