chore: fix line endings and formatting; add maritime_bench dataset

opencompass/configs/datasets/maritimebench/maritimebench_gen.py

@@ -0,0 +1,42 @@
+from opencompass.datasets import MaritimeBenchDataset
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.utils.text_postprocessors import first_option_postprocess
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+
+maritimebench_reader_cfg = dict(
+    input_columns=['question', 'A', 'B', 'C', 'D'],
+    output_column='answer',
+    train_split='test'  # 明确指定使用test分割
+)
+
+maritimebench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='请回答单选题。要求只输出选项，不输出解释，将选项放在<>里，直接输出答案。示例：\n\n题目：在船舶主推进动力装置中，传动轴系在运转中承受以下复杂的应力和负荷，但不包括______。\n选项：\nA. 电磁力\nB. 压拉应力\nC. 弯曲应力\nD. 扭应力\n答：<A> 当前题目:\n {question}\nA:{A}\nB:{B}\nC:{C}\nD:{D}')
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),  # 不使用上下文
+    inferencer=dict(type=GenInferencer)  # 添加推理器配置
+)
+
+maritimebench_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCD')
+)
+
+maritimebench_datasets = [
+    dict(
+        abbr='maritimebench',
+        type=MaritimeBenchDataset,
+        name='default',
+        path='opencompass/maritimebench',
+        reader_cfg=maritimebench_reader_cfg,
+        infer_cfg=maritimebench_infer_cfg,
+        eval_cfg=maritimebench_eval_cfg
+    )
+]

opencompass/datasets/__init__.py

@@ -85,6 +85,7 @@ from .llm_compression import LLMCompressionDataset  # noqa: F401, F403
 from .longbench import *  # noqa: F401, F403
 from .longbenchv2 import *  # noqa: F401, F403
 from .lveval import *  # noqa: F401, F403
+from .maritime_bench import *  # noqa: F401, F403
 from .mastermath2024v1 import *  # noqa: F401, F403
 from .math import *  # noqa: F401, F403
 from .math401 import *  # noqa: F401, F403

opencompass/datasets/maritime_bench.py

@@ -0,0 +1,64 @@
+import json
+import os.path as osp
+from os import environ
+
+import datasets
+from datasets import Dataset, DatasetDict
+
+from opencompass.registry import LOAD_DATASET
+from opencompass.utils import get_data_path
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class MaritimeBenchDataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str, name: str) -> datasets.Dataset:
+        path = get_data_path(path)
+        dataset = DatasetDict()
+        dataset_list = []
+
+        if environ.get('DATASET_SOURCE') == 'ModelScope':
+            from modelscope import MsDataset
+            for split in ['test']:
+                # 从 ModelScope 加载数据
+                ms_dataset = MsDataset.load(path,
+                                            subset_name=name,
+                                            split=split)
+
+                for line in ms_dataset:
+                    question = line['question']
+                    A = line['A']
+                    B = line['B']
+                    C = line['C']
+                    D = line['D']
+                    answer = line['answer']
+                    dataset_list.append({
+                        'question': question,
+                        'A': A,
+                        'B': B,
+                        'C': C,
+                        'D': D,
+                        'answer': answer,
+                    })
+            # dataset[split] = Dataset.from_list(dataset_list)
+        else:
+            for split in ['test']:
+                filename = osp.join(path, split, f'{name}_{split}.jsonl')
+                with open(filename, encoding='utf-8') as f:
+                    for line in f:
+                        data = json.loads(line)
+                        dataset_list.append({
+                            'question': data['question'],
+                            'A': data['A'],
+                            'B': data['B'],
+                            'C': data['C'],
+                            'D': data['D'],
+                            'answer': data['answer']
+                        })
+
+        dataset[split] = Dataset.from_list(dataset_list)
+
+        return dataset

opencompass/utils/datasets_info.py

@@ -420,6 +420,11 @@ DATASETS_MAPPING = {
         "hf_id": "",
         "local": "./data/OlympiadBench",
     },
+    "opencompass/maritimebench": {
+        "ms_id": "HiDolphin/MaritimeBench",
+        "hf_id": "",
+        "local": "./data/maritimebench",
+    },
 }
 
 DATASETS_URL = {