add support for matbench

dataset-index.yml

@@ -110,6 +110,12 @@
     paper: ''
     configpath: opencompass/configs/datasets/mastermath2024v1/mastermath2024v1_gen.py
     configpath_llmjudge: ''
+- matbench:
+    name: matbench
+    category: Material
+    paper: 'https://www.nature.com/articles/s41524-020-00406-3'
+    configpath: opencompass/configs/datasets/matbench/matbench_gen.py
+    configpath_llmjudge: ''
 - medbench:
     name: MedBench
     category: Knowledge / Medicine

opencompass/configs/datasets/matbench/matbench_gen.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets.matbench.matbench import MatbenchDataset, MatbenchEvaluator_regression, MatbenchEvaluator_classification
+
+
+
+matbench_reader_cfg = dict(
+    input_columns=['problem'], output_column='answer')
+
+
+matbench_tasks =  ['matbench_steels','matbench_expt_gap', 'matbench_expt_is_metal','matbench_glass']
+
+matbench_datasets = []
+
+for task in matbench_tasks:
+    if task in ['matbench_expt_is_metal','matbench_glass']:
+        matbench_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt=f'{{problem}} Please present your answer by yes or no, do not output anything else.')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer))
+
+        matbench_eval_cfg = dict(
+            evaluator=dict(type=MatbenchEvaluator_classification),
+            pred_role='BOT')
+
+    elif task in ['matbench_steels','matbench_expt_gap']:
+        matbench_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt=f'{{problem}} Please present your answer by one float number, do not output anything else.')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer))
+
+
+        matbench_eval_cfg = dict(
+            evaluator=dict(type=MatbenchEvaluator_regression),
+            pred_role='BOT')
+
+
+    matbench_datasets.append(
+        dict(
+            type=MatbenchDataset,
+            path=f'opencompass/Matbench',
+            abbr=task,
+            reader_cfg=matbench_reader_cfg,
+            infer_cfg=matbench_infer_cfg,
+            eval_cfg=matbench_eval_cfg))
+

opencompass/datasets/__init__.py

@@ -86,6 +86,7 @@ from .longbench import *  # noqa: F401, F403
 from .longbenchv2 import *  # noqa: F401, F403
 from .lveval import *  # noqa: F401, F403
 from .mastermath2024v1 import *  # noqa: F401, F403
+from .matbench import *  # noqa: F401, F403
 from .math import *  # noqa: F401, F403
 from .math401 import *  # noqa: F401, F403
 from .math_intern import *  # noqa: F401, F403

opencompass/datasets/matbench/__init__.py

@@ -0,0 +1,3 @@
+# flake8: noqa
+
+from .matbench import *  # noqa: F401, F403

opencompass/datasets/matbench/matbench.py

@@ -0,0 +1,82 @@
+import json
+
+from datasets import Dataset
+from sklearn.metrics import (accuracy_score, f1_score, precision_score,
+                             recall_score)
+
+from opencompass.datasets.matbench.post_process import (parse_float_answer,
+                                                        parse_true_false_answer
+                                                        )
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
+
+from ..base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class MatbenchDataset(BaseDataset):
+
+    @staticmethod
+    def load(path):
+        dataset = []
+        with open(path, 'r', encoding='utf-8') as file:
+            data = json.load(file)
+            for item in data:
+                dataset.append({
+                    'problem': item['problem'],
+                    'answer': item['answer'],
+                })
+        dataset = Dataset.from_list(dataset)
+        return dataset
+
+
+@ICL_EVALUATORS.register_module()
+class MatbenchEvaluator_regression(BaseEvaluator):
+
+    def score(self, predictions, references):
+        mae_sum = 0
+        count = 0
+        details = []
+        for pred, ref in zip(predictions, references):
+            pred = parse_float_answer(pred)
+            detail = {'pred': pred, 'answer': ref, 'error': None}
+            count += 1
+            try:
+                error = abs(float(pred) - float(ref))
+                mae_sum += error
+                detail['error'] = error
+            except Exception as e:
+                detail['error'] = str(e)
+            details.append(detail)
+        mae = mae_sum / count if count > 0 else 0
+        result = {'mae': mae, 'details': details}
+        return result
+
+
+@ICL_EVALUATORS.register_module()
+class MatbenchEvaluator_classification(BaseEvaluator):
+
+    def score(self, predictions, references):
+        details = []
+        predictions_parsed = []
+        for pred, ref in zip(predictions, references):
+            pred = parse_true_false_answer(pred)
+            detail = {'pred': pred, 'answer': ref, 'correct': False}
+            if pred == ref:
+                detail['correct'] = True
+            details.append(detail)
+            predictions_parsed.append(pred)
+        accuracy = accuracy_score(references, predictions_parsed)
+        precision = precision_score(references,
+                                    predictions_parsed,
+                                    average='binary')
+        recall = recall_score(references, predictions_parsed, average='binary')
+        f1 = f1_score(references, predictions_parsed, average='binary')
+
+        return {
+            'accuracy': accuracy,
+            'precision': precision,
+            'recall': recall,
+            'f1_score': f1,
+            'details': details
+        }

opencompass/datasets/matbench/post_process.py

@@ -0,0 +1,25 @@
+# flake8: noqa
+
+import re
+
+
+def parse_float_answer(raw_string, option=''):
+    number_pattern = re.compile(r'[-+]?\d+(\.\d+)?([eE][-+]?\d+)?')
+
+    # Search for the first match
+    match = number_pattern.search(raw_string)
+    if match:
+        # Extract the matched number and convert it to float
+        return float(match.group())
+    else:
+        # Return None if no number is found
+        return 0
+
+
+def parse_true_false_answer(raw_string, option=''):
+    if 'yes' in raw_string.lower():
+        return True
+    elif 'no' in raw_string.lower():
+        return False
+    else:
+        return True

opencompass/utils/datasets_info.py

@@ -27,6 +27,11 @@ DATASETS_MAPPING = {
         "hf_id": "opencompass/ai2_arc",
         "local": "./data/ARC/ARC-e/ARC-Easy-Dev.jsonl",
     },
+    "opencompass/Matbench": {
+    # "ms_id": "opencompass/Matbench",
+    # "hf_id": "opencompass/Matbench",
+    "local": "./data/Matbench",
+    },
     # BBH
     "opencompass/bbh": {
         "ms_id": "opencompass/bbh",