OpenCompass/opencompass/datasets/subjective_cmp.py
bittersweet1999 1fe152b3e8
[Feature] Support AlignmentBench infer and judge (#697)
* alignmentbench infer and judge

* alignmentbench

* alignmentbench done

* alignment all done

* alignment all done
2023-12-13 19:59:30 +08:00

34 lines
1000 B
Python

import json
import os.path as osp
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET
from .base import BaseDataset
@LOAD_DATASET.register_module()
class SubjectiveCmpDataset(BaseDataset):
def load(self, path: str, name: str):
filename = osp.join(path, f'{name}.json')
dataset = DatasetDict()
raw_data = []
with open(filename, 'r', encoding='utf-8') as f:
json_data = json.load(f)
for problem in json_data:
question = problem['question']
capability = problem['capability']
others = problem['others']
raw_data.append({
'question': question,
'capability': capability,
'others': others,
'judge': {
'capability': capability
}
})
dataset = Dataset.from_list(raw_data)
return dataset