OpenCompass/opencompass/datasets/rolebench.py

import json
import os

from datasets import Dataset, DatasetDict

from opencompass.registry import LOAD_DATASET
from opencompass.utils import get_data_path

from .base import BaseDataset


@LOAD_DATASET.register_module()
class RoleBenchBaseDataset(BaseDataset):

    @staticmethod
    def load_single(source_file, desc_list):
        with open(source_file, 'r', encoding='utf-8') as f:
            source_data = [json.loads(line) for line in f.readlines()]
        dataset = [{
            'role': item['role'],
            'desc': desc_list[item['role']],
            'question': item['question'],
            'answer': item['generated'][0]
        } for item in source_data]
        return dataset

    @staticmethod
    def load_desc(path):
        path = get_data_path(path, local_mode=True)
        with open(path, 'r', encoding='utf-8') as f:
            desc_list = json.load(f)
        return desc_list

    @staticmethod
    def load_dataset(path, desc_list):
        train_data_list = RoleBenchBaseDataset.load_single(
            os.path.join(path, 'general/train.jsonl'), desc_list)
        train_data_list.extend(
            RoleBenchBaseDataset.load_single(
                os.path.join(path, 'role_specific/train.jsonl'), desc_list))
        test_dataset = RoleBenchBaseDataset.load_single(
            os.path.join(path, 'general/test.jsonl'), desc_list)
        test_dataset.extend(
            RoleBenchBaseDataset.load_single(
                os.path.join(path, 'role_specific/test.jsonl'), desc_list))
        return Dataset.from_list(train_data_list).shuffle(
            seed=42), Dataset.from_list(test_dataset).shuffle(seed=42)


@LOAD_DATASET.register_module()
class InstructionGeneralizationEnglishDataset(RoleBenchBaseDataset):

    @staticmethod
    def load(path):
        path = get_data_path(path, local_mode=True)
        desc_list = RoleBenchBaseDataset.load_desc(
            os.path.join(path, 'profiles-eng/desc.json'))
        path = os.path.join(path, 'rolebench-eng/instruction-generalization')
        train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset(
            path, desc_list)
        return DatasetDict({'train': train_dataset, 'test': test_dataset})


@LOAD_DATASET.register_module()
class RoleGeneralizationEnglishDataset(RoleBenchBaseDataset):

    @staticmethod
    def load(path):
        path = get_data_path(path, local_mode=True)
        desc_list = RoleBenchBaseDataset.load_desc(
            os.path.join(path, 'profiles-eng/desc.json'))
        path = os.path.join(path, 'rolebench-eng/role-generalization')
        train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset(
            path, desc_list)
        return DatasetDict({'train': train_dataset, 'test': test_dataset})


@LOAD_DATASET.register_module()
class InstructionGeneralizationChineseDataset(RoleBenchBaseDataset):

    @staticmethod
    def load(path):
        desc_list = RoleBenchBaseDataset.load_desc(
            os.path.join(path, 'profiles-zh/desc.json'))
        path = os.path.join(path, 'rolebench-zh')
        train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset(
            path, desc_list)
        return DatasetDict({'train': train_dataset, 'test': test_dataset})