added rolebench dataset. (#633)

* added rolebench

* 修改了不合理的变量名

* 修改了评论中的变量名
This commit is contained in:
rolellm 2023-12-01 22:54:42 +08:00 committed by GitHub
parent f4bbff6537
commit e10f1c9139
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 207 additions and 0 deletions

View File

@ -0,0 +1,41 @@
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.openicl.icl_evaluator import RougeEvaluator
from opencompass.datasets.rolebench import InstructionGeneralizationEnglishDataset
instruction_generalization_eng_reader_cfg = dict(
input_columns=['role', 'desc', 'question'],
output_column='answer',
train_split='train',
test_split='test'
)
instruction_generalization_eng_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template=dict(
begin=[
dict(role='SYSTEM', fallback_role='HUMAN', prompt='You are {role}, your description is: {desc}. Now please answer some questions to accurately show your personality traits! Your speaking style should fully imitate the personality role assigned to you! Please do not expose that you are an artificial intelligence model or a language model, you must always remember that you are only assigned one personality role. Dont be verbose or too formal or polite when speaking.'),
],
round=[
dict(role='HUMAN', prompt='{question}'),
dict(role='BOT', prompt=''),
], )),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer, max_out_len=512)
)
instruction_generalization_eng_eval_cfg = dict(
evaluator=dict(type=RougeEvaluator),
pred_role='BOT'
)
instruction_generalization_eng_datasets = [
dict(
type=InstructionGeneralizationEnglishDataset,
path='ZenMoore/RoleBench',
reader_cfg=instruction_generalization_eng_reader_cfg,
infer_cfg=instruction_generalization_eng_infer_cfg,
eval_cfg=instruction_generalization_eng_eval_cfg)
]

View File

@ -0,0 +1,41 @@
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.openicl.icl_evaluator import RougeEvaluator
from opencompass.datasets.rolebench import InstructionGeneralizationChineseDataset
instruction_generalization_zh_reader_cfg = dict(
input_columns=['role', 'desc', 'question'],
output_column='answer',
train_split='train',
test_split='test'
)
instruction_generalization_zh_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template=dict(
begin=[
dict(role='SYSTEM', fallback_role='HUMAN', prompt='你是{role},你的特征描述是:{desc}。现在请你回答我的一些问题,以准确展现你的人格特征!你的说话风格要全面模仿被赋予的人格角色!请不要暴露你是人工智能模型或者语言模型,你要时刻记住你只被赋予的一个人格角色。说话不要嗦,也不要太过于正式或礼貌。'),
],
round=[
dict(role='HUMAN', prompt='{question}'),
dict(role='BOT', prompt=''),
], )),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer, max_out_len=512)
)
instruction_generalization_zh_eval_cfg = dict(
evaluator=dict(type=RougeEvaluator),
pred_role='BOT'
)
instruction_generalization_zh_datasets = [
dict(
type=InstructionGeneralizationChineseDataset,
path='ZenMoore/RoleBench',
reader_cfg=instruction_generalization_zh_reader_cfg,
infer_cfg=instruction_generalization_zh_infer_cfg,
eval_cfg=instruction_generalization_zh_eval_cfg)
]

View File

@ -0,0 +1,41 @@
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.openicl.icl_evaluator import RougeEvaluator
from opencompass.datasets.rolebench import RoleGeneralizationEnglishDataset
role_generalization_eng_reader_cfg = dict(
input_columns=['role', 'desc', 'question'],
output_column='answer',
train_split='train',
test_split='test'
)
role_generalization_eng_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template=dict(
begin=[
dict(role='SYSTEM', fallback_role='HUMAN', prompt='You are {role}, your description is: {desc}. Now please answer some questions to accurately show your personality traits! Your speaking style should fully imitate the personality role assigned to you! Please do not expose that you are an artificial intelligence model or a language model, you must always remember that you are only assigned one personality role. Dont be verbose or too formal or polite when speaking.'),
],
round=[
dict(role='HUMAN', prompt='{question}'),
dict(role='BOT', prompt=''),
], )),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer, max_out_len=512)
)
role_generalization_eng_eval_cfg = dict(
evaluator=dict(type=RougeEvaluator),
pred_role='BOT'
)
role_generalization_eng_datasets = [
dict(
type=RoleGeneralizationEnglishDataset,
path='ZenMoore/RoleBench',
reader_cfg=role_generalization_eng_reader_cfg,
infer_cfg=role_generalization_eng_infer_cfg,
eval_cfg=role_generalization_eng_eval_cfg)
]

View File

@ -0,0 +1,84 @@
import json
import os
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET
from .base import BaseDataset
@LOAD_DATASET.register_module()
class RoleBenchBaseDataset(BaseDataset):
@staticmethod
def load_single(source_file, desc_list):
with open(source_file, 'r', encoding='utf-8') as f:
source_data = [json.loads(line) for line in f.readlines()]
dataset = [{
'role': item['role'],
'desc': desc_list[item['role']],
'question': item['question'],
'answer': item['generated'][0]
} for item in source_data]
return dataset
@staticmethod
def load_desc(path):
with open(path, 'r', encoding='utf-8') as f:
desc_list = json.load(f)
return desc_list
@staticmethod
def load_dataset(path, desc_list):
train_data_list = RoleBenchBaseDataset.load_single(
os.path.join(path, 'general/train.jsonl'), desc_list)
train_data_list.extend(
RoleBenchBaseDataset.load_single(
os.path.join(path, 'role_specific/train.jsonl'), desc_list))
test_dataset = RoleBenchBaseDataset.load_single(
os.path.join(path, 'general/test.jsonl'), desc_list)
test_dataset.extend(
RoleBenchBaseDataset.load_single(
os.path.join(path, 'role_specific/test.jsonl'), desc_list))
return Dataset.from_list(train_data_list).shuffle(
seed=42), Dataset.from_list(test_dataset).shuffle(seed=42)
@LOAD_DATASET.register_module()
class InstructionGeneralizationEnglishDataset(RoleBenchBaseDataset):
@staticmethod
def load(path):
desc_list = RoleBenchBaseDataset.load_desc(
os.path.join(path, 'profiles-eng/desc.json'))
path = os.path.join(path, 'rolebench-eng/instruction-generalization')
train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset(
path, desc_list)
return DatasetDict({'train': train_dataset, 'test': test_dataset})
@LOAD_DATASET.register_module()
class RoleGeneralizationEnglishDataset(RoleBenchBaseDataset):
@staticmethod
def load(path):
desc_list = RoleBenchBaseDataset.load_desc(
os.path.join(path, 'profiles-eng/desc.json'))
path = os.path.join(path, 'rolebench-eng/role-generalization')
train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset(
path, desc_list)
return DatasetDict({'train': train_dataset, 'test': test_dataset})
@LOAD_DATASET.register_module()
class InstructionGeneralizationChineseDataset(RoleBenchBaseDataset):
@staticmethod
def load(path):
desc_list = RoleBenchBaseDataset.load_desc(
os.path.join(path, 'profiles-zh/desc.json'))
path = os.path.join(path, 'rolebench-zh')
train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset(
path, desc_list)
return DatasetDict({'train': train_dataset, 'test': test_dataset})