jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Fix] add do sample demo for subjective dataset (#873)

Browse Source 
* add do sample demo for subjective dataset

* fix strings

* format

---------

Co-authored-by: Leymore <zfz-960727@163.com>
This commit is contained in:
bittersweet1999 2024-02-05 15:55:58 +08:00 committed by GitHub
parent 7ad1168062
commit 32b5948f4e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 446 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
configs/eval_subjective_alignbench.py
View File

@ -1,16 +1,7 @@
from mmengine.config import read_base from mmengine.config import read_base
with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .models.judge_llm.auto_j.hf_autoj_eng_13b import models as hf_autoj
from .models.judge_llm.judgelm.hf_judgelm_33b_v1 import models as hf_judgelm
from .models.judge_llm.pandalm.hf_pandalm_7b_v1 import models as hf_pandalm
from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets
datasets = [*subjective_datasets] with read_base():
from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
from opencompass.models.openai_api import OpenAIAllesAPIN from opencompass.models.openai_api import OpenAIAllesAPIN
@ -23,10 +14,42 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import AlignmentBenchSummarizer from opencompass.summarizers import AlignmentBenchSummarizer
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
# -------------Inferen Stage ---------------------------------------- # -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
models = [*hf_chatglm3_6b]#, *hf_chatglm3_6b, *hf_internlm_chat_20b, *hf_qwen_7b_chat, *hf_qwen_14b_chat] datasets = [*subjective_datasets]
infer = dict( infer = dict(
partitioner=dict(type=NaivePartitioner), partitioner=dict(type=NaivePartitioner),
@ -35,51 +58,39 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ---------------------------------------- # -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration ## ------------- JudgeLLM Configuration
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAIAllesAPIN, path='gpt-4-1106-preview', type=OpenAIAllesAPIN,
key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well path='gpt-4-1106-preview',
url='xxxx', key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template, url='xxxx',
query_per_second=16, meta_template=api_meta_template,
max_out_len=2048, query_per_second=16,
max_seq_len=2048, max_out_len=2048,
batch_size=8, max_seq_len=2048,
temperature = 0 batch_size=8,
temperature=0,
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(
type=SubjectiveNaivePartitioner, type=SubjectiveNaivePartitioner, mode='singlescore', models=models
mode='singlescore',
models = models
), ),
runner=dict( runner=dict(
type=LocalRunner, type=LocalRunner,
max_num_workers=2, max_num_workers=2,
task=dict( task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model),
type=SubjectiveEvalTask, ),
judge_cfg=judge_model
)),
) )
summarizer = dict( summarizer = dict(type=AlignmentBenchSummarizer, judge_type='general')
type=AlignmentBenchSummarizer, judge_type = 'general'
)
work_dir = 'outputs/alignment_bench/' work_dir = 'outputs/alignment_bench/'

106
configs/eval_subjective_alpacaeval.py
View File

@ -1,16 +1,9 @@
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_7b import models as hf_internlm_chat_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .datasets.subjective.alpaca_eval.alpacav1_judgeby_gpt4 import subjective_datasets as alpacav1 from .datasets.subjective.alpaca_eval.alpacav1_judgeby_gpt4 import subjective_datasets as alpacav1
from .datasets.subjective.alpaca_eval.alpacav2_judgeby_gpt4 import subjective_datasets as alpacav2 from .datasets.subjective.alpaca_eval.alpacav2_judgeby_gpt4 import subjective_datasets as alpacav2
datasets = [*alpacav2]
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
from opencompass.models.openai_api import OpenAI, OpenAIAllesAPIN from opencompass.models.openai_api import OpenAI, OpenAIAllesAPIN
from opencompass.partitioners import NaivePartitioner, SizePartitioner from opencompass.partitioners import NaivePartitioner, SizePartitioner
@ -22,18 +15,59 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import AlpacaSummarizer from opencompass.summarizers import AlpacaSummarizer
models = [*hf_qwen_7b_chat, *hf_chatglm3_6b]
api_meta_template = dict( api_meta_template = dict(
round=[ round=[
dict(role='HUMAN', api_role='HUMAN'), dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True) dict(role='BOT', api_role='BOT', generate=True),
],
reserved_roles=[
dict(role='SYSTEM', api_role='SYSTEM'),
], ],
reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
) )
# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
datasets = [*alpacav2]
gpt4 = dict(
abbr='gpt4-turbo',
type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=1,
max_out_len=2048,
max_seq_len=4096,
batch_size=4,
retry=20,
temperature=1,
) # Re-inference gpt4's predictions or you can choose to use the pre-commited gpt4's predictions
infer = dict( infer = dict(
partitioner=dict(type=NaivePartitioner), partitioner=dict(type=NaivePartitioner),
runner=dict( runner=dict(
@ -41,42 +75,40 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAI, path='gpt-4-1106-preview', type=OpenAI,
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well path='gpt-4-1106-preview',
meta_template=api_meta_template, key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
query_per_second=1, meta_template=api_meta_template,
max_out_len=1024, query_per_second=1,
max_seq_len=4096, max_out_len=1024,
batch_size=2, max_seq_len=4096,
retry=20, batch_size=2,
temperature = 0 retry=20,
temperature=0,
) )
## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(
type=SubjectiveSizePartitioner, type=SubjectiveSizePartitioner, max_task_size=1000, mode='m2n', base_models=[gpt4], compare_models=models
max_task_size=1000,
mode='m2n',
base_models = [*hf_chatglm3_6b],
compare_models = [*hf_qwen_7b_chat]
), ),
runner=dict( runner=dict(
type=SlurmSequentialRunner, type=SlurmSequentialRunner,
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict( task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model),
type=SubjectiveEvalTask, ),
judge_cfg=judge_model
)),
) )
work_dir = 'outputs/alpaca/' work_dir = 'outputs/alpaca/'
summarizer = dict( summarizer = dict(type=AlpacaSummarizer, judge_type='v2')
type=AlpacaSummarizer, judge_type='v2'
)

125
configs/eval_subjective_compassarena.py
View File

@ -1,8 +1,8 @@
from os import getenv as gv from os import getenv as gv
from opencompass.models import HuggingFaceCausalLM from opencompass.models import HuggingFaceCausalLM
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.chatglm.hf_chatglm3_6b_32k import models as chatglm3_6b_32k_model
from .datasets.subjective.compassarena.compassarena_compare import subjective_datasets from .datasets.subjective.compassarena.compassarena_compare import subjective_datasets
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
@ -16,56 +16,85 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import CompassArenaSummarizer from opencompass.summarizers import CompassArenaSummarizer
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
],
reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
)
# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
datasets = [*subjective_datasets]
gpt4 = dict(
abbr='gpt4-turbo',
type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=1,
max_out_len=2048,
max_seq_len=4096,
batch_size=4,
retry=20,
temperature=1,
) # Re-inference gpt4's predictions or you can choose to use the pre-commited gpt4's predictions
infer = dict( infer = dict(
#partitioner=dict(type=NaivePartitioner),
partitioner=dict(type=SizePartitioner, strategy='split', max_task_size=10000), partitioner=dict(type=SizePartitioner, strategy='split', max_task_size=10000),
runner=dict( runner=dict(
type=SlurmSequentialRunner, type=SlurmSequentialRunner,
partition='llm_dev2', partition='llm_dev2',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
api_meta_template = dict( # -------------Evalation Stage ----------------------------------------
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
gpt4 = dict(
abbr='gpt4-turbo',
type=OpenAI, path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=1,
max_out_len=2048,
max_seq_len=4096,
batch_size=4,
retry=20,
temperature = 1
)
models = [*chatglm3_6b_32k_model]
datasets = [*subjective_datasets]
work_dir = 'outputs/compass_arena_debug/'
# -------------Inferen Stage ----------------------------------------
## ------------- JudgeLLM Configuration
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAI, path='gpt-4-1106-preview', type=OpenAI,
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well path='gpt-4-1106-preview',
meta_template=api_meta_template, key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
query_per_second=1, meta_template=api_meta_template,
max_out_len=1024, query_per_second=1,
max_seq_len=4096, max_out_len=1024,
batch_size=2, max_seq_len=4096,
retry=20, batch_size=2,
temperature = 0 retry=20,
temperature=0,
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
@ -75,22 +104,18 @@ eval = dict(
strategy='split', strategy='split',
max_task_size=10000, max_task_size=10000,
mode='m2n', mode='m2n',
base_models = [gpt4], base_models=[gpt4],
compare_models = [*chatglm3_6b_32k_model] compare_models=models,
), ),
runner=dict( runner=dict(
type=SlurmSequentialRunner, type=SlurmSequentialRunner,
partition='llm_dev2', partition='llm_dev2',
quotatype='auto', quotatype='auto',
max_num_workers=32, max_num_workers=32,
task=dict( task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model),
type=SubjectiveEvalTask, ),
judge_cfg=judge_model
)),
) )
work_dir = 'outputs/compass_arena_debug/'
summarizer = dict( summarizer = dict(type=CompassArenaSummarizer, summary_type='half_add')
type=CompassArenaSummarizer,
summary_type='half_add'
)

125
configs/eval_subjective_corev2.py
View File

@ -1,16 +1,9 @@
from mmengine.config import read_base from mmengine.config import read_base
with read_base(): with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm2_chat_7b import models as internlm2_7b
from .models.hf_internlm.hf_internlm2_chat_20b import models as internlm2_20b
from .datasets.subjective.subjective_cmp.subjective_corev2 import subjective_datasets from .datasets.subjective.subjective_cmp.subjective_corev2 import subjective_datasets
datasets = [*subjective_datasets] from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3, OpenAI
from opencompass.models import HuggingFaceCausalLM, HuggingFace, OpenAI
from opencompass.partitioners import NaivePartitioner, SizePartitioner from opencompass.partitioners import NaivePartitioner, SizePartitioner
from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
from opencompass.partitioners.sub_size import SubjectiveSizePartitioner from opencompass.partitioners.sub_size import SubjectiveSizePartitioner
@ -19,18 +12,62 @@ from opencompass.runners import SlurmSequentialRunner
from opencompass.tasks import OpenICLInferTask from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import Corev2Summarizer from opencompass.summarizers import Corev2Summarizer
models = [*internlm2_7b, *internlm2_20b]
api_meta_template = dict( api_meta_template = dict(
round=[ round=[
dict(role='HUMAN', api_role='HUMAN'), dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True) dict(role='BOT', api_role='BOT', generate=True),
], ],
reserved_roles=[ reserved_roles=[
dict(role='SYSTEM', api_role='SYSTEM'), dict(role='SYSTEM', api_role='SYSTEM'),
], ],
) )
# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
datasets = [*subjective_datasets]
gpt4 = dict(
abbr='gpt4-turbo',
type=OpenAI,
path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=1,
max_out_len=2048,
max_seq_len=4096,
batch_size=4,
retry=20,
temperature=1,
) # Re-inference gpt4's predictions or you can choose to use the pre-commited gpt4's predictions
infer = dict( infer = dict(
partitioner=dict(type=SizePartitioner, max_task_size=500), partitioner=dict(type=SizePartitioner, max_task_size=500),
runner=dict( runner=dict(
@ -38,61 +75,41 @@ infer = dict(
partition='llm_dev2', partition='llm_dev2',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ----------------------------------------
_meta_template = dict( ## ------------- JudgeLLM Configuration
round=[ judge_model = dict(
dict(role="HUMAN", begin='\n<|im_start|>user\n', end='<|im_end|>'), abbr='GPT4-Turbo',
dict(role="BOT", begin="\n<|im_start|>assistant\n", end='<|im_end|>', generate=True), type=OpenAI,
], path='gpt-4-1106-preview',
key='', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template,
query_per_second=1,
max_out_len=1024,
max_seq_len=4096,
batch_size=2,
retry=20,
temperature=0,
) )
## ------------- Evaluation Configuration
judge_model = dict(
type=HuggingFaceCausalLM,
abbr='qwen-7b-chat-hf',
path="Qwen/Qwen-7B-Chat",
tokenizer_path='Qwen/Qwen-7B-Chat',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
use_fast=False,),
pad_token_id=151643,
max_out_len=2048,
max_seq_len=2048,
batch_size=8,
meta_template=_meta_template,
run_cfg=dict(num_gpus=1, num_procs=1),
)
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(
type=SubjectiveSizePartitioner, type=SubjectiveSizePartitioner, mode='m2n', max_task_size=500, base_models=[gpt4], compare_models=models
mode='m2n',
max_task_size=500,
base_models = [*internlm2_7b],
compare_models = [*internlm2_20b]
), ),
runner=dict( runner=dict(
type=SlurmSequentialRunner, type=SlurmSequentialRunner,
partition='llm_dev2', partition='llm_dev2',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict( task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model),
type=SubjectiveEvalTask, ),
judge_cfg=judge_model
)),
) )
work_dir = 'outputs/corev2/'
summarizer = dict( summarizer = dict(type=Corev2Summarizer, match_method='smart')
type=Corev2Summarizer,
match_method='smart', work_dir = 'outputs/corev2/'
)

99
configs/eval_subjective_creationbench.py
View File

@ -1,16 +1,7 @@
from mmengine.config import read_base from mmengine.config import read_base
with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .models.judge_llm.auto_j.hf_autoj_eng_13b import models as hf_autoj
from .models.judge_llm.judgelm.hf_judgelm_33b_v1 import models as hf_judgelm
from .models.judge_llm.pandalm.hf_pandalm_7b_v1 import models as hf_pandalm
from .datasets.subjective.creationbench.creationbench_judgeby_gpt4_withref import subjective_datasets
datasets = [*subjective_datasets] with read_base():
from .datasets.subjective.creationbench.creationbench_judgeby_gpt4_withref import subjective_datasets
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
from opencompass.models.openai_api import OpenAIAllesAPIN from opencompass.models.openai_api import OpenAIAllesAPIN
@ -23,10 +14,42 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import CreationBenchSummarizer from opencompass.summarizers import CreationBenchSummarizer
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
# -------------Inferen Stage ---------------------------------------- # -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
models = [*hf_chatglm3_6b]#, *hf_chatglm3_6b, *hf_internlm_chat_20b, *hf_qwen_7b_chat, *hf_qwen_14b_chat] datasets = [*subjective_datasets]
infer = dict( infer = dict(
partitioner=dict(type=NaivePartitioner), partitioner=dict(type=NaivePartitioner),
@ -35,51 +58,33 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ---------------------------------------- # -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration ## ------------- JudgeLLM Configuration
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
judge_model = dict( judge_model = dict(
abbr='GPT4-Turbo', abbr='GPT4-Turbo',
type=OpenAIAllesAPIN, path='gpt-4-1106-preview', type=OpenAIAllesAPIN,
key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well path='gpt-4-1106-preview',
url='xxxx', key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
meta_template=api_meta_template, url='xxxx',
query_per_second=16, meta_template=api_meta_template,
max_out_len=2048, query_per_second=16,
max_seq_len=2048, max_out_len=2048,
batch_size=8, max_seq_len=2048,
temperature = 0 batch_size=8,
temperature=0,
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(type=SubjectiveNaivePartitioner, mode='singlescore', models=models),
type=SubjectiveNaivePartitioner, runner=dict(type=LocalRunner, max_num_workers=2, task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model)),
mode='singlescore',
models = models
),
runner=dict(
type=LocalRunner,
max_num_workers=2,
task=dict(
type=SubjectiveEvalTask,
judge_cfg=judge_model
)),
) )
summarizer = dict( summarizer = dict(type=CreationBenchSummarizer, judge_type='general')
type=CreationBenchSummarizer, judge_type = 'general'
)
work_dir = 'outputs/creationbench/' work_dir = 'outputs/creationbench/'

106
configs/eval_subjective_judge_pandalm.py
View File

@ -1,13 +1,7 @@
from mmengine.config import read_base from mmengine.config import read_base
with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets
datasets = [*subjective_datasets] with read_base():
from .datasets.subjective.alignbench.alignbench_judgeby_critiquellm import subjective_datasets
from opencompass.models import HuggingFaceCausalLM, HuggingFace, OpenAIAllesAPIN, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, OpenAIAllesAPIN, HuggingFaceChatGLM3
from opencompass.partitioners import NaivePartitioner from opencompass.partitioners import NaivePartitioner
@ -18,10 +12,42 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import AlignmentBenchSummarizer from opencompass.summarizers import AlignmentBenchSummarizer
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
# -------------Inferen Stage ---------------------------------------- # -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
dict(
type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template,
max_out_len=2048,
max_seq_len=4096,
batch_size=1,
run_cfg=dict(num_gpus=1, num_procs=1),
)
]
models = [*hf_baichuan2_7b]#, *hf_chatglm3_6b, *hf_internlm_chat_20b, *hf_qwen_7b_chat, *hf_qwen_14b_chat] datasets = [*subjective_datasets]
infer = dict( infer = dict(
partitioner=dict(type=NaivePartitioner), partitioner=dict(type=NaivePartitioner),
@ -30,55 +56,37 @@ infer = dict(
partition='llmeval', partition='llmeval',
quotatype='auto', quotatype='auto',
max_num_workers=256, max_num_workers=256,
task=dict(type=OpenICLInferTask)), task=dict(type=OpenICLInferTask),
),
) )
# -------------Evalation Stage ---------------------------------------- # -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration ## ------------- JudgeLLM Configuration
api_meta_template = dict(
round=[
dict(role='HUMAN', api_role='HUMAN'),
dict(role='BOT', api_role='BOT', generate=True),
]
)
judge_model = dict( judge_model = dict(
type=HuggingFaceCausalLM, type=HuggingFaceCausalLM,
abbr='pandalm-7b-v1-hf', abbr='pandalm-7b-v1-hf',
path="WeOpenML/PandaLM-7B-v1", path='WeOpenML/PandaLM-7B-v1',
tokenizer_path='WeOpenML/PandaLM-7B-v1', tokenizer_path='WeOpenML/PandaLM-7B-v1',
tokenizer_kwargs=dict(padding_side='left', tokenizer_kwargs=dict(
truncation_side='left', padding_side='left',
trust_remote_code=True, truncation_side='left',
use_fast=False,), trust_remote_code=True,
max_out_len=512, use_fast=False,
max_seq_len=2048, ),
batch_size=8, max_out_len=512,
model_kwargs=dict(device_map='auto', trust_remote_code=True), max_seq_len=2048,
run_cfg=dict(num_gpus=1, num_procs=1), batch_size=8,
) model_kwargs=dict(device_map='auto', trust_remote_code=True),
run_cfg=dict(num_gpus=1, num_procs=1),
)
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(type=SubjectiveNaivePartitioner, mode='singlescore', models=models),
type=SubjectiveNaivePartitioner, runner=dict(type=LocalRunner, max_num_workers=2, task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model)),
mode='singlescore',
models = [*hf_baichuan2_7b]
),
runner=dict(
type=LocalRunner,
max_num_workers=2,
task=dict(
type=SubjectiveEvalTask,
judge_cfg=judge_model
)),
) )
summarizer = dict( summarizer = dict(type=AlignmentBenchSummarizer)
type=AlignmentBenchSummarizer,
)
work_dir = 'outputs/pandalm' work_dir = 'outputs/pandalm'

150
configs/eval_subjective_mtbench.py
View File

@ -1,17 +1,9 @@
from mmengine.config import read_base from mmengine.config import read_base
with read_base():
from .models.qwen.hf_qwen_7b_chat import models as hf_qwen_7b_chat
from .models.qwen.hf_qwen_14b_chat import models as hf_qwen_14b_chat
from .models.chatglm.hf_chatglm3_6b import models as hf_chatglm3_6b
from .models.baichuan.hf_baichuan2_7b_chat import models as hf_baichuan2_7b
from .models.hf_internlm.hf_internlm_chat_20b import models as hf_internlm_chat_20b
from .models.judge_llm.auto_j.hf_autoj_eng_13b import models as hf_autoj
from .models.judge_llm.judgelm.hf_judgelm_33b_v1 import models as hf_judgelm
from .models.judge_llm.pandalm.hf_pandalm_7b_v1 import models as hf_pandalm
from .datasets.subjective.multiround.mtbench_single_judge import subjective_datasets
#from .datasets.subjective.multiround.mtbench_pair_judge import subjective_datasets
datasets = [*subjective_datasets] with read_base():
from .datasets.subjective.multiround.mtbench_single_judge import subjective_datasets
# from .datasets.subjective.multiround.mtbench_pair_judge import subjective_datasets
from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3 from opencompass.models import HuggingFaceCausalLM, HuggingFace, HuggingFaceChatGLM3
from opencompass.models.openai_api import OpenAIAllesAPIN from opencompass.models.openai_api import OpenAIAllesAPIN
@ -24,24 +16,6 @@ from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import MTBenchSummarizer from opencompass.summarizers import MTBenchSummarizer
# -------------Inferen Stage ----------------------------------------
models = [*hf_chatglm3_6b, *hf_qwen_7b_chat]
infer = dict(
partitioner=dict(type=SizePartitioner, max_task_size=100),
runner=dict(
type=SlurmSequentialRunner,
partition='llmeval',
quotatype='auto',
max_num_workers=256,
task=dict(type=OpenICLInferTask)),
)
# -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration
api_meta_template = dict( api_meta_template = dict(
round=[ round=[
dict(role='HUMAN', api_role='HUMAN'), dict(role='HUMAN', api_role='HUMAN'),
@ -49,68 +23,82 @@ api_meta_template = dict(
] ]
) )
judge_model = dict( # -------------Inference Stage ----------------------------------------
abbr='GPT4-Turbo', # For subjective evaluation, we often set do sample for models
type=OpenAIAllesAPIN, path='gpt-4-1106-preview', models = [
key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well dict(
url='xxxx', type=HuggingFaceChatGLM3,
abbr='chatglm3-6b-hf',
path='THUDM/chatglm3-6b',
tokenizer_path='THUDM/chatglm3-6b',
model_kwargs=dict(
device_map='auto',
trust_remote_code=True,
),
tokenizer_kwargs=dict(
padding_side='left',
truncation_side='left',
trust_remote_code=True,
),
generation_kwargs=dict(
do_sample=True,
),
meta_template=api_meta_template, meta_template=api_meta_template,
query_per_second=16,
max_out_len=2048, max_out_len=2048,
max_seq_len=2048, max_seq_len=4096,
batch_size=8, batch_size=1,
temperature = 0 run_cfg=dict(num_gpus=1, num_procs=1),
)
]
datasets = [*subjective_datasets]
infer = dict(
partitioner=dict(type=SizePartitioner, max_task_size=100),
runner=dict(
type=SlurmSequentialRunner,
partition='llmeval',
quotatype='auto',
max_num_workers=256,
task=dict(type=OpenICLInferTask),
),
)
# -------------Evalation Stage ----------------------------------------
## ------------- JudgeLLM Configuration
judge_model = dict(
abbr='GPT4-Turbo',
type=OpenAIAllesAPIN,
path='gpt-4-0613',
key='xxxx', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
url='xxxx',
meta_template=api_meta_template,
query_per_second=16,
max_out_len=1024,
max_seq_len=2048,
batch_size=8,
temperature=0,
) )
## ------------- Evaluation Configuration ## ------------- Evaluation Configuration
''' # ## pair evaluation
## pair evaluation # eval = dict(
eval = dict( # partitioner=dict(
partitioner=dict( # type=SubjectiveSizePartitioner, max_task_size=100, mode='m2n', base_models=[gpt4], compare_models=models
type=SubjectiveSizePartitioner, # ),
max_task_size=100, # runner=dict(type=LocalRunner, max_num_workers=32, task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model)),
mode='m2n', # )
base_models = [*hf_chatglm3_6b, ],
compare_models = models
),
runner=dict(
type=SlurmSequentialRunner,
partition='llmeval',
quotatype='auto',
max_num_workers=32,
task=dict(
type=SubjectiveEvalTask,
judge_cfg=judge_model
)),
)
summarizer = dict( # summarizer = dict(type=MTBenchSummarizer, judge_type='pair')
type=MTBenchSummarizer, judge_type='pair'
)
'''
## single evaluation ## single evaluation
eval = dict( eval = dict(
partitioner=dict( partitioner=dict(type=SubjectiveSizePartitioner, max_task_size=100, mode='singlescore', models=models),
type=SubjectiveSizePartitioner, runner=dict(type=LocalRunner, max_num_workers=32, task=dict(type=SubjectiveEvalTask, judge_cfg=judge_model)),
max_task_size=100,
mode='singlescore',
models = models
),
runner=dict(
type=SlurmSequentialRunner,
partition='llmeval',
quotatype='auto',
max_num_workers=32,
task=dict(
type=SubjectiveEvalTask,
judge_cfg=judge_model
)),
) )
summarizer = dict( summarizer = dict(type=MTBenchSummarizer, judge_type='single')
type=MTBenchSummarizer, judge_type='single'
)
work_dir = 'outputs/mtbench/' work_dir = 'outputs/mtbench/'