diff --git a/configs/eval_internlm_7b.py b/configs/eval_internlm_7b.py new file mode 100644 index 00000000..539a3ff9 --- /dev/null +++ b/configs/eval_internlm_7b.py @@ -0,0 +1,9 @@ +from mmengine.config import read_base + +with read_base(): + # choose a list of datasets + from .datasets.collections.base_medium import datasets + # choose a model of interest + from .models.hf_internlm_7b import models + # and output the results in a choosen format + from .summarizers.medium import summarizer diff --git a/configs/models/classic/gpt-3.5-turbo.py b/configs/models/gpt_3.5_turbo.py similarity index 100% rename from configs/models/classic/gpt-3.5-turbo.py rename to configs/models/gpt_3.5_turbo.py diff --git a/configs/models/hf_baichuan_7b.py b/configs/models/hf_baichuan_7b.py new file mode 100644 index 00000000..34fedd2d --- /dev/null +++ b/configs/models/hf_baichuan_7b.py @@ -0,0 +1,20 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='baichuan-7b-hf', + path="baichuan-inc/baichuan-7B", + tokenizer_path='baichuan-inc/baichuan-7B', + tokenizer_kwargs=dict(padding_side='left', + truncation_side='left', + trust_remote_code=True, + use_fast=False,), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(device_map='auto', trust_remote_code=True), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_chatglm2_6b.py b/configs/models/hf_chatglm2_6b.py new file mode 100644 index 00000000..ad3c00f6 --- /dev/null +++ b/configs/models/hf_chatglm2_6b.py @@ -0,0 +1,21 @@ +from opencompass.models import HuggingFace + + +models = [ + dict( + type=HuggingFace, + abbr='chatglm2-6b-hf', + path='THUDM/chatglm2-6b', + tokenizer_path='THUDM/chatglm2-6b', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='a6d54fac46dff2db65d53416c207a4485ca6bd40'), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_chatglm_6b.py b/configs/models/hf_chatglm_6b.py new file mode 100644 index 00000000..f51f00a1 --- /dev/null +++ b/configs/models/hf_chatglm_6b.py @@ -0,0 +1,21 @@ +from opencompass.models import HuggingFace + + +models = [ + dict( + type=HuggingFace, + abbr='chatglm-6b-hf', + path='THUDM/chatglm-6b', + tokenizer_path='THUDM/chatglm-6b', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='1d240ba371910e9282298d4592532d7f0f3e9f3e'), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_falcon_7b.py b/configs/models/hf_falcon_7b.py new file mode 100644 index 00000000..cab4f61a --- /dev/null +++ b/configs/models/hf_falcon_7b.py @@ -0,0 +1,21 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='falcon-7b-hf', + path='tiiuae/falcon-7b', + tokenizer_path='tiiuae/falcon-7b', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='2f5c3cd4eace6be6c0f12981f377fb35e5bf6ee5'), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_internlm_7b.py b/configs/models/hf_internlm_7b.py new file mode 100644 index 00000000..aea4208e --- /dev/null +++ b/configs/models/hf_internlm_7b.py @@ -0,0 +1,21 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='internlm-7b-hf', + path="internlm-7b", + tokenizer_path='internlm-7b', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + use_fast=False, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(trust_remote_code=True, device_map='auto'), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_internlm_chat_7b.py b/configs/models/hf_internlm_chat_7b.py new file mode 100644 index 00000000..6491cd97 --- /dev/null +++ b/configs/models/hf_internlm_chat_7b.py @@ -0,0 +1,29 @@ +from opencompass.models import HuggingFaceCausalLM + + +_meta_template = dict( + round=[ + dict(role='HUMAN', begin='<|User|>:', end='\n'), + dict(role='BOT', begin='<|Bot|>:', end='\n', generate=True), + ], +) + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='internlm-chat-7b-hf', + path="internlm-chat-7b", + tokenizer_path='internlm-chat-7b', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + use_fast=False, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + meta_template=_meta_template, + model_kwargs=dict(trust_remote_code=True, device_map='auto'), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_llama_13b.py b/configs/models/hf_llama_13b.py new file mode 100644 index 00000000..d1db1574 --- /dev/null +++ b/configs/models/hf_llama_13b.py @@ -0,0 +1,21 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + # LLaMA 13B + dict( + type=HuggingFaceCausalLM, + path="decapoda-research/llama-13b-hf", + tokenizer_path='decapoda-research/llama-13b-hf', + tokenizer_kwargs=dict(padding_side='left', + truncation_side='left', + use_fast=False, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(device_map='auto'), + batch_padding=False, # if false, inference with for-loop without batch padding + run_cfg=dict(num_gpus=2, num_procs=1), + ) +] diff --git a/configs/models/hf_llama_30b.py b/configs/models/hf_llama_30b.py new file mode 100644 index 00000000..ed6b38e9 --- /dev/null +++ b/configs/models/hf_llama_30b.py @@ -0,0 +1,22 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + # LLaMA 30B + dict( + type=HuggingFaceCausalLM, + abbr='llama-30b-hf', + path="decapoda-research/llama-30b-hf", + tokenizer_path='decapoda-research/llama-30b-hf', + tokenizer_kwargs=dict(padding_side='left', + truncation_side='left', + use_fast=False, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(device_map='auto'), + batch_padding=False, # if false, inference with for-loop without batch padding + run_cfg=dict(num_gpus=4, num_procs=1), + ) +] diff --git a/configs/models/hf_llama_65b.py b/configs/models/hf_llama_65b.py new file mode 100644 index 00000000..410c3001 --- /dev/null +++ b/configs/models/hf_llama_65b.py @@ -0,0 +1,22 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + # LLaMA 65B + dict( + type=HuggingFaceCausalLM, + abbr='llama-65b-hf', + path="decapoda-research/llama-65b-hf", + tokenizer_path='decapoda-research/llama-65b-hf', + tokenizer_kwargs=dict(padding_side='left', + truncation_side='left', + use_fast=False, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(device_map='auto'), + batch_padding=False, # if false, inference with for-loop without batch padding + run_cfg=dict(num_gpus=8, num_procs=1), + ) +] diff --git a/configs/models/classic/hf_llama.py b/configs/models/hf_llama_7b.py similarity index 85% rename from configs/models/classic/hf_llama.py rename to configs/models/hf_llama_7b.py index 64bcec55..e77d04db 100644 --- a/configs/models/classic/hf_llama.py +++ b/configs/models/hf_llama_7b.py @@ -5,6 +5,7 @@ models = [ # LLaMA 7B dict( type=HuggingFaceCausalLM, + abbr='llama-7b-hf', path="decapoda-research/llama-7b-hf", tokenizer_path='decapoda-research/llama-7b-hf', tokenizer_kwargs=dict(padding_side='left', @@ -15,8 +16,7 @@ models = [ max_seq_len=2048, batch_size=8, model_kwargs=dict(device_map='auto'), - batch_padding=False, # if false, inference with for-loop without batch padding - run_cfg=dict(num_gpus=2, num_procs=1), + batch_padding=False, # if false, inference with for-loop without batch padding + run_cfg=dict(num_gpus=1, num_procs=1), ) - ] diff --git a/configs/models/hf_moss_moon_003_base.py b/configs/models/hf_moss_moon_003_base.py new file mode 100644 index 00000000..fc43d0eb --- /dev/null +++ b/configs/models/hf_moss_moon_003_base.py @@ -0,0 +1,21 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='moss-moon-003-base-hf', + path='fnlp/moss-moon-003-base', + tokenizer_path='fnlp/moss-moon-003-base', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='5e406ca0ebbdea11cc3b12aa5932995c692568ac'), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_moss_moon_003_sft.py b/configs/models/hf_moss_moon_003_sft.py new file mode 100644 index 00000000..ef5110df --- /dev/null +++ b/configs/models/hf_moss_moon_003_sft.py @@ -0,0 +1,21 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='moss-moon-003-sft-hf', + path='fnlp/moss-moon-003-sft', + tokenizer_path='fnlp/moss-moon-003-sft', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='7119d446173035561f40977fb9cb999995bb7517'), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_mpt_7b.py b/configs/models/hf_mpt_7b.py new file mode 100644 index 00000000..0eadbad0 --- /dev/null +++ b/configs/models/hf_mpt_7b.py @@ -0,0 +1,27 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='mpt-7b-hf', + path='mosaicml/mpt-7b', + tokenizer_path='mosaicml/mpt-7b', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + use_fast=True + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict( + device_map='auto', + trust_remote_code=True, + max_seq_len=4096, + revision='68e1a8e0ebb9b30f3c45c1ef6195980f29063ae2', + ), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_mpt_instruct_7b.py b/configs/models/hf_mpt_instruct_7b.py new file mode 100644 index 00000000..6cb04d9d --- /dev/null +++ b/configs/models/hf_mpt_instruct_7b.py @@ -0,0 +1,27 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='mpt-instruct-7b-hf', + path="mosaicml/mpt-7b-instruct", + tokenizer_path="mosaicml/mpt-7b-instruct", + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + use_fast=True + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict( + device_map='auto', + trust_remote_code=True, + max_seq_len=4096, + revision='68e1a8e0ebb9b30f3c45c1ef6195980f29063ae2', + ), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +] diff --git a/configs/models/hf_tigerbot_7b_base.py b/configs/models/hf_tigerbot_7b_base.py new file mode 100644 index 00000000..b3d5c028 --- /dev/null +++ b/configs/models/hf_tigerbot_7b_base.py @@ -0,0 +1,21 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='tigerbot-base-7b-hf', + path='TigerResearch/tigerbot-7b-base', + tokenizer_path='TigerResearch/tigerbot-7b-base', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(trust_remote_code=True, device_map='auto', revision='5f34fd223586e9efb8eb0e3bc667f03581886992'), + run_cfg=dict(num_gpus=1, num_procs=1), + ), +] diff --git a/configs/models/classic/tigerbot-7b-sft.py b/configs/models/hf_tigerbot_7b_sft.py similarity index 88% rename from configs/models/classic/tigerbot-7b-sft.py rename to configs/models/hf_tigerbot_7b_sft.py index b5ea2860..d4a792d7 100644 --- a/configs/models/classic/tigerbot-7b-sft.py +++ b/configs/models/hf_tigerbot_7b_sft.py @@ -1,17 +1,17 @@ from opencompass.models import HuggingFaceCausalLM + _meta_template = dict( round=[ - dict(role='HUMAN', begin='\n\n### Instruction:\n:'), + dict(role='HUMAN', begin='\n\n### Instruction:\n:'), dict(role='BOT', begin='\n\n### Response:\n:', generate=True), ], ) - models = [ dict( type=HuggingFaceCausalLM, - abbr='TigerBot-SFT', + abbr='tigerbot-sft-7b-hf', path="TigerResearch/tigerbot-7b-sft", tokenizer_path='TigerResearch/tigerbot-7b-sft', tokenizer_kwargs=dict( diff --git a/configs/models/hf_vicuna_v1.3_13b.py b/configs/models/hf_vicuna_v1.3_13b.py new file mode 100644 index 00000000..bc3a6e01 --- /dev/null +++ b/configs/models/hf_vicuna_v1.3_13b.py @@ -0,0 +1,22 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='vicuna-13b-hf', + path="lmsys/vicuna-13b-v1.3", + tokenizer_path='lmsys/vicuna-13b-v1.3', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + use_fast=False, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(device_map='auto'), + batch_padding=False, # if false, inference with for-loop without batch padding + run_cfg=dict(num_gpus=2, num_procs=1) + ) +] diff --git a/configs/models/hf_vicuna_v1.3_33b.py b/configs/models/hf_vicuna_v1.3_33b.py new file mode 100644 index 00000000..c29e3839 --- /dev/null +++ b/configs/models/hf_vicuna_v1.3_33b.py @@ -0,0 +1,22 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='vicuna-33b-hf', + path="lmsys/vicuna-33b-v1.3", + tokenizer_path='lmsys/vicuna-33b-v1.3', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + use_fast=False, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(device_map='auto'), + batch_padding=False, # if false, inference with for-loop without batch padding + run_cfg=dict(num_gpus=4, num_procs=1) + ) +] diff --git a/configs/models/hf_vicuna_v1.3_7b.py b/configs/models/hf_vicuna_v1.3_7b.py new file mode 100644 index 00000000..005f0760 --- /dev/null +++ b/configs/models/hf_vicuna_v1.3_7b.py @@ -0,0 +1,22 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='vicuna-7b-hf', + path="lmsys/vicuna-7b-v1.3", + tokenizer_path='lmsys/vicuna-7b-v1.3', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + use_fast=False, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict(device_map='auto'), + batch_padding=False, # if false, inference with for-loop without batch padding + run_cfg=dict(num_gpus=1, num_procs=1) + ) +] diff --git a/configs/models/hf_wizardlm_7b.py b/configs/models/hf_wizardlm_7b.py new file mode 100644 index 00000000..cf6fb0c6 --- /dev/null +++ b/configs/models/hf_wizardlm_7b.py @@ -0,0 +1,24 @@ +from opencompass.models import HuggingFaceCausalLM + + +models = [ + dict( + type=HuggingFaceCausalLM, + abbr='wizardlm-7b-hf', + path='TheBloke/wizardLM-7B-HF', + tokenizer_path='TheBloke/wizardLM-7B-HF', + tokenizer_kwargs=dict( + padding_side='left', + truncation_side='left', + trust_remote_code=True, + ), + max_out_len=100, + max_seq_len=2048, + batch_size=8, + model_kwargs=dict( + device_map='auto', + trust_remote_code=True, + ), + run_cfg=dict(num_gpus=1, num_procs=1), + ) +]