From 6997990c932e4b0eed32029450ad7e9fb750e022 Mon Sep 17 00:00:00 2001 From: Songyang Zhang Date: Thu, 12 Sep 2024 23:35:30 +0800 Subject: [PATCH] [Feature] Update Models (#1518) * Update Models * Update * Update humanevalx * Update * Update --- .../models/hf_llama/hf_llama3_1_70b_instruct.py | 13 +++++++++++++ .../models/hf_llama/hf_llama3_1_8b_instruct.py | 13 +++++++++++++ .../hf_llama/lmdeploy_llama3_1_70b_instruct.py | 16 ++++++++++++++++ .../hf_llama/lmdeploy_llama3_1_8b_instruct.py | 2 +- .../mistral/vllm_mixtral_large_instruct_2407.py | 15 +++++++++++++++ configs/models/openbmb/hf_minicpm3_4b.py | 15 +++++++++++++++ configs/models/phi/hf_phi_3_5_MoE_instruct.py | 12 ++++++++++++ configs/models/phi/hf_phi_3_5_mini_instruct.py | 12 ++++++++++++ configs/summarizers/groups/humanevalx.py | 5 +++++ .../models/hf_llama/hf_llama3_1_70b_instruct.py | 13 +++++++++++++ .../models/hf_llama/hf_llama3_1_8b_instruct.py | 13 +++++++++++++ .../hf_llama/lmdeploy_llama3_1_70b_instruct.py | 16 ++++++++++++++++ .../hf_llama/lmdeploy_llama3_1_8b_instruct.py | 2 +- .../mistral/vllm_mixtral_large_instruct_2407.py | 15 +++++++++++++++ .../configs/models/openbmb/hf_minicpm3_4b.py | 15 +++++++++++++++ .../models/phi/hf_phi_3_5_MoE_instruct.py | 12 ++++++++++++ .../models/phi/hf_phi_3_5_mini_instruct.py | 12 ++++++++++++ .../configs/summarizers/groups/humanevalx.py | 5 +++++ opencompass/datasets/ds1000.py | 9 +++++++-- opencompass/datasets/humanevalx.py | 9 ++++++--- opencompass/utils/datasets_info.py | 8 ++++++++ 21 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 configs/models/hf_llama/hf_llama3_1_70b_instruct.py create mode 100644 configs/models/hf_llama/hf_llama3_1_8b_instruct.py create mode 100644 configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py create mode 100644 configs/models/mistral/vllm_mixtral_large_instruct_2407.py create mode 100644 configs/models/openbmb/hf_minicpm3_4b.py create mode 100644 configs/models/phi/hf_phi_3_5_MoE_instruct.py create mode 100644 configs/models/phi/hf_phi_3_5_mini_instruct.py create mode 100644 configs/summarizers/groups/humanevalx.py create mode 100644 opencompass/configs/models/hf_llama/hf_llama3_1_70b_instruct.py create mode 100644 opencompass/configs/models/hf_llama/hf_llama3_1_8b_instruct.py create mode 100644 opencompass/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py create mode 100644 opencompass/configs/models/mistral/vllm_mixtral_large_instruct_2407.py create mode 100644 opencompass/configs/models/openbmb/hf_minicpm3_4b.py create mode 100644 opencompass/configs/models/phi/hf_phi_3_5_MoE_instruct.py create mode 100644 opencompass/configs/models/phi/hf_phi_3_5_mini_instruct.py create mode 100644 opencompass/configs/summarizers/groups/humanevalx.py diff --git a/configs/models/hf_llama/hf_llama3_1_70b_instruct.py b/configs/models/hf_llama/hf_llama3_1_70b_instruct.py new file mode 100644 index 00000000..4a17de93 --- /dev/null +++ b/configs/models/hf_llama/hf_llama3_1_70b_instruct.py @@ -0,0 +1,13 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='llama-3_1-70b-instruct-hf', + path='meta-llama/Meta-Llama-3.1-70B-Instruct', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=4), + stop_words=['<|end_of_text|>', '<|eot_id|>'], + ) +] diff --git a/configs/models/hf_llama/hf_llama3_1_8b_instruct.py b/configs/models/hf_llama/hf_llama3_1_8b_instruct.py new file mode 100644 index 00000000..3ae0d5f3 --- /dev/null +++ b/configs/models/hf_llama/hf_llama3_1_8b_instruct.py @@ -0,0 +1,13 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='llama-3_1-8b-instruct-hf', + path='meta-llama/Meta-Llama-3.1-8B-Instruct', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=1), + stop_words=['<|end_of_text|>', '<|eot_id|>'], + ) +] diff --git a/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py b/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py new file mode 100644 index 00000000..23f9bc2a --- /dev/null +++ b/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py @@ -0,0 +1,16 @@ +from opencompass.models import TurboMindModelwithChatTemplate + +models = [ + dict( + type=TurboMindModelwithChatTemplate, + abbr='llama-3_1-70b-instruct-turbomind', + path='meta-llama/Meta-Llama-3.1-70B-Instruct', + engine_config=dict(max_batch_size=16, tp=4), + gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024), + max_seq_len=7168, + max_out_len=1024, + batch_size=16, + run_cfg=dict(num_gpus=4), + stop_words=['<|end_of_text|>', '<|eot_id|>'], + ) +] diff --git a/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py b/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py index b7dedb72..429dfec7 100644 --- a/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py +++ b/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py @@ -3,7 +3,7 @@ from opencompass.models import TurboMindModelwithChatTemplate models = [ dict( type=TurboMindModelwithChatTemplate, - abbr='llama-3.1-8b-instruct-turbomind', + abbr='llama-3_1-8b-instruct-turbomind', path='meta-llama/Meta-Llama-3.1-8B-Instruct', engine_config=dict(max_batch_size=16, tp=1), gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024), diff --git a/configs/models/mistral/vllm_mixtral_large_instruct_2407.py b/configs/models/mistral/vllm_mixtral_large_instruct_2407.py new file mode 100644 index 00000000..5f67f294 --- /dev/null +++ b/configs/models/mistral/vllm_mixtral_large_instruct_2407.py @@ -0,0 +1,15 @@ +from opencompass.models import VLLMwithChatTemplate + + +models = [ + dict( + type=VLLMwithChatTemplate, + abbr='mixtral-large-instruct-2407-vllm', + path='mistralai/Mistral-Large-Instruct-2407', + model_kwargs=dict(tensor_parallel_size=8), + max_out_len=256, + batch_size=16, + generation_kwargs=dict(temperature=0), + run_cfg=dict(num_gpus=8), + ) +] diff --git a/configs/models/openbmb/hf_minicpm3_4b.py b/configs/models/openbmb/hf_minicpm3_4b.py new file mode 100644 index 00000000..730324ea --- /dev/null +++ b/configs/models/openbmb/hf_minicpm3_4b.py @@ -0,0 +1,15 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='MiniCPM3-4B-hf', + path='openbmb/MiniCPM3-4B', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=1), + model_kwargs=dict( + torch_dtype='torch.bfloat16', + ), + ) +] diff --git a/configs/models/phi/hf_phi_3_5_MoE_instruct.py b/configs/models/phi/hf_phi_3_5_MoE_instruct.py new file mode 100644 index 00000000..40153f8c --- /dev/null +++ b/configs/models/phi/hf_phi_3_5_MoE_instruct.py @@ -0,0 +1,12 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='phi-3-5-MoE-instruct-hf', + path='microsoft/Phi-3.5-MoE-instruct', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=1), + ) +] diff --git a/configs/models/phi/hf_phi_3_5_mini_instruct.py b/configs/models/phi/hf_phi_3_5_mini_instruct.py new file mode 100644 index 00000000..f02e4c6f --- /dev/null +++ b/configs/models/phi/hf_phi_3_5_mini_instruct.py @@ -0,0 +1,12 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='phi-3-5-mini-instruct-hf', + path='microsoft/Phi-3.5-mini-instruct', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=1), + ) +] diff --git a/configs/summarizers/groups/humanevalx.py b/configs/summarizers/groups/humanevalx.py new file mode 100644 index 00000000..e4c008be --- /dev/null +++ b/configs/summarizers/groups/humanevalx.py @@ -0,0 +1,5 @@ +humanevalx_summary_groups = [] + +_humanevalx_all = ['python', 'cpp', 'go', 'java', 'js'] +_humanevalx_all = ['humanevalx-' + d for d in _humanevalx_all] +humanevalx_summary_groups.append({'name': 'humanevalx', 'subsets': _humanevalx_all}) diff --git a/opencompass/configs/models/hf_llama/hf_llama3_1_70b_instruct.py b/opencompass/configs/models/hf_llama/hf_llama3_1_70b_instruct.py new file mode 100644 index 00000000..4a17de93 --- /dev/null +++ b/opencompass/configs/models/hf_llama/hf_llama3_1_70b_instruct.py @@ -0,0 +1,13 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='llama-3_1-70b-instruct-hf', + path='meta-llama/Meta-Llama-3.1-70B-Instruct', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=4), + stop_words=['<|end_of_text|>', '<|eot_id|>'], + ) +] diff --git a/opencompass/configs/models/hf_llama/hf_llama3_1_8b_instruct.py b/opencompass/configs/models/hf_llama/hf_llama3_1_8b_instruct.py new file mode 100644 index 00000000..3ae0d5f3 --- /dev/null +++ b/opencompass/configs/models/hf_llama/hf_llama3_1_8b_instruct.py @@ -0,0 +1,13 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='llama-3_1-8b-instruct-hf', + path='meta-llama/Meta-Llama-3.1-8B-Instruct', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=1), + stop_words=['<|end_of_text|>', '<|eot_id|>'], + ) +] diff --git a/opencompass/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py b/opencompass/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py new file mode 100644 index 00000000..23f9bc2a --- /dev/null +++ b/opencompass/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py @@ -0,0 +1,16 @@ +from opencompass.models import TurboMindModelwithChatTemplate + +models = [ + dict( + type=TurboMindModelwithChatTemplate, + abbr='llama-3_1-70b-instruct-turbomind', + path='meta-llama/Meta-Llama-3.1-70B-Instruct', + engine_config=dict(max_batch_size=16, tp=4), + gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024), + max_seq_len=7168, + max_out_len=1024, + batch_size=16, + run_cfg=dict(num_gpus=4), + stop_words=['<|end_of_text|>', '<|eot_id|>'], + ) +] diff --git a/opencompass/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py b/opencompass/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py index b7dedb72..429dfec7 100644 --- a/opencompass/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py +++ b/opencompass/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py @@ -3,7 +3,7 @@ from opencompass.models import TurboMindModelwithChatTemplate models = [ dict( type=TurboMindModelwithChatTemplate, - abbr='llama-3.1-8b-instruct-turbomind', + abbr='llama-3_1-8b-instruct-turbomind', path='meta-llama/Meta-Llama-3.1-8B-Instruct', engine_config=dict(max_batch_size=16, tp=1), gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024), diff --git a/opencompass/configs/models/mistral/vllm_mixtral_large_instruct_2407.py b/opencompass/configs/models/mistral/vllm_mixtral_large_instruct_2407.py new file mode 100644 index 00000000..5f67f294 --- /dev/null +++ b/opencompass/configs/models/mistral/vllm_mixtral_large_instruct_2407.py @@ -0,0 +1,15 @@ +from opencompass.models import VLLMwithChatTemplate + + +models = [ + dict( + type=VLLMwithChatTemplate, + abbr='mixtral-large-instruct-2407-vllm', + path='mistralai/Mistral-Large-Instruct-2407', + model_kwargs=dict(tensor_parallel_size=8), + max_out_len=256, + batch_size=16, + generation_kwargs=dict(temperature=0), + run_cfg=dict(num_gpus=8), + ) +] diff --git a/opencompass/configs/models/openbmb/hf_minicpm3_4b.py b/opencompass/configs/models/openbmb/hf_minicpm3_4b.py new file mode 100644 index 00000000..730324ea --- /dev/null +++ b/opencompass/configs/models/openbmb/hf_minicpm3_4b.py @@ -0,0 +1,15 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='MiniCPM3-4B-hf', + path='openbmb/MiniCPM3-4B', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=1), + model_kwargs=dict( + torch_dtype='torch.bfloat16', + ), + ) +] diff --git a/opencompass/configs/models/phi/hf_phi_3_5_MoE_instruct.py b/opencompass/configs/models/phi/hf_phi_3_5_MoE_instruct.py new file mode 100644 index 00000000..40153f8c --- /dev/null +++ b/opencompass/configs/models/phi/hf_phi_3_5_MoE_instruct.py @@ -0,0 +1,12 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='phi-3-5-MoE-instruct-hf', + path='microsoft/Phi-3.5-MoE-instruct', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=1), + ) +] diff --git a/opencompass/configs/models/phi/hf_phi_3_5_mini_instruct.py b/opencompass/configs/models/phi/hf_phi_3_5_mini_instruct.py new file mode 100644 index 00000000..f02e4c6f --- /dev/null +++ b/opencompass/configs/models/phi/hf_phi_3_5_mini_instruct.py @@ -0,0 +1,12 @@ +from opencompass.models import HuggingFacewithChatTemplate + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='phi-3-5-mini-instruct-hf', + path='microsoft/Phi-3.5-mini-instruct', + max_out_len=1024, + batch_size=8, + run_cfg=dict(num_gpus=1), + ) +] diff --git a/opencompass/configs/summarizers/groups/humanevalx.py b/opencompass/configs/summarizers/groups/humanevalx.py new file mode 100644 index 00000000..e4c008be --- /dev/null +++ b/opencompass/configs/summarizers/groups/humanevalx.py @@ -0,0 +1,5 @@ +humanevalx_summary_groups = [] + +_humanevalx_all = ['python', 'cpp', 'go', 'java', 'js'] +_humanevalx_all = ['humanevalx-' + d for d in _humanevalx_all] +humanevalx_summary_groups.append({'name': 'humanevalx', 'subsets': _humanevalx_all}) diff --git a/opencompass/datasets/ds1000.py b/opencompass/datasets/ds1000.py index ed55216b..2f267a15 100644 --- a/opencompass/datasets/ds1000.py +++ b/opencompass/datasets/ds1000.py @@ -366,7 +366,7 @@ class DS1000ServiceEvaluator(BaseEvaluator): def __init__(self, lib: str, ip_address='localhost', - port=5000, + port='', timeout=600) -> None: assert lib in _LIBRARY_NAME_LIST, ( f' lib must be in {_LIBRARY_NAME_LIST}') @@ -421,9 +421,14 @@ class DS1000ServiceEvaluator(BaseEvaluator): Returns: tuple[bool, str]: Whether the access is successful and the output. """ + if self.port: + eval_server_url = f'{self.ip_address}:{self.port}/evaluate' + else: + eval_server_url = f'{self.ip_address}/evaluate' + exec_result = subprocess.run([ 'curl', '-X', 'POST', '-F', f'file=@{file_path}', - f'{self.ip_address}:{self.port}/evaluate' + f'{eval_server_url}' ], timeout=self.timeout, capture_output=True) diff --git a/opencompass/datasets/humanevalx.py b/opencompass/datasets/humanevalx.py index 03901503..369df95c 100644 --- a/opencompass/datasets/humanevalx.py +++ b/opencompass/datasets/humanevalx.py @@ -76,7 +76,7 @@ class HumanevalXEvaluator(BaseEvaluator): def __init__(self, language, ip_address='localhost', - port=5000, + port='', retry=2, timeout=600) -> None: assert language in _LANGUAGE_NAME_DICT.keys(), ( @@ -141,10 +141,13 @@ class HumanevalXEvaluator(BaseEvaluator): f'\nError Information: {output}') def _code_eval_service(self, file_path): + if self.port: + eval_server_url = f'{self.ip_address}:{self.port}/evaluate' + else: + eval_server_url = f'{self.ip_address}/evaluate' exec_result = subprocess.run([ 'curl', '-X', 'POST', '-F', f'file=@{file_path}', '-F', - f'dataset=humanevalx/{self.language}', - f'{self.ip_address}:{self.port}/evaluate' + f'dataset=humanevalx/{self.language}', f'{eval_server_url}' ], timeout=self.timeout, capture_output=True) diff --git a/opencompass/utils/datasets_info.py b/opencompass/utils/datasets_info.py index cb8475f8..bf414bf5 100644 --- a/opencompass/utils/datasets_info.py +++ b/opencompass/utils/datasets_info.py @@ -340,6 +340,14 @@ DATASETS_URL = { "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/humaneval.zip", "md5": "88b1b89dc47b7121c81da6bcd85a69c3", }, + "/humanevalx": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/humanevalx.zip", + "md5": "22930355c03fb73fb5bae14b50f1deb9", + }, + "/ds1000_data": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/ds1000_data.zip", + "md5": "1a4990aec04a2fd73ccfad12e2d43b43", + }, "/drop_simple_eval/": { "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/drop_simple_eval.zip", "md5": "c912afe5b4a63509851cf16e6b91830e",