[Feature] Update Models (#1518)

* Update Models

* Update

* Update humanevalx

* Update

* Update

configs/models/hf_llama/hf_llama3_1_70b_instruct.py

@@ -0,0 +1,13 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='llama-3_1-70b-instruct-hf',
+        path='meta-llama/Meta-Llama-3.1-70B-Instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+        stop_words=['<|end_of_text|>', '<|eot_id|>'],
+    )
+]

configs/models/hf_llama/hf_llama3_1_8b_instruct.py

@@ -0,0 +1,13 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='llama-3_1-8b-instruct-hf',
+        path='meta-llama/Meta-Llama-3.1-8B-Instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+        stop_words=['<|end_of_text|>', '<|eot_id|>'],
+    )
+]

configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py

@@ -0,0 +1,16 @@
+from opencompass.models import TurboMindModelwithChatTemplate
+
+models = [
+    dict(
+        type=TurboMindModelwithChatTemplate,
+        abbr='llama-3_1-70b-instruct-turbomind',
+        path='meta-llama/Meta-Llama-3.1-70B-Instruct',
+        engine_config=dict(max_batch_size=16, tp=4),
+        gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024),
+        max_seq_len=7168,
+        max_out_len=1024,
+        batch_size=16,
+        run_cfg=dict(num_gpus=4),
+        stop_words=['<|end_of_text|>', '<|eot_id|>'],
+    )
+]

configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py

@@ -3,7 +3,7 @@ from opencompass.models import TurboMindModelwithChatTemplate
 models = [
     dict(
         type=TurboMindModelwithChatTemplate,
-        abbr='llama-3.1-8b-instruct-turbomind',
+        abbr='llama-3_1-8b-instruct-turbomind',
         path='meta-llama/Meta-Llama-3.1-8B-Instruct',
         engine_config=dict(max_batch_size=16, tp=1),
         gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024),

configs/models/mistral/vllm_mixtral_large_instruct_2407.py

@@ -0,0 +1,15 @@
+from opencompass.models import VLLMwithChatTemplate
+
+
+models = [
+    dict(
+        type=VLLMwithChatTemplate,
+        abbr='mixtral-large-instruct-2407-vllm',
+        path='mistralai/Mistral-Large-Instruct-2407',
+        model_kwargs=dict(tensor_parallel_size=8),
+        max_out_len=256,
+        batch_size=16,
+        generation_kwargs=dict(temperature=0),
+        run_cfg=dict(num_gpus=8),
+    )
+]

configs/models/openbmb/hf_minicpm3_4b.py

@@ -0,0 +1,15 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='MiniCPM3-4B-hf',
+        path='openbmb/MiniCPM3-4B',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+        model_kwargs=dict(
+            torch_dtype='torch.bfloat16',
+        ),
+    )
+]

configs/models/phi/hf_phi_3_5_MoE_instruct.py

@@ -0,0 +1,12 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='phi-3-5-MoE-instruct-hf',
+        path='microsoft/Phi-3.5-MoE-instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+    )
+]

configs/models/phi/hf_phi_3_5_mini_instruct.py

@@ -0,0 +1,12 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='phi-3-5-mini-instruct-hf',
+        path='microsoft/Phi-3.5-mini-instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+    )
+]

configs/summarizers/groups/humanevalx.py

@@ -0,0 +1,5 @@
+humanevalx_summary_groups = []
+
+_humanevalx_all = ['python', 'cpp', 'go', 'java', 'js']
+_humanevalx_all = ['humanevalx-' + d for d in _humanevalx_all]
+humanevalx_summary_groups.append({'name': 'humanevalx', 'subsets': _humanevalx_all})

opencompass/configs/models/hf_llama/hf_llama3_1_70b_instruct.py

@@ -0,0 +1,13 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='llama-3_1-70b-instruct-hf',
+        path='meta-llama/Meta-Llama-3.1-70B-Instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=4),
+        stop_words=['<|end_of_text|>', '<|eot_id|>'],
+    )
+]

opencompass/configs/models/hf_llama/hf_llama3_1_8b_instruct.py

@@ -0,0 +1,13 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='llama-3_1-8b-instruct-hf',
+        path='meta-llama/Meta-Llama-3.1-8B-Instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+        stop_words=['<|end_of_text|>', '<|eot_id|>'],
+    )
+]

opencompass/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py

@@ -0,0 +1,16 @@
+from opencompass.models import TurboMindModelwithChatTemplate
+
+models = [
+    dict(
+        type=TurboMindModelwithChatTemplate,
+        abbr='llama-3_1-70b-instruct-turbomind',
+        path='meta-llama/Meta-Llama-3.1-70B-Instruct',
+        engine_config=dict(max_batch_size=16, tp=4),
+        gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024),
+        max_seq_len=7168,
+        max_out_len=1024,
+        batch_size=16,
+        run_cfg=dict(num_gpus=4),
+        stop_words=['<|end_of_text|>', '<|eot_id|>'],
+    )
+]

opencompass/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py

@@ -3,7 +3,7 @@ from opencompass.models import TurboMindModelwithChatTemplate
 models = [
     dict(
         type=TurboMindModelwithChatTemplate,
-        abbr='llama-3.1-8b-instruct-turbomind',
+        abbr='llama-3_1-8b-instruct-turbomind',
         path='meta-llama/Meta-Llama-3.1-8B-Instruct',
         engine_config=dict(max_batch_size=16, tp=1),
         gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024),

opencompass/configs/models/mistral/vllm_mixtral_large_instruct_2407.py

@@ -0,0 +1,15 @@
+from opencompass.models import VLLMwithChatTemplate
+
+
+models = [
+    dict(
+        type=VLLMwithChatTemplate,
+        abbr='mixtral-large-instruct-2407-vllm',
+        path='mistralai/Mistral-Large-Instruct-2407',
+        model_kwargs=dict(tensor_parallel_size=8),
+        max_out_len=256,
+        batch_size=16,
+        generation_kwargs=dict(temperature=0),
+        run_cfg=dict(num_gpus=8),
+    )
+]

opencompass/configs/models/openbmb/hf_minicpm3_4b.py

@@ -0,0 +1,15 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='MiniCPM3-4B-hf',
+        path='openbmb/MiniCPM3-4B',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+        model_kwargs=dict(
+            torch_dtype='torch.bfloat16',
+        ),
+    )
+]

opencompass/configs/models/phi/hf_phi_3_5_MoE_instruct.py

@@ -0,0 +1,12 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='phi-3-5-MoE-instruct-hf',
+        path='microsoft/Phi-3.5-MoE-instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+    )
+]

opencompass/configs/models/phi/hf_phi_3_5_mini_instruct.py

@@ -0,0 +1,12 @@
+from opencompass.models import HuggingFacewithChatTemplate
+
+models = [
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='phi-3-5-mini-instruct-hf',
+        path='microsoft/Phi-3.5-mini-instruct',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+    )
+]

opencompass/configs/summarizers/groups/humanevalx.py

@@ -0,0 +1,5 @@
+humanevalx_summary_groups = []
+
+_humanevalx_all = ['python', 'cpp', 'go', 'java', 'js']
+_humanevalx_all = ['humanevalx-' + d for d in _humanevalx_all]
+humanevalx_summary_groups.append({'name': 'humanevalx', 'subsets': _humanevalx_all})

opencompass/datasets/ds1000.py

@@ -366,7 +366,7 @@ class DS1000ServiceEvaluator(BaseEvaluator):
     def __init__(self,
                  lib: str,
                  ip_address='localhost',
-                 port=5000,
+                 port='',
                  timeout=600) -> None:
         assert lib in _LIBRARY_NAME_LIST, (
             f' lib must be in {_LIBRARY_NAME_LIST}')
@@ -421,9 +421,14 @@ class DS1000ServiceEvaluator(BaseEvaluator):
         Returns:
             tuple[bool, str]: Whether the access is successful and the output.
         """
+        if self.port:
+            eval_server_url = f'{self.ip_address}:{self.port}/evaluate'
+        else:
+            eval_server_url = f'{self.ip_address}/evaluate'
+
         exec_result = subprocess.run([
             'curl', '-X', 'POST', '-F', f'file=@{file_path}',
-            f'{self.ip_address}:{self.port}/evaluate'
+            f'{eval_server_url}'
         ],
                                      timeout=self.timeout,
                                      capture_output=True)

opencompass/datasets/humanevalx.py

@@ -76,7 +76,7 @@ class HumanevalXEvaluator(BaseEvaluator):
     def __init__(self,
                  language,
                  ip_address='localhost',
-                 port=5000,
+                 port='',
                  retry=2,
                  timeout=600) -> None:
         assert language in _LANGUAGE_NAME_DICT.keys(), (
@@ -141,10 +141,13 @@ class HumanevalXEvaluator(BaseEvaluator):
                 f'\nError Information: {output}')
 
     def _code_eval_service(self, file_path):
+        if self.port:
+            eval_server_url = f'{self.ip_address}:{self.port}/evaluate'
+        else:
+            eval_server_url = f'{self.ip_address}/evaluate'
         exec_result = subprocess.run([
             'curl', '-X', 'POST', '-F', f'file=@{file_path}', '-F',
-            f'dataset=humanevalx/{self.language}',
-            f'{self.ip_address}:{self.port}/evaluate'
+            f'dataset=humanevalx/{self.language}', f'{eval_server_url}'
         ],
                                      timeout=self.timeout,
                                      capture_output=True)

opencompass/utils/datasets_info.py

@@ -340,6 +340,14 @@ DATASETS_URL = {
         "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/humaneval.zip",
         "md5": "88b1b89dc47b7121c81da6bcd85a69c3",
     },
+    "/humanevalx": {
+        "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/humanevalx.zip",
+        "md5": "22930355c03fb73fb5bae14b50f1deb9",
+    },
+    "/ds1000_data": {
+        "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/ds1000_data.zip",
+        "md5": "1a4990aec04a2fd73ccfad12e2d43b43",
+    },
     "/drop_simple_eval/": {
         "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/drop_simple_eval.zip",
         "md5": "c912afe5b4a63509851cf16e6b91830e",