diff --git a/examples/eval_OpenHuEval_HuMatchingFIB.py b/examples/eval_OpenHuEval_HuMatchingFIB.py index 2bf1d62d..ab48c4da 100644 --- a/examples/eval_OpenHuEval_HuMatchingFIB.py +++ b/examples/eval_OpenHuEval_HuMatchingFIB.py @@ -4,11 +4,15 @@ with read_base(): from opencompass.configs.datasets.OpenHuEval.HuMatchingFIB.HuMatchingFIB import hu_matching_fib_datasets # from opencompass.configs.models.openai.gpt_4o_mini_20240718 import models as gpt_4o_mini_20240718_model + from opencompass.configs.models.deepseek.deepseek_v3_api_siliconflow import models as deepseek_v3_api_siliconflow_model + # from opencompass.configs.models.deepseek.deepseek_v3_api import models as deepseek_v3_api_model + # from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b_instruct import models as lmdeploy_qwen2_5_7b_instruct_model # from opencompass.configs.models.hf_internlm.lmdeploy_internlm3_8b_instruct import models as lmdeploy_internlm3_8b_instruct_model # from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import models as lmdeploy_qwq_32b_preview_model - from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model + from opencompass.configs.models.deepseek.deepseek_r1_siliconflow import models as deepseek_r1_siliconflow_model + # from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model # from opencompass.configs.models.openai.o3_mini_2025_01_31 import models as o3_mini_2025_01_31_model datasets = hu_matching_fib_datasets diff --git a/opencompass/configs/models/deepseek/deepseek_r1_siliconflow.py b/opencompass/configs/models/deepseek/deepseek_r1_siliconflow.py new file mode 100644 index 00000000..dbc3afed --- /dev/null +++ b/opencompass/configs/models/deepseek/deepseek_r1_siliconflow.py @@ -0,0 +1,15 @@ +from opencompass.models import OpenAISDK + +models = [ + dict( + abbr='deepseek_r1_api_siliconflow', + type=OpenAISDK, + path='deepseek-ai/DeepSeek-R1', + key='ENV_SILICONFLOW', + openai_api_base='https://api.siliconflow.cn/v1/', + query_per_second=0.1, + max_completion_tokens=8192, + batch_size=1, + verbose=True, + ), +] diff --git a/opencompass/configs/models/deepseek/deepseek_v3_api.py b/opencompass/configs/models/deepseek/deepseek_v3_api.py new file mode 100644 index 00000000..f0590ed9 --- /dev/null +++ b/opencompass/configs/models/deepseek/deepseek_v3_api.py @@ -0,0 +1,16 @@ +from opencompass.models import OpenAISDK + +models = [ + dict( + abbr='deepseek_v3_api', + type=OpenAISDK, + path='deepseek-chat', + key='ENV_DEEPSEEK', + openai_api_base='https://api.deepseek.com/v1/', + query_per_second=1, + max_out_len=2048, + max_seq_len=4096, + batch_size=8, + verbose=True, + ), +] diff --git a/opencompass/configs/models/deepseek/deepseek_v3_api_siliconflow.py b/opencompass/configs/models/deepseek/deepseek_v3_api_siliconflow.py new file mode 100644 index 00000000..1a88adbd --- /dev/null +++ b/opencompass/configs/models/deepseek/deepseek_v3_api_siliconflow.py @@ -0,0 +1,16 @@ +from opencompass.models import OpenAISDK + +models = [ + dict( + abbr='deepseek_v3_api_siliconflow', + type=OpenAISDK, + path='deepseek-ai/DeepSeek-V3', + key='ENV_SILICONFLOW', + openai_api_base='https://api.siliconflow.cn/v1/', + query_per_second=0.1, + max_out_len=2048, + max_seq_len=4096, + batch_size=8, + verbose=True, + ), +] diff --git a/opencompass/models/openai_api.py b/opencompass/models/openai_api.py index 24a170d6..e239c6ae 100644 --- a/opencompass/models/openai_api.py +++ b/opencompass/models/openai_api.py @@ -31,6 +31,8 @@ O1_MODEL_LIST = [ 'o1-preview', 'o1-mini', 'o3-mini-2025-01-31', + 'deepseek-ai/DeepSeek-R1', + 'DeepSeek-R1', ] @@ -127,6 +129,14 @@ class OpenAI(BaseAPIModel): if 'OPENAI_API_KEY' not in os.environ: raise ValueError('OpenAI API key is not set.') self.keys = os.getenv('OPENAI_API_KEY').split(',') + elif key == 'ENV_SILICONFLOW': + if 'SILICONFLOW_API_KEY' not in os.environ: + raise ValueError('Siliconflow API key is not set.') + self.keys = os.getenv('SILICONFLOW_API_KEY').split(',') + elif key == 'ENV_DEEPSEEK': + if 'DEEPSEEK_API_KEY' not in os.environ: + raise ValueError('Deepseek API key is not set.') + self.keys = os.getenv('DEEPSEEK_API_KEY').split(',') else: self.keys = [key] else: @@ -626,7 +636,7 @@ class OpenAISDK(OpenAI): f"'max_token' is unsupported for model {self.path}") self.logger.warning( f'We use max_completion_tokens: ' - f'{self.max_completion_tokens}for this query') + f'{self.max_completion_tokens} for this query') query_data = dict( model=self.path, max_completion_tokens=self.max_completion_tokens,