From 794ab7c3727f39193b9ac9e5f2b27df24d2c8ae3 Mon Sep 17 00:00:00 2001 From: wujiang Date: Sun, 2 Feb 2025 15:52:36 +0800 Subject: [PATCH] add & update openai models --- examples/eval_OpenHuEval_HuMatchingFIB.py | 9 +++++--- .../models/openai/gpt_4o_mini_20240718.py | 1 + .../models/openai/o1_mini_2024_09_12.py | 2 ++ .../models/openai/o1_preview_2024_09_12.py | 1 + .../models/openai/o3_mini_2025_01_31.py | 22 +++++++++++++++++++ opencompass/models/openai_api.py | 2 ++ 6 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 opencompass/configs/models/openai/o3_mini_2025_01_31.py diff --git a/examples/eval_OpenHuEval_HuMatchingFIB.py b/examples/eval_OpenHuEval_HuMatchingFIB.py index 06294e34..2bf1d62d 100644 --- a/examples/eval_OpenHuEval_HuMatchingFIB.py +++ b/examples/eval_OpenHuEval_HuMatchingFIB.py @@ -3,10 +3,13 @@ from mmengine.config import read_base with read_base(): from opencompass.configs.datasets.OpenHuEval.HuMatchingFIB.HuMatchingFIB import hu_matching_fib_datasets - from opencompass.configs.models.openai.gpt_4o_mini_20240718 import models as gpt_4o_mini_20240718_model - from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b_instruct import models as lmdeploy_qwen2_5_7b_instruct_model - from opencompass.configs.models.hf_internlm.lmdeploy_internlm3_8b_instruct import models as lmdeploy_internlm3_8b_instruct_model + # from opencompass.configs.models.openai.gpt_4o_mini_20240718 import models as gpt_4o_mini_20240718_model + # from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b_instruct import models as lmdeploy_qwen2_5_7b_instruct_model + # from opencompass.configs.models.hf_internlm.lmdeploy_internlm3_8b_instruct import models as lmdeploy_internlm3_8b_instruct_model + # from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import models as lmdeploy_qwq_32b_preview_model + from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model + # from opencompass.configs.models.openai.o3_mini_2025_01_31 import models as o3_mini_2025_01_31_model datasets = hu_matching_fib_datasets models = sum([v for k, v in locals().items() if k.endswith('_model')], []) diff --git a/opencompass/configs/models/openai/gpt_4o_mini_20240718.py b/opencompass/configs/models/openai/gpt_4o_mini_20240718.py index 36bfa610..8073bfbc 100644 --- a/opencompass/configs/models/openai/gpt_4o_mini_20240718.py +++ b/opencompass/configs/models/openai/gpt_4o_mini_20240718.py @@ -12,6 +12,7 @@ models = [ path='gpt-4o-mini-2024-07-18', key= 'ENV', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well + openai_proxy_url='ENV', meta_template=api_meta_template, query_per_second=1, max_out_len=2048, diff --git a/opencompass/configs/models/openai/o1_mini_2024_09_12.py b/opencompass/configs/models/openai/o1_mini_2024_09_12.py index 331ecf31..ebbb9f30 100644 --- a/opencompass/configs/models/openai/o1_mini_2024_09_12.py +++ b/opencompass/configs/models/openai/o1_mini_2024_09_12.py @@ -12,6 +12,8 @@ models = [ path='o1-mini-2024-09-12', key= 'ENV', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well + openai_proxy_url='ENV', + verbose=True, meta_template=api_meta_template, query_per_second=1, batch_size=1, diff --git a/opencompass/configs/models/openai/o1_preview_2024_09_12.py b/opencompass/configs/models/openai/o1_preview_2024_09_12.py index 9dff1037..b989ee01 100644 --- a/opencompass/configs/models/openai/o1_preview_2024_09_12.py +++ b/opencompass/configs/models/openai/o1_preview_2024_09_12.py @@ -12,6 +12,7 @@ models = [ path='o1-preview-2024-09-12', key= 'ENV', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well + openai_proxy_url='ENV', meta_template=api_meta_template, query_per_second=1, batch_size=1, diff --git a/opencompass/configs/models/openai/o3_mini_2025_01_31.py b/opencompass/configs/models/openai/o3_mini_2025_01_31.py new file mode 100644 index 00000000..fd224f61 --- /dev/null +++ b/opencompass/configs/models/openai/o3_mini_2025_01_31.py @@ -0,0 +1,22 @@ +from opencompass.models import OpenAISDK + +api_meta_template = dict(round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True), +], ) + +models = [ + dict( + abbr='o3-mini-2025-01-31', + type=OpenAISDK, + path='o3-mini-2025-01-31', + key= + 'ENV', # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well + openai_proxy_url='ENV', + verbose=True, + meta_template=api_meta_template, + query_per_second=1, + batch_size=1, + temperature=1, + max_completion_tokens=8192), # you can change it for large reasoning inference cost, according to: https://platform.openai.com/docs/guides/reasoning +] diff --git a/opencompass/models/openai_api.py b/opencompass/models/openai_api.py index 2781d160..24a170d6 100644 --- a/opencompass/models/openai_api.py +++ b/opencompass/models/openai_api.py @@ -30,6 +30,7 @@ O1_MODEL_LIST = [ 'o1-mini-2024-09-12', 'o1-preview', 'o1-mini', + 'o3-mini-2025-01-31', ] @@ -579,6 +580,7 @@ class OpenAISDK(OpenAI): verbose=verbose, max_completion_tokens=max_completion_tokens, ) + key = random.choice(self.keys) from openai import OpenAI # support multiple api_base for acceleration