From a3bac3611a3e172f8ef5364a2b906e6766ed0398 Mon Sep 17 00:00:00 2001 From: wangjingchao Date: Thu, 13 Mar 2025 17:16:16 +0800 Subject: [PATCH] [Fix] Fix bugs when adding QwQ models --- examples/eval_OpenHuEval_HuLifeQA.py | 11 +++++++ examples/eval_OpenHuEval_HuMatchingFIB.py | 32 ++++++++++++++++++++ examples/eval_OpenHuEval_HuProverbRea_2CQ.py | 8 +++++ examples/eval_OpenHuEval_HuProverbRea_OE.py | 8 +++++ examples/eval_OpenHuEval_HuSimpleQA.py | 17 +++++++++++ examples/eval_OpenHuEval_HuStandardFIB.py | 9 ++++++ 6 files changed, 85 insertions(+) diff --git a/examples/eval_OpenHuEval_HuLifeQA.py b/examples/eval_OpenHuEval_HuLifeQA.py index 7e74a1e7..d4d4ef32 100644 --- a/examples/eval_OpenHuEval_HuLifeQA.py +++ b/examples/eval_OpenHuEval_HuLifeQA.py @@ -71,6 +71,17 @@ api_meta_template = dict( ] ) +<<<<<<< HEAD +======= +for model in deepseek_r1_api_aliyun_model: + model['return_reasoning_content'] = True + model['pred_postprocessor'] = {'open_hu_eval_*': {'type': 'rm__before_eval'}} + if model['abbr'].startswith('QwQ'): + model['pred_postprocessor'] = { + 'OpenHuEval_*': {'type': 'extract_qwq_answer_before_eval'} + } +del model +>>>>>>> 6b84df9... add_qwen_api_qwq_32b models = [ *gpt_4o_mini_20240718_model, diff --git a/examples/eval_OpenHuEval_HuMatchingFIB.py b/examples/eval_OpenHuEval_HuMatchingFIB.py index fa2f055a..9dba7501 100644 --- a/examples/eval_OpenHuEval_HuMatchingFIB.py +++ b/examples/eval_OpenHuEval_HuMatchingFIB.py @@ -1,6 +1,7 @@ from mmengine.config import read_base with read_base(): +<<<<<<< HEAD from opencompass.configs.datasets.OpenHuEval.HuMatchingFIB.HuMatchingFIB import ( HuMatchingFIB_datasets, ) @@ -66,6 +67,33 @@ with read_base(): from opencompass.configs.models.openai.o1_mini_2024_09_12 import ( models as o1_mini_2024_09_12_model, ) +======= + from opencompass.configs.datasets.OpenHuEval.HuMatchingFIB.HuMatchingFIB import HuMatchingFIB_datasets + + from opencompass.configs.models.openai.gpt_4o_mini_20240718 import models as gpt_4o_mini_20240718_model + from opencompass.configs.models.openai.gpt_4o_2024_11_20 import models as gpt_4o_2024_11_20_model + from opencompass.configs.models.deepseek.deepseek_v3_api_aliyun import models as deepseek_v3_api_aliyun_model + + from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b_instruct import models as lmdeploy_qwen2_5_7b_instruct_model + from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_32b_instruct import models as lmdeploy_qwen2_5_32b_instruct_model + from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_14b_instruct import models as lmdeploy_qwen2_5_14b_instruct_model + from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_72b_instruct import models as lmdeploy_qwen2_5_72b_instruct_model + from opencompass.configs.models.hf_llama.lmdeploy_llama3_1_8b_instruct import models as lmdeploy_llama3_1_8b_instruct_model + from opencompass.configs.models.hf_llama.lmdeploy_llama3_1_70b_instruct import models as lmdeploy_llama3_1_70b_instruct_model + from opencompass.configs.models.hf_llama.llama3_3_70b_api_siliconflow import models as llama3_3_70b_api_siliconflow_model + + from opencompass.configs.models.hf_internlm.lmdeploy_internlm3_8b_instruct import models as lmdeploy_internlm3_8b_instruct_model + + from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import models as lmdeploy_qwq_32b_preview_model + from opencompass.configs.models.qwq.qwq_32b import models as qwq_32b_model + from opencompass.configs.models.qwq.qwq_plus_2025_03_05 import models as qwq_plus_2025_03_05_model + from opencompass.configs.models.deepseek.deepseek_r1_api_aliyun import models as deepseek_r1_api_aliyun_model + from opencompass.configs.models.deepseek.deepseek_r1_distill_llama_8b_api_aliyun import models as deepseek_r1_distill_llama_8b_api_aliyun_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_qwen_32b_instruct import models as deepseek_r1_distill_qwen_32b_instruct_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_qwen_14b_instruct import models as deepseek_r1_distill_qwen_14b_instruct_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_llama_70b_instruct import models as deepseek_r1_distill_llama_70b_instruct_model + from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model +>>>>>>> 6b84df9... add_qwen_api_qwq_32b # from opencompass.configs.models.openai.o3_mini_2025_01_31 import models as o3_mini_2025_01_31_model datasets = HuMatchingFIB_datasets @@ -85,6 +113,10 @@ for model in models: } del model +<<<<<<< HEAD +======= + +>>>>>>> 6b84df9... add_qwen_api_qwq_32b work_dir = ( './outputs/' + __file__.split('/')[-1].split('.')[0] + '/' ) # do NOT modify this line, yapf: disable, pylint: disable diff --git a/examples/eval_OpenHuEval_HuProverbRea_2CQ.py b/examples/eval_OpenHuEval_HuProverbRea_2CQ.py index c2ab2601..7bb77c07 100644 --- a/examples/eval_OpenHuEval_HuProverbRea_2CQ.py +++ b/examples/eval_OpenHuEval_HuProverbRea_2CQ.py @@ -32,6 +32,7 @@ with read_base(): models as lmdeploy_internlm3_8b_instruct_model, ) +<<<<<<< HEAD from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import ( models as lmdeploy_qwq_32b_preview_model, ) @@ -45,6 +46,13 @@ with read_base(): from opencompass.configs.models.openai.o1_mini_2024_09_12 import ( models as o1_mini_2024_09_12_model, ) +======= + from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import models as lmdeploy_qwq_32b_preview_model + from opencompass.configs.models.qwq.qwq_32b import models as qwq_32b_model + from opencompass.configs.models.qwq.qwq_plus_2025_03_05 import models as qwq_plus_2025_03_05_model + from opencompass.configs.models.deepseek.deepseek_r1_api_aliyun import models as deepseek_r1_api_aliyun_model + from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model +>>>>>>> 6b84df9... add_qwen_api_qwq_32b # from opencompass.configs.models.openai.o3_mini_2025_01_31 import models as o3_mini_2025_01_31_model datasets = HuProverbRea_datasets diff --git a/examples/eval_OpenHuEval_HuProverbRea_OE.py b/examples/eval_OpenHuEval_HuProverbRea_OE.py index 4afabe67..a3ab8615 100644 --- a/examples/eval_OpenHuEval_HuProverbRea_OE.py +++ b/examples/eval_OpenHuEval_HuProverbRea_OE.py @@ -32,6 +32,7 @@ with read_base(): models as lmdeploy_internlm3_8b_instruct_model, ) +<<<<<<< HEAD from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import ( models as lmdeploy_qwq_32b_preview_model, ) @@ -45,6 +46,13 @@ with read_base(): from opencompass.configs.models.openai.o1_mini_2024_09_12 import ( models as o1_mini_2024_09_12_model, ) +======= + from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import models as lmdeploy_qwq_32b_preview_model + from opencompass.configs.models.qwq.qwq_32b import models as qwq_32b_model + from opencompass.configs.models.qwq.qwq_plus_2025_03_05 import models as qwq_plus_2025_03_05_model + from opencompass.configs.models.deepseek.deepseek_r1_api_aliyun import models as deepseek_r1_api_aliyun_model + from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model +>>>>>>> 6b84df9... add_qwen_api_qwq_32b # from opencompass.configs.models.openai.o3_mini_2025_01_31 import models as o3_mini_2025_01_31_model datasets = HuProverbRea_datasets diff --git a/examples/eval_OpenHuEval_HuSimpleQA.py b/examples/eval_OpenHuEval_HuSimpleQA.py index 54bb0f42..2c952c7a 100644 --- a/examples/eval_OpenHuEval_HuSimpleQA.py +++ b/examples/eval_OpenHuEval_HuSimpleQA.py @@ -42,6 +42,7 @@ with read_base(): # from opencompass.configs.models.hf_internlm.lmdeploy_internlm3_8b_instruct import models as lmdeploy_internlm3_8b_instruct_model +<<<<<<< HEAD from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import ( models as lmdeploy_qwq_32b_preview_model, ) @@ -67,6 +68,17 @@ with read_base(): from opencompass.configs.models.openai.o1_mini_2024_09_12 import ( models as o1_mini_2024_09_12_model, ) +======= + from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import models as lmdeploy_qwq_32b_preview_model + from opencompass.configs.models.qwq.qwq_32b import models as qwq_32b_model + from opencompass.configs.models.qwq.qwq_plus_2025_03_05 import models as qwq_plus_2025_03_05_model + from opencompass.configs.models.deepseek.deepseek_r1_api_aliyun import models as deepseek_r1_api_aliyun_model + from opencompass.configs.models.deepseek.deepseek_r1_distill_llama_8b_api_aliyun import models as deepseek_r1_distill_llama_8b_api_aliyun_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_qwen_32b_instruct import models as deepseek_r1_distill_qwen_32b_instruct_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_qwen_14b_instruct import models as deepseek_r1_distill_qwen_14b_instruct_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_llama_70b_instruct import models as deepseek_r1_distill_llama_70b_instruct_model + from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model +>>>>>>> 6b84df9... add_qwen_api_qwq_32b # from opencompass.configs.models.openai.o3_mini_2025_01_31 import models as o3_mini_2025_01_31_model from opencompass.models import OpenAI @@ -146,7 +158,12 @@ eval = dict( ), ) +<<<<<<< HEAD summarizer = dict(type=HuSimpleQASummarizer, prompt_languages=PROMPT_LANGUAGES) +======= +summarizer = dict(type=HuSimpleQASummarizer, + prompt_languages=PROMPT_LANGUAGES) +>>>>>>> 6b84df9... add_qwen_api_qwq_32b work_dir = ( './outputs/' + __file__.split('/')[-1].split('.')[0] + '/' diff --git a/examples/eval_OpenHuEval_HuStandardFIB.py b/examples/eval_OpenHuEval_HuStandardFIB.py index 26c4ee3c..55453a3b 100644 --- a/examples/eval_OpenHuEval_HuStandardFIB.py +++ b/examples/eval_OpenHuEval_HuStandardFIB.py @@ -32,6 +32,7 @@ with read_base(): models as lmdeploy_internlm3_8b_instruct_model, ) +<<<<<<< HEAD from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import ( models as lmdeploy_qwq_32b_preview_model, ) @@ -48,6 +49,14 @@ with read_base(): from opencompass.configs.models.openai.o1_mini_2024_09_12 import ( models as o1_mini_2024_09_12_model, ) +======= + from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import models as lmdeploy_qwq_32b_preview_model + from opencompass.configs.models.qwq.qwq_32b import models as qwq_32b_model + from opencompass.configs.models.qwq.qwq_plus_2025_03_05 import models as qwq_plus_2025_03_05_model + from opencompass.configs.models.deepseek.deepseek_r1_api_aliyun import models as deepseek_r1_api_aliyun_model + from opencompass.configs.models.deepseek.deepseek_r1_api_siliconflow import models as deepseek_r1_api_siliconflow_model + from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model +>>>>>>> 6b84df9... add_qwen_api_qwq_32b # from opencompass.configs.models.openai.o3_mini_2025_01_31 import models as o3_mini_2025_01_31_model datasets = HuStandardFIB_datasets