diff --git a/examples/eval_OpenHuEval_HuLifeQA.py b/examples/eval_OpenHuEval_HuLifeQA.py index 73545dd7..f4ed6d62 100644 --- a/examples/eval_OpenHuEval_HuLifeQA.py +++ b/examples/eval_OpenHuEval_HuLifeQA.py @@ -3,7 +3,7 @@ from mmengine.config import read_base with read_base(): from opencompass.configs.datasets.OpenHuEval.HuLifeQA import ( hu_life_qa_datasets, - task_group_new, + TASK_GROUP_NEW, ) from opencompass.configs.models.openai.gpt_4o_mini_20240718 import models as gpt_4o_mini_20240718_model @@ -119,7 +119,7 @@ eval = dict( summarizer = dict( type=WildBenchSingleSummarizer, - customized_task_group_new=task_group_new, + customized_task_group_new=TASK_GROUP_NEW, ) work_dir = ( diff --git a/examples/eval_OpenHuEval_HuSimpleQA.py b/examples/eval_OpenHuEval_HuSimpleQA.py index f2055432..e4ce9cd8 100644 --- a/examples/eval_OpenHuEval_HuSimpleQA.py +++ b/examples/eval_OpenHuEval_HuSimpleQA.py @@ -10,22 +10,21 @@ with read_base(): from opencompass.configs.models.deepseek.deepseek_v3_api_aliyun import models as deepseek_v3_api_aliyun_model from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b_instruct import models as lmdeploy_qwen2_5_7b_instruct_model - # from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_32b_instruct import models as lmdeploy_qwen2_5_32b_instruct_model - # from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_14b_instruct import models as lmdeploy_qwen2_5_14b_instruct_model + from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_32b_instruct import models as lmdeploy_qwen2_5_32b_instruct_model + from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_14b_instruct import models as lmdeploy_qwen2_5_14b_instruct_model from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_72b_instruct import models as lmdeploy_qwen2_5_72b_instruct_model from opencompass.configs.models.hf_llama.lmdeploy_llama3_1_8b_instruct import models as lmdeploy_llama3_1_8b_instruct_model from opencompass.configs.models.hf_llama.lmdeploy_llama3_1_70b_instruct import models as lmdeploy_llama3_1_70b_instruct_model - # from opencompass.configs.models.hf_llama.llama3_3_70b_api_siliconflow import models as llama3_3_70b_api_siliconflow_model - + from opencompass.configs.models.hf_llama.llama3_3_70b_api_siliconflow import models as llama3_3_70b_api_siliconflow_model # from opencompass.configs.models.hf_internlm.lmdeploy_internlm3_8b_instruct import models as lmdeploy_internlm3_8b_instruct_model from opencompass.configs.models.qwq.lmdeploy_qwq_32b_preview import models as lmdeploy_qwq_32b_preview_model from opencompass.configs.models.deepseek.deepseek_r1_api_aliyun import models as deepseek_r1_api_aliyun_model - # from opencompass.configs.models.deepseek.deepseek_r1_distill_llama_8b_api_aliyun import models as deepseek_r1_distill_llama_8b_api_aliyun_model - # from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_qwen_32b_instruct import models as deepseek_r1_distill_qwen_32b_instruct_model - # from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_qwen_14b_instruct import models as deepseek_r1_distill_qwen_14b_instruct_model - # from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_llama_70b_instruct import models as deepseek_r1_distill_llama_70b_instruct_model + from opencompass.configs.models.deepseek.deepseek_r1_distill_llama_8b_api_aliyun import models as deepseek_r1_distill_llama_8b_api_aliyun_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_qwen_32b_instruct import models as deepseek_r1_distill_qwen_32b_instruct_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_qwen_14b_instruct import models as deepseek_r1_distill_qwen_14b_instruct_model + from opencompass.configs.models.deepseek.lmdeploy_deepseek_r1_distill_llama_70b_instruct import models as deepseek_r1_distill_llama_70b_instruct_model from opencompass.configs.models.openai.o1_mini_2024_09_12 import models as o1_mini_2024_09_12_model # from opencompass.configs.models.openai.o3_mini_2025_01_31 import models as o3_mini_2025_01_31_model diff --git a/opencompass/configs/datasets/OpenHuEval/HuLifeQA.py b/opencompass/configs/datasets/OpenHuEval/HuLifeQA/HuLifeQA.py similarity index 55% rename from opencompass/configs/datasets/OpenHuEval/HuLifeQA.py rename to opencompass/configs/datasets/OpenHuEval/HuLifeQA/HuLifeQA.py index 5961ed14..405a42ca 100644 --- a/opencompass/configs/datasets/OpenHuEval/HuLifeQA.py +++ b/opencompass/configs/datasets/OpenHuEval/HuLifeQA/HuLifeQA.py @@ -4,14 +4,14 @@ from opencompass.openicl.icl_inferencer import ChatInferencer from opencompass.openicl.icl_prompt_template import PromptTemplate from opencompass.openicl.icl_retriever import ZeroRetriever +with read_base(): + from .HuLifeQA_setting import DATA_PATH, TASK_GROUP_NEW + hu_life_qa_reader_cfg = dict( input_columns=['dialogue', 'prompt'], output_column='judge', ) -data_path ='/mnt/hwfile/opendatalab/yanghaote/share/HuLifeQA_20250131.jsonl' - -hu_life_qa_datasets = [] hu_life_qa_infer_cfg = dict( prompt_template=dict( type=PromptTemplate, @@ -37,34 +37,15 @@ hu_life_qa_eval_cfg = dict( pred_role='BOT', ) +hu_life_qa_datasets = [] hu_life_qa_datasets.append( dict( abbr='open_hu_eval_hu_life_qa', type=WildBenchDataset, - path=data_path, + path=DATA_PATH, reader_cfg=hu_life_qa_reader_cfg, infer_cfg=hu_life_qa_infer_cfg, eval_cfg=hu_life_qa_eval_cfg, ) ) -task_group_new = { - 'life_culture_custom': 'life_culture_custom', - 'childbearing and education': 'life_culture_custom', - 'culture and community': 'life_culture_custom', - 'culture and customs': 'life_culture_custom', - 'food and drink': 'life_culture_custom', - 'health': 'life_culture_custom', - 'holidays': 'life_culture_custom', - 'home': 'life_culture_custom', - 'person': 'life_culture_custom', - 'transport': 'life_culture_custom', - 'science': 'life_culture_custom', - 'travel': 'life_culture_custom', - 'business_finance': 'business_finance', - 'business and finance': 'business_finance', - 'education_profession': 'education_profession', - 'public education and courses': 'education_profession', - 'politics_policy_law': 'politics_policy_law', - 'politics': 'politics_policy_law', -} diff --git a/opencompass/configs/datasets/OpenHuEval/HuLifeQA/HuLifeQA_setting.py b/opencompass/configs/datasets/OpenHuEval/HuLifeQA/HuLifeQA_setting.py new file mode 100644 index 00000000..b41145c3 --- /dev/null +++ b/opencompass/configs/datasets/OpenHuEval/HuLifeQA/HuLifeQA_setting.py @@ -0,0 +1,24 @@ +TASK_GROUP_NEW = { + 'life_culture_custom': 'life_culture_custom', + 'childbearing and education': 'life_culture_custom', + 'culture and community': 'life_culture_custom', + 'culture and customs': 'life_culture_custom', + 'food and drink': 'life_culture_custom', + 'health': 'life_culture_custom', + 'holidays': 'life_culture_custom', + 'home': 'life_culture_custom', + 'person': 'life_culture_custom', + 'transport': 'life_culture_custom', + 'science': 'life_culture_custom', + 'travel': 'life_culture_custom', + 'business_finance': 'business_finance', + 'business and finance': 'business_finance', + 'education_profession': 'education_profession', + 'public education and courses': 'education_profession', + 'politics_policy_law': 'politics_policy_law', + 'politics': 'politics_policy_law', +} + +OpenHuEval_Path = '/mnt/hwfile/opendatalab/MinerU4S/yanghaote/XYZ/OpenHuEval' +DATA_VERSION = '250131' +DATA_PATH = f'{OpenHuEval_Path}/data/HuLifeQA/HuLifeQA_{DATA_VERSION}/HuLifeQA.jsonl' \ No newline at end of file diff --git a/opencompass/configs/datasets/OpenHuEval/HuMatchingFIB/HuMatchingFIB_setting.py b/opencompass/configs/datasets/OpenHuEval/HuMatchingFIB/HuMatchingFIB_setting.py index 789aee39..e6e03fd4 100644 --- a/opencompass/configs/datasets/OpenHuEval/HuMatchingFIB/HuMatchingFIB_setting.py +++ b/opencompass/configs/datasets/OpenHuEval/HuMatchingFIB/HuMatchingFIB_setting.py @@ -35,6 +35,6 @@ A kérdés és az opciók: """, } -OpenHuEval_Path = '/mnt/hwfile/opendatalab/weixingjian/OpenHuEval' +OpenHuEval_Path = '/mnt/hwfile/opendatalab/MinerU4S/yanghaote/XYZ/OpenHuEval' DATA_VERSION = '250205' DATA_PATH = f'{OpenHuEval_Path}/data/HuMatchingFIB/HuMatchingFIB_{DATA_VERSION}/HuMatchingFIB.jsonl' diff --git a/opencompass/configs/datasets/OpenHuEval/HuProverbRea/HuProverbRea_setting.py b/opencompass/configs/datasets/OpenHuEval/HuProverbRea/HuProverbRea_setting.py index 09d76d29..a52a6208 100644 --- a/opencompass/configs/datasets/OpenHuEval/HuProverbRea/HuProverbRea_setting.py +++ b/opencompass/configs/datasets/OpenHuEval/HuProverbRea/HuProverbRea_setting.py @@ -72,6 +72,6 @@ judge_prompt_template = { } -OpenHuEval_Path = '/mnt/hwfile/opendatalab/wj/proj/polyglot_24July/OpenHuEval' +OpenHuEval_Path = '/mnt/hwfile/opendatalab/MinerU4S/yanghaote/XYZ/OpenHuEval' DATA_VERSION = '250127' DATA_PATH = f'{OpenHuEval_Path}/data/HuProverbRea/HuProverbRea_{DATA_VERSION}/HuProverbRea.jsonl' diff --git a/opencompass/configs/datasets/OpenHuEval/HuSimpleQA/HuSimpleQA_setting.py b/opencompass/configs/datasets/OpenHuEval/HuSimpleQA/HuSimpleQA_setting.py index 81e118b2..8719f152 100644 --- a/opencompass/configs/datasets/OpenHuEval/HuSimpleQA/HuSimpleQA_setting.py +++ b/opencompass/configs/datasets/OpenHuEval/HuSimpleQA/HuSimpleQA_setting.py @@ -17,6 +17,6 @@ Kérjük, magyar nyelven adja meg a legjobb választ erre a kérdésre, és 0-t """ } -OpenHuEval_Path = '/mnt/hwfile/opendatalab/weixingjian/OpenHuEval' +OpenHuEval_Path = '/mnt/hwfile/opendatalab/MinerU4S/yanghaote/XYZ/OpenHuEval' DATA_VERSION = '250208' DATA_PATH = f'{OpenHuEval_Path}/data/HuSimpleQA/HuSimpleQA_{DATA_VERSION}/HuSimpleQA.jsonl' diff --git a/opencompass/configs/datasets/OpenHuEval/HuStandardFIB/HuStandardFIB_setting.py b/opencompass/configs/datasets/OpenHuEval/HuStandardFIB/HuStandardFIB_setting.py index 998201b8..e87b0bd0 100644 --- a/opencompass/configs/datasets/OpenHuEval/HuStandardFIB/HuStandardFIB_setting.py +++ b/opencompass/configs/datasets/OpenHuEval/HuStandardFIB/HuStandardFIB_setting.py @@ -22,6 +22,6 @@ The questions are: 'Initial version, using 1shot, incontext, #0# as place holder, output in JSON format', } -OpenHuEval_Path = '/mnt/hwfile/opendatalab/weixingjian/OpenHuEval' +OpenHuEval_Path = '/mnt/hwfile/opendatalab/MinerU4S/yanghaote/XYZ/OpenHuEval' DATA_VERSION = '250205' DATA_PATH = f'{OpenHuEval_Path}/data/HuStandardFIB/HuStandardFIB_{DATA_VERSION}/HuStandardFIB.jsonl'