diff --git a/.github/scripts/eval_regression_base_fullbench.py b/.github/scripts/eval_regression_base_fullbench.py
index 11c2f514..028a41b6 100644
--- a/.github/scripts/eval_regression_base_fullbench.py
+++ b/.github/scripts/eval_regression_base_fullbench.py
@@ -66,6 +66,8 @@ with read_base():
from opencompass.configs.summarizers.groups.mmlu_pro import \
mmlu_pro_summary_groups # noqa: F401, E501
+ from ...volc import infer as volc_infer # noqa: F401, E501
+
race_datasets = [race_datasets[1]] # Only take RACE-High
humaneval_v2_datasets[0]['abbr'] = 'openai_humaneval_v2'
bbh_datasets = [
@@ -99,61 +101,66 @@ GaokaoBench_datasets = [
]
datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
+summary_groups = sum(
+ [v for k, v in locals().items() if k.endswith('_summary_groups')], [])
+summary_groups.append(
+ {
+ 'name': 'Mathbench',
+ 'subsets': ['mathbench-a (average)', 'mathbench-t (average)'],
+ }, )
+
summarizer = dict(
dataset_abbrs=[
+ 'Language',
['race-high', 'accuracy'],
['ARC-c', 'accuracy'],
['BoolQ', 'accuracy'],
- ['mmlu_pro', 'naive_average'],
- ['GPQA_diamond', 'accuracy'],
- ['cmmlu', 'naive_average'],
- ['mmlu', 'naive_average'],
+ ['triviaqa_wiki_1shot', 'score'],
+ ['nq_open_1shot', 'score'],
+ '',
+ 'General Reasoning',
['drop', 'accuracy'],
['bbh', 'naive_average'],
+ ['GPQA_diamond', 'accuracy'],
+ ['hellaswag', 'accuracy'],
+ ['TheoremQA', 'score'],
+ ['winogrande', 'accuracy'],
+ '',
+ 'Math Calculation',
+ ['gsm8k', 'accuracy'],
+ ['GaokaoBench', 'weighted_average'],
+ 'GaokaoBench_2010-2022_Math_II_MCQs',
+ 'GaokaoBench_2010-2022_Math_II_Fill-in-the-Blank',
['math', 'accuracy'],
+ ['Mathbench', 'naive_average'],
+ '',
+ 'Knowledge',
+ ['wikibench-wiki-single_choice_cncircular', 'perf_4'],
+ ['cmmlu', 'naive_average'],
+ ['mmlu', 'naive_average'],
+ ['mmlu_pro', 'naive_average'],
+ '',
+ 'Code',
['openai_humaneval', 'humaneval_pass@1'],
['openai_humaneval_v2', 'humaneval_pass@1'],
['sanitized_mbpp', 'score'],
- ['wikibench-wiki-single_choice_cncircular', 'perf_4'],
- ['gsm8k', 'accuracy'],
- ['GaokaoBench', 'weighted_average'],
- ['triviaqa_wiki_1shot', 'score'],
- ['nq_open_1shot', 'score'],
- ['winogrande', 'accuracy'],
- ['hellaswag', 'accuracy'],
- ['TheoremQA', 'score'],
+ '',
['dingo_en_192', 'score'],
['dingo_zh_170', 'score'],
- '###### MathBench-A: Application Part ######',
- 'college',
- 'high',
- 'middle',
- 'primary',
- 'arithmetic',
- 'mathbench-a (average)',
- '###### MathBench-T: Theory Part ######',
- 'college_knowledge',
- 'high_knowledge',
- 'middle_knowledge',
- 'primary_knowledge',
- 'mathbench-t (average)',
- '###### Overall: Average between MathBench-A and MathBench-T ######',
- 'Overall',
- '',
- 'bbh-logical_deduction_seven_objects',
- 'bbh-multistep_arithmetic_two',
'',
'mmlu',
'mmlu-stem',
'mmlu-social-science',
'mmlu-humanities',
['mmlu-other', 'accuracy'],
+ '',
'cmmlu',
'cmmlu-stem',
'cmmlu-social-science',
'cmmlu-humanities',
'cmmlu-other',
['cmmlu-china-specific', 'accuracy'],
+ '',
'mmlu_pro',
'mmlu_pro_biology',
'mmlu_pro_business',
@@ -169,9 +176,24 @@ summarizer = dict(
'mmlu_pro_physics',
'mmlu_pro_psychology',
'mmlu_pro_other',
+ '',
+ 'bbh-logical_deduction_seven_objects',
+ 'bbh-multistep_arithmetic_two',
+ '###### MathBench-A: Application Part ######',
+ 'college',
+ 'high',
+ 'middle',
+ 'primary',
+ 'arithmetic',
+ 'mathbench-a (average)',
+ '###### MathBench-T: Theory Part ######',
+ 'college_knowledge',
+ 'high_knowledge',
+ 'middle_knowledge',
+ 'primary_knowledge',
+ 'mathbench-t (average)',
],
- summary_groups=sum(
- [v for k, v in locals().items() if k.endswith('_summary_groups')], []),
+ summary_groups=summary_groups,
)
models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
diff --git a/.github/scripts/eval_regression_base.py b/.github/scripts/eval_regression_base_models.py
similarity index 80%
rename from .github/scripts/eval_regression_base.py
rename to .github/scripts/eval_regression_base_models.py
index 330c97e5..08413707 100644
--- a/.github/scripts/eval_regression_base.py
+++ b/.github/scripts/eval_regression_base_models.py
@@ -13,12 +13,22 @@ with read_base():
# read hf models - chat models
from opencompass.configs.models.chatglm.hf_glm4_9b import \
models as hf_glm4_9b_model # noqa: F401, E501
+ from opencompass.configs.models.chatglm.lmdeploy_glm4_9b import \
+ models as lmdeploy_glm4_9b_model # noqa: F401, E501
+ from opencompass.configs.models.deepseek.hf_deepseek_7b_base import \
+ models as hf_deepseek_7b_base_model # noqa: F401, E501
+ from opencompass.configs.models.deepseek.hf_deepseek_67b_base import \
+ models as hf_deepseek_67b_base_model # noqa: F401, E501
from opencompass.configs.models.deepseek.hf_deepseek_moe_16b_base import \
models as hf_deepseek_moe_16b_base_model # noqa: F401, E501
from opencompass.configs.models.deepseek.hf_deepseek_v2_lite import \
models as hf_deepseek_v2_lite_model # noqa: F401, E501
from opencompass.configs.models.deepseek.lmdeploy_deepseek_7b_base import \
models as lmdeploy_deepseek_7b_base_model # noqa: F401, E501
+ from opencompass.configs.models.deepseek.lmdeploy_deepseek_67b_base import \
+ models as lmdeploy_deepseek_67b_base_model # noqa: F401, E501
+ from opencompass.configs.models.deepseek.lmdeploy_deepseek_v2 import \
+ lmdeploy_deepseek_v2_model # noqa: F401, E501
from opencompass.configs.models.deepseek.vllm_deepseek_moe_16b_base import \
models as vllm_deepseek_moe_16b_base_model # noqa: F401, E501
from opencompass.configs.models.gemma.hf_gemma2_2b import \
@@ -29,6 +39,8 @@ with read_base():
models as hf_gemma_2b_model # noqa: F401, E501
from opencompass.configs.models.gemma.hf_gemma_7b import \
models as hf_gemma_7b_model # noqa: F401, E501
+ from opencompass.configs.models.gemma.lmdeploy_gemma_9b import \
+ models as lmdeploy_gemma_9b_model # noqa: F401, E501
from opencompass.configs.models.gemma.vllm_gemma_2b import \
models as vllm_gemma_2b_model # noqa: F401, E501
from opencompass.configs.models.gemma.vllm_gemma_7b import \
@@ -59,10 +71,14 @@ with read_base():
models as hf_llama3_1_8b_model # noqa: F401, E501
from opencompass.configs.models.hf_llama.hf_llama3_8b import \
models as hf_llama3_8b_model # noqa: F401, E501
+ from opencompass.configs.models.hf_llama.hf_llama3_70b import \
+ models as hf_llama3_70b_model # noqa: F401, E501
from opencompass.configs.models.hf_llama.lmdeploy_llama3_1_8b import \
models as lmdeploy_llama3_1_8b_model # noqa: F401, E501
from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b import \
models as lmdeploy_llama3_8b_model # noqa: F401, E501
+ from opencompass.configs.models.hf_llama.lmdeploy_llama3_70b import \
+ models as lmdeploy_llama3_70b_model # noqa: F401, E501
from opencompass.configs.models.mistral.hf_mistral_7b_v0_2 import \
models as hf_mistral_7b_v0_2_model # noqa: F401, E501
from opencompass.configs.models.mistral.hf_mistral_7b_v0_3 import \
@@ -73,10 +89,16 @@ with read_base():
models as hf_qwen_2_5_7b_model # noqa: F401, E501
from opencompass.configs.models.qwen2_5.hf_qwen_2_5_14b import \
models as hf_qwen_2_5_14b_model # noqa: F401, E501
+ from opencompass.configs.models.qwen2_5.hf_qwen_2_5_32b import \
+ models as hf_qwen_2_5_32b_model # noqa: F401, E501
from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_1_5b import \
models as lmdeploy_qwen2_5_1_5b_model # noqa: F401, E501
from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b import \
models as lmdeploy_qwen2_5_7b_model # noqa: F401, E501
+ from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_32b import \
+ models as lmdeploy_qwen2_5_32b_model # noqa: F401, E501
+ from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_72b import \
+ models as lmdeploy_qwen2_5_72b_model # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen1_5_moe_a2_7b import \
models as hf_qwen1_5_moe_a2_7b_model # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen2_0_5b import \
@@ -95,6 +117,10 @@ with read_base():
models as hf_yi_1_5_6b_model # noqa: F401, E501
from opencompass.configs.models.yi.hf_yi_1_5_9b import \
models as hf_yi_1_5_9b_model # noqa: F401, E501
+ from opencompass.configs.models.yi.lmdeploy_yi_1_5_9b import \
+ models as lmdeploy_yi_1_5_9b_model # noqa: F401, E501
+
+ from ...volc import infer as volc_infer # noqa: F401, E501
race_datasets = [race_datasets[1]]
models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
diff --git a/.github/scripts/eval_regression_chat.py b/.github/scripts/eval_regression_chat_models.py
similarity index 76%
rename from .github/scripts/eval_regression_chat.py
rename to .github/scripts/eval_regression_chat_models.py
index 7762e4f7..eeade13f 100644
--- a/.github/scripts/eval_regression_chat.py
+++ b/.github/scripts/eval_regression_chat_models.py
@@ -7,8 +7,6 @@ with read_base():
from opencompass.configs.datasets.race.race_gen import \
race_datasets # noqa: F401, E501
# read hf models - chat models
- from opencompass.configs.models.baichuan.hf_baichuan2_7b_chat import \
- models as hf_baichuan2_7b_chat_model # noqa: F401, E501
from opencompass.configs.models.chatglm.hf_glm4_9b_chat import \
models as hf_glm4_9b_chat_model # noqa: F401, E501
from opencompass.configs.models.chatglm.lmdeploy_glm4_9b_chat import \
@@ -17,22 +15,30 @@ with read_base():
models as vllm_glm4_9b_chat_model # noqa: F401, E501
from opencompass.configs.models.deepseek.hf_deepseek_7b_chat import \
models as hf_deepseek_7b_chat_model # noqa: F401, E501
+ from opencompass.configs.models.deepseek.hf_deepseek_67b_chat import \
+ models as hf_deepseek_67b_chat_model # noqa: F401, E501
from opencompass.configs.models.deepseek.hf_deepseek_moe_16b_chat import \
models as hf_deepseek_moe_16b_chat_model # noqa: F401, E501
from opencompass.configs.models.deepseek.hf_deepseek_v2_lite_chat import \
models as hf_deepseek_v2_lite_chat_model # noqa: F401, E501
+ from opencompass.configs.models.deepseek.lmdeploy_deepseek_v2_5_1210 import \
+ models as lmdeploy_deepseek_v2_5_1210_model # noqa: F401, E501
from opencompass.configs.models.deepseek.vllm_deepseek_7b_chat import \
models as vllm_deepseek_7b_chat_model # noqa: F401, E501
from opencompass.configs.models.gemma.hf_gemma2_2b_it import \
models as hf_gemma2_2b_it_model # noqa: F401, E501
from opencompass.configs.models.gemma.hf_gemma2_9b_it import \
models as hf_gemma2_9b_it_model # noqa: F401, E501
+ from opencompass.configs.models.gemma.hf_gemma2_27b_it import \
+ models as hf_gemma2_27b_it_model # noqa: F401, E501
from opencompass.configs.models.gemma.hf_gemma_2b_it import \
models as hf_gemma_2b_it_model # noqa: F401, E501
from opencompass.configs.models.gemma.hf_gemma_7b_it import \
models as hf_gemma_7b_it_model # noqa: F401, E501
from opencompass.configs.models.gemma.lmdeploy_gemma_9b_it import \
models as lmdeploy_gemma_9b_it_model # noqa: F401, E501
+ from opencompass.configs.models.gemma.lmdeploy_gemma_27b_it import \
+ models as lmdeploy_gemma_27b_it_model # noqa: F401, E501
from opencompass.configs.models.gemma.vllm_gemma_7b_it import \
models as vllm_gemma_7b_it_model # noqa: F401, E501
from opencompass.configs.models.hf_internlm.hf_internlm2_5_7b_chat import \
@@ -65,6 +71,8 @@ with read_base():
models as lmdeploy_llama3_1_8b_instruct_model # noqa: F401, E501
from opencompass.configs.models.hf_llama.lmdeploy_llama3_2_3b_instruct import \
models as lmdeploy_llama3_2_3b_instruct_model # noqa: F401, E501
+ from opencompass.configs.models.hf_llama.lmdeploy_llama3_3_70b_instruct import \
+ models as lmdeploy_llama3_3_70b_instruct_model # noqa: F401, E501
from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import \
models as lmdeploy_llama3_8b_instruct_model # noqa: F401, E501
from opencompass.configs.models.mistral.hf_mistral_7b_instruct_v0_2 import \
@@ -75,6 +83,13 @@ with read_base():
models as hf_mistral_nemo_instruct_2407_model # noqa: F401, E501
from opencompass.configs.models.mistral.hf_mistral_small_instruct_2409 import \
models as hf_mistral_small_instruct_2409_model # noqa: F401, E501
+ from opencompass.configs.models.mistral.hf_mixtral_8x7b_instruct_v0_1 import \
+ models as hf_mixtral_8x7b_instruct_v0_1_model # noqa: F401, E501
+ from opencompass.configs.models.mistral.hf_mixtral_8x22b_instruct_v0_1 import \
+ models as hf_mixtral_8x22b_instruct_v0_1_model # noqa: F401, E501
+ from opencompass.configs.models.mistral.lmdeploy_mistral_large_instruct_2411 import \
+ models as \
+ lmdeploy_mistral_large_instruct_2411_model # noqa: F401, E501
from opencompass.configs.models.mistral.lmdeploy_mistral_nemo_instruct_2407 import \
models as lmdeploy_mistral_nemo_instruct_2407_model # noqa: F401, E501
from opencompass.configs.models.mistral.lmdeploy_mistral_small_instruct_2409 import \
@@ -84,22 +99,28 @@ with read_base():
models as vllm_mistral_7b_instruct_v0_1_model # noqa: F401, E501
from opencompass.configs.models.mistral.vllm_mistral_7b_instruct_v0_2 import \
models as vllm_mistral_7b_instruct_v0_2_model # noqa: F401, E501
+ from opencompass.configs.models.nvidia.lmdeploy_nemotron_70b_instruct_hf import \
+ models as lmdeploy_nemotron_70b_instruct_hf_model # noqa: F401, E501
from opencompass.configs.models.openbmb.hf_minicpm3_4b import \
models as hf_minicpm3_4b_model # noqa: F401, E501
- from opencompass.configs.models.openbmb.hf_minicpm_2b_dpo_fp32 import \
- models as hf_minicpm_2b_dpo_fp32_model # noqa: F401, E501
- from opencompass.configs.models.openbmb.hf_minicpm_2b_sft_bf16 import \
- models as hf_minicpm_2b_sft_bf16_model # noqa: F401, E501
- from opencompass.configs.models.openbmb.hf_minicpm_2b_sft_fp32 import \
- models as hf_minicpm_2b_sft_fp32_model # noqa: F401, E501
from opencompass.configs.models.phi.hf_phi_3_mini_4k_instruct import \
models as hf_phi_3_mini_4k_instruct_model # noqa: F401, E501
from opencompass.configs.models.phi.hf_phi_3_small_8k_instruct import \
models as hf_phi_3_mini_8k_instruct_model # noqa: F401, E501
+ from opencompass.configs.models.qwen2_5.hf_qwen2_5_0_5b_instruct import \
+ models as hf_qwen2_5_0_5b_instruct_model # noqa: F401, E501
+ from opencompass.configs.models.qwen2_5.hf_qwen2_5_3b_instruct import \
+ models as hf_qwen2_5_3b_instruct_model # noqa: F401, E501
from opencompass.configs.models.qwen2_5.hf_qwen2_5_14b_instruct import \
models as hf_qwen2_5_14b_instruct_model # noqa: F401, E501
+ from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_0_5b_instruct import \
+ models as lmdeploy_qwen2_5_0_5b_instruct_model # noqa: F401, E501
+ from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_3b_instruct import \
+ models as lmdeploy_qwen2_5_3b_instruct_model # noqa: F401, E501
from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_14b_instruct import \
models as lmdeploy_qwen2_5_14b_instruct_model # noqa: F401, E501
+ from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_72b_instruct import \
+ models as lmdeploy_qwen2_5_72b_instruct_model # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen1_5_0_5b_chat import \
models as hf_qwen1_5_0_5b_chat_model # noqa: F401, E501
from opencompass.configs.models.qwen.hf_qwen2_1_5b_instruct import \
@@ -116,6 +137,14 @@ with read_base():
models as hf_yi_1_5_6b_chat_model # noqa: F401, E501
from opencompass.configs.models.yi.hf_yi_1_5_9b_chat import \
models as hf_yi_1_5_9b_chat_model # noqa: F401, E501
+ from opencompass.configs.models.yi.lmdeploy_yi_1_5_6b_chat import \
+ models as lmdeploy_yi_1_5_6b_chat_model # noqa: F401, E501
+ from opencompass.configs.models.yi.lmdeploy_yi_1_5_9b_chat import \
+ models as lmdeploy_yi_1_5_9b_chat_model # noqa: F401, E501
+ from opencompass.configs.models.yi.lmdeploy_yi_1_5_34b_chat import \
+ models as lmdeploy_yi_1_5_34b_chat_model # noqa: F401, E501
+
+ from ...volc import infer as volc_infer # noqa: F401, E501
race_datasets = [race_datasets[1]]
datasets = sum([v for k, v in locals().items() if k.endswith('_datasets')], [])
diff --git a/.github/scripts/eval_regression_chat_objective_fullbench.py b/.github/scripts/eval_regression_chat_obj_fullbench.py
similarity index 90%
rename from .github/scripts/eval_regression_chat_objective_fullbench.py
rename to .github/scripts/eval_regression_chat_obj_fullbench.py
index 368fe040..98588dee 100644
--- a/.github/scripts/eval_regression_chat_objective_fullbench.py
+++ b/.github/scripts/eval_regression_chat_obj_fullbench.py
@@ -7,8 +7,14 @@ with read_base():
aime2024_datasets # noqa: F401, E501
from opencompass.configs.datasets.ARC_c.ARC_c_cot_gen_926652 import \
ARC_c_datasets # noqa: F401, E501
+ # remove because of oom
+ # from opencompass.configs.datasets.ARC_Prize_Public_Evaluation.arc_prize_public_evaluation_gen_872059 import arc_prize_public_evaluation_datasets # noqa: F401, E501
from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import \
bbh_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.bigcodebench.bigcodebench_hard_complete_gen_faf748 import \
+ bigcodebench_hard_complete_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.bigcodebench.bigcodebench_hard_instruct_gen_8815eb import \
+ bigcodebench_hard_instruct_datasets # noqa: F401, E501
from opencompass.configs.datasets.cmmlu.cmmlu_0shot_cot_gen_305931 import \
cmmlu_datasets # noqa: F401, E501
from opencompass.configs.datasets.cmo_fib.cmo_fib_gen_ace24b import \
@@ -26,15 +32,17 @@ with read_base():
gsm8k_datasets # noqa: F401, E501
from opencompass.configs.datasets.hellaswag.hellaswag_10shot_gen_e42710 import \
hellaswag_datasets # noqa: F401, E501
- from opencompass.configs.datasets.humaneval.humaneval_openai_sample_evals_gen_159614 import \
+ from opencompass.configs.datasets.humaneval.humaneval_openai_sample_evals_gen_dcae0e import \
humaneval_datasets # noqa: F401, E501
- from opencompass.configs.datasets.humanevalx.humanevalx_gen_620cfa import \
+ from opencompass.configs.datasets.humanevalx.humanevalx_gen_3d84a3 import \
humanevalx_datasets # noqa: F401, E501
- from opencompass.configs.datasets.IFEval.IFEval_gen_3321a3 import \
+ from opencompass.configs.datasets.IFEval.IFEval_gen_353ae7 import \
ifeval_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.korbench.korbench_single_0_shot_gen import \
+ korbench_0shot_single_datasets # noqa: F401, E501
from opencompass.configs.datasets.livecodebench.livecodebench_gen_b2b0fd import \
LCB_datasets # noqa: F401, E501
- from opencompass.configs.datasets.math.math_0shot_gen_393424 import \
+ from opencompass.configs.datasets.math.math_0shot_gen_11c4b5 import \
math_datasets # noqa: F401, E501
from opencompass.configs.datasets.MathBench.mathbench_2024_gen_50a320 import \
mathbench_datasets # noqa: F401, E501
@@ -71,6 +79,7 @@ with read_base():
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat import \
models as lmdeploy_internlm2_5_7b_chat_model # noqa: F401, E501
# Summary Groups
+ # Summary Groups
from opencompass.configs.summarizers.groups.bbh import \
bbh_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.cmmlu import \
@@ -81,6 +90,8 @@ with read_base():
GaokaoBench_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.humanevalx import \
humanevalx_summary_groups # noqa: F401, E501
+ from opencompass.configs.summarizers.groups.korbench import \
+ korbench_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.mathbench_v1_2024 import \
mathbench_2024_summary_groups # noqa: F401, E501
from opencompass.configs.summarizers.groups.mmlu import \
@@ -96,6 +107,8 @@ with read_base():
from opencompass.configs.summarizers.mmmlu_lite import \
mmmlu_summary_groups # noqa: F401, E501
+ from ...volc import infer as volc_infer # noqa: F401, E501
+
# For HumanEval-X Evaluation
# Apply the evaluator ip_address and port
race_datasets = [race_datasets[1]]
@@ -185,6 +198,8 @@ summarizer = dict(
['hellaswag', 'accuracy'],
['TheoremQA', 'score'],
['musr_average', 'naive_average'],
+ ['korbench_single', 'naive_average'],
+ ['ARC_Prize_Public_Evaluation', 'accuracy'],
'',
'Math Calculation',
['gsm8k', 'accuracy'],
@@ -208,6 +223,8 @@ summarizer = dict(
['lcb_code_generation', 'pass@1'],
['lcb_code_execution', 'pass@1'],
['lcb_test_output', 'pass@1'],
+ ['bigcodebench_hard_instruct', 'pass@1'],
+ ['bigcodebench_hard_complete', 'pass@1'],
'',
'Agent',
['teval', 'naive_average'],
diff --git a/.github/scripts/eval_regression_chat_sub_fullbench.py b/.github/scripts/eval_regression_chat_sub_fullbench.py
new file mode 100644
index 00000000..6ef87752
--- /dev/null
+++ b/.github/scripts/eval_regression_chat_sub_fullbench.py
@@ -0,0 +1,182 @@
+from copy import deepcopy
+
+from mmengine.config import read_base
+
+from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
+from opencompass.runners import LocalRunner
+from opencompass.summarizers import DefaultSubjectiveSummarizer
+from opencompass.tasks.subjective_eval import SubjectiveEvalTask
+
+with read_base():
+ # read hf models - chat models
+ # Dataset
+ from opencompass.configs.datasets.chinese_simpleqa.chinese_simpleqa_gen import \
+ csimpleqa_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.SimpleQA.simpleqa_gen_0283c3 import \
+ simpleqa_datasets # noqa: F401, E501; noqa: F401, E501
+ from opencompass.configs.datasets.subjective.alignbench.alignbench_v1_1_judgeby_critiquellm_new import \
+ alignbench_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.subjective.alpaca_eval.alpacav2_judgeby_gpt4_new import \
+ alpacav2_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.subjective.arena_hard.arena_hard_compare_new import \
+ arenahard_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.subjective.compassarena.compassarena_compare_new import \
+ compassarena_datasets # noqa: F401, E501
+ # from opencompass.configs.datasets.subjective.fofo.fofo_bilingual_judge_new import fofo_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.subjective.followbench.followbench_llmeval_new import \
+ followbench_llmeval_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.subjective.multiround.mtbench101_judge_new import \
+ mtbench101_datasets # noqa: F401, E501
+ from opencompass.configs.datasets.subjective.wildbench.wildbench_pair_judge_new import \
+ wildbench_datasets # noqa: F401, E501
+ from opencompass.configs.models.hf_internlm.hf_internlm2_5_7b_chat import \
+ models as hf_internlm2_5_7b_chat_model # noqa: F401, E501
+ from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat import \
+ models as lmdeploy_internlm2_5_7b_chat_model # noqa: F401, E501
+
+ from ...volc import infer as volc_infer # noqa: F401, E501
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')
+ and 'mtbench101' not in k and 'wildbench' not in k), [])
+datasets += mtbench101_datasets # noqa: F401, E501
+datasets += wildbench_datasets # noqa: F401, E501
+
+api_meta_template = dict(
+ round=[
+ dict(role='HUMAN', api_role='HUMAN'),
+ dict(role='BOT', api_role='BOT', generate=True),
+ ],
+ reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
+)
+
+models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
+for m in models:
+ m['abbr'] = m['abbr'] + '_fullbench'
+ if 'turbomind' in m['abbr'] or 'lmdeploy' in m['abbr']:
+ m['engine_config']['max_batch_size'] = 1
+ m['batch_size'] = 1
+
+models = sorted(models, key=lambda x: x['run_cfg']['num_gpus'])
+
+judge_models = deepcopy([models[1]])
+judge_models[0]['abbr'] = judge_models[0]['abbr'] + '-judge'
+
+eval = dict(
+ partitioner=dict(
+ type=SubjectiveNaivePartitioner,
+ models=models,
+ judge_models=judge_models,
+ ),
+ runner=dict(type=LocalRunner,
+ max_num_workers=16,
+ task=dict(type=SubjectiveEvalTask)),
+)
+
+summary_groups = []
+summary_groups.append({
+ 'name': 'compassarena_language',
+ 'subsets': [
+ ['compassarena_language', '内容总结'],
+ ],
+})
+summary_groups.append({
+ 'name': 'compassarena_knowledge',
+ 'subsets': [
+ ['compassarena_knowledge', '生活常识_ZH'],
+ ],
+})
+summary_groups.append({
+ 'name': 'compassarena_reason_v2',
+ 'subsets': [
+ ['compassarena_reason_v2', 'reasoning'],
+ ],
+})
+summary_groups.append({
+ 'name': 'compassarena_math_v2',
+ 'subsets': [
+ ['compassarena_math_v2', '高等数学_ZH'],
+ ],
+})
+summary_groups.append({
+ 'name': 'compassarena_creationv2_zh',
+ 'subsets': [
+ ['compassarena_creationv2_zh', '内容扩写_ZH'],
+ ],
+})
+summary_groups.append({
+ 'name':
+ 'CompassArena',
+ 'subsets': [
+ 'compassarena_language',
+ 'compassarena_knowledge',
+ 'compassarena_reason_v2',
+ 'compassarena_math_v2',
+ 'compassarena_creationv2_zh',
+ ],
+})
+summary_groups.append({
+ 'name':
+ 'FoFo',
+ 'subsets': [['fofo_test_prompts', 'overall'],
+ ['fofo_test_prompts_cn', 'overall']],
+})
+summary_groups.append({
+ 'name':
+ 'Followbench',
+ 'subsets': [
+ ['followbench_llmeval_en', 'HSR_AVG'],
+ ['followbench_llmeval_en', 'SSR_AVG'],
+ ],
+})
+
+# Summarizer
+summarizer = dict(
+ dataset_abbrs=[
+ ['alignment_bench_v1_1', '总分'],
+ ['alpaca_eval', 'total'],
+ ['arenahard', 'score'],
+ ['Followbench', 'naive_average'],
+ ['CompassArena', 'naive_average'],
+ ['FoFo', 'naive_average'],
+ ['mtbench101', 'avg'],
+ ['wildbench', 'average'],
+ ['simpleqa', 'accuracy_given_attempted'],
+ ['chinese_simpleqa', 'given_attempted_accuracy'],
+ '',
+ ['alignment_bench_v1_1', '专业能力'],
+ ['alignment_bench_v1_1', '数学计算'],
+ ['alignment_bench_v1_1', '基本任务'],
+ ['alignment_bench_v1_1', '逻辑推理'],
+ ['alignment_bench_v1_1', '中文理解'],
+ ['alignment_bench_v1_1', '文本写作'],
+ ['alignment_bench_v1_1', '角色扮演'],
+ ['alignment_bench_v1_1', '综合问答'],
+ ['alpaca_eval', 'helpful_base'],
+ ['alpaca_eval', 'koala'],
+ ['alpaca_eval', 'oasst'],
+ ['alpaca_eval', 'selfinstruct'],
+ ['alpaca_eval', 'vicuna'],
+ ['compassarena_language', 'naive_average'],
+ ['compassarena_knowledge', 'naive_average'],
+ ['compassarena_reason_v2', 'naive_average'],
+ ['compassarena_math_v2', 'naive_average'],
+ ['compassarena_creationv2_zh', 'naive_average'],
+ ['fofo_test_prompts', 'overall'],
+ ['fofo_test_prompts_cn', 'overall'],
+ ['followbench_llmeval_en', 'HSR_AVG'],
+ ['followbench_llmeval_en', 'SSR_AVG'],
+ ['followbench_llmeval_en', 'HSR_L1'],
+ ['followbench_llmeval_en', 'HSR_L2'],
+ ['followbench_llmeval_en', 'HSR_L3'],
+ ['followbench_llmeval_en', 'HSR_L4'],
+ ['followbench_llmeval_en', 'HSR_L5'],
+ ['followbench_llmeval_en', 'SSR_L1'],
+ ['followbench_llmeval_en', 'SSR_L2'],
+ ['followbench_llmeval_en', 'SSR_L3'],
+ ['followbench_llmeval_en', 'SSR_L4'],
+ ['followbench_llmeval_en', 'SSR_L5'],
+ ['simpleqa', 'f1'],
+ ],
+ type=DefaultSubjectiveSummarizer,
+ summary_groups=summary_groups,
+)
diff --git a/.github/scripts/eval_regression_chat_subjective_fullbench.py b/.github/scripts/eval_regression_chat_subjective_fullbench.py
deleted file mode 100644
index 8a6ef8fd..00000000
--- a/.github/scripts/eval_regression_chat_subjective_fullbench.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from copy import deepcopy
-
-from mmengine.config import read_base
-
-from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
-from opencompass.runners import LocalRunner
-from opencompass.summarizers import SubjectiveSummarizer
-from opencompass.tasks.subjective_eval import SubjectiveEvalTask
-
-with read_base():
- # read hf models - chat models
- # Dataset
- from opencompass.configs.datasets.subjective.alignbench.alignbench_v1_1_judgeby_critiquellm import \
- alignbench_datasets # noqa: F401, E501
- from opencompass.configs.datasets.subjective.alpaca_eval.alpacav2_judgeby_gpt4 import \
- alpacav2_datasets # noqa: F401, E501
- from opencompass.configs.datasets.subjective.arena_hard.arena_hard_compare import \
- arenahard_datasets # noqa: F401, E501
- from opencompass.configs.datasets.subjective.compassarena.compassarena_compare import \
- compassarena_datasets # noqa: F401, E501
- from opencompass.configs.datasets.subjective.fofo.fofo_bilingual_judge import \
- fofo_datasets # noqa: F401, E501
- from opencompass.configs.datasets.subjective.followbench.followbench_llmeval import \
- followbench_llmeval_datasets # noqa: F401, E501
- from opencompass.configs.datasets.subjective.multiround.mtbench101_judge import \
- mtbench101_datasets # noqa: F401, E501
- from opencompass.configs.datasets.subjective.wildbench.wildbench_pair_judge import \
- wildbench_datasets # noqa: F401, E501
- from opencompass.configs.models.hf_internlm.hf_internlm2_5_7b_chat import \
- models as hf_internlm2_5_7b_chat_model # noqa: F401, E501
- from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat import \
- models as lmdeploy_internlm2_5_7b_chat_model # noqa: F401, E501
-
-summarizer = dict(type=SubjectiveSummarizer, function='subjective')
-
-datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')
- and 'mtbench101' not in k and 'wildbench' not in k), [])
-datasets += mtbench101_datasets # noqa: F401, E501
-datasets += wildbench_datasets # noqa: F401, E501
-
-api_meta_template = dict(
- round=[
- dict(role='HUMAN', api_role='HUMAN'),
- dict(role='BOT', api_role='BOT', generate=True),
- ],
- reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
-)
-
-models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
-for m in models:
- m['abbr'] = m['abbr'] + '_fullbench'
- if 'turbomind' in m['abbr'] or 'lmdeploy' in m['abbr']:
- m['engine_config']['max_batch_size'] = 1
- m['batch_size'] = 1
-
-models = sorted(models, key=lambda x: x['run_cfg']['num_gpus'])
-
-judge_models = deepcopy([models[1]])
-judge_models[0]['abbr'] = judge_models[0]['abbr'] + '-judge'
-
-eval = dict(
- partitioner=dict(
- type=SubjectiveNaivePartitioner,
- models=models,
- judge_models=judge_models,
- ),
- runner=dict(type=LocalRunner,
- max_num_workers=16,
- task=dict(type=SubjectiveEvalTask)),
-)
diff --git a/.github/scripts/oc_score_assert.py b/.github/scripts/oc_score_assert.py
index 179dec27..6ad6e295 100644
--- a/.github/scripts/oc_score_assert.py
+++ b/.github/scripts/oc_score_assert.py
@@ -6,47 +6,19 @@ import yaml
output_path = 'regression_result_daily'
-chat_model_list = [
- 'baichuan2-7b-chat-hf', 'glm-4-9b-chat-hf', 'glm-4-9b-chat-turbomind',
- 'glm-4-9b-chat-vllm', 'deepseek-7b-chat-hf', 'deepseek-moe-16b-chat-hf',
- 'deepseek-7b-chat-vllm', 'gemma2-2b-it-hf', 'gemma2-9b-it-hf',
- 'gemma-2b-it-hf', 'gemma-7b-it-hf', 'gemma-2-9b-it-turbomind',
- 'gemma-7b-it-vllm', 'internlm2_5-7b-chat-hf',
- 'internlm2_5-7b-chat-turbomind', 'internlm2-chat-1.8b-turbomind',
- 'internlm2-chat-1.8b-sft-turbomind', 'internlm2-chat-7b-lmdeploy',
- 'internlm2-chat-7b-sft-turbomind', 'internlm2-chat-7b-vllm',
- 'llama-3_1-8b-instruct-hf', 'llama-3_2-3b-instruct-hf',
- 'llama-3-8b-instruct-hf', 'llama-3_1-8b-instruct-turbomind',
- 'llama-3_2-3b-instruct-turbomind', 'llama-3-8b-instruct-turbomind',
- 'mistral-7b-instruct-v0.2-hf', 'mistral-7b-instruct-v0.3-hf',
- 'mistral-nemo-instruct-2407-hf', 'mistral-nemo-instruct-2407-turbomind',
- 'mistral-7b-instruct-v0.1-vllm', 'mistral-7b-instruct-v0.2-vllm',
- 'MiniCPM3-4B-hf', 'minicpm-2b-dpo-fp32-hf', 'minicpm-2b-sft-bf16-hf',
- 'minicpm-2b-sft-fp32-hf', 'phi-3-mini-4k-instruct-hf',
- 'qwen1.5-0.5b-chat-hf', 'qwen2-1.5b-instruct-hf', 'qwen2-7b-instruct-hf',
- 'qwen2-1.5b-instruct-turbomind', 'qwen2-7b-instruct-turbomind',
- 'qwen1.5-0.5b-chat-vllm', 'yi-1.5-6b-chat-hf', 'yi-1.5-9b-chat-hf',
- 'deepseek-v2-lite-chat-hf', 'internlm2_5-20b-chat-hf',
- 'internlm2_5-20b-chat-turbomind', 'mistral-small-instruct-2409-hf',
- 'mistral-small-instruct-2409-turbomind', 'qwen2.5-14b-instruct-hf',
- 'qwen2.5-14b-instruct-turbomind'
-]
-base_model_list = [
- 'glm-4-9b-hf', 'deepseek-moe-16b-base-hf', 'deepseek-7b-base-turbomind',
- 'deepseek-moe-16b-base-vllm', 'gemma2-2b-hf', 'gemma2-9b-hf',
- 'gemma-2b-hf', 'gemma-7b-hf', 'gemma-2b-vllm', 'gemma-7b-vllm',
- 'internlm2_5-7b-hf', 'internlm2-7b-hf', 'internlm2-base-7b-hf',
- 'internlm2-1.8b-turbomind', 'internlm2_5-7b-turbomind',
- 'internlm2-7b-turbomind', 'internlm2-base-7b-turbomind', 'llama-2-7b-hf',
- 'llama-3_1-8b-hf', 'llama-3-8b-hf', 'llama-3.1-8b-turbomind',
- 'llama-3-8b-turbomind', 'mistral-7b-v0.2-hf', 'mistral-7b-v0.3-hf',
- 'mistral-7b-v0.2-vllm', 'qwen2.5-7b-hf', 'qwen2.5-1.5b-turbomind',
- 'qwen2.5-7b-turbomind', 'qwen1.5-moe-a2.7b-hf', 'qwen2-0.5b-hf',
- 'qwen2-1.5b-hf', 'qwen2-7b-hf', 'qwen2-1.5b-turbomind',
- 'qwen2-7b-turbomind', 'qwen1.5-0.5b-vllm', 'yi-1.5-6b-hf', 'yi-1.5-9b-hf',
- 'deepseek-v2-lite-hf', 'internlm2-20b-hf', 'internlm2-base-20b-hf',
- 'internlm2-20b-turbomind', 'qwen2.5-14b-hf'
-]
+
+def model_list(type):
+ config_path = '.github/scripts/oc_score_baseline_testrange.yaml'
+ with open(config_path) as f:
+ config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+ return config.get(type).keys()
+
+
+def dataset_list(model, type):
+ config_path = '.github/scripts/oc_score_baseline_fullbench.yaml'
+ with open(config_path) as f:
+ config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+ return config.get(model).get(type).keys()
@pytest.fixture()
@@ -88,35 +60,39 @@ def result_scores():
@pytest.mark.usefixtures('result_scores')
@pytest.mark.usefixtures('baseline_scores_testrange')
-@pytest.mark.chat
+@pytest.mark.chat_models
class TestChat:
"""Test cases for chat model."""
- @pytest.mark.parametrize('model, dataset',
- [(p1, p2) for p1 in chat_model_list
- for p2 in ['gsm8k', 'race-high']])
+ @pytest.mark.parametrize(
+ 'model, dataset', [(p1, p2) for p1 in model_list('chat')
+ for p2 in ['gsm8k_accuracy', 'race-high_accuracy']])
def test_model_dataset_score(self, baseline_scores_testrange,
result_scores, model, dataset):
- base_score = baseline_scores_testrange.get(model).get(dataset)
+ base_score = baseline_scores_testrange.get('chat').get(model).get(
+ dataset)
result_score = result_scores.get(model).get(dataset)
assert_score(model, result_score, base_score)
@pytest.mark.usefixtures('result_scores')
@pytest.mark.usefixtures('baseline_scores_testrange')
-@pytest.mark.base
+@pytest.mark.base_models
class TestBase:
"""Test cases for base model."""
- @pytest.mark.parametrize(
- 'model, dataset',
- [(p1, p2) for p1 in base_model_list
- for p2 in ['gsm8k', 'GPQA_diamond', 'race-high', 'winogrande']])
+ @pytest.mark.parametrize('model, dataset',
+ [(p1, p2) for p1 in model_list('base') for p2 in [
+ 'gsm8k_accuracy', 'GPQA_diamond_accuracy',
+ 'race-high_accuracy', 'winogrande_accuracy'
+ ]])
def test_model_dataset_score(self, baseline_scores_testrange,
result_scores, model, dataset):
- if model in ['gemma-2b-vllm', 'gemma-7b-vllm'] and dataset != 'gsm8k':
+ if model in ['gemma-2b-vllm', 'gemma-7b-vllm'
+ ] and dataset != 'gsm8k_accuracy':
return
- base_score = baseline_scores_testrange.get(model).get(dataset)
+ base_score = baseline_scores_testrange.get('base').get(model).get(
+ dataset)
result_score = result_scores.get(model).get(dataset)
assert_score(model, result_score, base_score)
@@ -130,21 +106,11 @@ class TestChatObjFullbench:
@pytest.mark.parametrize('model, dataset', [(p1, p2) for p1 in [
'internlm2_5-7b-chat-hf_fullbench',
'internlm2_5-7b-chat-turbomind_fullbench'
- ] for p2 in [
- 'race-high', 'ARC-c', 'BoolQ', 'triviaqa_wiki_1shot', 'nq_open_1shot',
- 'IFEval', 'drop', 'GPQA_diamond', 'hellaswag', 'TheoremQA',
- 'musr_average', 'gsm8k', 'math', 'cmo_fib', 'aime2024',
- 'wikibench-wiki-single_choice_cncircular', 'sanitized_mbpp', 'ds1000',
- 'lcb_code_generation', 'lcb_code_execution', 'lcb_test_output',
- 'bbh-logical_deduction_seven_objects', 'bbh-multistep_arithmetic_two',
- 'mmlu-other', 'cmmlu-china-specific', 'mmlu_pro_math', 'ds1000_Pandas',
- 'ds1000_Numpy', 'ds1000_Tensorflow', 'ds1000_Scipy', 'ds1000_Sklearn',
- 'ds1000_Pytorch', 'ds1000_Matplotlib', 'openai_mmmlu_lite_AR-XY',
- 'college', 'college_knowledge'
- ]])
+ ] for p2 in dataset_list('internlm2_5-7b-chat-hf_fullbench', 'objective')])
def test_model_dataset_score(self, baseline_scores_fullbench,
result_scores, model, dataset):
- base_score = baseline_scores_fullbench.get(model).get(dataset)
+ base_score = baseline_scores_fullbench.get(model).get('objective').get(
+ dataset)
result_score = result_scores.get(model).get(dataset)
assert_score(model, result_score, base_score)
@@ -158,22 +124,12 @@ class TestChatSubFullbench:
@pytest.mark.parametrize('model, dataset', [(p1, p2) for p1 in [
'internlm2_5-7b-chat-hf_fullbench',
'internlm2_5-7b-chat-turbomind_fullbench'
- ] for p2 in [
- 'Alignbench总分', 'Alignbench专业能力', 'AlpacaEvaltotal',
- 'AlpacaEvalhelpful_base', 'CompassArenacompassarena_language',
- 'CompassArenacompassarena_knowledge',
- 'CompassArenacompassarena_reason_v2',
- 'CompassArenacompassarena_math_v2',
- 'CompassArenacompassarena_creationv2_zh', 'Fofofofo_test_prompts',
- 'followbenchHSR_AVG', 'followbenchSSR_AVG', 'followbenchHSR_L1',
- 'followbenchHSR_L2', 'followbenchHSR_L3', 'followbenchHSR_L4',
- 'followbenchHSR_L5', 'followbenchSSR_L1', 'followbenchSSR_L2',
- 'followbenchSSR_L3', 'followbenchSSR_L4', 'followbenchSSR_L5',
- 'MTBench101average', 'Wildbenchscore'
- ]])
+ ] for p2 in dataset_list('internlm2_5-7b-chat-hf_fullbench', 'subjective')]
+ )
def test_model_dataset_score(self, baseline_scores_fullbench,
result_scores, model, dataset):
- base_score = baseline_scores_fullbench.get(model).get(dataset)
+ base_score = baseline_scores_fullbench.get(model).get(
+ 'subjective').get(dataset)
result_score = result_scores.get(model).get(dataset)
assert_score(model, result_score, base_score)
@@ -184,20 +140,15 @@ class TestChatSubFullbench:
class TestBaseFullbench:
"""Test cases for chat model."""
- @pytest.mark.parametrize('model, dataset', [(p1, p2) for p1 in [
- 'internlm2_5-7b-hf_fullbench', 'internlm2_5-7b-turbomind_fullbench'
- ] for p2 in [
- 'race-high', 'ARC-c', 'BoolQ', 'drop', 'GPQA_diamond', 'math',
- 'wikibench-wiki-single_choice_cncircular', 'sanitized_mbpp', 'gsm8k',
- 'triviaqa_wiki_1shot', 'nq_open_1shot', 'winogrande', 'hellaswag',
- 'TheoremQA', 'dingo_en_192', 'dingo_zh_170', 'college',
- 'college_knowledge', 'bbh-logical_deduction_seven_objects',
- 'bbh-multistep_arithmetic_two', 'mmlu-other', 'cmmlu-china-specific',
- 'mmlu_pro_math'
- ]])
+ @pytest.mark.parametrize(
+ 'model, dataset',
+ [(p1, p2) for p1 in
+ ['internlm2_5-7b-hf_fullbench', 'internlm2_5-7b-turbomind_fullbench']
+ for p2 in dataset_list('internlm2_5-7b-hf_fullbench', 'objective')])
def test_model_dataset_score(self, baseline_scores_fullbench,
result_scores, model, dataset):
- base_score = baseline_scores_fullbench.get(model).get(dataset)
+ base_score = baseline_scores_fullbench.get(model).get('objective').get(
+ dataset)
result_score = result_scores.get(model).get(dataset)
assert_score(model, result_score, base_score)
@@ -209,40 +160,109 @@ class TestApibench:
"""Test cases for chat model."""
@pytest.mark.parametrize('model, dataset',
- [('lmdeploy-api-test', 'race-middle'),
- ('lmdeploy-api-test', 'race-high'),
- ('lmdeploy-api-test', 'gsm8k')])
+ [('lmdeploy-api-test', 'race-middle_accuracy'),
+ ('lmdeploy-api-test', 'race-high_accuracy'),
+ ('lmdeploy-api-test', 'gsm8k_accuracy')])
def test_api(self, baseline_scores, result_scores, model, dataset):
base_score = baseline_scores.get(model).get(dataset)
result_score = result_scores.get(model).get(dataset)
assert_score(model + '_batch', result_score, base_score)
+@pytest.mark.usefixtures('result_scores')
+@pytest.mark.usefixtures('baseline_scores_fullbench')
+@pytest.mark.volc_fullbench
+class TestVolcFullbench:
+ """Test cases for chat model."""
+
+ @pytest.mark.parametrize(
+ 'model, dataset',
+ [(p1, p2) for p1 in ['internlm2_5-7b-chat-turbomind']
+ for p2 in dataset_list('internlm2_5-7b-chat-turbomind', 'objective')])
+ @pytest.mark.chat_objective
+ def test_chat_objective(self, baseline_scores_fullbench, result_scores,
+ model, dataset):
+ base_score = baseline_scores_fullbench.get(model).get('objective').get(
+ dataset)
+ result_score = result_scores.get(model).get(dataset)
+ assert_score(model + '_batch', result_score, base_score)
+
+ @pytest.mark.parametrize('model, dataset', [
+ (p1, p2) for p1 in ['internlm2_5-7b-chat-turbomind']
+ for p2 in dataset_list('internlm2_5-7b-chat-turbomind', 'subjective')
+ ])
+ @pytest.mark.chat_subjective
+ def test_chat_subjective(self, baseline_scores_fullbench, result_scores,
+ model, dataset):
+ base_score = baseline_scores_fullbench.get(model).get(
+ 'subjective').get(dataset)
+ result_score = result_scores.get(model).get(dataset)
+ assert_score(model + '_batch', result_score, base_score)
+
+ @pytest.mark.parametrize(
+ 'model, dataset',
+ [(p1, p2) for p1 in ['internlm2_5-7b-turbomind']
+ for p2 in dataset_list('internlm2_5-7b-turbomind', 'objective')])
+ @pytest.mark.base_objective
+ def test_base_objective(self, baseline_scores_fullbench, result_scores,
+ model, dataset):
+ base_score = baseline_scores_fullbench.get(model).get('objective').get(
+ dataset)
+ result_score = result_scores.get(model).get(dataset)
+ assert_score(model + '_batch', result_score, base_score)
+
+ @pytest.mark.parametrize(
+ 'model, dataset',
+ [(p1, p2) for p1 in ['internlm2_5-7b-turbomind']
+ for p2 in dataset_list('internlm2_5-7b-turbomind', 'long_context')])
+ @pytest.mark.base_long_context
+ def test_base_long_context(self, baseline_scores_fullbench, result_scores,
+ model, dataset):
+ base_score = baseline_scores_fullbench.get(model).get(
+ 'long_context').get(dataset)
+ result_score = result_scores.get(model).get(dataset)
+ assert_score(model + '_batch', result_score, base_score)
+
+ @pytest.mark.parametrize(
+ 'model, dataset',
+ [(p1, p2)
+ for p1 in ['internlm2_5-7b-chat-1m-turbomind'] for p2 in dataset_list(
+ 'internlm2_5-7b-chat-1m-turbomind', 'long_context')])
+ @pytest.mark.chat_long_context
+ def test_chat_long_context(self, baseline_scores_fullbench, result_scores,
+ model, dataset):
+ base_score = baseline_scores_fullbench.get(model).get(
+ 'long_context').get(dataset)
+ result_score = result_scores.get(model).get(dataset)
+ assert_score(model + '_batch', result_score, base_score)
+
+
@pytest.mark.usefixtures('result_scores')
@pytest.mark.usefixtures('baseline_scores')
class TestCmdCase:
@pytest.mark.case1
@pytest.mark.parametrize('model, dataset',
- [('internlm2_5-7b-hf', 'race-middle'),
- ('internlm2_5-7b-hf', 'race-high'),
- ('internlm2_5-7b-hf', 'demo_gsm8k'),
- ('internlm2-1.8b-hf', 'race-middle'),
- ('internlm2-1.8b-hf', 'race-high'),
- ('internlm2-1.8b-hf', 'demo_gsm8k')])
+ [('internlm2_5-7b-hf', 'race-middle_accuracy'),
+ ('internlm2_5-7b-hf', 'race-high_accuracy'),
+ ('internlm2_5-7b-hf', 'demo_gsm8k_accuracy'),
+ ('internlm2-1.8b-hf', 'race-middle_accuracy'),
+ ('internlm2-1.8b-hf', 'race-high_accuracy'),
+ ('internlm2-1.8b-hf', 'demo_gsm8k_accuracy')])
def test_cmd_case1(self, baseline_scores, result_scores, model, dataset):
base_score = baseline_scores.get(model).get(dataset)
result_score = result_scores.get(model).get(dataset)
assert_score(model, result_score, base_score)
@pytest.mark.case2
- @pytest.mark.parametrize('model, dataset',
- [('internlm2_5-7b-chat-lmdeploy', 'race-middle'),
- ('internlm2_5-7b-chat-lmdeploy', 'race-high'),
- ('internlm2_5-7b-chat-lmdeploy', 'demo_gsm8k'),
- ('internlm2-chat-1.8b-lmdeploy', 'race-middle'),
- ('internlm2-chat-1.8b-lmdeploy', 'race-high'),
- ('internlm2-chat-1.8b-lmdeploy', 'demo_gsm8k')])
+ @pytest.mark.parametrize(
+ 'model, dataset',
+ [('internlm2_5-7b-chat-lmdeploy', 'race-middle_accuracy'),
+ ('internlm2_5-7b-chat-lmdeploy', 'race-high_accuracy'),
+ ('internlm2_5-7b-chat-lmdeploy', 'demo_gsm8k_accuracy'),
+ ('internlm2-chat-1.8b-lmdeploy', 'race-middle_accuracy'),
+ ('internlm2-chat-1.8b-lmdeploy', 'race-high_accuracy'),
+ ('internlm2-chat-1.8b-lmdeploy', 'demo_gsm8k_accuracy')])
def test_cmd_case2(self, baseline_scores, result_scores, model, dataset):
base_score = baseline_scores.get(model).get(dataset)
result_score = result_scores.get(model).get(dataset)
@@ -250,19 +270,19 @@ class TestCmdCase:
@pytest.mark.case3
@pytest.mark.parametrize('model, dataset',
- [('internlm2_5-7b_hf', 'race-middle'),
- ('internlm2_5-7b_hf', 'race-high'),
- ('internlm2_5-7b_hf', 'demo_gsm8k')])
+ [('internlm2_5-7b_hf', 'race-middle_accuracy'),
+ ('internlm2_5-7b_hf', 'race-high_accuracy'),
+ ('internlm2_5-7b_hf', 'demo_gsm8k_accuracy')])
def test_cmd_case3(self, baseline_scores, result_scores, model, dataset):
base_score = baseline_scores.get(model).get(dataset)
result_score = result_scores.get(model).get(dataset)
assert_score(model, result_score, base_score)
@pytest.mark.case4
- @pytest.mark.parametrize('model, dataset',
- [('internlm2_5-7b-chat_hf', 'race-middle'),
- ('internlm2_5-7b-chat_hf', 'race-high'),
- ('internlm2_5-7b-chat_hf', 'demo_gsm8k')])
+ @pytest.mark.parametrize(
+ 'model, dataset', [('internlm2_5-7b-chat_hf', 'race-middle_accuracy'),
+ ('internlm2_5-7b-chat_hf', 'race-high_accuracy'),
+ ('internlm2_5-7b-chat_hf', 'demo_gsm8k_accuracy')])
def test_cmd_case4(self, baseline_scores, result_scores, model, dataset):
base_score = baseline_scores.get(model).get(dataset)
result_score = result_scores.get(model).get(dataset)
@@ -310,8 +330,7 @@ def find_csv_files(directory):
csv_files = []
for root, dirs, files in os.walk(directory):
for file in files:
- if file.endswith('.csv') and (file.startswith('summary') or
- file.startswith('Subjective_all')):
+ if file.endswith('.csv') and file.startswith('summary'):
csv_files.append(os.path.join(root, file))
csv_files_with_time = {f: os.path.getctime(f) for f in csv_files}
@@ -324,24 +343,15 @@ def read_csv_file(file_path):
with open(file_path, 'r') as csvfile:
reader = csv.DictReader(csvfile)
filtered_data = []
- if 'Subjective_all' not in file_path:
- for row in reader:
- if row['metric'] is not None and 'bpb' not in row['metric']:
- filtered_row = {
- k: v
- for k, v in row.items()
- if k not in ['version', 'metric', 'mode']
- }
- filtered_data.append(filtered_row)
- else:
- for row in reader:
- if row['Detailed Scores'] is not None:
- filtered_row = row
- filtered_row['dataset'] = filtered_row[
- 'Dataset'] + filtered_row['Detailed Scores']
- del filtered_row['Dataset']
- del filtered_row['Detailed Scores']
- filtered_data.append(filtered_row)
+ for row in reader:
+ if row['metric'] is not None and 'bpb' not in row[
+ 'metric'] and '_' != row['metric']:
+ filtered_row = row
+ filtered_row['dataset'] = row['dataset'] + '_' + row['metric']
+ del filtered_row['version']
+ del filtered_row['metric']
+ del filtered_row['mode']
+ filtered_data.append(filtered_row)
result = {}
for data in filtered_data:
diff --git a/.github/scripts/oc_score_baseline.yaml b/.github/scripts/oc_score_baseline.yaml
index 40cb1087..a8e40891 100644
--- a/.github/scripts/oc_score_baseline.yaml
+++ b/.github/scripts/oc_score_baseline.yaml
@@ -1,34 +1,34 @@
internlm2_5-7b-hf:
- demo_gsm8k: 42.19
- race-middle: 91.78
- race-high: 90.02
+ demo_gsm8k_accuracy: 42.19
+ race-middle_accuracy: 91.78
+ race-high_accuracy: 90.02
internlm2_5-7b_hf:
- demo_gsm8k: 42.19
- race-middle: 91.78
- race-high: 90.02
+ demo_gsm8k_accuracy: 42.19
+ race-middle_accuracy: 91.78
+ race-high_accuracy: 90.02
internlm2-1.8b-hf:
- demo_gsm8k: 15.62
- race-middle: 71.66
- race-high: 66.38
+ demo_gsm8k_accuracy: 15.62
+ race-middle_accuracy: 71.66
+ race-high_accuracy: 66.38
internlm2_5-7b-chat-lmdeploy:
- demo_gsm8k: 84.38
- race-middle: 92.76
- race-high: 90.54
+ demo_gsm8k_accuracy: 89.06
+ race-middle_accuracy: 92.76
+ race-high_accuracy: 90.54
internlm2-chat-1.8b-lmdeploy:
- demo_gsm8k: 31
- race-middle: 81.34
- race-high: 73.96
+ demo_gsm8k_accuracy: 32
+ race-middle_accuracy: 81.34
+ race-high_accuracy: 73.96
internlm2_5-7b-chat_hf:
- demo_gsm8k: 87.50
- race-middle: 92.76
- race-high: 90.48
+ demo_gsm8k_accuracy: 87.50
+ race-middle_accuracy: 92.76
+ race-high_accuracy: 90.48
lmdeploy-api-test:
- gsm8k: 83.78
- race-middle: 92.41
- race-high: 90.37
+ gsm8k_accuracy: 83.78
+ race-middle_accuracy: 92.41
+ race-high_accuracy: 90.37
diff --git a/.github/scripts/oc_score_baseline_fullbench.yaml b/.github/scripts/oc_score_baseline_fullbench.yaml
index 49393e05..568ed5fd 100644
--- a/.github/scripts/oc_score_baseline_fullbench.yaml
+++ b/.github/scripts/oc_score_baseline_fullbench.yaml
@@ -1,173 +1,456 @@
internlm2_5-7b-chat-hf_fullbench:
- race-high: 93.75
- ARC-c: 93.75
- BoolQ: 81.25
- triviaqa_wiki_1shot: 50
- nq_open_1shot: 25
- IFEval: 50
- drop: 81.25
- GPQA_diamond: 25
- hellaswag: 87.5
- TheoremQA: 18.75
- musr_average: 39.58
- gsm8k: 56.25
- math: 75
- cmo_fib: 6.25
- aime2024: 6.25
- wikibench-wiki-single_choice_cncircular: 50
- sanitized_mbpp: 68.75
- ds1000: 16.96
- lcb_code_generation: 12.5
- lcb_code_execution: 43.75
- lcb_test_output: 18.75
- bbh-logical_deduction_seven_objects: 50
- bbh-multistep_arithmetic_two: 68.75
- mmlu-other: 72.6
- cmmlu-china-specific: 76.25
- mmlu_pro_math: 25
- ds1000_Pandas: 12.5
- ds1000_Numpy: 0
- ds1000_Tensorflow: 12.5
- ds1000_Scipy: 18.75
- ds1000_Sklearn: 18.75
- ds1000_Pytorch: 12.5
- ds1000_Matplotlib: 43.75
- openai_mmmlu_lite_AR-XY: 37.5
- college: 12.5
- college_knowledge: 87.5
- Alignbench总分: 0.65
- Alignbench专业能力: 7.83
- AlpacaEvaltotal: 0
- AlpacaEvalhelpful_base: 0
- CompassArenacompassarena_language: 60
- CompassArenacompassarena_knowledge: 56
- CompassArenacompassarena_reason_v2: 50
- CompassArenacompassarena_math_v2: 53.5
- CompassArenacompassarena_creationv2_zh: 48.75
- Fofofofo_test_prompts: 1
- followbenchHSR_AVG: 1
- followbenchSSR_AVG: 1
- followbenchHSR_L1: 1
- followbenchHSR_L2: 1
- followbenchHSR_L3: 1
- followbenchHSR_L4: 1
- followbenchHSR_L5: 1
- followbenchSSR_L1: 1
- followbenchSSR_L2: 1
- followbenchSSR_L3: 1
- followbenchSSR_L4: 1
- followbenchSSR_L5: 1
- MTBench101average: 8.1
- Wildbenchscore: -3.3333333333333335
+ objective:
+ race-high_accuracy: 93.75
+ ARC-c_accuracy: 93.75
+ BoolQ_accuracy: 81.25
+ triviaqa_wiki_1shot_score: 50
+ nq_open_1shot_score: 25
+ IFEval_Prompt-level-strict-accuracy: 50
+ drop_accuracy: 81.25
+ GPQA_diamond_accuracy: 25
+ hellaswag_accuracy: 87.5
+ TheoremQA_score: 18.75
+ musr_average_naive_average: 39.58
+ korbench_single_naive_average: 40
+ gsm8k_accuracy: 62.50
+ math_accuracy: 75
+ cmo_fib_accuracy: 6.25
+ aime2024_accuracy: 6.25
+ wikibench-wiki-single_choice_cncircular_perf_4: 50
+ sanitized_mbpp_score: 68.75
+ ds1000_naive_average: 16.96
+ lcb_code_generation_pass@1: 12.5
+ lcb_code_execution_pass@1: 43.75
+ lcb_test_output_pass@1: 18.75
+ bbh-logical_deduction_seven_objects_score: 50
+ bbh-multistep_arithmetic_two_score: 68.75
+ mmlu-other_naive_average: 72.6
+ cmmlu-china-specific_naive_average: 76.25
+ mmlu_pro_math_accuracy: 25
+ ds1000_Pandas_accuracy: 12.5
+ ds1000_Numpy_accuracy: 0
+ ds1000_Tensorflow_accuracy: 12.5
+ ds1000_Scipy_accuracy: 18.75
+ ds1000_Sklearn_accuracy: 18.75
+ ds1000_Pytorch_accuracy: 12.5
+ ds1000_Matplotlib_accuracy: 43.75
+ openai_mmmlu_lite_AR-XY_accuracy: 37.5
+ college_naive_average: 12.5
+ college_knowledge_naive_average: 87.5
+ subjective:
+ alignment_bench_v1_1_总分: 0.66
+ alpaca_eval_total: 20
+ arenahard_score: 50
+ Followbench_naive_average: 1
+ CompassArena_naive_average: 44.00
+ mtbench101_avg: 7.8
+ wildbench_average: -12.78
+ simpleqa_accuracy_given_attempted: 0
+ chinese_simpleqa_given_attempted_accuracy: 1
+ alignment_bench_v1_1_专业能力: 7.90
+ alignment_bench_v1_1_数学计算: 0
+ alignment_bench_v1_1_基本任务: 0
+ alignment_bench_v1_1_逻辑推理: 0
+ alignment_bench_v1_1_中文理解: 0
+ alignment_bench_v1_1_文本写作: 0
+ alignment_bench_v1_1_角色扮演: 0
+ alignment_bench_v1_1_综合问答: 0
+ alpaca_eval_helpful_base: 20
+ compassarena_language_naive_average: 35
+ compassarena_knowledge_naive_average: 55
+ compassarena_reason_v2_naive_average: 45.00
+ compassarena_math_v2_naive_average: 55
+ compassarena_creationv2_zh_naive_average: 30
+ followbench_llmeval_en_HSR_AVG: 1
+ followbench_llmeval_en_SSR_AVG: 1
+ followbench_llmeval_en_HSR_L1: 1
+ followbench_llmeval_en_HSR_L2: 1
+ followbench_llmeval_en_HSR_L3: 1
+ followbench_llmeval_en_HSR_L4: 1
+ followbench_llmeval_en_HSR_L5: 1
+ followbench_llmeval_en_SSR_L1: 1
+ followbench_llmeval_en_SSR_L2: 1
+ followbench_llmeval_en_SSR_L3: 1
+ followbench_llmeval_en_SSR_L4: 1
+ followbench_llmeval_en_SSR_L5: 1
+ simpleqa_f1: 0
internlm2_5-7b-chat-turbomind_fullbench:
- race-high: 93.75
- ARC-c: 87.5
- BoolQ: 68.75
- triviaqa_wiki_1shot: 50
- nq_open_1shot: 25
- IFEval: 50
- drop: 75
- hellaswag: 81.25
- TheoremQA: 6.25
- musr_average: 37.5
- gsm8k: 68.75
- math: 75
- GPQA_diamond: 25
- cmo_fib: 6.25
- aime2024: 6.25
- wikibench-wiki-single_choice_cncircular: 25
- sanitized_mbpp: 68.75
- ds1000: 13.39
- lcb_code_generation: 12.5
- lcb_code_execution: 43.75
- lcb_test_output: 12.5
- bbh-logical_deduction_seven_objects: 56.25
- bbh-multistep_arithmetic_two: 68.75
- mmlu-other: 74.04
- cmmlu-china-specific: 76.25
- mmlu_pro_math: 25
- ds1000_Pandas: 0
- ds1000_Numpy: 0
- ds1000_Tensorflow: 12.5
- ds1000_Scipy: 18.75
- ds1000_Sklearn: 18.75
- ds1000_Pytorch: 6.25
- ds1000_Matplotlib: 37.5
- openai_mmmlu_lite_AR-XY: 37.5
- college: 0
- college_knowledge: 87.5
- Alignbench总分: 0.64
- Alignbench专业能力: 7.6
- AlpacaEvaltotal: 10
- AlpacaEvalhelpful_base: 10
- CompassArenacompassarena_language: 59
- CompassArenacompassarena_knowledge: 57
- CompassArenacompassarena_reason_v2: 49.5
- CompassArenacompassarena_math_v2: 51
- CompassArenacompassarena_creationv2_zh: 43.75
- Fofofofo_test_prompts: 1
- followbenchHSR_AVG: 1
- followbenchSSR_AVG: 1
- followbenchHSR_L1: 1
- followbenchHSR_L2: 1
- followbenchHSR_L3: 1
- followbenchHSR_L4: 1
- followbenchHSR_L5: 1
- followbenchSSR_L1: 1
- followbenchSSR_L2: 1
- followbenchSSR_L3: 1
- followbenchSSR_L4: 1
- followbenchSSR_L5: 1
- MTBench101average: 8.1
- Wildbenchscore: -8.333333333333334
+ objective:
+ race-high_accuracy: 93.75
+ ARC-c_accuracy: 93.75
+ BoolQ_accuracy: 68.75
+ triviaqa_wiki_1shot_score: 50
+ nq_open_1shot_score: 25
+ IFEval_Prompt-level-strict-accuracy: 56.25
+ drop_accuracy: 81.25
+ GPQA_diamond_accuracy: 31.25
+ hellaswag_accuracy: 81.25
+ TheoremQA_score: 6.25
+ musr_average_naive_average: 39.58
+ korbench_single_naive_average: 37.50
+ gsm8k_accuracy: 68.75
+ math_accuracy: 68.75
+ cmo_fib_accuracy: 6.25
+ aime2024_accuracy: 6.25
+ wikibench-wiki-single_choice_cncircular_perf_4: 50.00
+ sanitized_mbpp_score: 68.75
+ ds1000_naive_average: 16.96
+ lcb_code_generation_pass@1: 12.5
+ lcb_code_execution_pass@1: 43.75
+ lcb_test_output_pass@1: 25.00
+ bbh-logical_deduction_seven_objects_score: 50.00
+ bbh-multistep_arithmetic_two_score: 68.75
+ mmlu-other_naive_average: 69.71
+ cmmlu-china-specific_naive_average: 75.83
+ mmlu_pro_math_accuracy: 31.25
+ ds1000_Pandas_accuracy: 0
+ ds1000_Numpy_accuracy: 0
+ ds1000_Tensorflow_accuracy: 12.5
+ ds1000_Scipy_accuracy: 18.75
+ ds1000_Sklearn_accuracy: 18.75
+ ds1000_Pytorch_accuracy: 18.75
+ ds1000_Matplotlib_accuracy: 50.00
+ openai_mmmlu_lite_AR-XY_accuracy: 37.5
+ college_naive_average: 12.50
+ college_knowledge_naive_average: 87.5
+ subjective:
+ alignment_bench_v1_1_总分: 0.70
+ alpaca_eval_total: 0
+ arenahard_score: 50
+ Followbench_naive_average: 1
+ CompassArena_naive_average: 38
+ mtbench101_avg: 7.80
+ wildbench_average: -4.86
+ simpleqa_accuracy_given_attempted: 0
+ chinese_simpleqa_given_attempted_accuracy: 1
+ alignment_bench_v1_1_专业能力: 8.4
+ alignment_bench_v1_1_数学计算: 0
+ alignment_bench_v1_1_基本任务: 0
+ alignment_bench_v1_1_逻辑推理: 0
+ alignment_bench_v1_1_中文理解: 0
+ alignment_bench_v1_1_文本写作: 0
+ alignment_bench_v1_1_角色扮演: 0
+ alignment_bench_v1_1_综合问答: 0
+ alpaca_eval_helpful_base: 0
+ compassarena_language_naive_average: 35
+ compassarena_knowledge_naive_average: 50
+ compassarena_reason_v2_naive_average: 30
+ compassarena_math_v2_naive_average: 50
+ compassarena_creationv2_zh_naive_average: 25
+ followbench_llmeval_en_HSR_AVG: 1
+ followbench_llmeval_en_SSR_AVG: 1
+ followbench_llmeval_en_HSR_L1: 1
+ followbench_llmeval_en_HSR_L2: 1
+ followbench_llmeval_en_HSR_L3: 1
+ followbench_llmeval_en_HSR_L4: 1
+ followbench_llmeval_en_HSR_L5: 1
+ followbench_llmeval_en_SSR_L1: 1
+ followbench_llmeval_en_SSR_L2: 1
+ followbench_llmeval_en_SSR_L3: 1
+ followbench_llmeval_en_SSR_L4: 1
+ followbench_llmeval_en_SSR_L5: 1
+ simpleqa_f1: 0
internlm2_5-7b-hf_fullbench:
- race-high: 100
- ARC-c: 68.75
- BoolQ: 87.5
- GPQA_diamond: 62.5
- drop: 62.5
- math: 12.5
- wikibench-wiki-single_choice_cncircular: 25
- sanitized_mbpp: 56.25
- gsm8k: 37.5
- triviaqa_wiki_1shot: 43.75
- nq_open_1shot: 43.75
- winogrande: 75
- hellaswag: 93.75
- TheoremQA: 25
- dingo_en_192: 37.5
- dingo_zh_170: 100
- college: 12.5
- college_knowledge: 87.5
- bbh-logical_deduction_seven_objects: 43.75
- bbh-multistep_arithmetic_two: 56.25
- mmlu-other: 76.92
- cmmlu-china-specific: 84.17
- mmlu_pro_math: 18.75
+ objective:
+ race-high_accuracy: 100
+ ARC-c_accuracy: 68.75
+ BoolQ_accuracy: 87.5
+ triviaqa_wiki_1shot_score: 43.75
+ nq_open_1shot_score: 43.75
+ drop_accuracy: 62.5
+ GPQA_diamond_accuracy: 62.5
+ hellaswag_accuracy: 93.75
+ TheoremQA_score: 25
+ winogrande_accuracy: 75
+ gsm8k_accuracy: 37.5
+ GaokaoBench_2010-2022_Math_II_MCQs_score: 62.5
+ GaokaoBench_2010-2022_Math_II_Fill-in-the-Blank_score: 0
+ math_accuracy: 12.5
+ wikibench-wiki-single_choice_cncircular_perf_4: 25
+ sanitized_mbpp_score: 56.25
+ dingo_en_192_score: 37.5
+ dingo_zh_170_score: 100
+ mmlu-other_accuracy: 76.92
+ cmmlu-china-specific_accuracy: 84.17
+ mmlu_pro_math_accuracy: 18.75
+ bbh-logical_deduction_seven_objects_score: 43.75
+ bbh-multistep_arithmetic_two_score: 56.25
+ college_naive_average: 12.5
+ college_knowledge_naive_average: 87.5
internlm2_5-7b-turbomind_fullbench:
- race-high: 100
- ARC-c: 68.75
- BoolQ: 87.5
- GPQA_diamond: 62.5
- drop: 62.5
- math: 18.75
- wikibench-wiki-single_choice_cncircular: 25
- sanitized_mbpp: 56.25
- gsm8k: 68.75
- triviaqa_wiki_1shot: 43.75
- nq_open_1shot: 43.75
- winogrande: 87.5
- hellaswag: 93.75
- TheoremQA: 31.25
- dingo_en_192: 43.75
- dingo_zh_170: 100
- college: 12.5
- college_knowledge: 87.5
- bbh-logical_deduction_seven_objects: 50
- bbh-multistep_arithmetic_two: 56.25
- mmlu-other: 76.92
- cmmlu-china-specific: 84.17
- mmlu_pro_math: 18.75
+ objective:
+ race-high_accuracy: 100
+ ARC-c_accuracy: 68.75
+ BoolQ_accuracy: 87.5
+ triviaqa_wiki_1shot_score: 43.75
+ nq_open_1shot_score: 43.75
+ drop_accuracy: 62.5
+ GPQA_diamond_accuracy: 62.5
+ hellaswag_accuracy: 93.75
+ TheoremQA_score: 25.00
+ winogrande_accuracy: 87.5
+ gsm8k_accuracy: 62.50
+ GaokaoBench_2010-2022_Math_II_MCQs_score: 81.25
+ GaokaoBench_2010-2022_Math_II_Fill-in-the-Blank_score: 0
+ math_accuracy: 18.75
+ wikibench-wiki-single_choice_cncircular_perf_4: 25
+ sanitized_mbpp_score: 62.50
+ dingo_en_192_score: 31.25
+ dingo_zh_170_score: 93.75
+ mmlu-other_accuracy: 76.92
+ cmmlu-china-specific_accuracy: 84.17
+ mmlu_pro_math_accuracy: 18.75
+ bbh-logical_deduction_seven_objects_score: 50
+ bbh-multistep_arithmetic_two_score: 56.25
+ college_naive_average: 12.5
+ college_knowledge_naive_average: 87.5
+
+internlm2_5-7b-turbomind:
+ objective:
+ race-high_accuracy: 89.28
+ ARC-c_accuracy: 52.2
+ BoolQ_accuracy: 89.72
+ triviaqa_wiki_1shot_score: 65.88
+ nq_open_1shot_score: 34.82
+ drop_accuracy: 68.1
+ bbh_naive_average: 72.15
+ GPQA_diamond_accuracy: 32.83
+ hellaswag_accuracy: 88.36
+ TheoremQA_score: 25
+ winogrande_accuracy: 81.29
+ gsm8k_accuracy: 74.68
+ GaokaoBench_weighted_average: 58.19
+ math_accuracy: 33.98
+ Mathbench_naive_average: 48.38
+ wikibench-wiki-single_choice_cncircular_perf_4: 29.1
+ cmmlu_naive_average: 78.94
+ mmlu_naive_average: 71.44
+ mmlu_pro_naive_average: 38.18
+ openai_humaneval_humaneval_pass@1: 59.76
+ openai_humaneval_v2_humaneval_pass@1: 51.22
+ sanitized_mbpp_score: 55.25
+ dingo_en_192_score: 60.94
+ dingo_zh_170_score: 67.65
+ mmlu-stem_naive_average: 63.72
+ mmlu-social-science_naive_average: 80.15
+ mmlu-humanities_naive_average: 74.27
+ mmlu-other_naive_average: 71.85
+ cmmlu-stem_naive_average: 67.07
+ cmmlu-social-science_naive_average: 81.49
+ cmmlu-humanities_naive_average: 85.84
+ cmmlu-other_naive_average: 82.69
+ cmmlu-china-specific_naive_average: 79.88
+ mmlu_pro_biology_accuracy: 58.58
+ mmlu_pro_business_accuracy: 28.01
+ mmlu_pro_chemistry_accuracy: 22.79
+ mmlu_pro_computer_science_accuracy: 39.02
+ mmlu_pro_economics_accuracy: 53.08
+ mmlu_pro_engineering_accuracy: 25.7
+ mmlu_pro_health_accuracy: 46.94
+ mmlu_pro_history_accuracy: 43.04
+ mmlu_pro_law_accuracy: 29.7
+ mmlu_pro_math_accuracy: 24.2
+ mmlu_pro_philosophy_accuracy: 42.48
+ mmlu_pro_physics_accuracy: 26.02
+ mmlu_pro_psychology_accuracy: 52.76
+ mmlu_pro_other_accuracy: 42.21
+ college_naive_average: 10.67
+ high_naive_average: 6.67
+ middle_naive_average: 26.67
+ primary_naive_average: 60
+ arithmetic_naive_average: 55
+ mathbench-a (average)_naive_average: 31.8
+ college_knowledge_naive_average: 62.34
+ high_knowledge_naive_average: 59.83
+ middle_knowledge_naive_average: 71.15
+ primary_knowledge_naive_average: 66.55
+ mathbench-t (average)_naive_average: 64.97
+ long_context:
+ Single-Needle-Retrieval(S-RT)-32000_naive_average: 100
+ Single-Needle-Retrieval-EN-32000_naive_average: 100
+ Single-Needle-Retrieval-ZH-32000_naive_average: 100
+ Single-Needle-Retrieval(S-RT)-100000_naive_average: 100
+ Single-Needle-Retrieval-EN-100000_naive_average: 100
+ Single-Needle-Retrieval-ZH-100000_naive_average: 100
+ Single-Needle-Retrieval(S-RT)-200000_naive_average: 100
+ Single-Needle-Retrieval-EN-200000_naive_average: 100
+ Single-Needle-Retrieval-ZH-200000_naive_average: 100
+ longbench_naive_average: 46.19
+ longbench_zh_naive_average: 49.3
+ longbench_en_naive_average: 43.97
+ longbench_single-document-qa_naive_average: 42.84
+ longbench_multi-document-qa_naive_average: 37.29
+ longbench_summarization_naive_average: 23.21
+ longbench_few-shot-learning_naive_average: 61.67
+ longbench_synthetic-tasks_naive_average: 60.05
+ longbench_code-completion_naive_average: 52.09
+
+internlm2_5-7b-chat-turbomind:
+ objective:
+ race-high_accuracy: 86.16
+ ARC-c_accuracy: 90.17
+ BoolQ_accuracy: 87.89
+ triviaqa_wiki_1shot_score: 64.91
+ nq_open_1shot_score: 22.69
+ mmmlu_lite_naive_average: 44.96
+ IFEval_Prompt-level-strict-accuracy: 58.04
+ drop_accuracy: 77.68
+ bbh_naive_average: 73.14
+ GPQA_diamond_accuracy: 25.76
+ hellaswag_accuracy: 94.79
+ TheoremQA_score: 21.5
+ musr_average_naive_average: 51.03
+ korbench_single_naive_average: 31.92
+ ARC_Prize_Public_Evaluation_accuracy: 0.01
+ gsm8k_accuracy: 86.73
+ GaokaoBench_weighted_average: 77.89
+ math_accuracy: 61.5
+ cmo_fib_accuracy: 12.5
+ aime2024_accuracy: 3.33
+ Mathbench_naive_average: 65.17
+ wikibench-wiki-single_choice_cncircular_perf_4: 31.55
+ cmmlu_naive_average: 74.14
+ mmlu_naive_average: 70.52
+ mmlu_pro_naive_average: 44.98
+ openai_humaneval_humaneval_pass@1: 70.73
+ sanitized_mbpp_score: 63.81
+ humanevalx_naive_average: 38.17
+ ds1000_naive_average: 14.15
+ lcb_code_generation_pass@1: 17.75
+ lcb_code_execution_pass@1: 32.57
+ lcb_test_output_pass@1: 24.89
+ bigcodebench_hard_instruct_pass@1: 0.08
+ bigcodebench_hard_complete_pass@1: 0.06
+ teval_naive_average: 80.03
+ qa_dingo_cn_score: 99.01
+ mmlu-stem_naive_average: 68.2
+ mmlu-social-science_naive_average: 76.11
+ mmlu-humanities_naive_average: 68.71
+ mmlu-other_naive_average: 70.56
+ cmmlu-stem_naive_average: 66.27
+ cmmlu-social-science_naive_average: 75.7
+ cmmlu-humanities_naive_average: 77.7
+ cmmlu-other_naive_average: 77.71
+ cmmlu-china-specific_naive_average: 72.94
+ mmlu_pro_biology_accuracy: 66.25
+ mmlu_pro_business_accuracy: 48.42
+ mmlu_pro_chemistry_accuracy: 35.25
+ mmlu_pro_computer_science_accuracy: 47.56
+ mmlu_pro_economics_accuracy: 55.92
+ mmlu_pro_engineering_accuracy: 30.44
+ mmlu_pro_health_accuracy: 45.97
+ mmlu_pro_history_accuracy: 41.21
+ mmlu_pro_law_accuracy: 25.79
+ mmlu_pro_math_accuracy: 54.03
+ mmlu_pro_philosophy_accuracy: 36.47
+ mmlu_pro_physics_accuracy: 37.41
+ mmlu_pro_psychology_accuracy: 58.77
+ mmlu_pro_other_accuracy: 46.21
+ humanevalx-python_pass@1: 53.66
+ humanevalx-cpp_pass@1: 24.39
+ humanevalx-go_pass@1: 0
+ humanevalx-java_pass@1: 57.93
+ humanevalx-js_pass@1: 54.88
+ ds1000_Pandas_accuracy: 12.03
+ ds1000_Numpy_accuracy: 4.09
+ ds1000_Tensorflow_accuracy: 11.11
+ ds1000_Scipy_accuracy: 8.49
+ ds1000_Sklearn_accuracy: 6.96
+ ds1000_Pytorch_accuracy: 7.35
+ ds1000_Matplotlib_accuracy: 49.03
+ openai_mmmlu_lite_AR-XY_accuracy: 17.89
+ openai_mmmlu_lite_BN-BD_accuracy: 27.58
+ openai_mmmlu_lite_DE-DE_accuracy: 51.16
+ openai_mmmlu_lite_ES-LA_accuracy: 56.84
+ openai_mmmlu_lite_FR-FR_accuracy: 57.96
+ openai_mmmlu_lite_HI-IN_accuracy: 33.68
+ openai_mmmlu_lite_ID-ID_accuracy: 51.02
+ openai_mmmlu_lite_IT-IT_accuracy: 50.46
+ openai_mmmlu_lite_JA-JP_accuracy: 50.53
+ openai_mmmlu_lite_KO-KR_accuracy: 45.05
+ openai_mmmlu_lite_PT-BR_accuracy: 57.68
+ openai_mmmlu_lite_SW-KE_accuracy: 32.77
+ openai_mmmlu_lite_YO-NG_accuracy: 31.79
+ openai_mmmlu_lite_ZH-CN_accuracy: 65.05
+ college_naive_average: 20.33
+ high_naive_average: 47.67
+ middle_naive_average: 62
+ primary_naive_average: 72
+ arithmetic_naive_average: 62.33
+ mathbench-a (average)_naive_average: 52.87
+ college_knowledge_naive_average: 70.57
+ high_knowledge_naive_average: 70.13
+ middle_knowledge_naive_average: 81.17
+ primary_knowledge_naive_average: 88.01
+ mathbench-t (average)_naive_average: 77.47
+ subjective:
+ alignment_bench_v1_1_总分: 5.68
+ alpaca_eval_total: 25.96
+ arenahard_score: 17.15
+ Followbench_naive_average: 0.81
+ CompassArena_naive_average: 34.61
+ FoFo_naive_average: 0.38
+ mtbench101_avg: 8.01
+ wildbench_average: -15.69
+ simpleqa_accuracy_given_attempted: 0.04
+ chinese_simpleqa_given_attempted_accuracy: 0.34
+ alignment_bench_v1_1_专业能力: 6.05
+ alignment_bench_v1_1_数学计算: 5.87
+ alignment_bench_v1_1_基本任务: 6.01
+ alignment_bench_v1_1_逻辑推理: 4.48
+ alignment_bench_v1_1_中文理解: 6.17
+ alignment_bench_v1_1_文本写作: 6.06
+ alignment_bench_v1_1_角色扮演: 6.3
+ alignment_bench_v1_1_综合问答: 6.45
+ alpaca_eval_helpful_base: 17.83
+ alpaca_eval_koala: 28.21
+ alpaca_eval_oasst: 23.4
+ alpaca_eval_selfinstruct: 30.95
+ alpaca_eval_vicuna: 25
+ compassarena_language_naive_average: 52.5
+ compassarena_knowledge_naive_average: 36
+ compassarena_reason_v2_naive_average: 35
+ compassarena_math_v2_naive_average: 19.91
+ compassarena_creationv2_zh_naive_average: 29.64
+ fofo_test_prompts_overall: 0.35
+ fofo_test_prompts_cn_overall: 0.41
+ followbench_llmeval_en_HSR_AVG: 0.73
+ followbench_llmeval_en_SSR_AVG: 0.88
+ followbench_llmeval_en_HSR_L1: 0.94
+ followbench_llmeval_en_HSR_L2: 0.77
+ followbench_llmeval_en_HSR_L3: 0.73
+ followbench_llmeval_en_HSR_L4: 0.68
+ followbench_llmeval_en_HSR_L5: 0.54
+ followbench_llmeval_en_SSR_L1: 0.94
+ followbench_llmeval_en_SSR_L2: 0.88
+ followbench_llmeval_en_SSR_L3: 0.87
+ followbench_llmeval_en_SSR_L4: 0.87
+ followbench_llmeval_en_SSR_L5: 0.85
+ simpleqa_f1: 0.04
+
+internlm2_5-7b-chat-1m-turbomind:
+ long_context:
+ ruler_8k_naive_average: 88.53
+ ruler_32k_naive_average: 83.84
+ ruler_128k_naive_average: 70.94
+ NeedleBench-Overall-Score-8K_weighted_average: 91.89
+ NeedleBench-Overall-Score-32K_weighted_average: 91.42
+ NeedleBench-Overall-Score-128K_weighted_average: 88.57
+ longbench_naive_average: 46.44
+ longbench_zh_naive_average: 45.19
+ longbench_en_naive_average: 45.71
+ babilong_0k_naive_average: 79.3
+ babilong_4k_naive_average: 67
+ babilong_16k_naive_average: 52.7
+ babilong_32k_naive_average: 48.9
+ babilong_128k_naive_average: 40.8
+ babilong_256k_naive_average: 23.5
+ longbench_single-document-qa_naive_average: 43.56
+ longbench_multi-document-qa_naive_average: 46.24
+ longbench_summarization_naive_average: 24.32
+ longbench_few-shot-learning_naive_average: 51.67
+ longbench_synthetic-tasks_naive_average: 66.83
+ longbench_code-completion_naive_average: 45.99
diff --git a/.github/scripts/oc_score_baseline_testrange.yaml b/.github/scripts/oc_score_baseline_testrange.yaml
index 68f6660a..131fd2ea 100644
--- a/.github/scripts/oc_score_baseline_testrange.yaml
+++ b/.github/scripts/oc_score_baseline_testrange.yaml
@@ -1,459 +1,468 @@
-baichuan2-7b-chat-hf:
- gsm8k: 18.75
- race-high: 78.12
-
-glm-4-9b-chat-hf:
- gsm8k: 68.75
- race-high: 90.62
-
-glm-4-9b-chat-turbomind:
- gsm8k: 75.00
- race-high: 90.62
-
-glm-4-9b-chat-vllm:
- gsm8k: 65.62
- race-high: 90.62
-
-deepseek-7b-chat-hf:
- gsm8k: 46.88
- race-high: 81.25
-
-deepseek-moe-16b-chat-hf:
- gsm8k: 50
- race-high: 68.75
-
-deepseek-7b-chat-vllm:
- gsm8k: 43.75
- race-high: 75
-
-gemma2-2b-it-hf:
- gsm8k: 50
- race-high: 71.88
-
-gemma2-9b-it-hf:
- gsm8k: 71.88
- race-high: 84.38
-
-gemma-2b-it-hf:
- gsm8k: 3.12
- race-high: 40.62
-
-gemma-7b-it-hf:
- gsm8k: 40.62
- race-high: 68.75
-
-gemma-2-9b-it-turbomind:
- gsm8k: 65.62
- race-high: 84.38
-
-gemma-7b-it-vllm:
- gsm8k: 34.38
- race-high: 68.75
-
-internlm2_5-7b-chat-hf:
- gsm8k: 84.38
- race-high: 90.62
-
-internlm2_5-7b-chat-turbomind:
- gsm8k: 84.38
- race-high: 90.62
-
-internlm2-chat-1.8b-turbomind:
- gsm8k: 25
- race-high: 84.38
-
-internlm2-chat-1.8b-sft-turbomind:
- gsm8k: 21.88
- race-high: 84.38
-
-internlm2-chat-7b-lmdeploy:
- gsm8k: 53.12
- race-high: 84.38
-
-internlm2-chat-7b-sft-turbomind:
- gsm8k: 50
- race-high: 90.62
-
-internlm2-chat-7b-vllm:
- gsm8k: 43.75
- race-high: 87.5
-
-llama-3_1-8b-instruct-hf:
- gsm8k: 84.38
- race-high: 90.62
-
-llama-3_2-3b-instruct-hf:
- gsm8k: 65.62
- race-high: 81.25
-
-llama-3-8b-instruct-hf:
- gsm8k: 68.75
- race-high: 87.5
-
-llama-3_1-8b-instruct-turbomind:
- gsm8k: 78.12
- race-high: 90.62
-
-llama-3_2-3b-instruct-turbomind:
- gsm8k: 62.50
- race-high: 81.25
-
-llama-3-8b-instruct-turbomind:
- gsm8k: 68.75
- race-high: 87.5
-
-mistral-7b-instruct-v0.2-hf:
- gsm8k: 40.62
- race-high: 75
-
-mistral-7b-instruct-v0.3-hf:
- gsm8k: 40.62
- race-high: 75
-
-mistral-nemo-instruct-2407-hf:
- gsm8k: 75
- race-high: 81.25
-
-mistral-nemo-instruct-2407-turbomind:
- gsm8k: 68.75
- race-high: 87.50
-
-mistral-7b-instruct-v0.1-vllm:
- gsm8k: 34.38
- race-high: 68.75
-
-mistral-7b-instruct-v0.2-vllm:
- gsm8k: 43.75
- race-high: 75
-
-MiniCPM3-4B-hf:
- gsm8k: 68.75
- race-high: 84.38
-
-minicpm-2b-dpo-fp32-hf:
- gsm8k: 56.25
- race-high: 53.12
-
-minicpm-2b-sft-bf16-hf:
- gsm8k: 46.88
- race-high: 65.62
-
-minicpm-2b-sft-fp32-hf:
- gsm8k: 46.88
- race-high: 65.62
-
-phi-3-mini-4k-instruct-hf:
- gsm8k: 56.25
- race-high: 84.38
-
-qwen1.5-0.5b-chat-hf:
- gsm8k: 0
- race-high: 53.12
-
-qwen2-1.5b-instruct-hf:
- gsm8k: 62.5
- race-high: 84.38
-
-qwen2-7b-instruct-hf:
- gsm8k: 68.75
- race-high: 90.62
-
-qwen2-1.5b-instruct-turbomind:
- gsm8k: 62.50
- race-high: 84.38
-
-qwen2-7b-instruct-turbomind:
- gsm8k: 81.25
- race-high: 87.5
-
-qwen1.5-0.5b-chat-vllm:
- gsm8k: 3.12
- race-high: 53.12
-
-yi-1.5-6b-chat-hf:
- gsm8k: 65.62
- race-high: 84.38
-
-yi-1.5-9b-chat-hf:
- gsm8k: 75
- race-high: 93.75
-
-deepseek-v2-lite-chat-hf:
- gsm8k: 43.75
- race-high: 71.88
-
-internlm2_5-20b-chat-hf:
- gsm8k: 84.38
- race-high: 87.5
-
-internlm2_5-20b-chat-turbomind:
- gsm8k: 84.38
- race-high: 87.5
-
-mistral-small-instruct-2409-hf:
- gsm8k: 81.25
- race-high: 87.50
-
-mistral-small-instruct-2409-turbomind:
- gsm8k: 78.12
- race-high: 87.50
-
-qwen2.5-14b-instruct-hf:
- gsm8k: 71.88
- race-high: 96.88
-
-qwen2.5-14b-instruct-turbomind:
- gsm8k: 71.88
- race-high: 93.75
-
-glm-4-9b-hf:
- gsm8k: 68.75
- GPQA_diamond: 31.25
- race-high: 93.75
- winogrande: 84.38
-
-deepseek-moe-16b-base-hf:
- gsm8k: 21.88
- GPQA_diamond: 0
- race-high: 21.88
- winogrande: 65.62
-
-deepseek-7b-base-turbomind:
- gsm8k: 21.88
- GPQA_diamond: 0
- race-high: 46.88
- winogrande: 84.38
-
-deepseek-moe-16b-base-vllm:
- gsm8k: 21.88
- GPQA_diamond: 0
- race-high: 25
- winogrande: 68.75
-
-gemma2-2b-hf:
- gsm8k: 31.25
- GPQA_diamond: 3.12
- race-high: 56.25
- winogrande: 71.88
-
-gemma2-9b-hf:
- gsm8k: 68.75
- GPQA_diamond: 0
- race-high: 81.25
- winogrande: 84.38
-
-gemma-2b-hf:
- gsm8k: 18.75
- GPQA_diamond: 3.12
- race-high: 25
- winogrande: 53.12
-
-gemma-7b-hf:
- gsm8k: 56.25
- GPQA_diamond: 6.25
- race-high: 65.62
- winogrande: 78.12
-
-gemma-2b-vllm:
- gsm8k: 15.62
- GPQA_diamond: 6.25
- race-high:
- winogrande:
-
-gemma-7b-vllm:
- gsm8k: 53.12
- GPQA_diamond: 6.25
- race-high:
- winogrande:
-
-internlm2_5-7b-hf:
- gsm8k: 37.5
- GPQA_diamond: 25
- race-high: 93.75
- winogrande: 71.88
-
-internlm2-7b-hf:
- gsm8k: 53.12
- GPQA_diamond: 18.75
- race-high: 62.5
- winogrande: 78.12
-
-internlm2-base-7b-hf:
- gsm8k: 3.12
- GPQA_diamond: 21.88
- race-high: 75
- winogrande: 65.62
-
-internlm2-1.8b-turbomind:
- gsm8k: 12.5
- GPQA_diamond: 12.5
- race-high: 71.88
- winogrande: 75
-
-internlm2_5-7b-turbomind:
- gsm8k: 68.75
- GPQA_diamond: 31.25
- race-high: 93.75
- winogrande: 84.38
-
-internlm2-7b-turbomind:
- gsm8k: 56.25
- GPQA_diamond: 21.88
- race-high: 75
- winogrande: 81.25
-
-internlm2-base-7b-turbomind:
- gsm8k: 40.62
- GPQA_diamond: 28.12
- race-high: 84.38
- winogrande: 71.88
-
-llama-2-7b-hf:
- gsm8k: 21.88
- GPQA_diamond: 21.88
- race-high: 40.62
- winogrande: 71.88
-
-llama-3_1-8b-hf:
- gsm8k: 78.12
- GPQA_diamond: 25
- race-high: 90.62
- winogrande: 62.5
-
-llama-3-8b-hf:
- gsm8k: 46.88
- GPQA_diamond: 6.25
- race-high: 65.62
- winogrande: 65.62
-
-llama-3.1-8b-turbomind:
- gsm8k: 56.25
- GPQA_diamond: 6.25
- race-high: 78.12
- winogrande: 78.12
-
-llama-3-8b-turbomind:
- gsm8k: 50
- GPQA_diamond: 9.38
- race-high: 65.62
- winogrande: 78.12
-
-mistral-7b-v0.2-hf:
- gsm8k: 31.25
- GPQA_diamond: 6.25
- race-high: 62.5
- winogrande: 59.38
-
-mistral-7b-v0.3-hf:
- gsm8k: 31.25
- GPQA_diamond: 6.25
- race-high: 62.5
- winogrande: 59.38
-
-mistral-7b-v0.2-vllm:
- gsm8k: 34.38
- GPQA_diamond: 6.25
- race-high: 62.5
- winogrande: 65.62
-
-qwen2.5-7b-hf:
- gsm8k: 81.25
- GPQA_diamond: 18.75
- race-high: 87.5
- winogrande: 71.88
-
-qwen2.5-1.5b-turbomind:
- gsm8k: 71.88
- GPQA_diamond: 15.62
- race-high: 78.12
- winogrande: 71.88
-
-qwen2.5-7b-turbomind:
- gsm8k: 71.88
- GPQA_diamond: 25
- race-high: 87.5
- winogrande: 71.88
-
-qwen1.5-moe-a2.7b-hf:
- gsm8k: 62.5
- GPQA_diamond: 18.75
- race-high: 84.38
- winogrande: 75
-
-qwen2-0.5b-hf:
- gsm8k: 25
- GPQA_diamond: 0
- race-high: 40.62
- winogrande: 62.5
-
-qwen2-1.5b-hf:
- gsm8k: 59.38
- GPQA_diamond: 9.38
- race-high: 81.25
- winogrande: 62.5
-
-qwen2-7b-hf:
- gsm8k: 68.75
- GPQA_diamond: 9.38
- race-high: 87.5
- winogrande: 68.75
-
-qwen2-1.5b-turbomind:
- gsm8k: 62.50
- GPQA_diamond: 6.25
- race-high: 81.25
- winogrande: 75
-
-qwen2-7b-turbomind:
- gsm8k: 68.75
- GPQA_diamond: 12.5
- race-high: 87.5
- winogrande: 71.88
-
-qwen1.5-0.5b-vllm:
- gsm8k: 9.38
- GPQA_diamond: 0
- race-high: 56.25
- winogrande: 62.5
-
-yi-1.5-6b-hf:
- gsm8k: 62.5
- GPQA_diamond: 3.12
- race-high: 87.5
- winogrande: 62.5
-
-yi-1.5-9b-hf:
- gsm8k: 75
- GPQA_diamond: 40.62
- race-high: 87.5
- winogrande: 59.38
-
-deepseek-v2-lite-hf:
- gsm8k: 28.12
- GPQA_diamond: 21.88
- race-high: 59.38
- winogrande: 75
-
-internlm2-20b-hf:
- gsm8k: 56.25
- GPQA_diamond: 15.62
- race-high: 68.75
- winogrande: 75
-
-internlm2-base-20b-hf:
- gsm8k: 12.5
- GPQA_diamond: 9.38
- race-high: 84.38
- winogrande: 65.62
-
-internlm2-20b-turbomind:
- gsm8k: 68.75
- GPQA_diamond: 15.62
- race-high: 68.75
- winogrande: 81.25
-
-qwen2.5-14b-hf:
- gsm8k: 75
- GPQA_diamond: 37.5
- race-high: 93.75
- winogrande: 84.38
+chat:
+ glm-4-9b-chat-hf:
+ gsm8k_accuracy: 68.75
+ race-high_accuracy: 90.62
+ glm-4-9b-chat-turbomind:
+ gsm8k_accuracy: 71.88
+ race-high_accuracy: 90.62
+ glm-4-9b-chat-vllm:
+ gsm8k_accuracy: 65.62
+ race-high_accuracy: 90.62
+ deepseek-7b-chat-hf:
+ gsm8k_accuracy: 46.88
+ race-high_accuracy: 81.25
+ deepseek-moe-16b-chat-hf:
+ gsm8k_accuracy: 50
+ race-high_accuracy: 68.75
+ deepseek-7b-chat-vllm:
+ gsm8k_accuracy: 43.75
+ race-high_accuracy: 75
+ gemma2-2b-it-hf:
+ gsm8k_accuracy: 50
+ race-high_accuracy: 71.88
+ gemma2-9b-it-hf:
+ gsm8k_accuracy: 71.88
+ race-high_accuracy: 84.38
+ gemma-2b-it-hf:
+ gsm8k_accuracy: 3.12
+ race-high_accuracy: 40.62
+ gemma-7b-it-hf:
+ gsm8k_accuracy: 40.62
+ race-high_accuracy: 68.75
+ gemma-2-9b-it-turbomind:
+ gsm8k_accuracy: 71.88
+ race-high_accuracy: 84.38
+ gemma-2-27b-it-turbomind:
+ gsm8k_accuracy: 78.12
+ race-high_accuracy: 93.75
+ gemma-7b-it-vllm:
+ gsm8k_accuracy: 34.38
+ race-high_accuracy: 68.75
+ internlm2_5-7b-chat-hf:
+ gsm8k_accuracy: 84.38
+ race-high_accuracy: 90.62
+ internlm2_5-7b-chat-turbomind:
+ gsm8k_accuracy: 87.50
+ race-high_accuracy: 90.62
+ internlm2-chat-1.8b-turbomind:
+ gsm8k_accuracy: 28.12
+ race-high_accuracy: 84.38
+ internlm2-chat-1.8b-sft-turbomind:
+ gsm8k_accuracy: 21.88
+ race-high_accuracy: 84.38
+ internlm2-chat-7b-lmdeploy:
+ gsm8k_accuracy: 53.12
+ race-high_accuracy: 84.38
+ internlm2-chat-7b-sft-turbomind:
+ gsm8k_accuracy: 53.12
+ race-high_accuracy: 90.62
+ internlm2-chat-7b-vllm:
+ gsm8k_accuracy: 56.25
+ race-high_accuracy: 84.38
+ llama-3_1-8b-instruct-hf:
+ gsm8k_accuracy: 84.38
+ race-high_accuracy: 90.62
+ llama-3_2-3b-instruct-hf:
+ gsm8k_accuracy: 68.75
+ race-high_accuracy: 81.25
+ llama-3-8b-instruct-hf:
+ gsm8k_accuracy: 68.75
+ race-high_accuracy: 87.5
+ llama-2-7b-chat-turbomind:
+ gsm8k_accuracy: 18.75
+ race-high_accuracy: 46.88
+ llama-3_1-8b-instruct-turbomind:
+ gsm8k_accuracy: 78.12
+ race-high_accuracy: 90.62
+ llama-3_2-3b-instruct-turbomind:
+ gsm8k_accuracy: 71.88
+ race-high_accuracy: 81.25
+ llama-3-8b-instruct-turbomind:
+ gsm8k_accuracy: 71.88
+ race-high_accuracy: 87.5
+ mistral-7b-instruct-v0.2-hf:
+ gsm8k_accuracy: 40.62
+ race-high_accuracy: 75
+ mistral-7b-instruct-v0.3-hf:
+ gsm8k_accuracy: 40.62
+ race-high_accuracy: 75
+ mistral-nemo-instruct-2407-hf:
+ gsm8k_accuracy: 75
+ race-high_accuracy: 81.25
+ mistral-nemo-instruct-2407-turbomind:
+ gsm8k_accuracy: 65.62
+ race-high_accuracy: 87.50
+ mistral-7b-instruct-v0.1-vllm:
+ gsm8k_accuracy: 34.38
+ race-high_accuracy: 68.75
+ mistral-7b-instruct-v0.2-vllm:
+ gsm8k_accuracy: 43.75
+ race-high_accuracy: 75
+ MiniCPM3-4B-hf:
+ gsm8k_accuracy: 68.75
+ race-high_accuracy: 84.38
+ phi-3-mini-4k-instruct-hf:
+ gsm8k_accuracy: 56.25
+ race-high_accuracy: 84.38
+ phi-3-small-8k-instruct-hf:
+ gsm8k_accuracy: 0
+ race-high_accuracy: 0
+ qwen2.5-0.5b-instruct-hf:
+ gsm8k_accuracy: 34.38
+ race-high_accuracy: 46.88
+ qwen2.5-3b-instruct-hf :
+ gsm8k_accuracy: 53.12
+ race-high_accuracy: 90.62
+ qwen2.5-0.5b-instruct-turbomind:
+ gsm8k_accuracy: 28.12
+ race-high_accuracy: 50
+ qwen2.5-3b-instruct-turbomind:
+ gsm8k_accuracy: 59.38
+ race-high_accuracy: 90.62
+ qwen1.5-0.5b-chat-hf:
+ gsm8k_accuracy: 0
+ race-high_accuracy: 53.12
+ qwen2-1.5b-instruct-hf:
+ gsm8k_accuracy: 62.5
+ race-high_accuracy: 84.38
+ qwen2-7b-instruct-hf:
+ gsm8k_accuracy: 68.75
+ race-high_accuracy: 90.62
+ qwen2-1.5b-instruct-turbomind:
+ gsm8k_accuracy: 53.12
+ race-high_accuracy: 84.38
+ qwen2-7b-instruct-turbomind:
+ gsm8k_accuracy: 81.25
+ race-high_accuracy: 90.62
+ qwen1.5-0.5b-chat-vllm:
+ gsm8k_accuracy: 3.12
+ race-high_accuracy: 53.12
+ yi-1.5-6b-chat-hf:
+ gsm8k_accuracy: 65.62
+ race-high_accuracy: 84.38
+ yi-1.5-9b-chat-hf:
+ gsm8k_accuracy: 75
+ race-high_accuracy: 93.75
+ yi-1.5-6b-chat-turbomind:
+ gsm8k_accuracy: 62.5
+ race-high_accuracy: 84.38
+ yi-1.5-9b-chat-turbomind:
+ gsm8k_accuracy: 71.88
+ race-high_accuracy: 93.75
+ deepseek-v2-lite-chat-hf:
+ gsm8k_accuracy: 46.88
+ race-high_accuracy: 71.88
+ gemma2-27b-it-hf:
+ gsm8k_accuracy: 75
+ race-high_accuracy: 93.75
+ internlm2_5-20b-chat-hf:
+ gsm8k_accuracy: 84.38
+ race-high_accuracy: 87.5
+ internlm2_5-20b-chat-turbomind:
+ gsm8k_accuracy: 87.50
+ race-high_accuracy: 87.5
+ mistral-small-instruct-2409-hf:
+ gsm8k_accuracy: 81.25
+ race-high_accuracy: 87.50
+ mistral-small-instruct-2409-turbomind:
+ gsm8k_accuracy: 81.25
+ race-high_accuracy: 87.50
+ qwen2.5-14b-instruct-hf:
+ gsm8k_accuracy: 71.88
+ race-high_accuracy: 96.88
+ qwen2.5-14b-instruct-turbomind:
+ gsm8k_accuracy: 68.75
+ race-high_accuracy: 93.75
+ yi-1.5-34b-chat-turbomind:
+ gsm8k_accuracy: 78.12
+ race-high_accuracy: 93.75
+ deepseek-67b-chat-hf:
+ gsm8k_accuracy: 71.88
+ race-high_accuracy: 78.12
+ llama-3_3-70b-instruct-turbomind:
+ gsm8k_accuracy: 93.75
+ race-high_accuracy: 87.5
+ mixtral-8x7b-instruct-v0.1-hf:
+ gsm8k_accuracy: 56.25
+ race-high_accuracy: 81.25
+ mixtral-large-instruct-2411-turbomind:
+ gsm8k_accuracy: 90.62
+ race-high_accuracy: 93.75
+ nvidia-3_1-Nemotron-70b-instruct-HF-turbomind:
+ gsm8k_accuracy: 87.5
+ race-high_accuracy: 46.88
+ qwen2.5-72b-instruct-turbomind:
+ gsm8k_accuracy: 75
+ race-high_accuracy: 93.75
+ deepseek-v2_5-1210-turbomind:
+ gsm8k_accuracy: 90.62
+ race-high_accuracy: 84.38
+ mixtral-8x22b-instruct-v0.1-hf:
+ gsm8k_accuracy: 81.25
+ race-high_accuracy: 81.25
+base:
+ glm-4-9b-hf:
+ gsm8k_accuracy: 68.75
+ GPQA_diamond_accuracy: 31.25
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 84.38
+ glm-4-9b-turbomind:
+ gsm8k_accuracy: 62.5
+ GPQA_diamond_accuracy: 28.12
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 84.38
+ deepseek-7b-base-hf:
+ gsm8k_accuracy: 25
+ GPQA_diamond_accuracy: 0
+ race-high_accuracy: 46.88
+ winogrande_accuracy: 71.88
+ deepseek-moe-16b-base-hf:
+ gsm8k_accuracy: 21.88
+ GPQA_diamond_accuracy: 0
+ race-high_accuracy: 21.88
+ winogrande_accuracy: 65.62
+ deepseek-7b-base-turbomind:
+ gsm8k_accuracy: 21.88
+ GPQA_diamond_accuracy: 0
+ race-high_accuracy: 46.88
+ winogrande_accuracy: 84.38
+ deepseek-moe-16b-base-vllm:
+ gsm8k_accuracy: 21.88
+ GPQA_diamond_accuracy: 0
+ race-high_accuracy: 25
+ winogrande_accuracy: 68.75
+ gemma2-2b-hf:
+ gsm8k_accuracy: 28.12
+ GPQA_diamond_accuracy: 3.12
+ race-high_accuracy: 56.25
+ winogrande_accuracy: 71.88
+ gemma2-9b-hf:
+ gsm8k_accuracy: 68.75
+ GPQA_diamond_accuracy: 0
+ race-high_accuracy: 81.25
+ winogrande_accuracy: 84.38
+ gemma-2b-hf:
+ gsm8k_accuracy: 18.75
+ GPQA_diamond_accuracy: 3.12
+ race-high_accuracy: 25
+ winogrande_accuracy: 53.12
+ gemma-7b-hf:
+ gsm8k_accuracy: 56.25
+ GPQA_diamond_accuracy: 6.25
+ race-high_accuracy: 65.62
+ winogrande_accuracy: 78.12
+ gemma-2b-vllm:
+ gsm8k_accuracy: 15.62
+ GPQA_diamond_accuracy: 3.12
+ race-high_accuracy:
+ winogrande_accuracy:
+ gemma-7b-vllm:
+ gsm8k_accuracy: 53.12
+ GPQA_diamond_accuracy: 9.38
+ race-high_accuracy:
+ winogrande_accuracy:
+ internlm2_5-7b-hf:
+ gsm8k_accuracy: 37.5
+ GPQA_diamond_accuracy: 25
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 71.88
+ internlm2-7b-hf:
+ gsm8k_accuracy: 53.12
+ GPQA_diamond_accuracy: 18.75
+ race-high_accuracy: 62.5
+ winogrande_accuracy: 78.12
+ internlm2-base-7b-hf:
+ gsm8k_accuracy: 3.12
+ GPQA_diamond_accuracy: 21.88
+ race-high_accuracy: 75
+ winogrande_accuracy: 65.62
+ internlm2-1.8b-turbomind:
+ gsm8k_accuracy: 12.5
+ GPQA_diamond_accuracy: 9.38
+ race-high_accuracy: 71.88
+ winogrande_accuracy: 78.12
+ internlm2_5-7b-turbomind:
+ gsm8k_accuracy: 62.50
+ GPQA_diamond_accuracy: 34.38
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 87.50
+ internlm2-7b-turbomind:
+ gsm8k_accuracy: 53.12
+ GPQA_diamond_accuracy: 21.88
+ race-high_accuracy: 71.88
+ winogrande_accuracy: 84.38
+ internlm2-base-7b-turbomind:
+ gsm8k_accuracy: 37.50
+ GPQA_diamond_accuracy: 28.12
+ race-high_accuracy: 81.25
+ winogrande_accuracy: 75
+ llama-2-7b-hf:
+ gsm8k_accuracy: 21.88
+ GPQA_diamond_accuracy: 21.88
+ race-high_accuracy: 40.62
+ winogrande_accuracy: 71.88
+ llama-3_1-8b-hf:
+ gsm8k_accuracy: 78.12
+ GPQA_diamond_accuracy: 25
+ race-high_accuracy: 90.62
+ winogrande_accuracy: 62.5
+ llama-3-8b-hf:
+ gsm8k_accuracy: 46.88
+ GPQA_diamond_accuracy: 6.25
+ race-high_accuracy: 65.62
+ winogrande_accuracy: 65.62
+ llama-3.1-8b-turbomind:
+ gsm8k_accuracy: 56.25
+ GPQA_diamond_accuracy: 9.38
+ race-high_accuracy: 78.12
+ winogrande_accuracy: 78.12
+ llama-3-8b-turbomind:
+ gsm8k_accuracy: 50
+ GPQA_diamond_accuracy: 12.50
+ race-high_accuracy: 65.62
+ winogrande_accuracy: 78.12
+ mistral-7b-v0.2-hf:
+ gsm8k_accuracy: 31.25
+ GPQA_diamond_accuracy: 6.25
+ race-high_accuracy: 62.5
+ winogrande_accuracy: 59.38
+ mistral-7b-v0.3-hf:
+ gsm8k_accuracy: 31.25
+ GPQA_diamond_accuracy: 6.25
+ race-high_accuracy: 62.5
+ winogrande_accuracy: 59.38
+ mistral-7b-v0.2-vllm:
+ gsm8k_accuracy: 34.38
+ GPQA_diamond_accuracy: 6.25
+ race-high_accuracy: 62.5
+ winogrande_accuracy: 65.62
+ qwen2.5-7b-hf:
+ gsm8k_accuracy: 81.25
+ GPQA_diamond_accuracy: 18.75
+ race-high_accuracy: 87.5
+ winogrande_accuracy: 71.88
+ qwen2.5-1.5b-turbomind:
+ gsm8k_accuracy: 62.50
+ GPQA_diamond_accuracy: 12.50
+ race-high_accuracy: 78.12
+ winogrande_accuracy: 68.75
+ qwen2.5-7b-turbomind:
+ gsm8k_accuracy: 75.00
+ GPQA_diamond_accuracy: 25
+ race-high_accuracy: 87.5
+ winogrande_accuracy: 71.88
+ qwen1.5-moe-a2.7b-hf:
+ gsm8k_accuracy: 62.5
+ GPQA_diamond_accuracy: 18.75
+ race-high_accuracy: 84.38
+ winogrande_accuracy: 75
+ qwen2-0.5b-hf:
+ gsm8k_accuracy: 25
+ GPQA_diamond_accuracy: 0
+ race-high_accuracy: 40.62
+ winogrande_accuracy: 62.5
+ qwen2-1.5b-hf:
+ gsm8k_accuracy: 59.38
+ GPQA_diamond_accuracy: 9.38
+ race-high_accuracy: 81.25
+ winogrande_accuracy: 62.5
+ qwen2-7b-hf:
+ gsm8k_accuracy: 68.75
+ GPQA_diamond_accuracy: 9.38
+ race-high_accuracy: 87.5
+ winogrande_accuracy: 68.75
+ qwen2-1.5b-turbomind:
+ gsm8k_accuracy: 56.25
+ GPQA_diamond_accuracy: 9.38
+ race-high_accuracy: 81.25
+ winogrande_accuracy: 75
+ qwen2-7b-turbomind:
+ gsm8k_accuracy: 75.00
+ GPQA_diamond_accuracy: 12.5
+ race-high_accuracy: 87.5
+ winogrande_accuracy: 71.88
+ qwen1.5-0.5b-vllm:
+ gsm8k_accuracy: 9.38
+ GPQA_diamond_accuracy: 0
+ race-high_accuracy: 56.25
+ winogrande_accuracy: 62.5
+ yi-1.5-6b-hf:
+ gsm8k_accuracy: 62.5
+ GPQA_diamond_accuracy: 3.12
+ race-high_accuracy: 87.5
+ winogrande_accuracy: 62.5
+ yi-1.5-9b-hf:
+ gsm8k_accuracy: 75
+ GPQA_diamond_accuracy: 40.62
+ race-high_accuracy: 87.5
+ winogrande_accuracy: 59.38
+ yi-1.5-9b-turbomind:
+ gsm8k_accuracy: 78.12
+ GPQA_diamond_accuracy: 40.62
+ race-high_accuracy: 87.5
+ winogrande_accuracy: 71.88
+ deepseek-v2-lite-hf:
+ gsm8k_accuracy: 31.25
+ GPQA_diamond_accuracy: 28.12
+ race-high_accuracy: 59.38
+ winogrande_accuracy: 71.88
+ internlm2-20b-hf:
+ gsm8k_accuracy: 56.25
+ GPQA_diamond_accuracy: 15.62
+ race-high_accuracy: 68.75
+ winogrande_accuracy: 75
+ internlm2-base-20b-hf:
+ gsm8k_accuracy: 12.5
+ GPQA_diamond_accuracy: 9.38
+ race-high_accuracy: 84.38
+ winogrande_accuracy: 65.62
+ internlm2-20b-turbomind:
+ gsm8k_accuracy: 71.88
+ GPQA_diamond_accuracy: 15.62
+ race-high_accuracy: 68.75
+ winogrande_accuracy: 81.25
+ qwen2.5-14b-hf:
+ gsm8k_accuracy: 75
+ GPQA_diamond_accuracy: 37.5
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 84.38
+ qwen2.5-32b-hf:
+ gsm8k_accuracy: 87.5
+ GPQA_diamond_accuracy: 31.25
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 78.12
+ qwen2.5-32b-turbomind:
+ gsm8k_accuracy: 84.38
+ GPQA_diamond_accuracy: 28.12
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 81.25
+ deepseek-67b-base-hf:
+ gsm8k_accuracy: 59.38
+ GPQA_diamond_accuracy: 31.25
+ race-high_accuracy: 81.25
+ winogrande_accuracy: 90.62
+ deepseek-67b-base-turbomind:
+ gsm8k_accuracy: 56.25
+ GPQA_diamond_accuracy: 28.12
+ race-high_accuracy: 81.25
+ winogrande_accuracy: 84.38
+ llama-3-70b-turbomind:
+ gsm8k_accuracy: 59.38
+ GPQA_diamond_accuracy: 9.38
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 84.38
+ qwen2.5-72b-turbomind:
+ gsm8k_accuracy: 84.38
+ GPQA_diamond_accuracy: 34.38
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 87.5
+ deepseek-v2-turbomind:
+ gsm8k_accuracy: 62.5
+ GPQA_diamond_accuracy: 3.12
+ race-high_accuracy: 81.25
+ winogrande_accuracy: 68.75
+ llama-3-70b-hf:
+ gsm8k_accuracy: 62.5
+ GPQA_diamond_accuracy: 3.12
+ race-high_accuracy: 93.75
+ winogrande_accuracy: 84.38
diff --git a/.github/workflows/daily-run-test.yml b/.github/workflows/daily-run-test.yml
index 1d7a1189..6c308c56 100644
--- a/.github/workflows/daily-run-test.yml
+++ b/.github/workflows/daily-run-test.yml
@@ -28,39 +28,39 @@ on:
description: 'Set branch or tag or commit id. Default is "main"'
type: string
default: 'main'
- regression_func:
+ regression_func_volc:
required: true
description: 'regression functions'
type: string
- default: "['chat_models','base_models', 'chat_obj_fullbench', 'chat_sub_fullbench', 'base_fullbench','cmd', 'api']"
- cuda_env:
+ default: "['chat_models','base_models', 'chat_obj_fullbench', 'base_fullbench']"
+ regression_func_local:
required: true
- description: "regression conda env, eg. ['dsw_cu11','dsw_cu12']"
+ description: 'regression functions'
type: string
- default: "['dsw_cu12']"
+ default: "['cmd', 'api', 'chat_sub_fullbench']"
+ fullbench_eval:
+ required: true
+ description: 'fullbench volc functions'
+ type: string
+ default: "['base_objective','chat_objective','chat_subjective','base_long_context','chat_long_context']"
schedule:
- - cron: '15 16 * * *'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
+ - cron: '15 14 * * *'
env:
- CONDA_ENV: opencompass_regression
- PIP_CACHE_PATH: /cpfs01/user/qa-llm-cicd/.cache/pip
- HF_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub
- HUGGINGFACE_HUB_CACHE: /cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub
- HF_HUB_CACHE: /cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub
- COMPASS_DATA_CACHE: /cpfs01/shared/public/llmeval/compass_data_cache
HF_DATASETS_OFFLINE: 1
HF_EVALUATE_OFFLINE: 1
TRANSFORMERS_OFFLINE: 1
VLLM_USE_MODELSCOPE: false
LMDEPLOY_USE_MODELSCOPE: false
HF_HUB_OFFLINE: 1
- TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas
- REPORT_ROOT: /cpfs01/shared/public/qa-llm-cicd/report
OUTPUT_FOLDER: cuda12.1_dist_${{ github.run_id }}
+ CONDA_PATH: /fs-computility/llm/qa-llm-cicd/miniconda3
+ PIP_CACHE_PATH: /fs-computility/llm/qa-llm-cicd/.cache/pip
+ REPORT_ROOT: /fs-computility/llm/qa-llm-cicd/eval_report/regression
+ COMPASS_DATA_CACHE: /fs-computility/llm/shared/llmeval/datasets/compass_data_cache
+ HUGGINGFACE_HUB_CACHE: /fs-computility/llm/shared/llmeval/models/opencompass_hf_hub
+ HF_HUB_CACHE: /fs-computility/llm/shared/llmeval/models/opencompass_hf_hub
+ CONDA_ENV: regression_test
jobs:
build-pypi:
@@ -124,11 +124,7 @@ jobs:
prepare_env:
if: ${{!cancelled()}}
needs: ['build-pypi', 'build-pypi-lmdeploy']
- strategy:
- fail-fast: false
- matrix:
- cuda_env: ${{ fromJSON(inputs.cuda_env || '["dsw_cu12"]')}}
- runs-on: ${{ matrix.cuda_env }}
+ runs-on: volc_cu12
environment: 'prod'
timeout-minutes: 240 #4hours
steps:
@@ -144,71 +140,52 @@ jobs:
- name: Remove Conda Env
if: always()
run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda env remove -y --name ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
+ . /fs-computility/llm/qa-llm-cicd/miniconda3/bin/activate
+ conda env remove -y --name ${{env.CONDA_ENV}}
conda info --envs
- - name: Prepare - create conda env and install torch - cu11
- if: ${{matrix.cuda_env == 'dsw_cu11'}}
- uses: nick-fields/retry@v3
- id: retry1
- with:
- max_attempts: 3
- timeout_minutes: 40
- command: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda create -y --name ${{env.CONDA_ENV}}_${{ matrix.cuda_env }} python=3.10
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
- pip install -r /cpfs01/shared/public/qa-llm-cicd/requirements-cu11.txt --cache-dir ${{env.PIP_CACHE_PATH}}
- pip install opencompass*.whl --cache-dir ${{env.PIP_CACHE_PATH}}
- pip install /cpfs01/user/qa-llm-cicd/packages/lmdeploy-0.6.1+cu118-cp310-cp310-manylinux2014_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}}
- pip install /cpfs01/user/qa-llm-cicd/packages/vllm-0.6.1.post1+cu118-cp310-cp310-manylinux1_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}}
- pip uninstall torch torchvision torchaudio -y
- pip install torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1 --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118
- FLASH_ATTENTION_FORCE_BUILD=TRUE pip install /cpfs01/user/qa-llm-cicd/packages/flash_attn-2.7.0.post2+cu11torch2.5cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
- pip install /cpfs01/user/qa-llm-cicd/packages/xformers-0.0.28.post3-cp310-cp310-manylinux_2_28_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}}
- conda info --envs
- pip list
- name: Prepare - create conda env and install torch - cu12
- if: ${{matrix.cuda_env == 'dsw_cu12'}}
uses: nick-fields/retry@v3
- id: retry2
with:
- max_attempts: 3
- timeout_minutes: 40
+ max_attempts: 1
+ timeout_minutes: 240
command: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda create -y --name ${{env.CONDA_ENV}}_${{ matrix.cuda_env }} python=3.10
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
- pip install -r /cpfs01/shared/public/qa-llm-cicd/requirements-cu12.txt --cache-dir ${{env.PIP_CACHE_PATH}}
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda create -y --name ${{env.CONDA_ENV}} python=3.10
+ conda activate ${{env.CONDA_ENV}}
+ pip install -r /fs-computility/llm/qa-llm-cicd/config/requirements.txt --cache-dir ${{env.PIP_CACHE_PATH}}
pip install opencompass*.whl --cache-dir ${{env.PIP_CACHE_PATH}}
pip install opencompass[lmdeploy] --cache-dir ${{env.PIP_CACHE_PATH}}
pip install opencompass[vllm] --cache-dir ${{env.PIP_CACHE_PATH}}
pip install torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1 --cache-dir ${{env.PIP_CACHE_PATH}}
- FLASH_ATTENTION_FORCE_BUILD=TRUE pip install /cpfs01/user/qa-llm-cicd/packages/flash_attn-2.7.0.post2+cu12torch2.5cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
- pip install /cpfs01/user/qa-llm-cicd/packages/xformers-0.0.28.post3-cp310-cp310-manylinux_2_28_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}}
- conda info --envs
- pip list
+ FLASH_ATTENTION_FORCE_BUILD=TRUE pip install /fs-computility/llm/qa-llm-cicd/packages/flash_attn-2.7.0.post2+cu12torch2.5cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
+ pip install xformers --index-url https://download.pytorch.org/whl/cu121 --cache-dir ${{env.PIP_CACHE_PATH}}
+ cp -r /root/nltk_data ${{env.CONDA_PATH}}/envs/${{env.CONDA_ENV}}/nltk_data
- name: Prepare - reinstall lmdeploy - cu12
- if: ${{matrix.cuda_env == 'dsw_cu12' && inputs.build_lmdeploy}}
+ if: ${{inputs.build_lmdeploy}}
uses: actions/download-artifact@v4
with:
name: my-artifact-${{ github.run_id }}-py310
- name: Prepare - reinstall lmdeploy - cu12
- if: ${{matrix.cuda_env == 'dsw_cu12' && inputs.build_lmdeploy}}
+ if: ${{inputs.build_lmdeploy}}
run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
pip install lmdeploy-*.whl --no-deps
+ - name: conda env
+ run: |
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
+ conda info --envs
+ pip list
- daily_run_test:
+ daily_run_test_volc:
if: ${{!cancelled()}}
needs: prepare_env
strategy:
fail-fast: false
matrix:
- cuda_env: ${{ fromJSON(inputs.cuda_env || '["dsw_cu12"]')}}
- regression_func: ${{fromJSON(github.event.inputs.regression_func || '["chat_models","base_models","chat_obj_fullbench","chat_sub_fullbench","base_fullbench","cmd","api"]')}}
- runs-on: ${{ matrix.cuda_env }}
+ regression_func: ${{fromJSON(github.event.inputs.regression_func_volc || '["chat_models","base_models","chat_obj_fullbench","base_fullbench"]')}}
+ runs-on: volc_cu12_daily
environment: 'prod'
timeout-minutes: 240 #4hours
steps:
@@ -217,97 +194,147 @@ jobs:
with:
repository: ${{ github.event.inputs.repo_org || 'open-compass/opencompass' }}
ref: ${{github.event.inputs.repo_ref || 'main'}}
- - name: Prepare - prepare data and hf model
+ - name: conda env
run: |
- rm -rf ~/.cache/huggingface/hub -f && mkdir ~/.cache -p && mkdir ~/.cache/huggingface -p
- ln -s ${{env.HF_CACHE_PATH}} ~/.cache/huggingface/hub
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
+ conda info --envs
+ pip list
+ - name: modify config
+ if: matrix.regression_func != 'chat_sub_fullbench'
+ run: |
+ cp -r /fs-computility/llm/qa-llm-cicd/ocplayground/template/configs_cluster/volc.py .
+ cat /fs-computility/llm/qa-llm-cicd/config/test_config.txt >> .github/scripts/eval_regression_${{matrix.regression_func}}.py
+ - name: Run test
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 1
+ timeout_minutes: 120
+ command: |
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
+ conda info --envs
+ opencompass .github/scripts/eval_regression_${{matrix.regression_func}}.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/${{matrix.regression_func}} --reuse --dump-eval-details
+ rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/${{matrix.regression_func}}/*/summary regression_result_daily
+ python -m pytest -m ${{matrix.regression_func}} -s -v --color=yes .github/scripts/oc_score_assert.py
+
+
+ daily_run_test_local:
+ if: ${{!cancelled()}}
+ needs: prepare_env
+ strategy:
+ fail-fast: false
+ matrix:
+ regression_func: ${{fromJSON(github.event.inputs.regression_func_local || '["cmd","api","chat_sub_fullbench"]')}}
+ runs-on: volc_cu12_local
+ environment: 'prod'
+ timeout-minutes: 240 #4hours
+ steps:
+ - name: Clone repository
+ uses: actions/checkout@v2
+ with:
+ repository: ${{ github.event.inputs.repo_org || 'open-compass/opencompass' }}
+ ref: ${{github.event.inputs.repo_ref || 'main'}}
+ - name: conda env
+ run: |
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
+ conda info --envs
+ pip list
+ - name: modify config
+ if: matrix.regression_func == 'chat_sub_fullbench'
+ run: |
+ cp -r /fs-computility/llm/qa-llm-cicd/ocplayground/template/configs_cluster/volc.py .
+ cat /fs-computility/llm/qa-llm-cicd/config/test_config_sub.txt >> .github/scripts/eval_regression_${{matrix.regression_func}}.py
- name: Run command testcase
if: matrix.regression_func == 'cmd'
run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
conda info --envs
export from_tf=TRUE
python tools/list_configs.py internlm2_5 mmlu
- opencompass --models hf_internlm2_5_7b hf_internlm2_1_8b --datasets race_ppl demo_gsm8k_chat_gen --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd1_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd1_${{ matrix.cuda_env }}/*/summary regression_result_daily
+ opencompass --models hf_internlm2_5_7b hf_internlm2_1_8b --datasets race_ppl demo_gsm8k_chat_gen --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd1 --reuse --max-num-workers 2 --dump-eval-details
+ rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd1/*/summary regression_result_daily
python -m pytest -m case1 -s -v --color=yes .github/scripts/oc_score_assert.py
- opencompass --models hf_internlm2_5_7b_chat hf_internlm2_chat_1_8b --datasets race_gen demo_gsm8k_chat_gen -a lmdeploy --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd2_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd2_${{ matrix.cuda_env }}/*/summary regression_result_daily
+ opencompass --models hf_internlm2_5_7b_chat hf_internlm2_chat_1_8b --datasets race_gen demo_gsm8k_chat_gen -a lmdeploy --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd2 --reuse --max-num-workers 2 --dump-eval-details
+ rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd2/*/summary regression_result_daily
python -m pytest -m case2 -s -v --color=yes .github/scripts/oc_score_assert.py
- opencompass --datasets race_ppl demo_gsm8k_chat_gen --hf-type base --hf-path internlm/internlm2_5-7b --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd3_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd3_${{ matrix.cuda_env }}/*/summary regression_result_daily
+ opencompass --datasets race_ppl demo_gsm8k_chat_gen --hf-type base --hf-path internlm/internlm2_5-7b --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd3 --reuse --max-num-workers 2 --dump-eval-details
+ rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd3/*/summary regression_result_daily
python -m pytest -m case3 -s -v --color=yes .github/scripts/oc_score_assert.py
- opencompass --datasets race_gen demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-7b-chat --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd4_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd4_${{ matrix.cuda_env }}/*/summary regression_result_daily
+ opencompass --datasets race_gen demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-7b-chat --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd4 --reuse --max-num-workers 2 --dump-eval-details
+ rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/cmd4/*/summary regression_result_daily
python -m pytest -m case4 -s -v --color=yes .github/scripts/oc_score_assert.py
- - name: Run chat model test
- if: matrix.regression_func == 'chat_models'
- run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
- conda info --envs
- opencompass .github/scripts/eval_regression_chat.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/chat_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/chat_${{ matrix.cuda_env }}/*/summary regression_result_daily
- python -m pytest -m chat -s -v --color=yes .github/scripts/oc_score_assert.py
- - name: Run base model test
- if: matrix.regression_func == 'base_models'
- run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
- conda info --envs
- opencompass .github/scripts/eval_regression_base.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/base_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/base_${{ matrix.cuda_env }}/*/summary regression_result_daily
- python -m pytest -m base -s -v --color=yes .github/scripts/oc_score_assert.py
- - name: Run chat model test - fullbench
- if: matrix.regression_func == 'chat_obj_fullbench'
- run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
- conda info --envs
- opencompass .github/scripts/eval_regression_chat_objective_fullbench.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/chat_obj_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/chat_obj_${{ matrix.cuda_env }}/*/summary regression_result_daily
- python -m pytest -m chat_obj_fullbench -s -v --color=yes .github/scripts/oc_score_assert.py
- - name: Run chat model test - fullbench
- if: matrix.regression_func == 'chat_sub_fullbench'
- env:
- COMPASS_DATA_CACHE: /cpfs01/shared/public/llmeval/compass_data_cache_subset
- run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
- conda info --envs
- opencompass .github/scripts/eval_regression_chat_subjective_fullbench.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/chat_sub_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/chat_sub_${{ matrix.cuda_env }}/*/summary regression_result_daily
- python -m pytest -m chat_sub_fullbench -s -v --color=yes .github/scripts/oc_score_assert.py
- - name: Run base model test - fullbench
- if: matrix.regression_func == 'base_fullbench'
- run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
- conda info --envs
- opencompass .github/scripts/eval_regression_base_fullbench.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/base_full_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/base_full_${{ matrix.cuda_env }}/*/summary regression_result_daily
- python -m pytest -m base_fullbench -s -v --color=yes .github/scripts/oc_score_assert.py
- name: Run model test - api
if: matrix.regression_func == 'api'
run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}_${{ matrix.cuda_env }}
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
conda info --envs
lmdeploy serve api_server internlm/internlm2_5-7b-chat --max-batch-size 256 --model-name internlm2 > ${{env.REPORT_ROOT}}/${{ github.run_id }}/restful.log 2>&1 &
echo "restful_pid=$!" >> "$GITHUB_ENV"
sleep 120s
- opencompass .github/scripts/eval_regression_api.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/api_${{ matrix.cuda_env }} --reuse --max-num-workers 2 --dump-eval-details
- rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/api_${{ matrix.cuda_env }}/*/summary regression_result_daily
+ opencompass .github/scripts/eval_regression_api.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/api --reuse --max-num-workers 2 --dump-eval-details
+ rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/api/*/summary regression_result_daily
python -m pytest -m api -s -v --color=yes .github/scripts/oc_score_assert.py
- name: Run model test - api kill
if: always() && matrix.regression_func == 'api'
run: |
kill -15 "$restful_pid"
+ - name: Run testcase
+ if: matrix.regression_func == 'chat_sub_fullbench'
+ env:
+ COMPASS_DATA_CACHE: /fs-computility/llm/shared/llmeval/datasets/compass_data_cache_subset
+ run: |
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
+ conda info --envs
+ export from_tf=TRUE
+ opencompass .github/scripts/eval_regression_${{matrix.regression_func}}.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/${{matrix.regression_func}} --reuse --dump-eval-details
+ rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/${{matrix.regression_func}}/*/summary regression_result_daily
+ python -m pytest -m ${{matrix.regression_func}} -s -v --color=yes .github/scripts/oc_score_assert.py
+
+ fullbench_run_test:
+ if: ${{!cancelled()}}
+ needs: prepare_env
+ strategy:
+ fail-fast: false
+ matrix:
+ function_type: ${{fromJSON(github.event.inputs.fullbench_eval || '["base_objective","chat_objective","chat_subjective","base_long_context","chat_long_context"]')}}
+ runs-on: volc_cu12
+ environment: 'prod'
+ timeout-minutes: 360 #6hours
+ steps:
+ - name: Clone repository
+ uses: actions/checkout@v2
+ with:
+ repository: ${{ github.event.inputs.repo_org || 'open-compass/opencompass' }}
+ ref: ${{github.event.inputs.repo_ref || 'main'}}
+ - name: conda env
+ run: |
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
+ conda info --envs
+ pip list
+ - name: Run testcase
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 1
+ timeout_minutes: 240
+ command: |
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
+ conda info --envs
+ export from_tf=TRUE
+ opencompass /fs-computility/llm/qa-llm-cicd/ocplayground/template/regression/eval_${{ matrix.function_type }}.py --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/${{ matrix.function_type }} --reuse
+ rm regression_result_daily -f && ln -s ${{env.REPORT_ROOT}}/${{ github.run_id }}/${{ matrix.function_type }}/*/summary regression_result_daily
+ python -m pytest -m ${{ matrix.function_type }} -s -v --color=yes .github/scripts/oc_score_assert.py
+
notify_to_feishu:
if: ${{ always() && !cancelled() && contains(needs.*.result, 'failure') && (github.ref_name == 'develop' || github.ref_name == 'main') }}
- needs: [daily_run_test]
+ needs: [daily_run_test_volc, daily_run_test_local, fullbench_run_test]
environment: 'prod'
timeout-minutes: 5
runs-on: self-hosted
diff --git a/.github/workflows/pr-run-test.yml b/.github/workflows/pr-run-test.yml
index 561ef750..dfd9a7e5 100644
--- a/.github/workflows/pr-run-test.yml
+++ b/.github/workflows/pr-run-test.yml
@@ -18,18 +18,23 @@ concurrency:
cancel-in-progress: true
env:
- CONDA_ENV: opencompass_
- USERSPACE_PREFIX: /cpfs01/user/qa-llm-cicd
- HF_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub
+ CONDA_ENV: pr_test
HF_DATASETS_OFFLINE: 1
+ HF_EVALUATE_OFFLINE: 1
TRANSFORMERS_OFFLINE: 1
- HF_HUB_OFFLINE: 1
VLLM_USE_MODELSCOPE: false
LMDEPLOY_USE_MODELSCOPE: false
+ HF_HUB_OFFLINE: 1
+ CONDA_PATH: /fs-computility/llm/qa-llm-cicd/miniconda3
+ PIP_CACHE_PATH: /fs-computility/llm/qa-llm-cicd/.cache/pip
+ REPORT_ROOT: /fs-computility/llm/qa-llm-cicd/eval_report/prtest
+ COMPASS_DATA_CACHE: /fs-computility/llm/shared/llmeval/datasets/compass_data_cache
+ HUGGINGFACE_HUB_CACHE: /fs-computility/llm/shared/llmeval/models/opencompass_hf_hub
+ HF_HUB_CACHE: /fs-computility/llm/shared/llmeval/models/opencompass_hf_hub
jobs:
pr_run_test:
- runs-on: self-hosted
+ runs-on: volc_cu12_local
environment: 'prod'
timeout-minutes: 30
steps:
@@ -37,54 +42,55 @@ jobs:
uses: actions/checkout@v2
- name: Prepare - Install opencompass
run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}${{ runner.name }}
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
python3 -m pip uninstall opencompass -y
- python3 -m pip install -e . --cache-dir ${{env.USERSPACE_PREFIX}}/.cache/pip
+ python3 -m pip install -e . --cache-dir ${{env.PIP_CACHE_PATH}}
conda info --envs
- - name: Prepare - prepare data and hf model
+ - name: conda env
run: |
- cp -r ${{env.USERSPACE_PREFIX}}/data .
- rm -rf ~/.cache/huggingface/hub -f && mkdir ~/.cache -p && mkdir ~/.cache/huggingface -p
- ln -s ${{env.HF_CACHE_PATH}} ~/.cache/huggingface/hub
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
+ conda info --envs
+ pip list
+ lmdeploy check_env
- name: Run test
run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}${{ runner.name }}
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
conda info --envs
rm -rf regression_result
- opencompass --models hf_internlm2_5_20b_chat --datasets demo_gsm8k_chat_gen --work-dir regression_result1 --debug
- opencompass --models hf_internlm2_5_7b_chat --datasets demo_gsm8k_chat_gen --work-dir regression_result2 --debug --max-num-workers 2
- opencompass --models hf_internlm2_5_7b_chat --datasets demo_gsm8k_chat_gen -a lmdeploy --work-dir regression_result3 --debug --max-num-workers 2
+ opencompass --models hf_internlm2_5_20b_chat --datasets demo_gsm8k_chat_gen --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/regression_result1 --debug
+ opencompass --models hf_internlm2_5_7b_chat --datasets demo_gsm8k_chat_gen --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/regression_result2 --debug --max-num-workers 2
+ opencompass --models hf_internlm2_5_7b_chat --datasets demo_gsm8k_chat_gen -a lmdeploy --work-dir ${{env.REPORT_ROOT}}/${{ github.run_id }}/regression_result3 --debug --max-num-workers 2
- name: Get result
run: |
- score=$(sed -n '$p' regression_result1/*/summary/*.csv | awk -F ',' '{print $NF}')
+ score=$(sed -n '$p' ${{env.REPORT_ROOT}}/${{ github.run_id }}/regression_result1/*/summary/*.csv | awk -F ',' '{print $NF}')
if (( ${score%.*} >= 88 && ${score%.*} <= 89 )); then
echo "score is $score between 88 and 89"
else
echo "score is $score not between 88 and 89"
exit 1
fi
- score=$(sed -n '$p' regression_result2/*/summary/*.csv | awk -F ',' '{print $NF}')
+ score=$(sed -n '$p' ${{env.REPORT_ROOT}}/${{ github.run_id }}/regression_result2/*/summary/*.csv | awk -F ',' '{print $NF}')
if (( ${score%.*} >= 87 && ${score%.*} <= 88 )); then
echo "score is $score between 87 and 88"
else
echo "score is $score not between 87 and 88"
exit 1
fi
- score=$(sed -n '$p' regression_result3/*/summary/*.csv | awk -F ',' '{print $NF}')
- if (( ${score%.*} >= 87 && ${score%.*} <= 89 )); then
- echo "score is $score between 87 and 89"
+ score=$(sed -n '$p' ${{env.REPORT_ROOT}}/${{ github.run_id }}/regression_result3/*/summary/*.csv | awk -F ',' '{print $NF}')
+ if (( ${score%.*} >= 87 && ${score%.*} <= 91 )); then
+ echo "score is $score between 87 and 91"
else
- echo "score is $score not between 87 and 89"
+ echo "score is $score not between 87 and 91"
exit 1
fi
- rm -rf regression_result1 & rm -rf regression_result2 & rm -rf regression_result3
- name: Uninstall opencompass
if: always()
run: |
- . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate
- conda activate ${{env.CONDA_ENV}}${{ runner.name }}
+ . ${{env.CONDA_PATH}}/bin/activate
+ conda activate ${{env.CONDA_ENV}}
python3 -m pip uninstall opencompass -y
conda info --envs
diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml
index e8c7c826..e59abbca 100644
--- a/.github/workflows/publish-to-pypi.yml
+++ b/.github/workflows/publish-to-pypi.yml
@@ -1,21 +1,26 @@
name: deploy
-on: push
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
+on:
+ push:
+ workflow_dispatch:
+ inputs:
+ confirm_publish:
+ description: 'Type YES to confirm publishing to PyPI'
+ required: true
+ type: string
jobs:
build-n-publish:
runs-on: ubuntu-latest
- if: startsWith(github.event.ref, 'refs/tags')
+ if: |
+ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') ||
+ (github.event_name == 'workflow_dispatch' && inputs.confirm_publish == 'YES')
steps:
- uses: actions/checkout@v2
- - name: Set up Python 3.7
- uses: actions/setup-python@v1
+ - name: Set up Python 3.10
+ uses: actions/setup-python@v4
with:
- python-version: 3.7
+ python-version: '3.10'
- name: Build lagent
run: |
pip install wheel
diff --git a/README.md b/README.md
index 1a65dfb5..6d8cabe5 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,8 @@ Just like a compass guides us on our journey, OpenCompass will guide you through
We provide [OpenCompass Leaderboard](https://rank.opencompass.org.cn/home) for the community to rank all public models and API models. If you would like to join the evaluation, please provide the model repository URL or a standard API interface to the email address `opencompass@pjlab.org.cn`.
+You can also refer to [CompassAcademic](configs/eval_academic_leaderboard_202412.py) to quickly reproduce the leaderboard results. The currently selected datasets include Knowledge Reasoning (MMLU-Pro/GPQA Diamond), Logical Reasoning (BBH), Mathematical Reasoning (MATH-500, AIME), Code Generation (LiveCodeBench, HumanEval), and Instruction Following (IFEval)."
+
🔝Back to top
## 🛠️ Installation
diff --git a/README_zh-CN.md b/README_zh-CN.md
index e2ec3f0c..21c0d666 100644
--- a/README_zh-CN.md
+++ b/README_zh-CN.md
@@ -77,6 +77,8 @@
我们将陆续提供开源模型和 API 模型的具体性能榜单，请见 [OpenCompass Leaderboard](https://rank.opencompass.org.cn/home) 。如需加入评测，请提供模型仓库地址或标准的 API 接口至邮箱 `opencompass@pjlab.org.cn`.
+你也可以参考[CompassAcademic](configs/eval_academic_leaderboard_202412.py)，快速地复现榜单的结果，目前选取的数据集包括 综合知识推理 (MMLU-Pro/GPQA Diamond) ,逻辑推理 (BBH) ,数学推理 (MATH-500, AIME) ,代码生成 (LiveCodeBench, HumanEval) ,指令跟随 (IFEval) 。
+
🔝返回顶部
## 🛠️ 安装指南
diff --git a/configs/datasets/ruler/ruler_128k_gen.py b/configs/datasets/ruler/ruler_128k_gen.py
index 4f302ad2..02dea12a 100644
--- a/configs/datasets/ruler/ruler_128k_gen.py
+++ b/configs/datasets/ruler/ruler_128k_gen.py
@@ -1,3 +1,5 @@
+import os
+
from mmengine.config import read_base
with read_base():
@@ -12,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 128]
abbr_suffixs = ['128k']
@@ -25,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/configs/datasets/ruler/ruler_16k_gen.py b/configs/datasets/ruler/ruler_16k_gen.py
index faab3cca..e6841fff 100644
--- a/configs/datasets/ruler/ruler_16k_gen.py
+++ b/configs/datasets/ruler/ruler_16k_gen.py
@@ -1,3 +1,4 @@
+import os
from mmengine.config import read_base
@@ -13,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 16]
abbr_suffixs = ['16k']
@@ -26,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/configs/datasets/ruler/ruler_1m_gen.py b/configs/datasets/ruler/ruler_1m_gen.py
index 0b20375c..bf4aa75e 100644
--- a/configs/datasets/ruler/ruler_1m_gen.py
+++ b/configs/datasets/ruler/ruler_1m_gen.py
@@ -1,3 +1,4 @@
+import os
from mmengine.config import read_base
@@ -13,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 1024]
abbr_suffixs = ['1m']
@@ -26,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/configs/datasets/ruler/ruler_32k_gen.py b/configs/datasets/ruler/ruler_32k_gen.py
index ab02cb4a..1d0ac63a 100644
--- a/configs/datasets/ruler/ruler_32k_gen.py
+++ b/configs/datasets/ruler/ruler_32k_gen.py
@@ -1,3 +1,4 @@
+import os
from mmengine.config import read_base
@@ -13,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 32]
abbr_suffixs = ['32k']
@@ -26,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/configs/datasets/ruler/ruler_4k_gen.py b/configs/datasets/ruler/ruler_4k_gen.py
index f0031507..75186b41 100644
--- a/configs/datasets/ruler/ruler_4k_gen.py
+++ b/configs/datasets/ruler/ruler_4k_gen.py
@@ -1,3 +1,5 @@
+import os
+
from mmengine.config import read_base
with read_base():
@@ -12,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 4]
abbr_suffixs = ['4k']
@@ -25,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/configs/datasets/ruler/ruler_64k_gen.py b/configs/datasets/ruler/ruler_64k_gen.py
index 709260d6..8ba25c1e 100644
--- a/configs/datasets/ruler/ruler_64k_gen.py
+++ b/configs/datasets/ruler/ruler_64k_gen.py
@@ -1,3 +1,5 @@
+import os
+
from mmengine.config import read_base
with read_base():
@@ -12,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 64]
abbr_suffixs: list[str] = ['64k']
@@ -25,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/configs/datasets/ruler/ruler_8k_gen.py b/configs/datasets/ruler/ruler_8k_gen.py
index 8c9a4ad8..bcbfb533 100644
--- a/configs/datasets/ruler/ruler_8k_gen.py
+++ b/configs/datasets/ruler/ruler_8k_gen.py
@@ -1,3 +1,4 @@
+import os
from mmengine.config import read_base
@@ -13,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 8]
abbr_suffixs = ['8k']
@@ -26,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/configs/datasets/ruler/ruler_niah_gen.py b/configs/datasets/ruler/ruler_niah_gen.py
index bb6e79a5..b338b03d 100644
--- a/configs/datasets/ruler/ruler_niah_gen.py
+++ b/configs/datasets/ruler/ruler_niah_gen.py
@@ -1,9 +1,7 @@
+from opencompass.datasets.ruler.ruler_niah import RulerNiahDataset, RulerNiahEvaluator
+from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
-from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets.ruler.ruler_niah import RulerNiahDataset
-from opencompass.datasets.ruler.ruler_niah import RulerNiahEvaluator
-
# Ruler Dataset settings
niah_configurations = [
@@ -92,10 +90,7 @@ for index, config in enumerate(niah_configurations):
'type': RulerNiahDataset,
'base_path': base_path,
'file_path': file_path,
- # 'tokenizer_model': model_path,
'tokens_to_generate': 128,
- # 'max_seq_length': max_seq_len,
- # 'num_samples': NUM_SAMPLES,
'type_haystack': config['type_haystack'],
'type_needle_k': config['type_needle_k'],
'type_needle_v': config['type_needle_v'],
diff --git a/configs/eval_academic_leaderboard_202412.py b/configs/eval_academic_leaderboard_202412.py
index 0a9e19a5..378a462c 100644
--- a/configs/eval_academic_leaderboard_202412.py
+++ b/configs/eval_academic_leaderboard_202412.py
@@ -10,12 +10,10 @@ from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
#######################################################################
with read_base():
# Datasets Part
- ## Core Set
# Knowledge
from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_cot_gen_08c1de import (
mmlu_pro_datasets,
)
-
# General Reasoning
from opencompass.configs.datasets.gpqa.gpqa_openai_simple_evals_gen_5aeece import (
gpqa_datasets,
@@ -23,22 +21,19 @@ with read_base():
from opencompass.configs.datasets.bbh.bbh_0shot_nocot_gen_925fc4 import (
bbh_datasets,
)
- from opencompass.configs.datasets.humaneval.humaneval_openai_sample_evals_gen_159614 import (
+ from opencompass.configs.datasets.humaneval.humaneval_openai_sample_evals_gen_dcae0e import (
humaneval_datasets,
)
# Instruction Following
- from opencompass.configs.datasets.IFEval.IFEval_gen_3321a3 import (
+ from opencompass.configs.datasets.IFEval.IFEval_gen_353ae7 import (
ifeval_datasets,
)
- from opencompass.configs.datasets.livecodebench.livecodebench_gen_6966bc import (
+ from opencompass.configs.datasets.livecodebench.livecodebench_gen_a4f90b import (
LCBCodeGeneration_dataset,
)
# Math
- from opencompass.configs.datasets.cmo_fib.cmo_fib_gen_ace24b import (
- cmo_fib_datasets,
- )
from opencompass.configs.datasets.aime2024.aime2024_gen_6e39a4 import (
aime2024_datasets,
)
@@ -77,7 +72,6 @@ core_summary_groups = [
['IFEval', 'Prompt-level-strict-accuracy'],
['bbh', 'naive_average'],
['math_prm800k_500', 'accuracy'],
- ['cmo_fib', 'accuracy'],
['aime2024', 'accuracy'],
['GPQA_diamond', 'accuracy'],
['mmlu_pro', 'naive_average'],
@@ -101,7 +95,6 @@ summarizer = dict(
'',
'Math Calculation',
['math_prm800k_500', 'accuracy'],
- ['cmo_fib', 'accuracy'],
['aime2024', 'accuracy'],
'',
'Knowledge',
diff --git a/opencompass/configs/datasets/livecodebench/livecodebench_gen_a4f90b.py b/opencompass/configs/datasets/livecodebench/livecodebench_gen_a4f90b.py
new file mode 100644
index 00000000..ceb3514f
--- /dev/null
+++ b/opencompass/configs/datasets/livecodebench/livecodebench_gen_a4f90b.py
@@ -0,0 +1,164 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+ LCBCodeGenerationDataset,
+ LCBCodeExecutionDataset,
+ LCBTestOutputPredictionDataset,
+ LCBCodeGenerationEvaluator,
+ LCBCodeExecutionEvaluator,
+ LCBTestOutputEvaluator
+)
+from opencompass.datasets.livecodebench import TestOutputPromptConstants
+
+
+lcb_code_generation_reader_cfg = dict(
+ input_columns=[
+ 'question_content',
+ 'format_prompt',
+ ],
+ # output_column='evaluation_sample',
+ output_column='question_id',
+)
+
+SYSTEM_MESSAGE_GENERIC = f'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.'
+
+prompt_template = '### Question:\n{question_content}\n\n{format_prompt}' + \
+ '### Answer: (use the provided format with backticks)\n\n'
+
+
+# Code Generation Tasks
+lcb_code_generation_infer_cfg = dict(
+ prompt_template=dict(
+ type=PromptTemplate,
+ template=dict(
+ round=[
+ dict(
+ role='HUMAN',
+ prompt=prompt_template
+ )
+ ]
+ )
+ ),
+ retriever=dict(type=ZeroRetriever),
+ inferencer=dict(type=GenInferencer)
+)
+
+lcb_code_generation_eval_cfg = dict(
+ evaluator=dict(
+ type=LCBCodeGenerationEvaluator,
+ num_process_evaluate=4,
+ timeout=6,
+ ),
+ pred_role='BOT',
+)
+
+LCBCodeGeneration_dataset = dict(
+ type=LCBCodeGenerationDataset,
+ abbr='lcb_code_generation',
+ path='opencompass/code_generation_lite',
+ reader_cfg=lcb_code_generation_reader_cfg,
+ infer_cfg=lcb_code_generation_infer_cfg,
+ eval_cfg=lcb_code_generation_eval_cfg
+)
+
+# Code Execution Dataset
+lcb_code_execution_reader_cfg = dict(
+ input_columns=[
+ 'prompt',
+ ],
+ output_column='evaluation_sample',
+)
+
+lcb_code_execution_infer_cfg = dict(
+ prompt_template=dict(
+ type=PromptTemplate,
+ template=dict(
+ begin=[
+ dict(
+ role='SYSTEM',
+ fallback_role='HUMAN',
+ prompt='You are an expert at Python programming, code execution, test case generation, and fuzzing.'
+ ),
+ ],
+ round=[
+ dict(
+ role='HUMAN',
+ prompt='{prompt}'
+ )
+ ]
+ )
+ ),
+ retriever=dict(type=ZeroRetriever),
+ inferencer=dict(type=GenInferencer)
+)
+
+lcb_code_execution_eval_cfg = dict(
+ evaluator=dict(
+ type=LCBCodeExecutionEvaluator,
+ ),
+ pred_role='BOT',
+)
+
+LCBCodeExecution_dataset = dict(
+ type=LCBCodeExecutionDataset,
+ abbr='lcb_code_execution',
+ path='opencompass/execution-v2',
+ reader_cfg=lcb_code_execution_reader_cfg,
+ infer_cfg=lcb_code_execution_infer_cfg,
+ eval_cfg=lcb_code_execution_eval_cfg,
+)
+
+# TestOuputput Dataset
+lcb_test_output_reader_cfg = dict(
+ input_columns=[
+ 'prompt',
+ ],
+ output_column='evaluation_sample',
+)
+
+system_prompt = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.'
+
+lcb_test_output_infer_cfg = dict(
+ prompt_template=dict(
+ type=PromptTemplate,
+ template=dict(
+ # begin=[
+ # dict(
+ # role='SYSTEM',
+ # prompt=system_prompt
+ # ),
+ # ],
+ round=[
+ dict(
+ role='HUMAN',
+ prompt='{prompt}'
+ )
+ ]
+ )
+ ),
+ retriever=dict(type=ZeroRetriever),
+ inferencer=dict(type=GenInferencer)
+)
+
+lcb_test_output_eval_cfg = dict(
+ evaluator=dict(
+ type=LCBTestOutputEvaluator,
+ ),
+ pred_role='BOT',
+)
+
+LCBTestOutput_dataset = dict(
+ type=LCBTestOutputPredictionDataset,
+ abbr='lcb_test_output',
+ path='opencompass/test_generation',
+ reader_cfg=lcb_test_output_reader_cfg,
+ infer_cfg=lcb_test_output_infer_cfg,
+ eval_cfg=lcb_test_output_eval_cfg,
+)
+
+LCB_datasets = [
+ LCBCodeGeneration_dataset,
+ LCBCodeExecution_dataset,
+ LCBTestOutput_dataset,
+]
diff --git a/opencompass/configs/datasets/math/math_prm800k_500_0shot_cot_gen.py b/opencompass/configs/datasets/math/math_prm800k_500_0shot_cot_gen.py
index d4690bb3..6c71a60f 100644
--- a/opencompass/configs/datasets/math/math_prm800k_500_0shot_cot_gen.py
+++ b/opencompass/configs/datasets/math/math_prm800k_500_0shot_cot_gen.py
@@ -23,7 +23,7 @@ math_infer_cfg = dict(
),
),
retriever=dict(type=ZeroRetriever),
- inferencer=dict(type=GenInferencer, max_out_len=1024),
+ inferencer=dict(type=GenInferencer),
)
# postprocess v2
diff --git a/opencompass/configs/datasets/ruler/ruler_128k_gen.py b/opencompass/configs/datasets/ruler/ruler_128k_gen.py
index 4f302ad2..02dea12a 100644
--- a/opencompass/configs/datasets/ruler/ruler_128k_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_128k_gen.py
@@ -1,3 +1,5 @@
+import os
+
from mmengine.config import read_base
with read_base():
@@ -12,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 128]
abbr_suffixs = ['128k']
@@ -25,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/opencompass/configs/datasets/ruler/ruler_16k_gen.py b/opencompass/configs/datasets/ruler/ruler_16k_gen.py
index faab3cca..e6841fff 100644
--- a/opencompass/configs/datasets/ruler/ruler_16k_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_16k_gen.py
@@ -1,3 +1,4 @@
+import os
from mmengine.config import read_base
@@ -13,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 16]
abbr_suffixs = ['16k']
@@ -26,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/opencompass/configs/datasets/ruler/ruler_1m_gen.py b/opencompass/configs/datasets/ruler/ruler_1m_gen.py
index 0b20375c..bf4aa75e 100644
--- a/opencompass/configs/datasets/ruler/ruler_1m_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_1m_gen.py
@@ -1,3 +1,4 @@
+import os
from mmengine.config import read_base
@@ -13,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 1024]
abbr_suffixs = ['1m']
@@ -26,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/opencompass/configs/datasets/ruler/ruler_32k_gen.py b/opencompass/configs/datasets/ruler/ruler_32k_gen.py
index ab02cb4a..1d0ac63a 100644
--- a/opencompass/configs/datasets/ruler/ruler_32k_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_32k_gen.py
@@ -1,3 +1,4 @@
+import os
from mmengine.config import read_base
@@ -13,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 32]
abbr_suffixs = ['32k']
@@ -26,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/opencompass/configs/datasets/ruler/ruler_4k_gen.py b/opencompass/configs/datasets/ruler/ruler_4k_gen.py
index f0031507..75186b41 100644
--- a/opencompass/configs/datasets/ruler/ruler_4k_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_4k_gen.py
@@ -1,3 +1,5 @@
+import os
+
from mmengine.config import read_base
with read_base():
@@ -12,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 4]
abbr_suffixs = ['4k']
@@ -25,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/opencompass/configs/datasets/ruler/ruler_64k_gen.py b/opencompass/configs/datasets/ruler/ruler_64k_gen.py
index 709260d6..8ba25c1e 100644
--- a/opencompass/configs/datasets/ruler/ruler_64k_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_64k_gen.py
@@ -1,3 +1,5 @@
+import os
+
from mmengine.config import read_base
with read_base():
@@ -12,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 64]
abbr_suffixs: list[str] = ['64k']
@@ -25,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/opencompass/configs/datasets/ruler/ruler_8k_gen.py b/opencompass/configs/datasets/ruler/ruler_8k_gen.py
index 8c9a4ad8..bcbfb533 100644
--- a/opencompass/configs/datasets/ruler/ruler_8k_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_8k_gen.py
@@ -1,3 +1,4 @@
+import os
from mmengine.config import read_base
@@ -13,6 +14,7 @@ import_ds = sum((cwe, fwe, niah, qa, vt), [])
# Evaluation config
NUM_SAMPLES = 100 # Change to the number of samples you need
+tokenizer_model = os.environ.get('TOKENIZER_MODEL', 'gpt-4')
# Change the context lengths to be tested
max_seq_lens = [1024 * 8]
abbr_suffixs = ['8k']
@@ -26,4 +28,5 @@ for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES
tmp_dataset['max_seq_length'] = max_seq_len
+ tmp_dataset['tokenizer_model'] = tokenizer_model
ruler_datasets.append(tmp_dataset)
diff --git a/opencompass/configs/datasets/ruler/ruler_niah_gen.py b/opencompass/configs/datasets/ruler/ruler_niah_gen.py
index bb6e79a5..b338b03d 100644
--- a/opencompass/configs/datasets/ruler/ruler_niah_gen.py
+++ b/opencompass/configs/datasets/ruler/ruler_niah_gen.py
@@ -1,9 +1,7 @@
+from opencompass.datasets.ruler.ruler_niah import RulerNiahDataset, RulerNiahEvaluator
+from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
-from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets.ruler.ruler_niah import RulerNiahDataset
-from opencompass.datasets.ruler.ruler_niah import RulerNiahEvaluator
-
# Ruler Dataset settings
niah_configurations = [
@@ -92,10 +90,7 @@ for index, config in enumerate(niah_configurations):
'type': RulerNiahDataset,
'base_path': base_path,
'file_path': file_path,
- # 'tokenizer_model': model_path,
'tokens_to_generate': 128,
- # 'max_seq_length': max_seq_len,
- # 'num_samples': NUM_SAMPLES,
'type_haystack': config['type_haystack'],
'type_needle_k': config['type_needle_k'],
'type_needle_v': config['type_needle_v'],
diff --git a/opencompass/datasets/bigcodebench/bigcodebench.py b/opencompass/datasets/bigcodebench/bigcodebench.py
index 6f215ede..8d8e06f3 100644
--- a/opencompass/datasets/bigcodebench/bigcodebench.py
+++ b/opencompass/datasets/bigcodebench/bigcodebench.py
@@ -163,6 +163,8 @@ class BigCodeBenchEvaluator(BaseEvaluator):
logger.info('Read timeout error. Retrying in 4s...')
time.sleep(4)
+ if 'pass@1' in pass_at_k.keys():
+ pass_at_k['pass@1'] *= 100
dump_results = {'details': results}
dump_results.update(pass_at_k)
diff --git a/opencompass/models/openai_api.py b/opencompass/models/openai_api.py
index 0827497f..51faf7be 100644
--- a/opencompass/models/openai_api.py
+++ b/opencompass/models/openai_api.py
@@ -147,7 +147,7 @@ class OpenAI(BaseAPIModel):
self.path = path
self.max_completion_tokens = max_completion_tokens
self.logger.warning(
- f'Max Completion tokens for {path} is :{max_completion_tokens}')
+ f'Max Completion tokens for {path} is {max_completion_tokens}')
def generate(self,
inputs: List[PromptType],
@@ -278,7 +278,7 @@ class OpenAI(BaseAPIModel):
self.logger.warning(
f"'max_token' is unsupported for model {self.path}")
self.logger.warning(
- f'We use max_completion_tokens:'
+ f'We use max_completion_tokens: '
f'{self.max_completion_tokens}for this query')
data = dict(
model=self.path,
@@ -588,13 +588,12 @@ class OpenAISDK(OpenAI):
self.logger.warning(
f"'max_token' is unsupported for model {self.path}")
self.logger.warning(
- f'We use max_completion_tokens:'
+ f'We use max_completion_tokens: '
f'{self.max_completion_tokens}for this query')
query_data = dict(
model=self.path,
max_completion_tokens=self.max_completion_tokens,
n=1,
- temperature=self.temperature,
messages=messages,
extra_body=self.extra_body,
)
@@ -636,8 +635,8 @@ class OpenAISDK(OpenAI):
if (status_code is not None
and status_code in self.status_code_mappings):
error_message = self.status_code_mappings[status_code]
- self.logger.info(f'Status Code: {status_code},\n'
- f'Original Error Message: {e},\n'
+ self.logger.info(f'Status Code: {status_code}, \n'
+ f'Original Error Message: {e}, \n'
f'Return Message: {error_message} ')
return error_message
else:
diff --git a/opencompass/runners/dlc.py b/opencompass/runners/dlc.py
index 22a189d5..03045870 100644
--- a/opencompass/runners/dlc.py
+++ b/opencompass/runners/dlc.py
@@ -335,7 +335,7 @@ class DLCRunner(BaseRunner):
pass
# Lark Report when failed
- if return_code == -1:
+ if return_code == -1 and self.lark_reporter is not None:
content = f'DLC job failed. Task name: {task_name}'
self.lark_reporter.post(title='DLC job failed', content=content)
diff --git a/opencompass/runners/local.py b/opencompass/runners/local.py
index 4ce37f2a..1b14ad17 100644
--- a/opencompass/runners/local.py
+++ b/opencompass/runners/local.py
@@ -207,9 +207,14 @@ class LocalRunner(BaseRunner):
task_name = task.name
+ pwd = os.getcwd()
# Dump task config to file
mmengine.mkdir_or_exist('tmp/')
- param_file = f'tmp/{os.getpid()}_{index}_params.py'
+ # Using uuid to avoid filename conflict
+ import uuid
+ uuid_str = str(uuid.uuid4())
+ param_file = f'{pwd}/tmp/{uuid_str}_params.py'
+
try:
task.cfg.dump(param_file)
tmpl = get_command_template(gpu_ids)
@@ -236,5 +241,8 @@ class LocalRunner(BaseRunner):
logger.error(f'task {task_name} fail, see\n{out_path}')
finally:
# Clean up
- os.remove(param_file)
+ if not self.keep_tmp_file:
+ os.remove(param_file)
+ else:
+ pass
return task_name, result.returncode
diff --git a/opencompass/runners/volc.py b/opencompass/runners/volc.py
index b5f38223..a01f7cac 100644
--- a/opencompass/runners/volc.py
+++ b/opencompass/runners/volc.py
@@ -227,20 +227,20 @@ class VOLCRunner(BaseRunner):
task_status = os.popen(ask_cmd).read()
pattern = r'(?<=\[{"Status":").*(?="}\])'
match = re.search(pattern, task_status)
- if match:
- task_status = match.group()
- else:
- task_status = 'Exception'
if self.debug:
print(task_status)
logs = os.popen(log_cmd).read()
with open(log_path, 'w', encoding='utf-8') as f:
f.write(logs)
- if task_status in [
- 'Success', 'Failed', 'Cancelled', 'Exception',
- 'Killing', 'SuccessHolding', 'FailedHolding'
- ]:
- break
+ if match:
+ task_status = match.group()
+ if task_status in [
+ 'Success', 'Failed', 'Cancelled', 'Exception',
+ 'Killing', 'SuccessHolding', 'FailedHolding',
+ 'Killed'
+ ]:
+ break
+ # If pattern not found or command failed, sleep and retry
time.sleep(poll_interval)
else:
task_status = 'Exception'
diff --git a/opencompass/summarizers/default_subjective.py b/opencompass/summarizers/default_subjective.py
index 7d9133bd..b365ba3d 100644
--- a/opencompass/summarizers/default_subjective.py
+++ b/opencompass/summarizers/default_subjective.py
@@ -4,6 +4,7 @@ import functools
import getpass
import math
import os.path as osp
+from collections import OrderedDict
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -110,6 +111,7 @@ class DefaultSubjectiveSummarizer:
if not osp.exists(filepath):
continue
result = mmengine.load(filepath)
+ result = OrderedDict(sorted(result.items()))
result.pop('details', None)
if idx == 0:
raw_results[model_abbr][dataset_abbr] = result
diff --git a/requirements/runtime.txt b/requirements/runtime.txt
index fb87ca1a..47133f21 100644
--- a/requirements/runtime.txt
+++ b/requirements/runtime.txt
@@ -2,7 +2,7 @@ absl-py
accelerate>=0.19.0
cpm_kernels
datasets>=2.12.0
-einops==0.5.0
+einops>=0.5.0
evaluate>=0.3.0
func_timeout
fuzzywuzzy
@@ -16,7 +16,7 @@ jieba
json5
jsonlines
mmengine-lite
-nltk==3.8
+nltk>=3.7
numpy>=1.23.4,<2.0.0
openai
OpenCC