diff --git a/configs/summarizers/groups/mathbench_v1_2024_lang.py b/configs/summarizers/groups/mathbench_v1_2024_lang.py index 3693f7f4..0495b106 100644 --- a/configs/summarizers/groups/mathbench_v1_2024_lang.py +++ b/configs/summarizers/groups/mathbench_v1_2024_lang.py @@ -1,22 +1,66 @@ +# Language specific summarizer groups for MathBench mathbench_2024_summary_groups = [ + # mathbench-a average with subsets {'name': 'college', 'subsets': [['mathbench-college-single_choice_cn', 'perf_4'], ['mathbench-college-single_choice_en', 'perf_4']]}, {'name': 'high', 'subsets': [['mathbench-high-single_choice_cn', 'perf_4'], ['mathbench-high-single_choice_en', 'perf_4']]}, {'name': 'middle', 'subsets': [['mathbench-middle-single_choice_cn', 'perf_4'], ['mathbench-middle-single_choice_en', 'perf_4']]}, {'name': 'primary', 'subsets': [['mathbench-primary-cloze_cn', 'accuracy'], ['mathbench-primary-cloze_en', 'accuracy']]}, {'name': 'arithmetic', 'subsets': [['mathbench-arithmetic-cloze_en', 'accuracy']]}, - {'name': 'mathbench-a-cn-average', 'subsets': ['mathbench-college-single_choice_cn', 'mathbench-high-single_choice_cn', 'mathbench-middle-single_choice_cn', 'mathbench-primary-cloze_cn']}, - {'name': 'mathbench-a-en-average', 'subsets': ['mathbench-college-single_choice_en', 'mathbench-high-single_choice_en', 'mathbench-middle-single_choice_en', 'mathbench-primary-cloze_en']}, {'name': 'mathbench-a (average)', 'subsets': ['college', 'high', 'middle', 'primary', 'arithmetic']}, + # mathbench-a language + {'name': 'mathbench-a-college-cn', 'subsets': [['mathbench-college-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-a-college-en', 'ssubsets': [['mathbench-college-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-a-high-cn', 'subsets': [['mathbench-high-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-a-high-en', 'subsets': [['mathbench-high-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-a-middle-cn', 'subsets': [['mathbench-middle-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-a-middle-en', 'subsets': [['mathbench-middle-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-a-primary-cn', 'subsets': [['mathbench-primary-cloze_cn', 'accuracy']]}, + {'name': 'mathbench-a-primary-en', 'subsets': [['mathbench-primary-cloze_en', 'accuracy']]}, + {'name': 'mathbench-a-arithmetic', 'subsets': [['mathbench-arithmetic-cloze_en', 'accuracy']]}, + {'name': 'mathbench-a-cn-average', 'subsets': ['mathbench-a-college-cn', 'mathbench-a-high-cn', 'mathbench-a-middle-cn', 'mathbench-a-primary-cn']}, + {'name': 'mathbench-a-en-average', 'subsets': ['mathbench-a-college-en', 'mathbench-a-high-en', 'mathbench-a-middle-en', 'mathbench-a-primary-en']}, + # mathbench-a average + {'name': 'mathbench-a (average)', 'subsets': ['mathbench-a-cn-average', 'mathbench-a-en-average']}, + + # mathbench-t average with subsets + {'name': 'college_knowledge', 'subsets': [['mathbench-college_knowledge-single_choice_cn', 'perf_4'], ['mathbench-college_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'high_knowledge', 'subsets': [['mathbench-high_knowledge-single_choice_cn', 'perf_4'], ['mathbench-high_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'middle_knowledge', 'subsets': [['mathbench-middle_knowledge-single_choice_cn', 'perf_4'], ['mathbench-middle_knowledge-single_choice_en', 'perf_4']]}, {'name': 'college_knowledge', 'subsets': [['mathbench-college_knowledge-single_choice_cn', 'perf_4'], ['mathbench-college_knowledge-single_choice_en', 'perf_4']]}, {'name': 'high_knowledge', 'subsets': [['mathbench-high_knowledge-single_choice_cn', 'perf_4'], ['mathbench-high_knowledge-single_choice_en', 'perf_4']]}, {'name': 'middle_knowledge', 'subsets': [['mathbench-middle_knowledge-single_choice_cn', 'perf_4'], ['mathbench-middle_knowledge-single_choice_en', 'perf_4']]}, {'name': 'primary_knowledge', 'subsets': [['mathbench-primary_knowledge-single_choice_cn', 'perf_4'], ['mathbench-primary_knowledge-single_choice_en', 'perf_4']]}, - {'name': 'mathbench-t-cn-average', 'subsets': ['mathbench-college_knowledge-single_choice_cn', 'mathbench-high_knowledge-single_choice_cn', 'mathbench-middle_knowledge-single_choice_cn', 'mathbench-primary_knowledge-single_choice_cn']}, - {'name': 'mathbench-t-en-average', 'subsets': ['mathbench-college_knowledge-single_choice_en', 'mathbench-high_knowledge-single_choice_en', 'mathbench-middle_knowledge-single_choice_en', 'mathbench-primary_knowledge-single_choice_en']}, - {'name': 'mathbench-t (average)', 'subsets': ['college_knowledge', 'high_knowledge', 'middle_knowledge', 'primary_knowledge']}, + # mathbench-t language + {'name': 'mathbench-t-college-cn', 'subsets': [['mathbench-college_knowledge-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-t-college-en', 'subsets': [['mathbench-college_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-t-high-cn', 'subsets': [['mathbench-high_knowledge-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-t-high-en', 'subsets': [['mathbench-high_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-t-middle-cn', 'subsets': [['mathbench-middle_knowledge-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-t-middle-en', 'subsets': [['mathbench-middle_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-t-primary-cn', 'subsets': [['mathbench-primary_knowledge-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-t-primary-en', 'subsets': [['mathbench-primary_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-t-cn-average', 'subsets': ['mathbench-t-college-cn', 'mathbench-t-high-cn', 'mathbench-t-middle-cn', 'mathbench-t-primary-cn']}, + {'name': 'mathbench-t-en-average', 'subsets': ['mathbench-t-college-en', 'mathbench-t-high-en', 'mathbench-t-middle-en', 'mathbench-t-primary-en']}, + # mathbench-t average + {'name': 'mathbench-t (average)', 'subsets': ['mathbench-t-cn-average', 'mathbench-t-en-average']}, + # overall cn + {'name': 'college-cn', 'subsets': ['mathbench-a-college-cn', 'mathbench-t-college-cn']}, + {'name': 'high-cn', 'subsets': ['mathbench-a-high-cn', 'mathbench-t-high-cn']}, + {'name': 'middle-cn', 'subsets': ['mathbench-a-middle-cn', 'mathbench-t-middle-cn']}, + {'name': 'primary-cn', 'subsets': ['mathbench-a-primary-cn', 'mathbench-t-primary-cn']}, + {'name': 'cn-avarage', 'subsets': ['college-cn', 'high-cn', 'middle-cn', 'primary-cn']}, + + # overall en + {'name': 'college-en', 'subsets': ['mathbench-a-college-en', 'mathbench-t-college-en']}, + {'name': 'high-en', 'subsets': ['mathbench-a-high-en', 'mathbench-t-high-en']}, + {'name': 'middle-en', 'subsets': ['mathbench-a-middle-en', 'mathbench-t-middle-en']}, + {'name': 'primary-en', 'subsets': ['mathbench-a-primary-en', 'mathbench-t-primary-en']}, + {'name': 'en-avarage', 'subsets': ['college-en', 'high-en', 'middle-en', 'primary-en']}, + + # overall {'name': 'Overall', 'subsets': ['mathbench-a (average)', 'mathbench-t (average)']}, ] @@ -25,33 +69,48 @@ summarizer = dict( dataset_abbrs = [ '########################################################', '###### MathBench-A-CN: Application Part (Chinese) ######', - 'mathbench-college-single_choice_cn', - 'mathbench-high-single_choice_cn', - 'mathbench-middle-single_choice_cn', - 'mathbench-primary-cloze_cn', + 'mathbench-a-college-cn', + 'mathbench-a-high-cn', + 'mathbench-a-middle-cn', + 'mathbench-a-primary-cn', 'mathbench-a-cn-average', - '###### MathBench-A-EN: Application Part (English) ######', - 'mathbench-college-single_choice_en', - 'mathbench-high-single_choice_en', - 'mathbench-middle-single_choice_en', - 'mathbench-primary-cloze_en', + 'mathbench-a-college-en', + 'mathbench-a-high-en', + 'mathbench-a-middle-en', + 'mathbench-a-primary-en', 'mathbench-a-en-average', - '###################################################', - '###### MathBench-T-CN: Theory Part (Chinese) ######', - 'mathbench-college_knowledge-single_choice_cn', - 'mathbench-high_knowledge-single_choice_cn', - 'mathbench-middle_knowledge-single_choice_cn', - 'mathbench-primary_knowledge-single_choice_cn', + '#########################################################', + '###### MathBench-T-CN: Theory Part (Chinese) ############', + 'mathbench-t-college-cn', + 'mathbench-t-high-cn', + 'mathbench-t-middle-cn', + 'mathbench-t-primary-cn', 'mathbench-t-cn-average', - '###### MathBench-T-EN: Theory Part (English) ######', - 'mathbench-college_knowledge-single_choice_en', - 'mathbench-high_knowledge-single_choice_en', - 'mathbench-middle_knowledge-single_choice_en', - 'mathbench-primary_knowledge-single_choice_en', + '###### MathBench-T-EN: Theory Part (English) ############', + 'mathbench-t-college-en', + 'mathbench-t-high-en', + 'mathbench-t-middle-en', + 'mathbench-t-primary-en', 'mathbench-t-en-average', + + '#########################################################', + '###### MathBench-CN ############', + 'college-cn', + 'high-cn', + 'middle-cn', + 'primary-cn', + 'cn-avarage', + + '###### MathBench-EN ############', + 'college-en', + 'high-en', + 'middle-en', + 'primary-en', + 'en-avarage', + '#########################################################', ], summary_groups=mathbench_2024_summary_groups, ) diff --git a/opencompass/configs/summarizers/groups/mathbench_v1_2024_lang.py b/opencompass/configs/summarizers/groups/mathbench_v1_2024_lang.py index 3693f7f4..0495b106 100644 --- a/opencompass/configs/summarizers/groups/mathbench_v1_2024_lang.py +++ b/opencompass/configs/summarizers/groups/mathbench_v1_2024_lang.py @@ -1,22 +1,66 @@ +# Language specific summarizer groups for MathBench mathbench_2024_summary_groups = [ + # mathbench-a average with subsets {'name': 'college', 'subsets': [['mathbench-college-single_choice_cn', 'perf_4'], ['mathbench-college-single_choice_en', 'perf_4']]}, {'name': 'high', 'subsets': [['mathbench-high-single_choice_cn', 'perf_4'], ['mathbench-high-single_choice_en', 'perf_4']]}, {'name': 'middle', 'subsets': [['mathbench-middle-single_choice_cn', 'perf_4'], ['mathbench-middle-single_choice_en', 'perf_4']]}, {'name': 'primary', 'subsets': [['mathbench-primary-cloze_cn', 'accuracy'], ['mathbench-primary-cloze_en', 'accuracy']]}, {'name': 'arithmetic', 'subsets': [['mathbench-arithmetic-cloze_en', 'accuracy']]}, - {'name': 'mathbench-a-cn-average', 'subsets': ['mathbench-college-single_choice_cn', 'mathbench-high-single_choice_cn', 'mathbench-middle-single_choice_cn', 'mathbench-primary-cloze_cn']}, - {'name': 'mathbench-a-en-average', 'subsets': ['mathbench-college-single_choice_en', 'mathbench-high-single_choice_en', 'mathbench-middle-single_choice_en', 'mathbench-primary-cloze_en']}, {'name': 'mathbench-a (average)', 'subsets': ['college', 'high', 'middle', 'primary', 'arithmetic']}, + # mathbench-a language + {'name': 'mathbench-a-college-cn', 'subsets': [['mathbench-college-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-a-college-en', 'ssubsets': [['mathbench-college-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-a-high-cn', 'subsets': [['mathbench-high-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-a-high-en', 'subsets': [['mathbench-high-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-a-middle-cn', 'subsets': [['mathbench-middle-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-a-middle-en', 'subsets': [['mathbench-middle-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-a-primary-cn', 'subsets': [['mathbench-primary-cloze_cn', 'accuracy']]}, + {'name': 'mathbench-a-primary-en', 'subsets': [['mathbench-primary-cloze_en', 'accuracy']]}, + {'name': 'mathbench-a-arithmetic', 'subsets': [['mathbench-arithmetic-cloze_en', 'accuracy']]}, + {'name': 'mathbench-a-cn-average', 'subsets': ['mathbench-a-college-cn', 'mathbench-a-high-cn', 'mathbench-a-middle-cn', 'mathbench-a-primary-cn']}, + {'name': 'mathbench-a-en-average', 'subsets': ['mathbench-a-college-en', 'mathbench-a-high-en', 'mathbench-a-middle-en', 'mathbench-a-primary-en']}, + # mathbench-a average + {'name': 'mathbench-a (average)', 'subsets': ['mathbench-a-cn-average', 'mathbench-a-en-average']}, + + # mathbench-t average with subsets + {'name': 'college_knowledge', 'subsets': [['mathbench-college_knowledge-single_choice_cn', 'perf_4'], ['mathbench-college_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'high_knowledge', 'subsets': [['mathbench-high_knowledge-single_choice_cn', 'perf_4'], ['mathbench-high_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'middle_knowledge', 'subsets': [['mathbench-middle_knowledge-single_choice_cn', 'perf_4'], ['mathbench-middle_knowledge-single_choice_en', 'perf_4']]}, {'name': 'college_knowledge', 'subsets': [['mathbench-college_knowledge-single_choice_cn', 'perf_4'], ['mathbench-college_knowledge-single_choice_en', 'perf_4']]}, {'name': 'high_knowledge', 'subsets': [['mathbench-high_knowledge-single_choice_cn', 'perf_4'], ['mathbench-high_knowledge-single_choice_en', 'perf_4']]}, {'name': 'middle_knowledge', 'subsets': [['mathbench-middle_knowledge-single_choice_cn', 'perf_4'], ['mathbench-middle_knowledge-single_choice_en', 'perf_4']]}, {'name': 'primary_knowledge', 'subsets': [['mathbench-primary_knowledge-single_choice_cn', 'perf_4'], ['mathbench-primary_knowledge-single_choice_en', 'perf_4']]}, - {'name': 'mathbench-t-cn-average', 'subsets': ['mathbench-college_knowledge-single_choice_cn', 'mathbench-high_knowledge-single_choice_cn', 'mathbench-middle_knowledge-single_choice_cn', 'mathbench-primary_knowledge-single_choice_cn']}, - {'name': 'mathbench-t-en-average', 'subsets': ['mathbench-college_knowledge-single_choice_en', 'mathbench-high_knowledge-single_choice_en', 'mathbench-middle_knowledge-single_choice_en', 'mathbench-primary_knowledge-single_choice_en']}, - {'name': 'mathbench-t (average)', 'subsets': ['college_knowledge', 'high_knowledge', 'middle_knowledge', 'primary_knowledge']}, + # mathbench-t language + {'name': 'mathbench-t-college-cn', 'subsets': [['mathbench-college_knowledge-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-t-college-en', 'subsets': [['mathbench-college_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-t-high-cn', 'subsets': [['mathbench-high_knowledge-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-t-high-en', 'subsets': [['mathbench-high_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-t-middle-cn', 'subsets': [['mathbench-middle_knowledge-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-t-middle-en', 'subsets': [['mathbench-middle_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-t-primary-cn', 'subsets': [['mathbench-primary_knowledge-single_choice_cn', 'perf_4']]}, + {'name': 'mathbench-t-primary-en', 'subsets': [['mathbench-primary_knowledge-single_choice_en', 'perf_4']]}, + {'name': 'mathbench-t-cn-average', 'subsets': ['mathbench-t-college-cn', 'mathbench-t-high-cn', 'mathbench-t-middle-cn', 'mathbench-t-primary-cn']}, + {'name': 'mathbench-t-en-average', 'subsets': ['mathbench-t-college-en', 'mathbench-t-high-en', 'mathbench-t-middle-en', 'mathbench-t-primary-en']}, + # mathbench-t average + {'name': 'mathbench-t (average)', 'subsets': ['mathbench-t-cn-average', 'mathbench-t-en-average']}, + # overall cn + {'name': 'college-cn', 'subsets': ['mathbench-a-college-cn', 'mathbench-t-college-cn']}, + {'name': 'high-cn', 'subsets': ['mathbench-a-high-cn', 'mathbench-t-high-cn']}, + {'name': 'middle-cn', 'subsets': ['mathbench-a-middle-cn', 'mathbench-t-middle-cn']}, + {'name': 'primary-cn', 'subsets': ['mathbench-a-primary-cn', 'mathbench-t-primary-cn']}, + {'name': 'cn-avarage', 'subsets': ['college-cn', 'high-cn', 'middle-cn', 'primary-cn']}, + + # overall en + {'name': 'college-en', 'subsets': ['mathbench-a-college-en', 'mathbench-t-college-en']}, + {'name': 'high-en', 'subsets': ['mathbench-a-high-en', 'mathbench-t-high-en']}, + {'name': 'middle-en', 'subsets': ['mathbench-a-middle-en', 'mathbench-t-middle-en']}, + {'name': 'primary-en', 'subsets': ['mathbench-a-primary-en', 'mathbench-t-primary-en']}, + {'name': 'en-avarage', 'subsets': ['college-en', 'high-en', 'middle-en', 'primary-en']}, + + # overall {'name': 'Overall', 'subsets': ['mathbench-a (average)', 'mathbench-t (average)']}, ] @@ -25,33 +69,48 @@ summarizer = dict( dataset_abbrs = [ '########################################################', '###### MathBench-A-CN: Application Part (Chinese) ######', - 'mathbench-college-single_choice_cn', - 'mathbench-high-single_choice_cn', - 'mathbench-middle-single_choice_cn', - 'mathbench-primary-cloze_cn', + 'mathbench-a-college-cn', + 'mathbench-a-high-cn', + 'mathbench-a-middle-cn', + 'mathbench-a-primary-cn', 'mathbench-a-cn-average', - '###### MathBench-A-EN: Application Part (English) ######', - 'mathbench-college-single_choice_en', - 'mathbench-high-single_choice_en', - 'mathbench-middle-single_choice_en', - 'mathbench-primary-cloze_en', + 'mathbench-a-college-en', + 'mathbench-a-high-en', + 'mathbench-a-middle-en', + 'mathbench-a-primary-en', 'mathbench-a-en-average', - '###################################################', - '###### MathBench-T-CN: Theory Part (Chinese) ######', - 'mathbench-college_knowledge-single_choice_cn', - 'mathbench-high_knowledge-single_choice_cn', - 'mathbench-middle_knowledge-single_choice_cn', - 'mathbench-primary_knowledge-single_choice_cn', + '#########################################################', + '###### MathBench-T-CN: Theory Part (Chinese) ############', + 'mathbench-t-college-cn', + 'mathbench-t-high-cn', + 'mathbench-t-middle-cn', + 'mathbench-t-primary-cn', 'mathbench-t-cn-average', - '###### MathBench-T-EN: Theory Part (English) ######', - 'mathbench-college_knowledge-single_choice_en', - 'mathbench-high_knowledge-single_choice_en', - 'mathbench-middle_knowledge-single_choice_en', - 'mathbench-primary_knowledge-single_choice_en', + '###### MathBench-T-EN: Theory Part (English) ############', + 'mathbench-t-college-en', + 'mathbench-t-high-en', + 'mathbench-t-middle-en', + 'mathbench-t-primary-en', 'mathbench-t-en-average', + + '#########################################################', + '###### MathBench-CN ############', + 'college-cn', + 'high-cn', + 'middle-cn', + 'primary-cn', + 'cn-avarage', + + '###### MathBench-EN ############', + 'college-en', + 'high-en', + 'middle-en', + 'primary-en', + 'en-avarage', + '#########################################################', ], summary_groups=mathbench_2024_summary_groups, ) diff --git a/opencompass/utils/text_postprocessors.py b/opencompass/utils/text_postprocessors.py index 855ee432..fc12e064 100644 --- a/opencompass/utils/text_postprocessors.py +++ b/opencompass/utils/text_postprocessors.py @@ -102,6 +102,9 @@ def first_option_postprocess(text: str, options: str, cushion=True) -> str: f'[Tt]he answer is option:?\s+\(?([{options}])\)?', f'[Tt]he correct answer is:?\s+\(?([{options}])\)?', f'[Tt]he correct answer is option:?\s+\(?([{options}])\)?', + f'[Tt]he correct answer is:?.*?boxed{{([{options}])}}', + f'[Tt]he correct option is:?.*?boxed{{([{options}])}}', + f'[Tt]he correct answer option is:?.*?boxed{{([{options}])}}', f'[Tt]he answer to the question is:?\s+\(?([{options}])\)?', f'^选项\s?([{options}])', f'^([{options}])\s?选?项',