From 8e7ad2e9819e8986b18fca813e98e5021595ba18 Mon Sep 17 00:00:00 2001 From: bittersweet1999 <148421775+bittersweet1999@users.noreply.github.com> Date: Fri, 12 Jul 2024 11:06:20 +0800 Subject: [PATCH 1/3] [Fix] add bc for alignbench summarizer (#1306) * fix pip version * fix pip version * fix alignbench * fix import error --- .../subjective/multiround/mtbench_single_judge_diff_temp.py | 2 +- opencompass/cli/main.py | 1 + opencompass/summarizers/subjective/alignmentbench.py | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/datasets/subjective/multiround/mtbench_single_judge_diff_temp.py b/configs/datasets/subjective/multiround/mtbench_single_judge_diff_temp.py index f5563f20..57ee2750 100644 --- a/configs/datasets/subjective/multiround/mtbench_single_judge_diff_temp.py +++ b/configs/datasets/subjective/multiround/mtbench_single_judge_diff_temp.py @@ -26,7 +26,7 @@ for _name in subjective_all_sets: template="""{dialogue}""", ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, temperature=temperature, do_sample=do_sample,infer_mode='every'), + inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=1024, temperature=temperature, do_sample=do_sample,infer_mode='every'), ) subjective_eval_cfg = dict( diff --git a/opencompass/cli/main.py b/opencompass/cli/main.py index bb47221e..d1539d5c 100644 --- a/opencompass/cli/main.py +++ b/opencompass/cli/main.py @@ -1,6 +1,7 @@ # flake8: noqa # yapf: disable import argparse +import copy import getpass import os import os.path as osp diff --git a/opencompass/summarizers/subjective/alignmentbench.py b/opencompass/summarizers/subjective/alignmentbench.py index 48fc7111..ce357d89 100644 --- a/opencompass/summarizers/subjective/alignmentbench.py +++ b/opencompass/summarizers/subjective/alignmentbench.py @@ -369,6 +369,9 @@ class AlignmentBenchSummarizer: if os.path.isdir(subdir_path): judged_answers, references = get_judgeanswer_and_reference( dataset, subdir_path, self.judge_function) + if len(judged_answers) == 0: + score_by_judgemodel[model] = None + continue if self.judge_type == 'general': get_dimension_results(judged_answers, references, fout, fout_flag, model) From 3aeabbc427b8084ea3276991348f911794a345f6 Mon Sep 17 00:00:00 2001 From: bittersweet1999 <148421775+bittersweet1999@users.noreply.github.com> Date: Fri, 12 Jul 2024 11:29:26 +0800 Subject: [PATCH 2/3] [Fix] update Faq (#1313) * fix pip version * fix pip version * update faq * update faq * update faq --------- Co-authored-by: Leymore --- docs/en/get_started/faq.md | 7 +++++++ docs/zh_cn/get_started/faq.md | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/docs/en/get_started/faq.md b/docs/en/get_started/faq.md index 570605c7..5d5117e1 100644 --- a/docs/en/get_started/faq.md +++ b/docs/en/get_started/faq.md @@ -119,3 +119,10 @@ If you have already download the checkpoints of the model, you can specify the l ```bash python run.py --datasets siqa_gen winograd_ppl --hf-type base --hf-path /path/to/model ``` + +## Dataset + +### How to build a new dataset? + +- For building new objective dataset: [new_dataset](../advanced_guides/new_dataset.md) +- For building new subjective dataset: [subjective_evaluation](../advanced_guides/subjective_evaluation.md) diff --git a/docs/zh_cn/get_started/faq.md b/docs/zh_cn/get_started/faq.md index 44e0f9ea..30d34676 100644 --- a/docs/zh_cn/get_started/faq.md +++ b/docs/zh_cn/get_started/faq.md @@ -119,3 +119,10 @@ OpenCompass 中的每个任务代表等待评估的特定模型和数据集部 ```bash python run.py --datasets siqa_gen winograd_ppl --hf-type base --hf-path /path/to/model ``` + +## 数据集 + +### 如何构建自己的评测数据集 + +- 客观数据集构建参见：[支持新数据集](../advanced_guides/new_dataset.md) +- 主观数据集构建参见：[主观评测指引](../advanced_guides/subjective_evaluation.md) From 0a1c89e6186aa3526da18a894ce65f1245023ea0 Mon Sep 17 00:00:00 2001 From: Xu Song Date: Tue, 16 Jul 2024 16:18:13 +0800 Subject: [PATCH 3/3] [Fix] Fix rouge evaluator of rolebench_zh (#1322) --- configs/datasets/rolebench/instruction_generalization_zh.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/datasets/rolebench/instruction_generalization_zh.py b/configs/datasets/rolebench/instruction_generalization_zh.py index 646aa7a1..5f53e98d 100644 --- a/configs/datasets/rolebench/instruction_generalization_zh.py +++ b/configs/datasets/rolebench/instruction_generalization_zh.py @@ -1,7 +1,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate from opencompass.openicl.icl_retriever import ZeroRetriever from opencompass.openicl.icl_inferencer import GenInferencer -from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.openicl.icl_evaluator import JiebaRougeEvaluator from opencompass.datasets.rolebench import InstructionGeneralizationChineseDataset instruction_generalization_zh_reader_cfg = dict( @@ -27,7 +27,7 @@ instruction_generalization_zh_infer_cfg = dict( ) instruction_generalization_zh_eval_cfg = dict( - evaluator=dict(type=RougeEvaluator), + evaluator=dict(type=JiebaRougeEvaluator), pred_role='BOT' )