diff --git a/configs/datasets/subjective/multiround/mtbench_single_judge_diff_temp.py b/configs/datasets/subjective/multiround/mtbench_single_judge_diff_temp.py index f5563f20..57ee2750 100644 --- a/configs/datasets/subjective/multiround/mtbench_single_judge_diff_temp.py +++ b/configs/datasets/subjective/multiround/mtbench_single_judge_diff_temp.py @@ -26,7 +26,7 @@ for _name in subjective_all_sets: template="""{dialogue}""", ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=512, temperature=temperature, do_sample=do_sample,infer_mode='every'), + inferencer=dict(type=ChatInferencer, max_seq_len=4096, max_out_len=1024, temperature=temperature, do_sample=do_sample,infer_mode='every'), ) subjective_eval_cfg = dict( diff --git a/opencompass/cli/main.py b/opencompass/cli/main.py index bb47221e..d1539d5c 100644 --- a/opencompass/cli/main.py +++ b/opencompass/cli/main.py @@ -1,6 +1,7 @@ # flake8: noqa # yapf: disable import argparse +import copy import getpass import os import os.path as osp diff --git a/opencompass/summarizers/subjective/alignmentbench.py b/opencompass/summarizers/subjective/alignmentbench.py index 48fc7111..ce357d89 100644 --- a/opencompass/summarizers/subjective/alignmentbench.py +++ b/opencompass/summarizers/subjective/alignmentbench.py @@ -369,6 +369,9 @@ class AlignmentBenchSummarizer: if os.path.isdir(subdir_path): judged_answers, references = get_judgeanswer_and_reference( dataset, subdir_path, self.judge_function) + if len(judged_answers) == 0: + score_by_judgemodel[model] = None + continue if self.judge_type == 'general': get_dimension_results(judged_answers, references, fout, fout_flag, model)