From decb621ff6616d0361d3932216e3aba8c8354250 Mon Sep 17 00:00:00 2001 From: yaoyingyy <932675001@qq.com> Date: Thu, 8 Aug 2024 16:08:26 +0800 Subject: [PATCH] [Fix] the issue where scores are negative in the Lawbench dataset evaluation(#1402) (#1403) --- .../lawbench/evaluation_functions/ljp_imprison.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/opencompass/datasets/lawbench/evaluation_functions/ljp_imprison.py b/opencompass/datasets/lawbench/evaluation_functions/ljp_imprison.py index 80b8ec2d..e051caf9 100644 --- a/opencompass/datasets/lawbench/evaluation_functions/ljp_imprison.py +++ b/opencompass/datasets/lawbench/evaluation_functions/ljp_imprison.py @@ -43,9 +43,12 @@ def compute_ljp_imprison(data_dict): score_list.append(abs(math.log(answer_digit + 1) - math.log(prediction_digit_month + 1))) else: score_list.append(math.log(216)) - - # compute the average of score_list (log distance) - log_distance = sum(score_list) / len(score_list) - # normalize the score to between 0 and 1 - log_distance = (math.log(216) - log_distance)/math.log(216) + + if abstentions == len(score_list): + log_distance = 0 + else: + # compute the average of score_list (log distance) + log_distance = sum(score_list) / len(score_list) + # normalize the score to between 0 and 1 + log_distance = (math.log(216) - log_distance)/math.log(216) return {"score": log_distance, "abstention_rate": abstentions/len(data_dict)}