jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Fix] the issue where scores are negative in the Lawbench dataset evaluation(#1402) (#1403)

Browse Source
This commit is contained in:
yaoyingyy 2024-08-08 16:08:26 +08:00 committed by GitHub
parent 818d72a650
commit decb621ff6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
opencompass/datasets/lawbench/evaluation_functions/ljp_imprison.py
View File

@ -43,9 +43,12 @@ def compute_ljp_imprison(data_dict):
score_list.append(abs(math.log(answer_digit + 1) - math.log(prediction_digit_month + 1)))
else:
score_list.append(math.log(216))
# compute the average of score_list (log distance)
log_distance = sum(score_list) / len(score_list)
# normalize the score to between 0 and 1
log_distance = (math.log(216) - log_distance)/math.log(216)
if abstentions == len(score_list):
log_distance = 0
else:
# compute the average of score_list (log distance)
log_distance = sum(score_list) / len(score_list)
# normalize the score to between 0 and 1
log_distance = (math.log(216) - log_distance)/math.log(216)
return {"score": log_distance, "abstention_rate": abstentions/len(data_dict)}