jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Feature] LLM Judge sanity check (#2012)

Browse Source 
* update

* update
This commit is contained in:
Linchen Xiao 2025-04-11 19:01:39 +08:00 committed by GitHub
parent 3f50b1dc49
commit 6a6a1a5c0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 22 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
opencompass/datasets/generic.py
View File

@ -11,17 +11,18 @@ def get_final_results(judged_answers,
is_correct_count = 0 is_correct_count = 0
is_incorrect_count = 0 is_incorrect_count = 0
is_not_attempted_count = 0 is_not_attempted_count = 0
attempted_judge_count = 0
details = [] details = []
for i, j, k in zip(judged_answers, references, origial_responses): for i, j, k in zip(judged_answers, references, origial_responses):
match = re.search(r'(A|B)', i) if i in ['A', 'B']:
grade_letter = match.group( attempted_judge_count += 1
0) if match else 'B' # Default to "INCORRECT" if no match grade_letter = i
detail = { detail = {
'pred': k, 'pred': k,
'ref': j, 'ref': j,
'origin_grade_response': i, 'origin_grade_response': i,
'grade_letter': grade_letter, 'grade_letter': grade_letter,
'correct': False 'correct': False,
} }
count += 1 count += 1
if grade_letter == 'A': if grade_letter == 'A':
@ -35,26 +36,32 @@ def get_final_results(judged_answers,
is_correct = is_correct_count / count is_correct = is_correct_count / count
is_incorrect = is_incorrect_count / count is_incorrect = is_incorrect_count / count
# is_not_attempted = is_not_attempted_count / count
is_given_attempted = is_correct + is_incorrect is_given_attempted = is_correct + is_incorrect
accuracy_given_attempted = is_correct / is_given_attempted \ loose_accuracy = is_correct / count
if is_given_attempted > 0 else 0 accuracy_given_attempted = (is_correct / is_given_attempted
f1 = 2 * accuracy_given_attempted * is_correct / ( if is_given_attempted > 0 else 0)
accuracy_given_attempted + is_correct) if (accuracy_given_attempted + attempted_judge_ratio = attempted_judge_count / count
is_correct) > 0 else 0
f1 = (2 * accuracy_given_attempted * is_correct /
(accuracy_given_attempted + is_correct) if
(accuracy_given_attempted + is_correct) > 0 else 0)
result = { result = {
# 'accuracy_given_attempted': accuracy_given_attempted, metric_name: loose_accuracy * 100,
metric_name: accuracy_given_attempted * 100, f'{metric_name}_given_attempted': accuracy_given_attempted * 100,
'f1': f1, 'f1': f1,
'details': details 'attempted_ratio': attempted_judge_ratio * 100,
'correct_count': is_correct_count,
'incorrect_count': is_incorrect_count,
'not_attempted_count': is_not_attempted_count,
'details': details,
} }
return result return result
def _generic_llmjudge_postprocess(judgement: str): def _generic_llmjudge_postprocess(judgement: str):
match = re.search(r'(A|B)', judgement) match = re.search(r'(A|B)', judgement)
grade_letter = match.group( grade_letter = (match.group(0) if match else 'unknown'
0) if match else 'B' # Default to "INCORRECT" if no match ) # Return 'unknown' if no match
return grade_letter return grade_letter