mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
48 lines
2.1 KiB
Python
48 lines
2.1 KiB
Python
|
# flake8: noqa: E501
|
||
|
|
from .common_answers import (common_false_list, common_start_false_dict,
|
||
|
|
common_start_true_dict, common_true_list)
|
||
|
|
|
||
|
|
|
||
|
|
def get_gt_label(item):
|
||
|
|
return item['gt_answer']
|
||
|
|
|
||
|
|
|
||
|
|
def get_pred_label(model_response, item, prompt_style, type):
|
||
|
|
model_response = model_response.strip().lower()
|
||
|
|
low_index = len(model_response)
|
||
|
|
start_str1_dict = common_start_true_dict
|
||
|
|
start_str2_dict = common_start_false_dict
|
||
|
|
start_option1_list, start_option2_list = [], []
|
||
|
|
# some of the model will give response containing the question,
|
||
|
|
# we usually preprocess the response to remove the question part,
|
||
|
|
# but sometimes due to the model's response format, some of the
|
||
|
|
# question part is not removed, so here we are checking the
|
||
|
|
# response with the question part as well.
|
||
|
|
for key1, key2 in zip(start_str1_dict.keys(), start_str2_dict.keys()):
|
||
|
|
for str1, str2 in zip(start_str1_dict[key1], start_str2_dict[key2]):
|
||
|
|
for i in range(key1, len(str1) + 1):
|
||
|
|
start_option1_list.append(str1[-i:])
|
||
|
|
for i in range(key2, len(str2) + 1):
|
||
|
|
start_option2_list.append(str2[-i:])
|
||
|
|
|
||
|
|
inner_option1_list = common_true_list
|
||
|
|
inner_option2_list = common_false_list
|
||
|
|
if model_response.startswith(tuple(start_option1_list)):
|
||
|
|
label = 1
|
||
|
|
elif model_response.startswith(tuple(start_option2_list)):
|
||
|
|
label = 0
|
||
|
|
elif any(model_response.find(option)>-1 and (low_index := min(low_index, model_response.find(option))) > -1 for option in inner_option1_list) \
|
||
|
|
or 'yes' in model_response and ('causes' in model_response or 'does cause' in model_response) \
|
||
|
|
or '是' in model_response and '会导致' in model_response:
|
||
|
|
label = 1
|
||
|
|
if any(option in model_response
|
||
|
|
and model_response.find(option) < low_index
|
||
|
|
for option in inner_option2_list):
|
||
|
|
label = 0
|
||
|
|
elif any(response in model_response for response in inner_option2_list) \
|
||
|
|
or '否' in model_response and '不会导致' in model_response:
|
||
|
|
label = 0
|
||
|
|
else:
|
||
|
|
return -1
|
||
|
|
return label
|