OpenCompass/opencompass/datasets/calm/evaluation/labeling/AS.py

from .common_answers import common_true_list, common_false_list, common_option_1_list, common_option_2_list, common_option_3_list, common_option_4_list, common_start_true_dict, common_start_false_dict, common_start_op1_dict, common_start_op2_dict, common_start_op3_dict, common_start_op4_dict    

def get_gt_label(item):
        return int(item["gt_answer"])

def get_pred_label(model_response, item, prompt_style, type): 
        model_response = model_response.strip().lower()
        low_index = len(model_response)
        Answer1 = item["option1"].strip().lower()
        Answer2 = item["option2"].strip().lower()
        Answer3 = item["option3"].strip().lower()
        start_str1_dict = {**common_start_op1_dict,
                            len(Answer1)-1:[f"答案（选项一或选项二或选项三？）：{Answer1[:-1]}",
                                        f"答案（选项一或选项二或选项三？）： {Answer1[:-1]}",
                                        f"answer (option 1 or 2 or 3?):{Answer1[:-1]}",
                                        f"answer (option 1 or 2 or 3?): {Answer1[:-1]}"]}
        start_str2_dict = {**common_start_op2_dict,
                        len(Answer2)-1:[f"答案（选项一或选项二或选项三？）：{Answer2[:-1]}",
                                        f"答案（选项一或选项二或选项三？）： {Answer2[:-1]}",
                                        f"answer (option 1 or 2 or 3?):{Answer2[:-1]}",
                                        f"answer (option 1 or 2 or 3?): {Answer2[:-1]}"]}
        start_str3_dict = {**common_start_op3_dict,
                        len(Answer3)-1:[f"答案（选项一或选项二或选项三？）：{Answer3[:-1]}",
                                        f"答案（选项一或选项二或选项三？）： {Answer3[:-1]}",
                                        f"answer (option 1 or 2 or 3?):{Answer3[:-1]}",
                                        f"answer (option 1 or 2 or 3?): {Answer3[:-1]}"]}
                            
        start_option1_list,start_option2_list,start_option3_list = [],[],[]
        # some of the model will give response containing the question, we usually preprocess the response to remove the question part, but sometimes due to the model's response format, some of the question part is not removed, so here we are checking the response with the question part as well.
        for key1, key2, key3 in zip(start_str1_dict.keys(), start_str2_dict.keys(), start_str3_dict.keys()):
            for str1, str2, str3 in zip(start_str1_dict[key1], start_str2_dict[key2], start_str3_dict[key3]):
                for i in range(key1, len(str1)+1):
                    start_option1_list.append(str1[-i:])
                for i in range(key2, len(str2)+1):
                    start_option2_list.append(str2[-i:])
                for i in range(key3, len(str3)+1):
                    start_option3_list.append(str3[-i:])

        inner_option1_list = ["answer (option 1 or 2 or 3 ?): {}".format(Answer1[:-1]),"(option 1 or 2 or 3?): {}".format({Answer1[:-1]})]+common_option_1_list
        inner_option2_list = ["answer (option 1 or 2 or 3 ?): {}".format(Answer2[:-1]),"(option 1 or 2 or 3?): {}".format({Answer2[:-1]})]+common_option_2_list
        inner_option3_list = ["answer (option 1 or 2 or 3 ?): {}".format(Answer3[:-1]),"(option 1 or 2 or 3?): {}".format({Answer3[:-1]})]+common_option_3_list

        if any(option in model_response for option in ["选项一或选项二","选项二或选项三","option 1 or option 2", "option2 or option 3"]) \
            or "option 1" in model_response and "option 2" in model_response and "option 3" in model_response \
            or "选项一" in model_response and "选项二" in model_response and "选项三" in model_response \
            or len(model_response) == 0:
            return -1
        elif model_response.startswith(tuple(start_option1_list)) \
            or any(Answer1 == option for option in [model_response]) \
            or len(Answer1) > 1 and len(model_response) > 0 and (model_response in Answer1):
            label = 1
        elif model_response.startswith(tuple(start_option2_list)) \
            or any(Answer2 == option for option in [model_response]) \
            or len(Answer2) > 1 and len(model_response) > 0 and (model_response in Answer2):
            label = 2
        elif model_response.startswith(tuple(start_option3_list)) \
            or any(Answer3 == option for option in [model_response]) \
            or len(Answer3) > 1 and len(model_response) > 0 and (model_response in Answer3):
            label = 3
        elif any(model_response.find(option)>-1 and (low_index:=min(low_index, model_response.find(option)))>-1 for option in inner_option1_list)\
            or "正确答案" in model_response and ("选项一" in model_response):
            label = 1
            if any(option in model_response and model_response.find(option) < low_index for option in inner_option2_list):
                label = 2
                if any(option in model_response and model_response.find(option) < low_index for option in inner_option3_list):
                    label = 3
        elif any(model_response.find(option) > -1 for option in inner_option2_list)\
            or "正确答案" in model_response and ("选项二" in model_response):
            label = 2
            if any(option in model_response and model_response.find(option) < low_index for option in inner_option3_list):
                label = 3
        elif any(model_response.find(option) > -1 for option in inner_option3_list)\
            or "正确答案" in model_response and ("选项三" in model_response):
            label = 3
        else:
            return -1
        return label