From cbba0a876f6a5783e10cef5004dc499a62584795 Mon Sep 17 00:00:00 2001 From: Myhs-phz Date: Wed, 5 Mar 2025 06:27:45 +0000 Subject: [PATCH] fix subjective processing --- opencompass/utils/result_station.py | 456 ++++++++++++++++++++-------- 1 file changed, 336 insertions(+), 120 deletions(-) diff --git a/opencompass/utils/result_station.py b/opencompass/utils/result_station.py index 0ca7afbf..96f0eebf 100644 --- a/opencompass/utils/result_station.py +++ b/opencompass/utils/result_station.py @@ -3,7 +3,9 @@ import os import os.path as osp import re -from opencompass.utils.abbr import dataset_abbr_from_cfg, model_abbr_from_cfg +from opencompass.utils.abbr import (dataset_abbr_from_cfg, + deal_with_judge_model_abbr, + model_abbr_from_cfg) def Save_To_Station(cfg, args): @@ -14,93 +16,226 @@ def Save_To_Station(cfg, args): station_path = cfg.get('station_path') work_dict = cfg['work_dir'] - model_list = [model_abbr_from_cfg(model) for model in cfg['models']] - dataset_list = [ - dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets'] - ] - # model_list = [i['abbr'] for i in cfg['models']] - # dataset_list = [i['abbr'] for i in cfg['datasets']] + # objective dataset processing + if 'judge_models' not in cfg.keys(): + model_list = [model_abbr_from_cfg(model) for model in cfg['models']] + dataset_list = [ + dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets'] + ] - rs_exist_results = [] - if 'rs_exist_results' in cfg.keys(): - rs_exist_results = cfg['rs_exist_results'] + rs_exist_results = [] + if 'rs_exist_results' in cfg.keys(): + rs_exist_results = cfg['rs_exist_results'] - for dataset in dataset_list: - result_path = osp.join(station_path, dataset) - if not osp.exists(result_path): - os.makedirs(result_path) + for dataset in dataset_list: + result_path = osp.join(station_path, dataset) + if not osp.exists(result_path): + os.makedirs(result_path) - for model in model_list: - if ([model, dataset] in rs_exist_results - and not args.station_overwrite): - continue - result_file_name = model + '.json' - if osp.exists( - osp.join(result_path, - result_file_name)) and not args.station_overwrite: - print('result of {} with {} already exists'.format( - dataset, model)) - continue - else: - # get result dict - local_result_path = osp.join(work_dict, 'results', model) - local_result_json = osp.join(local_result_path, - dataset + '.json') - if not osp.exists(local_result_json): - raise ValueError( - 'invalid file: {}'.format(local_result_json)) - with open(local_result_json, 'r') as f: - this_result = json.load(f) - f.close() - - # get prediction list - local_prediction_path = osp.join(work_dict, 'predictions', - model) - local_prediction_regex = \ - rf'^{re.escape(dataset)}(?:_\d+)?\.json$' - local_prediction_json = find_files_by_regex( - local_prediction_path, local_prediction_regex) - if not check_filenames(dataset, local_prediction_json): - raise ValueError( - 'invalid filelist: {}'.format(local_prediction_json)) - - this_prediction = [] - for prediction_json in local_prediction_json: - with open(osp.join(local_prediction_path, prediction_json), - 'r') as f: - this_prediction_load_json = json.load(f) + for model in model_list: + if ([model, dataset] in rs_exist_results + and not args.station_overwrite): + continue + result_file_name = model + '.json' + if osp.exists(osp.join( + result_path, + result_file_name)) and not args.station_overwrite: + print('result of {} with {} already exists'.format( + dataset, model)) + continue + else: + # get result dict + local_result_path = osp.join(work_dict, 'results', model) + local_result_json = osp.join(local_result_path, + dataset + '.json') + if not osp.exists(local_result_json): + if args.mode == 'viz': + continue + raise ValueError( + 'invalid file: {}'.format(local_result_json)) + with open(local_result_json, 'r') as f: + this_result = json.load(f) f.close() - for prekey in this_prediction_load_json.keys(): - this_prediction.append( - this_prediction_load_json[prekey]) - # get config dict - model_cfg = [ - i for i in cfg['models'] if model_abbr_from_cfg(i) == model - ][0] - dataset_cfg = [ - i for i in cfg['datasets'] - if dataset_abbr_from_cfg(i) == dataset - ][0] - this_cfg = {'models': model_cfg, 'datasets': dataset_cfg} + # get prediction list + local_prediction_path = osp.join(work_dict, 'predictions', + model) + local_prediction_regex = \ + rf'^{re.escape(dataset)}(?:_\d+)?\.json$' + local_prediction_json = find_files_by_regex( + local_prediction_path, local_prediction_regex) + if not check_filenames( + dataset, + local_prediction_json) and args.mode != 'viz': + raise ValueError('invalid filelist: {}'.format( + local_prediction_json)) - # dict combine - data_model_results = { - 'predictions': this_prediction, - 'results': this_result, - 'cfg': this_cfg - } - with open(osp.join(result_path, result_file_name), 'w') as f: - json.dump(data_model_results, - f, - ensure_ascii=False, - indent=4) - f.close() - print('successfully save result of {} with {} to the station'. - format(dataset, model)) + this_prediction = [] + for prediction_json in local_prediction_json: + with open( + osp.join(local_prediction_path, + prediction_json), 'r') as f: + this_prediction_load_json = json.load(f) + f.close() + for prekey in this_prediction_load_json.keys(): + this_prediction.append( + this_prediction_load_json[prekey]) - return True + # get config dict + model_cfg = [ + i for i in cfg['models'] + if model_abbr_from_cfg(i) == model + ][0] + dataset_cfg = [ + i for i in cfg['datasets'] + if dataset_abbr_from_cfg(i) == dataset + ][0] + this_cfg = {'models': model_cfg, 'datasets': dataset_cfg} + + # dict combine + data_model_results = { + 'predictions': this_prediction, + 'results': this_result, + 'cfg': this_cfg + } + with open(osp.join(result_path, result_file_name), + 'w') as f: + json.dump(data_model_results, + f, + ensure_ascii=False, + indent=4) + f.close() + print( + 'successfully save result of {} with {} to the station' + .format(dataset, model)) + return True + + # subjective processing + else: + model_list = [model for model in cfg['models']] + judge_list = [judge_model for judge_model in cfg['judge_models']] + model_pair_list = [[ + deal_with_judge_model_abbr(model, judge_model) + for judge_model in judge_list + ] for model in model_list] + + dataset_list = [[ + dataset_abbr_from_cfg(dataset), + [dataset_abbr_from_cfg(base) for base in dataset['given_pred']] + ] if 'base_models' in dataset.keys() and 'eval_cfg' in dataset.keys() + else [dataset_abbr_from_cfg(dataset), ['']] + for dataset in cfg['datasets']] + + rs_exist_results = [] + if 'rs_exist_results' in cfg.keys(): + rs_exist_results = cfg['rs_exist_results'] + + for pair_of_dataset_and_base in dataset_list: + dataset, base_list = pair_of_dataset_and_base[ + 0], pair_of_dataset_and_base[1] + + result_path = osp.join(station_path, dataset) + if not osp.exists(result_path): + os.makedirs(result_path) + + for base_model in base_list: + base_model_name = base_model + if base_model_name != '': + base_model_name += '_' + for model_pair_sub_list in model_pair_list: + for model_pair in model_pair_sub_list: + model = model_abbr_from_cfg(model_pair[0]) + model_result = model_abbr_from_cfg(model_pair) + if ([model, dataset] in rs_exist_results + and not args.station_overwrite): + continue + result_file_name = (base_model_name + model_result + + '.json') + if osp.exists(osp.join(result_path, result_file_name) + ) and not args.station_overwrite: + print('{} at {} already exists'.format( + result_file_name, result_path)) + continue + else: + # get result dict + local_result_path = osp.join( + work_dict, 'results', + base_model_name + model_result) + local_result_json = osp.join( + local_result_path, dataset + '.json') + if not osp.exists(local_result_json): + if args.mode == 'viz': + continue + raise ValueError('invalid file: {}'.format( + local_result_json)) + with open(local_result_json, 'r') as f: + this_result = json.load(f) + f.close() + + # get prediction list + local_prediction_path = osp.join( + work_dict, 'predictions', model) + local_prediction_regex = \ + rf'^{re.escape(dataset)}(?:_\d+)?\.json$' + local_prediction_json = find_files_by_regex( + local_prediction_path, local_prediction_regex) + if not check_filenames(dataset, + local_prediction_json + ) and args.mode != 'viz': + raise ValueError('invalid filelist: {}'.format( + local_prediction_json)) + + this_prediction = [] + for prediction_json in local_prediction_json: + with open( + osp.join(local_prediction_path, + prediction_json), 'r') as f: + this_prediction_load_json = json.load(f) + f.close() + for prekey in this_prediction_load_json.keys(): + this_prediction.append( + this_prediction_load_json[prekey]) + + # get config dict + model_cfg = [ + i for i in cfg['models'] + if model_abbr_from_cfg(i) == model + ][0] + dataset_cfg = [ + i for i in cfg['datasets'] + if dataset_abbr_from_cfg(i) == dataset + ][0] + judge_model_cfg = [ + i for i in cfg['judge_models'] + if 'judged-by--' + model_abbr_from_cfg(i) == + model_abbr_from_cfg(model_pair[1]) + ] + + this_cfg = { + 'models': model_cfg, + 'datasets': dataset_cfg, + 'judge_models': judge_model_cfg + } + + # dict combine + data_model_results = { + 'predictions': this_prediction, + 'results': this_result, + 'cfg': this_cfg + } + + with open(osp.join(result_path, result_file_name), + 'w') as f: + json.dump(data_model_results, + f, + ensure_ascii=False, + indent=4) + f.close() + print('successfully save result: {} at {} to the' + 'station'.format(result_file_name, + result_path)) + return True def Read_From_Station(cfg, args): @@ -111,52 +246,133 @@ def Read_From_Station(cfg, args): else: station_path = cfg.get('station_path') - model_list = [model_abbr_from_cfg(model) for model in cfg['models']] - dataset_list = [ - dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets'] - ] - # model_list = [i['abbr'] for i in cfg['models']] - # dataset_list = [i['abbr'] for i in cfg['datasets']] + # objective check + if 'judge_models' not in cfg.keys(): + model_list = [model_abbr_from_cfg(model) for model in cfg['models']] + dataset_list = [ + dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets'] + ] - existing_results_list = [] + existing_results_list = [] + result_local_path = osp.join(cfg['work_dir'], 'results') + if not osp.exists(result_local_path): + os.makedirs(result_local_path) - for model in model_list: for dataset in dataset_list: - result_file_path = osp.join(station_path, dataset, model + '.json') - if not osp.exists(result_file_path): - print('do not find result file: {} with {} at station'.format( - model, dataset)) + for model in model_list: + result_file_path = osp.join(station_path, dataset, + model + '.json') + if not osp.exists(result_file_path): + print('do not find result file: {} with {} at station'. + format(model, dataset)) + continue + else: + print('find result file: {} with {} at station'.format( + model, dataset)) + with open(result_file_path, 'r') as f: + download_json = json.load(f) + f.close() + existing_results_list.append({ + 'combination': [model, dataset], + 'file': + download_json + }) + + # save results to local + for i in existing_results_list: + this_result = i['file']['results'] + this_result_local_path = osp.join(result_local_path, + i['combination'][0]) + if not osp.exists(this_result_local_path): + os.makedirs(this_result_local_path) + this_result_local_file_path = osp.join( + this_result_local_path, i['combination'][1] + '.json') + if osp.exists(this_result_local_file_path): continue - else: - print('find result file: {} with {} at station'.format( - model, dataset)) - with open(result_file_path, 'r') as f: - download_json = json.load(f) - f.close() - existing_results_list.append({ - 'combination': [model, dataset], - 'file': download_json - }) + with open(this_result_local_file_path, 'w') as f: + json.dump(this_result, f, ensure_ascii=False, indent=4) + f.close() - # save results to local - result_local_path = osp.join(cfg['work_dir'], 'results') - if not osp.exists(result_local_path): - os.makedirs(result_local_path) - for i in existing_results_list: - this_result = i['file']['results'] - this_result_local_path = osp.join(result_local_path, - i['combination'][0]) - if not osp.exists(this_result_local_path): - os.makedirs(this_result_local_path) - this_result_local_file_path = osp.join(this_result_local_path, - i['combination'][1] + '.json') - if osp.exists(this_result_local_file_path): - continue - with open(this_result_local_file_path, 'w') as f: - json.dump(this_result, f, ensure_ascii=False, indent=4) - f.close() + return existing_results_list - return existing_results_list + # subjective check + else: + model_list = [model for model in cfg['models']] + judge_list = [judge_model for judge_model in cfg['judge_models']] + model_pair_list = [[ + deal_with_judge_model_abbr(model, judge_model) + for judge_model in judge_list + ] for model in model_list] + + dataset_list = [[ + dataset_abbr_from_cfg(dataset), + [dataset_abbr_from_cfg(base) for base in dataset['given_pred']] + ] if 'base_models' in dataset.keys() and 'eval_cfg' in dataset.keys() + else [dataset_abbr_from_cfg(dataset), ['']] + for dataset in cfg['datasets']] + + existing_results_list = [] + result_local_path = osp.join(cfg['work_dir'], 'results') + if not osp.exists(result_local_path): + os.makedirs(result_local_path) + + for pair_of_dataset_and_base in dataset_list: + dataset, base_list = pair_of_dataset_and_base[ + 0], pair_of_dataset_and_base[1] + + for model_pair_sub_list in model_pair_list: + result_file_path_list_origin = [] + for model_pair in model_pair_sub_list: + model_result = model_abbr_from_cfg(model_pair) + for base_model in base_list: + base_model_name = base_model + if base_model_name != '': + base_model_name += '_' + + result_file_path_list_origin.append( + osp.join(station_path, dataset, + base_model_name + model_result + '.json')) + + result_file_path_list = [ + result_file_path + for result_file_path in result_file_path_list_origin + if osp.exists(result_file_path) + ] + model = model_abbr_from_cfg(model_pair_sub_list[0][0]) + + # save all parts of results to local + for result_file_path in result_file_path_list: + with open(result_file_path, 'r') as f: + this_result = json.load(f)['results'] + f.close() + this_result_local_path = osp.join( + result_local_path, + osp.splitext(osp.basename(result_file_path))[0]) + if not osp.exists(this_result_local_path): + os.makedirs(this_result_local_path) + this_result_local_file_path = osp.join( + this_result_local_path, dataset + '.json') + if osp.exists(this_result_local_file_path): + continue + with open(this_result_local_file_path, 'w') as f: + json.dump(this_result, f, ensure_ascii=False, indent=4) + f.close() + + # check whether complete + if len(result_file_path_list) == len( + result_file_path_list_origin): + print('find complete results of {} with {} at station'. + format(model, dataset)) + existing_results_list.append({ + 'combination': [model, dataset], + 'file': + result_file_path_list + }) + else: + print('results of {} with {} at station is not complete'. + format(model, dataset)) + + return existing_results_list def find_files_by_regex(directory, pattern):