jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix subjective processing

Browse Source
This commit is contained in:
Myhs-phz 2025-03-05 06:27:45 +00:00
parent 30bb39076f
commit cbba0a876f
1 changed files with 336 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

270
opencompass/utils/result_station.py
View File

@ -3,7 +3,9 @@ import os
import os.path as osp import os.path as osp
import re import re
from opencompass.utils.abbr import dataset_abbr_from_cfg, model_abbr_from_cfg from opencompass.utils.abbr import (dataset_abbr_from_cfg,
deal_with_judge_model_abbr,
model_abbr_from_cfg)
def Save_To_Station(cfg, args): def Save_To_Station(cfg, args):
@ -14,14 +16,14 @@ def Save_To_Station(cfg, args):
station_path = cfg.get('station_path') station_path = cfg.get('station_path')
work_dict = cfg['work_dir'] work_dict = cfg['work_dir']
# objective dataset processing
if 'judge_models' not in cfg.keys():
model_list = [model_abbr_from_cfg(model) for model in cfg['models']] model_list = [model_abbr_from_cfg(model) for model in cfg['models']]
dataset_list = [ dataset_list = [
dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets'] dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets']
] ]
# model_list = [i['abbr'] for i in cfg['models']]
# dataset_list = [i['abbr'] for i in cfg['datasets']]
rs_exist_results = [] rs_exist_results = []
if 'rs_exist_results' in cfg.keys(): if 'rs_exist_results' in cfg.keys():
rs_exist_results = cfg['rs_exist_results'] rs_exist_results = cfg['rs_exist_results']
@ -36,8 +38,8 @@ def Save_To_Station(cfg, args):
and not args.station_overwrite): and not args.station_overwrite):
continue continue
result_file_name = model + '.json' result_file_name = model + '.json'
if osp.exists( if osp.exists(osp.join(
osp.join(result_path, result_path,
result_file_name)) and not args.station_overwrite: result_file_name)) and not args.station_overwrite:
print('result of {} with {} already exists'.format( print('result of {} with {} already exists'.format(
dataset, model)) dataset, model))
@ -48,6 +50,8 @@ def Save_To_Station(cfg, args):
local_result_json = osp.join(local_result_path, local_result_json = osp.join(local_result_path,
dataset + '.json') dataset + '.json')
if not osp.exists(local_result_json): if not osp.exists(local_result_json):
if args.mode == 'viz':
continue
raise ValueError( raise ValueError(
'invalid file: {}'.format(local_result_json)) 'invalid file: {}'.format(local_result_json))
with open(local_result_json, 'r') as f: with open(local_result_json, 'r') as f:
@ -61,14 +65,17 @@ def Save_To_Station(cfg, args):
rf'^{re.escape(dataset)}(?:_\d+)?\.json$' rf'^{re.escape(dataset)}(?:_\d+)?\.json$'
local_prediction_json = find_files_by_regex( local_prediction_json = find_files_by_regex(
local_prediction_path, local_prediction_regex) local_prediction_path, local_prediction_regex)
if not check_filenames(dataset, local_prediction_json): if not check_filenames(
raise ValueError( dataset,
'invalid filelist: {}'.format(local_prediction_json)) local_prediction_json) and args.mode != 'viz':
raise ValueError('invalid filelist: {}'.format(
local_prediction_json))
this_prediction = [] this_prediction = []
for prediction_json in local_prediction_json: for prediction_json in local_prediction_json:
with open(osp.join(local_prediction_path, prediction_json), with open(
'r') as f: osp.join(local_prediction_path,
prediction_json), 'r') as f:
this_prediction_load_json = json.load(f) this_prediction_load_json = json.load(f)
f.close() f.close()
for prekey in this_prediction_load_json.keys(): for prekey in this_prediction_load_json.keys():
@ -77,7 +84,8 @@ def Save_To_Station(cfg, args):
# get config dict # get config dict
model_cfg = [ model_cfg = [
i for i in cfg['models'] if model_abbr_from_cfg(i) == model i for i in cfg['models']
if model_abbr_from_cfg(i) == model
][0] ][0]
dataset_cfg = [ dataset_cfg = [
i for i in cfg['datasets'] i for i in cfg['datasets']
@ -91,15 +99,142 @@ def Save_To_Station(cfg, args):
'results': this_result, 'results': this_result,
'cfg': this_cfg 'cfg': this_cfg
} }
with open(osp.join(result_path, result_file_name), 'w') as f: with open(osp.join(result_path, result_file_name),
'w') as f:
json.dump(data_model_results, json.dump(data_model_results,
f, f,
ensure_ascii=False, ensure_ascii=False,
indent=4) indent=4)
f.close() f.close()
print('successfully save result of {} with {} to the station'. print(
format(dataset, model)) 'successfully save result of {} with {} to the station'
.format(dataset, model))
return True
# subjective processing
else:
model_list = [model for model in cfg['models']]
judge_list = [judge_model for judge_model in cfg['judge_models']]
model_pair_list = [[
deal_with_judge_model_abbr(model, judge_model)
for judge_model in judge_list
] for model in model_list]
dataset_list = [[
dataset_abbr_from_cfg(dataset),
[dataset_abbr_from_cfg(base) for base in dataset['given_pred']]
] if 'base_models' in dataset.keys() and 'eval_cfg' in dataset.keys()
else [dataset_abbr_from_cfg(dataset), ['']]
for dataset in cfg['datasets']]
rs_exist_results = []
if 'rs_exist_results' in cfg.keys():
rs_exist_results = cfg['rs_exist_results']
for pair_of_dataset_and_base in dataset_list:
dataset, base_list = pair_of_dataset_and_base[
0], pair_of_dataset_and_base[1]
result_path = osp.join(station_path, dataset)
if not osp.exists(result_path):
os.makedirs(result_path)
for base_model in base_list:
base_model_name = base_model
if base_model_name != '':
base_model_name += '_'
for model_pair_sub_list in model_pair_list:
for model_pair in model_pair_sub_list:
model = model_abbr_from_cfg(model_pair[0])
model_result = model_abbr_from_cfg(model_pair)
if ([model, dataset] in rs_exist_results
and not args.station_overwrite):
continue
result_file_name = (base_model_name + model_result +
'.json')
if osp.exists(osp.join(result_path, result_file_name)
) and not args.station_overwrite:
print('{} at {} already exists'.format(
result_file_name, result_path))
continue
else:
# get result dict
local_result_path = osp.join(
work_dict, 'results',
base_model_name + model_result)
local_result_json = osp.join(
local_result_path, dataset + '.json')
if not osp.exists(local_result_json):
if args.mode == 'viz':
continue
raise ValueError('invalid file: {}'.format(
local_result_json))
with open(local_result_json, 'r') as f:
this_result = json.load(f)
f.close()
# get prediction list
local_prediction_path = osp.join(
work_dict, 'predictions', model)
local_prediction_regex = \
rf'^{re.escape(dataset)}(?:_\d+)?\.json$'
local_prediction_json = find_files_by_regex(
local_prediction_path, local_prediction_regex)
if not check_filenames(dataset,
local_prediction_json
) and args.mode != 'viz':
raise ValueError('invalid filelist: {}'.format(
local_prediction_json))
this_prediction = []
for prediction_json in local_prediction_json:
with open(
osp.join(local_prediction_path,
prediction_json), 'r') as f:
this_prediction_load_json = json.load(f)
f.close()
for prekey in this_prediction_load_json.keys():
this_prediction.append(
this_prediction_load_json[prekey])
# get config dict
model_cfg = [
i for i in cfg['models']
if model_abbr_from_cfg(i) == model
][0]
dataset_cfg = [
i for i in cfg['datasets']
if dataset_abbr_from_cfg(i) == dataset
][0]
judge_model_cfg = [
i for i in cfg['judge_models']
if 'judged-by--' + model_abbr_from_cfg(i) ==
model_abbr_from_cfg(model_pair[1])
]
this_cfg = {
'models': model_cfg,
'datasets': dataset_cfg,
'judge_models': judge_model_cfg
}
# dict combine
data_model_results = {
'predictions': this_prediction,
'results': this_result,
'cfg': this_cfg
}
with open(osp.join(result_path, result_file_name),
'w') as f:
json.dump(data_model_results,
f,
ensure_ascii=False,
indent=4)
f.close()
print('successfully save result: {} at {} to the'
'station'.format(result_file_name,
result_path))
return True return True
@ -111,21 +246,25 @@ def Read_From_Station(cfg, args):
else: else:
station_path = cfg.get('station_path') station_path = cfg.get('station_path')
# objective check
if 'judge_models' not in cfg.keys():
model_list = [model_abbr_from_cfg(model) for model in cfg['models']] model_list = [model_abbr_from_cfg(model) for model in cfg['models']]
dataset_list = [ dataset_list = [
dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets'] dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets']
] ]
# model_list = [i['abbr'] for i in cfg['models']]
# dataset_list = [i['abbr'] for i in cfg['datasets']]
existing_results_list = [] existing_results_list = []
result_local_path = osp.join(cfg['work_dir'], 'results')
if not osp.exists(result_local_path):
os.makedirs(result_local_path)
for model in model_list:
for dataset in dataset_list: for dataset in dataset_list:
result_file_path = osp.join(station_path, dataset, model + '.json') for model in model_list:
result_file_path = osp.join(station_path, dataset,
model + '.json')
if not osp.exists(result_file_path): if not osp.exists(result_file_path):
print('do not find result file: {} with {} at station'.format( print('do not find result file: {} with {} at station'.
model, dataset)) format(model, dataset))
continue continue
else: else:
print('find result file: {} with {} at station'.format( print('find result file: {} with {} at station'.format(
@ -135,21 +274,19 @@ def Read_From_Station(cfg, args):
f.close() f.close()
existing_results_list.append({ existing_results_list.append({
'combination': [model, dataset], 'combination': [model, dataset],
'file': download_json 'file':
download_json
}) })
# save results to local # save results to local
result_local_path = osp.join(cfg['work_dir'], 'results')
if not osp.exists(result_local_path):
os.makedirs(result_local_path)
for i in existing_results_list: for i in existing_results_list:
this_result = i['file']['results'] this_result = i['file']['results']
this_result_local_path = osp.join(result_local_path, this_result_local_path = osp.join(result_local_path,
i['combination'][0]) i['combination'][0])
if not osp.exists(this_result_local_path): if not osp.exists(this_result_local_path):
os.makedirs(this_result_local_path) os.makedirs(this_result_local_path)
this_result_local_file_path = osp.join(this_result_local_path, this_result_local_file_path = osp.join(
i['combination'][1] + '.json') this_result_local_path, i['combination'][1] + '.json')
if osp.exists(this_result_local_file_path): if osp.exists(this_result_local_file_path):
continue continue
with open(this_result_local_file_path, 'w') as f: with open(this_result_local_file_path, 'w') as f:
@ -158,6 +295,85 @@ def Read_From_Station(cfg, args):
return existing_results_list return existing_results_list
# subjective check
else:
model_list = [model for model in cfg['models']]
judge_list = [judge_model for judge_model in cfg['judge_models']]
model_pair_list = [[
deal_with_judge_model_abbr(model, judge_model)
for judge_model in judge_list
] for model in model_list]
dataset_list = [[
dataset_abbr_from_cfg(dataset),
[dataset_abbr_from_cfg(base) for base in dataset['given_pred']]
] if 'base_models' in dataset.keys() and 'eval_cfg' in dataset.keys()
else [dataset_abbr_from_cfg(dataset), ['']]
for dataset in cfg['datasets']]
existing_results_list = []
result_local_path = osp.join(cfg['work_dir'], 'results')
if not osp.exists(result_local_path):
os.makedirs(result_local_path)
for pair_of_dataset_and_base in dataset_list:
dataset, base_list = pair_of_dataset_and_base[
0], pair_of_dataset_and_base[1]
for model_pair_sub_list in model_pair_list:
result_file_path_list_origin = []
for model_pair in model_pair_sub_list:
model_result = model_abbr_from_cfg(model_pair)
for base_model in base_list:
base_model_name = base_model
if base_model_name != '':
base_model_name += '_'
result_file_path_list_origin.append(
osp.join(station_path, dataset,
base_model_name + model_result + '.json'))
result_file_path_list = [
result_file_path
for result_file_path in result_file_path_list_origin
if osp.exists(result_file_path)
]
model = model_abbr_from_cfg(model_pair_sub_list[0][0])
# save all parts of results to local
for result_file_path in result_file_path_list:
with open(result_file_path, 'r') as f:
this_result = json.load(f)['results']
f.close()
this_result_local_path = osp.join(
result_local_path,
osp.splitext(osp.basename(result_file_path))[0])
if not osp.exists(this_result_local_path):
os.makedirs(this_result_local_path)
this_result_local_file_path = osp.join(
this_result_local_path, dataset + '.json')
if osp.exists(this_result_local_file_path):
continue
with open(this_result_local_file_path, 'w') as f:
json.dump(this_result, f, ensure_ascii=False, indent=4)
f.close()
# check whether complete
if len(result_file_path_list) == len(
result_file_path_list_origin):
print('find complete results of {} with {} at station'.
format(model, dataset))
existing_results_list.append({
'combination': [model, dataset],
'file':
result_file_path_list
})
else:
print('results of {} with {} at station is not complete'.
format(model, dataset))
return existing_results_list
def find_files_by_regex(directory, pattern): def find_files_by_regex(directory, pattern):