jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat

Browse Source
This commit is contained in:
Myhs-phz 2025-03-03 04:54:54 +00:00
parent c3ad4b5603
commit 8a52351e41
3 changed files with 88 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
opencompass/cli/main.py
View File

@ -141,6 +141,12 @@ def parse_args():
type=str, type=str,
default=None, default=None,
) )
parser.add_argument(
'--read-from-station',
help='Whether to save the evaluation results to the '
'data station.',
action='store_true',
)
# set srun args # set srun args
@ -256,8 +262,6 @@ def main():
else: else:
dirs = os.listdir(cfg.work_dir) dirs = os.listdir(cfg.work_dir)
dir_time_str = sorted(dirs)[-1] dir_time_str = sorted(dirs)[-1]
elif args.reuse == 'station':
Read_From_Station(cfg, args, dir_time_str)
else: else:
dir_time_str = args.reuse dir_time_str = args.reuse
logger.info(f'Reusing experiements from {dir_time_str}') logger.info(f'Reusing experiements from {dir_time_str}')
@ -280,6 +284,13 @@ def main():
# types cannot be serialized # types cannot be serialized
cfg = Config.fromfile(output_config_path, format_python_code=False) cfg = Config.fromfile(output_config_path, format_python_code=False)
# get existed results from station
if args.read_from_station:
existing_results_list = Read_From_Station(cfg, args)
rs_exist_results = [comb['combination'] for comb in existing_results_list]
cfg['rs_exist_results'] = rs_exist_results
# report to lark bot if specify --lark # report to lark bot if specify --lark
if not args.lark: if not args.lark:
cfg['lark_bot_url'] = None cfg['lark_bot_url'] = None
@ -371,12 +382,7 @@ def main():
# save to station # save to station
if args.save_to_station: if args.save_to_station:
if Save_To_Station(cfg, args): Save_To_Station(cfg, args)
logger.info('Successfully saved to station.')
else:
logger.warning('Failed to save result to station.')
# visualize # visualize
if args.mode in ['all', 'eval', 'viz']: if args.mode in ['all', 'eval', 'viz']:

20
opencompass/partitioners/base.py
View File

@ -102,6 +102,7 @@ class BasePartitioner:
return tasks return tasks
def parse_model_dataset_args(self, cfg: ConfigDict): def parse_model_dataset_args(self, cfg: ConfigDict):
#breakpoint()
models = cfg['models'] models = cfg['models']
datasets = cfg['datasets'] datasets = cfg['datasets']
@ -109,7 +110,24 @@ class BasePartitioner:
if 'model_dataset_combinations' in sig.parameters: if 'model_dataset_combinations' in sig.parameters:
combs = cfg.get('model_dataset_combinations', None) combs = cfg.get('model_dataset_combinations', None)
if combs is None: if combs is None:
combs = [{'models': models, 'datasets': datasets}] if 'rs_exist_results' in cfg.keys():
rs_exist_results = cfg['rs_exist_results']
combs = []
for model in models:
comb = {'models': [model], 'datasets': datasets}
combs.append(comb)
for i in range(len(combs)):
combs[i]['datasets'] = [
dataset for dataset in combs[i]['datasets'] if [
model_abbr_from_cfg(combs[i]['models'][0]),
dataset_abbr_from_cfg(dataset)
] not in rs_exist_results
]
combs = [
comb for comb in combs if len(comb['datasets']) != 0
]
else:
combs = [{'models': models, 'datasets': datasets}]
else: else:
# sanity check # sanity check
model_abbrs = [model_abbr_from_cfg(model) for model in models] model_abbrs = [model_abbr_from_cfg(model) for model in models]

112
opencompass/utils/result_station.py
View File

@ -3,6 +3,8 @@ import os
import os.path as osp import os.path as osp
import re import re
from opencompass.utils.abbr import dataset_abbr_from_cfg, model_abbr_from_cfg
def Save_To_Station(cfg, args): def Save_To_Station(cfg, args):
@ -14,8 +16,13 @@ def Save_To_Station(cfg, args):
station_path = args.station_path station_path = args.station_path
work_dict = cfg['work_dir'] work_dict = cfg['work_dir']
model_list = [i['abbr'] for i in cfg['models']] model_list = [model_abbr_from_cfg(model) for model in cfg['models']]
dataset_list = [i['abbr'] for i in cfg['datasets']] dataset_list = [
dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets']
]
# model_list = [i['abbr'] for i in cfg['models']]
# dataset_list = [i['abbr'] for i in cfg['datasets']]
for dataset in dataset_list: for dataset in dataset_list:
result_path = osp.join(station_path, dataset) result_path = osp.join(station_path, dataset)
@ -61,10 +68,21 @@ def Save_To_Station(cfg, args):
this_prediction.append( this_prediction.append(
this_prediction_load_json[prekey]) this_prediction_load_json[prekey])
# get config dict
model_cfg = [
i for i in cfg['models'] if model_abbr_from_cfg(i) == model
][0]
dataset_cfg = [
i for i in cfg['datasets']
if dataset_abbr_from_cfg(i) == dataset
][0]
this_cfg = {'models': model_cfg, 'datasets': dataset_cfg}
# dict combine # dict combine
data_model_results = { data_model_results = {
'predictions': this_prediction, 'predictions': this_prediction,
'results': this_result 'results': this_result,
'cfg': this_cfg
} }
with open(osp.join(result_path, result_file_name), 'w') as f: with open(osp.join(result_path, result_file_name), 'w') as f:
json.dump(data_model_results, json.dump(data_model_results,
@ -72,13 +90,13 @@ def Save_To_Station(cfg, args):
ensure_ascii=False, ensure_ascii=False,
indent=4) indent=4)
f.close() f.close()
print('result of {} with {} already exists'.format( print('successfully save result of {} with {} to the station'.
dataset, model)) format(dataset, model))
return True return True
def Read_From_Station(cfg, args, dir_time_str): def Read_From_Station(cfg, args):
assert args.station_path is not None or 'station_path' in cfg.keys( assert args.station_path is not None or 'station_path' in cfg.keys(
) and cfg['station_path'] is not None ) and cfg['station_path'] is not None
@ -87,70 +105,50 @@ def Read_From_Station(cfg, args, dir_time_str):
else: else:
station_path = args.station_path station_path = args.station_path
work_dict = osp.join(cfg.work_dir, dir_time_str) model_list = [model_abbr_from_cfg(model) for model in cfg['models']]
model_list = [i['abbr'] for i in cfg['models']] dataset_list = [
dataset_list = [i['abbr'] for i in cfg['datasets']] dataset_abbr_from_cfg(dataset) for dataset in cfg['datasets']
]
# model_list = [i['abbr'] for i in cfg['models']]
# dataset_list = [i['abbr'] for i in cfg['datasets']]
if not osp.exists(work_dict): existing_results_list = []
os.makedirs(work_dict)
local_prediction_path = osp.join(work_dict, 'predictions')
if not osp.exists(local_prediction_path):
os.makedirs(local_prediction_path)
local_result_path = osp.join(work_dict, 'results')
if not osp.exists(local_result_path):
os.makedirs(local_result_path)
for model in model_list: for model in model_list:
for dataset in dataset_list:
for data in dataset_list: result_file_path = osp.join(station_path, dataset, model + '.json')
result_file_path = osp.join(station_path, data, model + '.json')
if not osp.exists(result_file_path): if not osp.exists(result_file_path):
print('do not find result file: {} with {} at station'.format( print('do not find result file: {} with {} at station'.format(
model, data)) model, dataset))
continue continue
else: else:
print('find result file: {} with {} at station'.format( print('find result file: {} with {} at station'.format(
model, data)) model, dataset))
with open(result_file_path, 'r') as f: with open(result_file_path, 'r') as f:
download_json = json.load(f) download_json = json.load(f)
f.close() f.close()
existing_results_list.append({
'combination': [model, dataset],
'file': download_json
})
this_local_prediction_path = osp.join(local_prediction_path, # save results to local
model) result_local_path = osp.join(cfg['work_dir'], 'results')
if not osp.exists(this_local_prediction_path): if not osp.exists(result_local_path):
os.makedirs(this_local_prediction_path) os.makedirs(result_local_path)
this_local_result_path = osp.join(local_result_path, model) for i in existing_results_list:
if not osp.exists(this_local_result_path): this_result = i['file']['results']
os.makedirs(this_local_result_path) this_result_local_path = osp.join(result_local_path,
i['combination'][0])
if not osp.exists(this_result_local_path):
os.makedirs(this_result_local_path)
this_result_local_file_path = osp.join(this_result_local_path,
i['combination'][1] + '.json')
with open(this_result_local_file_path, 'w') as f:
json.dump(this_result, f, ensure_ascii=False, indent=4)
f.close()
this_local_prediction_path = osp.join( return existing_results_list
this_local_prediction_path, data + '.json')
this_local_result_path = osp.join(this_local_result_path,
data + '.json')
download_json_prediction = download_json['predictions']
download_json_result = download_json['results']
# save predictions
local_prediction = {}
for i in range(len(download_json_prediction)):
local_prediction[str(i)] = download_json_prediction[i]
with open(this_local_prediction_path, 'w') as f:
json.dump(local_prediction,
f,
ensure_ascii=False,
indent=4)
f.close()
# save results
with open(this_local_result_path, 'w') as f:
json.dump(download_json_result,
f,
ensure_ascii=False,
indent=4)
f.close()
return True
def find_files_by_regex(directory, pattern): def find_files_by_regex(directory, pattern):