jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Fix] Fix cli evaluation for multiple models (#1454)

Browse Source 
* update

* update
This commit is contained in:
Linchen Xiao 2024-08-23 17:15:36 +08:00 committed by GitHub
parent 2295a33a18
commit 94b6bd65fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
opencompass/utils/run.py
View File

@ -41,16 +41,18 @@ def match_cfg_file(workdir: Union[str, List[str]],
if len(files) != len(pattern):
nomatched = []
ambiguous = []
ambiguous_return_list = []
err_msg = ('The provided pattern matches 0 or more than one '
'config. Please verify your pattern and try again. '
'You may use tools/list_configs.py to list or '
'locate the configurations.\n')
for p in pattern:
files = _mf_with_multi_workdirs(workdir, p, fuzzy=False)
if len(files) == 0:
files_ = _mf_with_multi_workdirs(workdir, p, fuzzy=False)
if len(files_) == 0:
nomatched.append([p[:-3]])
elif len(files) > 1:
ambiguous.append([p[:-3], '\n'.join(f[1] for f in files)])
elif len(files_) > 1:
ambiguous.append([p[:-3], '\n'.join(f[1] for f in files_)])
ambiguous_return_list.append(files_[0])
if nomatched:
table = [['Not matched patterns'], *nomatched]
err_msg += tabulate.tabulate(table,
@ -58,12 +60,12 @@ def match_cfg_file(workdir: Union[str, List[str]],
tablefmt='psql')
if ambiguous:
table = [['Ambiguous patterns', 'Matched files'], *ambiguous]
warning_msg = 'Found ambiguous patterns, using the first matched config.'
warning_msg = 'Found ambiguous patterns, using the first matched config.\n'
warning_msg += tabulate.tabulate(table,
headers='firstrow',
tablefmt='psql')
logger.warning(warning_msg)
return [files[0]]
return ambiguous_return_list
raise ValueError(err_msg)
return files
@ -162,13 +164,13 @@ def get_config_from_arg(args) -> Config:
]
if args.models:
# model_dir = os.path.join(args.config_dir, 'models')
for model in match_cfg_file(models_dir, args.models):
logger.info(f'Loading {model[0]}: {model[1]}')
cfg = Config.fromfile(model[1])
if 'models' not in cfg:
raise ValueError(f'Config file {model[1]} does not contain "models" field')
models += cfg['models']
for model_arg in args.models:
for model in match_cfg_file(models_dir, [model_arg]):
logger.info(f'Loading {model[0]}: {model[1]}')
cfg = Config.fromfile(model[1])
if 'models' not in cfg:
raise ValueError(f'Config file {model[1]} does not contain "models" field')
models += cfg['models']
else:
if args.hf_type == 'chat':
mod = HuggingFacewithChatTemplate