mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
[Fix] SubSizePartition fix (#746)
* fix subjective_eval * subject_eval partition situation fixed * subject_eval partition situation fixed
This commit is contained in:
parent
0a525985e8
commit
db919f0191
@ -97,23 +97,42 @@ class SubjectiveEvalTask(BaseTask):
|
|||||||
for m in model_cfg
|
for m in model_cfg
|
||||||
]
|
]
|
||||||
|
|
||||||
# Load predictions
|
|
||||||
filename = get_infer_output_path(
|
|
||||||
model_cfg, dataset_cfg, osp.join(self.work_dir, 'predictions'))
|
|
||||||
# in case the prediction is partial
|
|
||||||
root, ext = osp.splitext(filename)
|
|
||||||
partial_filename = root + '_0' + ext
|
|
||||||
pred_strs = None
|
pred_strs = None
|
||||||
|
|
||||||
|
# There will be 5 situations, so we need to deal with them
|
||||||
|
# 1.There are no partitions in infer and judge stage
|
||||||
|
# 2.No partition in infer stage, but use partition in judge stage
|
||||||
|
# 3.Use partition in infer stage, but not use partition in judge stage
|
||||||
|
# 4.Use both partition, with same partition size
|
||||||
|
# 5.Use both partition, but different partition size
|
||||||
|
|
||||||
|
# If take SubjectSizePartition, get new filename without _0
|
||||||
|
if 'test_range' in dataset_cfg['reader_cfg']:
|
||||||
|
filename = get_infer_output_path(
|
||||||
|
model_cfg, dataset_cfg, osp.join(self.work_dir, 'predictions'))
|
||||||
|
root, ext = osp.splitext(filename)
|
||||||
|
filename = root[:-2] + ext
|
||||||
|
# If take SubjectNaivePartition, get filename
|
||||||
|
else:
|
||||||
|
filename = get_infer_output_path(
|
||||||
|
model_cfg, dataset_cfg, osp.join(self.work_dir, 'predictions'))
|
||||||
|
|
||||||
|
# Get partition name
|
||||||
|
root, ext = osp.splitext(filename)
|
||||||
|
partial_filename = root + '_0' + ext
|
||||||
|
|
||||||
|
# If no predictions get in predictions dir
|
||||||
if not osp.exists(osp.realpath(filename)) and not osp.exists(
|
if not osp.exists(osp.realpath(filename)) and not osp.exists(
|
||||||
osp.realpath(partial_filename)):
|
osp.realpath(partial_filename)):
|
||||||
return {'error': 'No predictions found.'}
|
return {'error': 'No predictions found.'}
|
||||||
else:
|
else:
|
||||||
|
# If use Naive partition in infer stage
|
||||||
if osp.exists(osp.realpath(filename)):
|
if osp.exists(osp.realpath(filename)):
|
||||||
preds = mmengine.load(filename)
|
preds = mmengine.load(filename)
|
||||||
pred_strs = [
|
pred_strs = [
|
||||||
preds[str(i)]['prediction'] for i in range(len(preds))
|
preds[str(i)]['prediction'] for i in range(len(preds))
|
||||||
]
|
]
|
||||||
|
# If use Size partition in infer stage
|
||||||
else:
|
else:
|
||||||
filename = partial_filename
|
filename = partial_filename
|
||||||
pred_strs = []
|
pred_strs = []
|
||||||
@ -125,6 +144,15 @@ class SubjectiveEvalTask(BaseTask):
|
|||||||
pred_strs += [
|
pred_strs += [
|
||||||
preds[str(i)]['prediction'] for i in range(len(preds))
|
preds[str(i)]['prediction'] for i in range(len(preds))
|
||||||
]
|
]
|
||||||
|
# Get all predictions in pred_strs
|
||||||
|
|
||||||
|
# If take SubjectSizePartition, get new pred_strs based on test_range
|
||||||
|
if 'test_range' in dataset_cfg['reader_cfg']:
|
||||||
|
test_range = dataset_cfg['reader_cfg']['test_range']
|
||||||
|
pred_strs = eval('pred_strs' + test_range)
|
||||||
|
# If take SubjectNaivePartition, get all pred_strs
|
||||||
|
else:
|
||||||
|
pred_strs = pred_strs
|
||||||
|
|
||||||
if ('pred_role' in eval_cfg and 'meta_template' in model_cfg
|
if ('pred_role' in eval_cfg and 'meta_template' in model_cfg
|
||||||
and not MODELS.get(model_cfg['type']).is_api):
|
and not MODELS.get(model_cfg['type']).is_api):
|
||||||
@ -150,6 +178,7 @@ class SubjectiveEvalTask(BaseTask):
|
|||||||
kwargs = pred_postprocessor or eval_cfg['pred_postprocessor']
|
kwargs = pred_postprocessor or eval_cfg['pred_postprocessor']
|
||||||
proc = TEXT_POSTPROCESSORS.get(kwargs.pop('type'))
|
proc = TEXT_POSTPROCESSORS.get(kwargs.pop('type'))
|
||||||
pred_strs = [proc(s, **kwargs) for s in pred_strs]
|
pred_strs = [proc(s, **kwargs) for s in pred_strs]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'model_name': model_abbr_from_cfg(model_cfg),
|
'model_name': model_abbr_from_cfg(model_cfg),
|
||||||
'model_preds': pred_strs
|
'model_preds': pred_strs
|
||||||
|
Loading…
Reference in New Issue
Block a user