jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Fix] SubSizePartition fix (#746)

Browse Source 
* fix subjective_eval

* subject_eval partition situation fixed

* subject_eval partition situation fixed
This commit is contained in:
bittersweet1999 2023-12-28 11:46:46 +08:00 committed by GitHub
parent 0a525985e8
commit db919f0191
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 58 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
opencompass/tasks/subjective_eval.py
View File

@ -97,23 +97,42 @@ class SubjectiveEvalTask(BaseTask):
for m in model_cfg for m in model_cfg
] ]
# Load predictions
filename = get_infer_output_path(
model_cfg, dataset_cfg, osp.join(self.work_dir, 'predictions'))
# in case the prediction is partial
root, ext = osp.splitext(filename)
partial_filename = root + '_0' + ext
pred_strs = None pred_strs = None
# There will be 5 situations, so we need to deal with them
# 1.There are no partitions in infer and judge stage
# 2.No partition in infer stage, but use partition in judge stage
# 3.Use partition in infer stage, but not use partition in judge stage
# 4.Use both partition, with same partition size
# 5.Use both partition, but different partition size
# If take SubjectSizePartition, get new filename without _0
if 'test_range' in dataset_cfg['reader_cfg']:
filename = get_infer_output_path(
model_cfg, dataset_cfg, osp.join(self.work_dir, 'predictions'))
root, ext = osp.splitext(filename)
filename = root[:-2] + ext
# If take SubjectNaivePartition, get filename
else:
filename = get_infer_output_path(
model_cfg, dataset_cfg, osp.join(self.work_dir, 'predictions'))
# Get partition name
root, ext = osp.splitext(filename)
partial_filename = root + '_0' + ext
# If no predictions get in predictions dir
if not osp.exists(osp.realpath(filename)) and not osp.exists( if not osp.exists(osp.realpath(filename)) and not osp.exists(
osp.realpath(partial_filename)): osp.realpath(partial_filename)):
return {'error': 'No predictions found.'} return {'error': 'No predictions found.'}
else: else:
# If use Naive partition in infer stage
if osp.exists(osp.realpath(filename)): if osp.exists(osp.realpath(filename)):
preds = mmengine.load(filename) preds = mmengine.load(filename)
pred_strs = [ pred_strs = [
preds[str(i)]['prediction'] for i in range(len(preds)) preds[str(i)]['prediction'] for i in range(len(preds))
] ]
# If use Size partition in infer stage
else: else:
filename = partial_filename filename = partial_filename
pred_strs = [] pred_strs = []
@ -125,6 +144,15 @@ class SubjectiveEvalTask(BaseTask):
pred_strs += [ pred_strs += [
preds[str(i)]['prediction'] for i in range(len(preds)) preds[str(i)]['prediction'] for i in range(len(preds))
] ]
# Get all predictions in pred_strs
# If take SubjectSizePartition, get new pred_strs based on test_range
if 'test_range' in dataset_cfg['reader_cfg']:
test_range = dataset_cfg['reader_cfg']['test_range']
pred_strs = eval('pred_strs' + test_range)
# If take SubjectNaivePartition, get all pred_strs
else:
pred_strs = pred_strs
if ('pred_role' in eval_cfg and 'meta_template' in model_cfg if ('pred_role' in eval_cfg and 'meta_template' in model_cfg
and not MODELS.get(model_cfg['type']).is_api): and not MODELS.get(model_cfg['type']).is_api):
@ -150,6 +178,7 @@ class SubjectiveEvalTask(BaseTask):
kwargs = pred_postprocessor or eval_cfg['pred_postprocessor'] kwargs = pred_postprocessor or eval_cfg['pred_postprocessor']
proc = TEXT_POSTPROCESSORS.get(kwargs.pop('type')) proc = TEXT_POSTPROCESSORS.get(kwargs.pop('type'))
pred_strs = [proc(s, **kwargs) for s in pred_strs] pred_strs = [proc(s, **kwargs) for s in pred_strs]
return { return {
'model_name': model_abbr_from_cfg(model_cfg), 'model_name': model_abbr_from_cfg(model_cfg),
'model_preds': pred_strs 'model_preds': pred_strs