diff --git a/opencompass/utils/run.py b/opencompass/utils/run.py index 0d44cff9..445c8451 100644 --- a/opencompass/utils/run.py +++ b/opencompass/utils/run.py @@ -95,7 +95,15 @@ def get_config_from_arg(args) -> Config: pad_token_id=args.pad_token_id, run_cfg=dict(num_gpus=args.num_gpus)) models.append(model) - return Config(dict(models=models, datasets=datasets), + + summarizer = None + if args.summarizer: + s = match_cfg_file('configs/summarizers/', [args.summarizer])[0] + get_logger().info(f'Loading {s[0]}: {s[1]}') + cfg = Config.fromfile(s[1]) + summarizer = cfg['summarizer'] + + return Config(dict(models=models, datasets=datasets, summarizer=summarizer), format_python_code=False) diff --git a/opencompass/utils/summarizer.py b/opencompass/utils/summarizer.py index aefa8c94..704c159b 100644 --- a/opencompass/utils/summarizer.py +++ b/opencompass/utils/summarizer.py @@ -36,7 +36,7 @@ class Summarizer: model_cfgs = self.cfg['models'] dataset_cfgs = self.cfg['datasets'] - summarizer_cfg = self.cfg.get('summarizer', {}) + summarizer_cfg = self.cfg.get('summarizer', {}) or {} # avoid 'summarizer' is in cfg but None work_dir = self.cfg['work_dir'] # pick up results diff --git a/run.py b/run.py index 0e3e15a3..00802209 100644 --- a/run.py +++ b/run.py @@ -38,9 +38,10 @@ def parse_args(): help='Whether or not enable multimodal evaluation', action='store_true', default=False) - # Add shortcut parameters (models and datasets) + # Add shortcut parameters (models, datasets and summarizer) parser.add_argument('--models', nargs='+', help='', default=None) parser.add_argument('--datasets', nargs='+', help='', default=None) + parser.add_argument('--summarizer', help='', default=None) # add general args parser.add_argument('--debug', help='Debug mode, in which scheduler will run tasks '