From 267401bded3936612182925de6b8384adfe3a7c5 Mon Sep 17 00:00:00 2001 From: so2liu Date: Mon, 18 Sep 2023 18:11:22 +0800 Subject: [PATCH] =?UTF-8?q?[Feat]=20add=20custom=20summarizer=20argument?= =?UTF-8?q?=20in=20CLI=20run=20mode=20=E5=9C=A8CLI=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E4=B8=AD=E6=B7=BB=E5=8A=A0=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89Summarizer=E5=8F=82=E6=95=B0=20(#411)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add custom summarizer in CLI run mode * feat: search local config by match_cfg_file --- opencompass/utils/run.py | 10 +++++++++- opencompass/utils/summarizer.py | 2 +- run.py | 3 ++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/opencompass/utils/run.py b/opencompass/utils/run.py index 0d44cff9..445c8451 100644 --- a/opencompass/utils/run.py +++ b/opencompass/utils/run.py @@ -95,7 +95,15 @@ def get_config_from_arg(args) -> Config: pad_token_id=args.pad_token_id, run_cfg=dict(num_gpus=args.num_gpus)) models.append(model) - return Config(dict(models=models, datasets=datasets), + + summarizer = None + if args.summarizer: + s = match_cfg_file('configs/summarizers/', [args.summarizer])[0] + get_logger().info(f'Loading {s[0]}: {s[1]}') + cfg = Config.fromfile(s[1]) + summarizer = cfg['summarizer'] + + return Config(dict(models=models, datasets=datasets, summarizer=summarizer), format_python_code=False) diff --git a/opencompass/utils/summarizer.py b/opencompass/utils/summarizer.py index aefa8c94..704c159b 100644 --- a/opencompass/utils/summarizer.py +++ b/opencompass/utils/summarizer.py @@ -36,7 +36,7 @@ class Summarizer: model_cfgs = self.cfg['models'] dataset_cfgs = self.cfg['datasets'] - summarizer_cfg = self.cfg.get('summarizer', {}) + summarizer_cfg = self.cfg.get('summarizer', {}) or {} # avoid 'summarizer' is in cfg but None work_dir = self.cfg['work_dir'] # pick up results diff --git a/run.py b/run.py index 0e3e15a3..00802209 100644 --- a/run.py +++ b/run.py @@ -38,9 +38,10 @@ def parse_args(): help='Whether or not enable multimodal evaluation', action='store_true', default=False) - # Add shortcut parameters (models and datasets) + # Add shortcut parameters (models, datasets and summarizer) parser.add_argument('--models', nargs='+', help='', default=None) parser.add_argument('--datasets', nargs='+', help='', default=None) + parser.add_argument('--summarizer', help='', default=None) # add general args parser.add_argument('--debug', help='Debug mode, in which scheduler will run tasks '