[Feature] Add SciCode summarizer config (#1514)

* [Feature] added SciCode  summarizer config and dataset config for with background evaluation

* fix lint issues

* removed unnecessary type in summarizer group

configs/datasets/scicode/scicode_wbg_gen_085b98.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import ChatInferencer
+from opencompass.datasets import SciCodeDataset, SciCodeEvaluator
+
+
+SciCode_reader_cfg = dict(input_columns=['prompt'], output_column=None)
+
+SciCode_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template='',
+        ),
+
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=ChatInferencer, infer_mode='every', max_out_len=4096))
+
+SciCode_eval_cfg = dict(
+    evaluator=dict(
+        type=SciCodeEvaluator,
+        dataset_path='./data/scicode',
+        with_bg=True
+    )
+)
+
+SciCode_datasets = [
+    dict(
+        abbr='SciCode_with_background',
+        type=SciCodeDataset,
+        path='./data/scicode',
+        with_bg=True,
+        reader_cfg=SciCode_reader_cfg,
+        infer_cfg=SciCode_infer_cfg,
+        eval_cfg=SciCode_eval_cfg
+    )
+]

configs/summarizers/groups/scicode.py

@@ -0,0 +1,23 @@
+scicode_summary_groups = [
+    {
+        'name': 'SciCode',
+        'subsets': [
+            ['SciCode', 'accuracy'],
+            ['SciCode', 'sub_accuracy'],
+        ]
+    },
+    {
+        'name': 'SciCode_with_background',
+        'subsets': [
+            ['SciCode_with_background', 'accuracy'],
+            ['SciCode_with_background', 'sub_accuracy'],
+        ]
+    },
+    {
+        'name': 'SciCode_wo_background',
+        'subsets': [
+            ['SciCode_wo_background', 'accuracy'],
+            ['SciCode_wo_background', 'sub_accuracy'],
+        ]
+    }
+]

configs/summarizers/scicode.py

@@ -0,0 +1,15 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .groups.scicode import scicode_summary_groups
+
+summarizer = dict(
+    dataset_abbrs=[
+        ['SciCode_with_background', 'accuracy'],
+        ['SciCode_with_background', 'sub_accuracy'],
+        ['SciCode_wo_background', 'accuracy'],
+        ['SciCode_wo_background', 'sub_accuracy'],
+    ],
+    summary_groups=sum(
+        [v for k, v in locals().items() if k.endswith('_summary_groups')], [])
+)

opencompass/configs/datasets/scicode/scicode_wbg_gen_085b98.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import ChatInferencer
+from opencompass.datasets import SciCodeDataset, SciCodeEvaluator
+
+
+SciCode_reader_cfg = dict(input_columns=['prompt'], output_column=None)
+
+SciCode_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template='',
+        ),
+
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=ChatInferencer, infer_mode='every', max_out_len=4096))
+
+SciCode_eval_cfg = dict(
+    evaluator=dict(
+        type=SciCodeEvaluator,
+        dataset_path='./data/scicode',
+        with_bg=True
+    )
+)
+
+SciCode_datasets = [
+    dict(
+        abbr='SciCode_with_background',
+        type=SciCodeDataset,
+        path='./data/scicode',
+        with_bg=True,
+        reader_cfg=SciCode_reader_cfg,
+        infer_cfg=SciCode_infer_cfg,
+        eval_cfg=SciCode_eval_cfg
+    )
+]

opencompass/configs/summarizers/groups/scicode.py

@@ -0,0 +1,23 @@
+scicode_summary_groups = [
+    {
+        'name': 'SciCode',
+        'subsets': [
+            ['SciCode', 'accuracy'],
+            ['SciCode', 'sub_accuracy'],
+        ]
+    },
+    {
+        'name': 'SciCode_with_background',
+        'subsets': [
+            ['SciCode_with_background', 'accuracy'],
+            ['SciCode_with_background', 'sub_accuracy'],
+        ]
+    },
+    {
+        'name': 'SciCode_wo_background',
+        'subsets': [
+            ['SciCode_wo_background', 'accuracy'],
+            ['SciCode_wo_background', 'sub_accuracy'],
+        ]
+    }
+]

opencompass/configs/summarizers/scicode.py

@@ -0,0 +1,15 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .groups.scicode import scicode_summary_groups
+
+summarizer = dict(
+    dataset_abbrs=[
+        ['SciCode_with_background', 'accuracy'],
+        ['SciCode_with_background', 'sub_accuracy'],
+        ['SciCode_wo_background', 'accuracy'],
+        ['SciCode_wo_background', 'sub_accuracy'],
+    ],
+    summary_groups=sum(
+        [v for k, v in locals().items() if k.endswith('_summary_groups')], [])
+)