Merge branch 'open-compass:main' into main

.github/workflows/link-check.yml

@@ -5,17 +5,22 @@ on:
     # check links at 01:30 a.m. every day
     - cron: '30 1 * * *'
 
+  workflow_dispatch: # allow manual trigger
+
 jobs:
   link-check:
     runs-on: ubuntu-latest
     steps:
       # - uses: actions/checkout@v3
 
-      - name: linkchecker
+      - name: Install linkchecker
         run: |
           pip install linkchecker
-          linkchecker https://opencompass.readthedocs.io/ --no-robots -t 30 --no-warnings |
+
-            --ignore-url https://opencompass\.readthedocs\.io/.*/static/images/opencompass_logo\.svg |
+      - name: Run linkchecker
-            --ignore-url https://opencompass\.readthedocs\.io/.*/_static/images/icon-menu-dots\.svg |
+        run: |
-            --ignore-url https://opencompass\.readthedocs\.io/policy |
+          linkchecker https://opencompass.readthedocs.io/ --no-robots -t 30 --no-warnings \
-            --ignore-url https://opencompass\.readthedocs\.io/(en|zh_CN)/[0-9a-f]{40}/.*
+            --ignore-url "https://opencompass.readthedocs.io/.*/static/images/opencompass_logo.svg" \
+            --ignore-url "https://opencompass.readthedocs.io/.*/_static/images/icon-menu-dots.svg" \
+            --ignore-url "https://opencompass.readthedocs.io/policy" \
+            --ignore-url "https://opencompass.readthedocs.io/(en|zh_CN)/[0-9a-f]{40}/.*"

configs/datasets/longbench/longbench2wikimqa/longbench_2wikimqa_gen_6b3efc.py

@@ -7,7 +7,7 @@ LongBench_2wikimqa_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_2wikimqa_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_2wikimqa_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_2wikimqa_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_2wikimqa_datasets = [
     dict(
         type=LongBench2wikimqaDataset,
         abbr='LongBench_2wikimqa',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='2wikimqa',
         reader_cfg=LongBench_2wikimqa_reader_cfg,
         infer_cfg=LongBench_2wikimqa_infer_cfg,
-        eval_cfg=LongBench_2wikimqa_eval_cfg)
+        eval_cfg=LongBench_2wikimqa_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchdureader/longbench_dureader_gen_c6c7e4.py

@@ -7,7 +7,7 @@ LongBench_dureader_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_dureader_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_dureader_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=128)
+    inferencer=dict(type=GenInferencer, max_out_len=128),
 )
 
 LongBench_dureader_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
+    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_dureader_datasets = [
     dict(
         type=LongBenchdureaderDataset,
         abbr='LongBench_dureader',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='dureader',
         reader_cfg=LongBench_dureader_reader_cfg,
         infer_cfg=LongBench_dureader_infer_cfg,
-        eval_cfg=LongBench_dureader_eval_cfg)
+        eval_cfg=LongBench_dureader_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchgov_report/longbench_gov_report_gen_54c5b0.py

@@ -7,7 +7,7 @@ LongBench_gov_report_reader_cfg = dict(
     input_columns=['context'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_gov_report_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_gov_report_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=512)
+    inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
 LongBench_gov_report_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator),
+    evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_gov_report_datasets = [
     dict(
         type=LongBenchgov_reportDataset,
         abbr='LongBench_gov_report',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='gov_report',
         reader_cfg=LongBench_gov_report_reader_cfg,
         infer_cfg=LongBench_gov_report_infer_cfg,
-        eval_cfg=LongBench_gov_report_eval_cfg)
+        eval_cfg=LongBench_gov_report_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchhotpotqa/longbench_hotpotqa_gen_6b3efc.py

@@ -7,7 +7,7 @@ LongBench_hotpotqa_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_hotpotqa_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_hotpotqa_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_hotpotqa_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_hotpotqa_datasets = [
     dict(
         type=LongBenchhotpotqaDataset,
         abbr='LongBench_hotpotqa',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='hotpotqa',
         reader_cfg=LongBench_hotpotqa_reader_cfg,
         infer_cfg=LongBench_hotpotqa_infer_cfg,
-        eval_cfg=LongBench_hotpotqa_eval_cfg)
+        eval_cfg=LongBench_hotpotqa_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchlcc/longbench_lcc_gen_6ba507.py

@@ -7,7 +7,7 @@ LongBench_lcc_reader_cfg = dict(
     input_columns=['context'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_lcc_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_lcc_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Please complete the code given below. \n{context}Next line of code:\n'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Please complete the code given below. \n{context}Next line of code:\n',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_lcc_eval_cfg = dict(
-    evaluator=dict(type=LongBenchCodeSimEvaluator),
+    evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_lcc_datasets = [
     dict(
         type=LongBenchlccDataset,
         abbr='LongBench_lcc',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='lcc',
         reader_cfg=LongBench_lcc_reader_cfg,
         infer_cfg=LongBench_lcc_infer_cfg,
-        eval_cfg=LongBench_lcc_eval_cfg)
+        eval_cfg=LongBench_lcc_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchlsht/longbench_lsht_gen_e8a339.py

@@ -1,13 +1,17 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchlshtDataset, lsht_postprocess
+from opencompass.datasets import (
+    LongBenchClassificationEvaluator,
+    LongBenchlshtDataset,
+    lsht_postprocess,
+)
 
 LongBench_lsht_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='all_labels',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_lsht_infer_cfg = dict(
@@ -15,10 +19,15 @@ LongBench_lsht_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_lsht_eval_cfg = dict(
@@ -31,9 +40,10 @@ LongBench_lsht_datasets = [
     dict(
         type=LongBenchlshtDataset,
         abbr='LongBench_lsht',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='lsht',
         reader_cfg=LongBench_lsht_reader_cfg,
         infer_cfg=LongBench_lsht_infer_cfg,
-        eval_cfg=LongBench_lsht_eval_cfg)
+        eval_cfg=LongBench_lsht_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchmulti_news/longbench_multi_news_gen_6f9da9.py

@@ -7,7 +7,7 @@ LongBench_multi_news_reader_cfg = dict(
     input_columns=['context'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_multi_news_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_multi_news_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=512)
+    inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
 LongBench_multi_news_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator),
+    evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_multi_news_datasets = [
     dict(
         type=LongBenchmulti_newsDataset,
         abbr='LongBench_multi_news',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='multi_news',
         reader_cfg=LongBench_multi_news_reader_cfg,
         infer_cfg=LongBench_multi_news_infer_cfg,
-        eval_cfg=LongBench_multi_news_eval_cfg)
+        eval_cfg=LongBench_multi_news_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchmultifieldqa_en/longbench_multifieldqa_en_gen_d3838e.py

@@ -7,7 +7,7 @@ LongBench_multifieldqa_en_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_multifieldqa_en_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_multifieldqa_en_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_multifieldqa_en_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_multifieldqa_en_datasets = [
     dict(
         type=LongBenchmultifieldqa_enDataset,
         abbr='LongBench_multifieldqa_en',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='multifieldqa_en',
         reader_cfg=LongBench_multifieldqa_en_reader_cfg,
         infer_cfg=LongBench_multifieldqa_en_infer_cfg,
-        eval_cfg=LongBench_multifieldqa_en_eval_cfg)
+        eval_cfg=LongBench_multifieldqa_en_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchmultifieldqa_zh/longbench_multifieldqa_zh_gen_e9a7ef.py

@@ -7,7 +7,7 @@ LongBench_multifieldqa_zh_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_multifieldqa_zh_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_multifieldqa_zh_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_multifieldqa_zh_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator, language='zh'),
+    evaluator=dict(type=LongBenchF1Evaluator, language='zh'), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_multifieldqa_zh_datasets = [
     dict(
         type=LongBenchmultifieldqa_zhDataset,
         abbr='LongBench_multifieldqa_zh',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='multifieldqa_zh',
         reader_cfg=LongBench_multifieldqa_zh_reader_cfg,
         infer_cfg=LongBench_multifieldqa_zh_infer_cfg,
-        eval_cfg=LongBench_multifieldqa_zh_eval_cfg)
+        eval_cfg=LongBench_multifieldqa_zh_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchmusique/longbench_musique_gen_6b3efc.py

@@ -7,7 +7,7 @@ LongBench_musique_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_musique_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_musique_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_musique_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_musique_datasets = [
     dict(
         type=LongBenchmusiqueDataset,
         abbr='LongBench_musique',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='musique',
         reader_cfg=LongBench_musique_reader_cfg,
         infer_cfg=LongBench_musique_infer_cfg,
-        eval_cfg=LongBench_musique_eval_cfg)
+        eval_cfg=LongBench_musique_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchnarrativeqa/longbench_narrativeqa_gen_a68305.py

@@ -7,7 +7,7 @@ LongBench_narrativeqa_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_narrativeqa_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_narrativeqa_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=128)
+    inferencer=dict(type=GenInferencer, max_out_len=128),
 )
 
 LongBench_narrativeqa_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_narrativeqa_datasets = [
     dict(
         type=LongBenchnarrativeqaDataset,
         abbr='LongBench_narrativeqa',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='narrativeqa',
         reader_cfg=LongBench_narrativeqa_reader_cfg,
         infer_cfg=LongBench_narrativeqa_infer_cfg,
-        eval_cfg=LongBench_narrativeqa_eval_cfg)
+        eval_cfg=LongBench_narrativeqa_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchpassage_count/longbench_passage_count_gen_dcdaab.py

@@ -7,7 +7,7 @@ LongBench_passage_count_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_passage_count_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_passage_count_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_passage_count_eval_cfg = dict(
-    evaluator=dict(type=LongBenchCountEvaluator),
+    evaluator=dict(type=LongBenchCountEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_passage_count_datasets = [
     dict(
         type=LongBenchpassage_countDataset,
         abbr='LongBench_passage_count',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='passage_count',
         reader_cfg=LongBench_passage_count_reader_cfg,
         infer_cfg=LongBench_passage_count_infer_cfg,
-        eval_cfg=LongBench_passage_count_eval_cfg)
+        eval_cfg=LongBench_passage_count_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchpassage_retrieval_en/longbench_passage_retrieval_en_gen_734db5.py

@@ -1,13 +1,16 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_enDataset
+from opencompass.datasets import (
+    LongBenchRetrievalEvaluator,
+    LongBenchpassage_retrieval_enDataset,
+)
 
 LongBench_passage_retrieval_en_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_passage_retrieval_en_infer_cfg = dict(
@@ -15,24 +18,29 @@ LongBench_passage_retrieval_en_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like \"Paragraph 1\", \"Paragraph 2\", etc.\n\nThe answer is: '),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_passage_retrieval_en_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRetrievalEvaluator),
+    evaluator=dict(type=LongBenchRetrievalEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_passage_retrieval_en_datasets = [
     dict(
         type=LongBenchpassage_retrieval_enDataset,
         abbr='LongBench_passage_retrieval_en',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='passage_retrieval_en',
         reader_cfg=LongBench_passage_retrieval_en_reader_cfg,
         infer_cfg=LongBench_passage_retrieval_en_infer_cfg,
-        eval_cfg=LongBench_passage_retrieval_en_eval_cfg)
+        eval_cfg=LongBench_passage_retrieval_en_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchpassage_retrieval_zh/longbench_passage_retrieval_zh_gen_01cca2.py

@@ -1,13 +1,16 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_zhDataset
+from opencompass.datasets import (
+    LongBenchRetrievalEvaluator,
+    LongBenchpassage_retrieval_zhDataset,
+)
 
 LongBench_passage_retrieval_zh_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_passage_retrieval_zh_infer_cfg = dict(
@@ -15,24 +18,29 @@ LongBench_passage_retrieval_zh_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是\"段落1\"，\"段落2\"等格式\n\n答案是：'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_passage_retrieval_zh_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'),
+    evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_passage_retrieval_zh_datasets = [
     dict(
         type=LongBenchpassage_retrieval_zhDataset,
         abbr='LongBench_passage_retrieval_zh',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='passage_retrieval_zh',
         reader_cfg=LongBench_passage_retrieval_zh_reader_cfg,
         infer_cfg=LongBench_passage_retrieval_zh_infer_cfg,
-        eval_cfg=LongBench_passage_retrieval_zh_eval_cfg)
+        eval_cfg=LongBench_passage_retrieval_zh_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchqasper/longbench_qasper_gen_6b3efc.py

@@ -7,7 +7,7 @@ LongBench_qasper_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_qasper_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_qasper_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_qasper_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_qasper_datasets = [
     dict(
         type=LongBenchqasperDataset,
         abbr='LongBench_qasper',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='qasper',
         reader_cfg=LongBench_qasper_reader_cfg,
         infer_cfg=LongBench_qasper_infer_cfg,
-        eval_cfg=LongBench_qasper_eval_cfg)
+        eval_cfg=LongBench_qasper_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchqmsum/longbench_qmsum_gen_d33331.py

@@ -7,7 +7,7 @@ LongBench_qmsum_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_qmsum_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_qmsum_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=512)
+    inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
 LongBench_qmsum_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator),
+    evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_qmsum_datasets = [
     dict(
         type=LongBenchqmsumDataset,
         abbr='LongBench_qmsum',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='qmsum',
         reader_cfg=LongBench_qmsum_reader_cfg,
         infer_cfg=LongBench_qmsum_infer_cfg,
-        eval_cfg=LongBench_qmsum_eval_cfg)
+        eval_cfg=LongBench_qmsum_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchrepobench/longbench_repobench_gen_6df953.py

@@ -7,7 +7,7 @@ LongBench_repobench_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_repobench_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_repobench_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Please complete the code given below. \n{context}{input}Next line of code:\n'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Please complete the code given below. \n{context}{input}Next line of code:\n',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_repobench_eval_cfg = dict(
-    evaluator=dict(type=LongBenchCodeSimEvaluator),
+    evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_repobench_datasets = [
     dict(
         type=LongBenchrepobenchDataset,
         abbr='LongBench_repobench-p',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='repobench-p',
         reader_cfg=LongBench_repobench_reader_cfg,
         infer_cfg=LongBench_repobench_infer_cfg,
-        eval_cfg=LongBench_repobench_eval_cfg)
+        eval_cfg=LongBench_repobench_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchsamsum/longbench_samsum_gen_f4416d.py

@@ -1,13 +1,17 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchRougeEvaluator, LongBenchsamsumDataset, samsum_postprocess
+from opencompass.datasets import (
+    LongBenchRougeEvaluator,
+    LongBenchsamsumDataset,
+    samsum_postprocess,
+)
 
 LongBench_samsum_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_samsum_infer_cfg = dict(
@@ -15,10 +19,15 @@ LongBench_samsum_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=128)
+    inferencer=dict(type=GenInferencer, max_out_len=128),
 )
 
 LongBench_samsum_eval_cfg = dict(
@@ -31,9 +40,10 @@ LongBench_samsum_datasets = [
     dict(
         type=LongBenchsamsumDataset,
         abbr='LongBench_samsum',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='samsum',
         reader_cfg=LongBench_samsum_reader_cfg,
         infer_cfg=LongBench_samsum_infer_cfg,
-        eval_cfg=LongBench_samsum_eval_cfg)
+        eval_cfg=LongBench_samsum_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchtrec/longbench_trec_gen_824187.py

@@ -1,13 +1,17 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchtrecDataset, trec_postprocess
+from opencompass.datasets import (
+    LongBenchClassificationEvaluator,
+    LongBenchtrecDataset,
+    trec_postprocess,
+)
 
 LongBench_trec_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='all_labels',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_trec_infer_cfg = dict(
@@ -15,10 +19,15 @@ LongBench_trec_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_trec_eval_cfg = dict(
@@ -31,9 +40,10 @@ LongBench_trec_datasets = [
     dict(
         type=LongBenchtrecDataset,
         abbr='LongBench_trec',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='trec',
         reader_cfg=LongBench_trec_reader_cfg,
         infer_cfg=LongBench_trec_infer_cfg,
-        eval_cfg=LongBench_trec_eval_cfg)
+        eval_cfg=LongBench_trec_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchtriviaqa/longbench_triviaqa_gen_d30cb9.py

@@ -1,13 +1,17 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchF1Evaluator, LongBenchtriviaqaDataset, triviaqa_postprocess
+from opencompass.datasets import (
+    LongBenchF1Evaluator,
+    LongBenchtriviaqaDataset,
+    triviaqa_postprocess,
+)
 
 LongBench_triviaqa_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_triviaqa_infer_cfg = dict(
@@ -15,10 +19,15 @@ LongBench_triviaqa_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_triviaqa_eval_cfg = dict(
@@ -31,9 +40,10 @@ LongBench_triviaqa_datasets = [
     dict(
         type=LongBenchtriviaqaDataset,
         abbr='LongBench_triviaqa',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='triviaqa',
         reader_cfg=LongBench_triviaqa_reader_cfg,
         infer_cfg=LongBench_triviaqa_infer_cfg,
-        eval_cfg=LongBench_triviaqa_eval_cfg)
+        eval_cfg=LongBench_triviaqa_eval_cfg,
+    )
 ]

configs/datasets/longbench/longbenchvcsum/longbench_vcsum_gen_f7a8ac.py

@@ -7,7 +7,7 @@ LongBench_vcsum_reader_cfg = dict(
     input_columns=['context'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_vcsum_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_vcsum_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=512)
+    inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
 LongBench_vcsum_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
+    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_vcsum_datasets = [
     dict(
         type=LongBenchvcsumDataset,
         abbr='LongBench_vcsum',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='vcsum',
         reader_cfg=LongBench_vcsum_reader_cfg,
         infer_cfg=LongBench_vcsum_infer_cfg,
-        eval_cfg=LongBench_vcsum_eval_cfg)
+        eval_cfg=LongBench_vcsum_eval_cfg,
+    )
 ]

configs/datasets/scicode/scicode_wbg_gen_085b98.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import ChatInferencer
+from opencompass.datasets import SciCodeDataset, SciCodeEvaluator
+
+
+SciCode_reader_cfg = dict(input_columns=['prompt'], output_column=None)
+
+SciCode_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template='',
+        ),
+
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=ChatInferencer, infer_mode='every', max_out_len=4096))
+
+SciCode_eval_cfg = dict(
+    evaluator=dict(
+        type=SciCodeEvaluator,
+        dataset_path='./data/scicode',
+        with_bg=True
+    )
+)
+
+SciCode_datasets = [
+    dict(
+        abbr='SciCode_with_background',
+        type=SciCodeDataset,
+        path='./data/scicode',
+        with_bg=True,
+        reader_cfg=SciCode_reader_cfg,
+        infer_cfg=SciCode_infer_cfg,
+        eval_cfg=SciCode_eval_cfg
+    )
+]

configs/datasets/subjective/fofo/fofo_bilingual_judge.py

@@ -15,7 +15,7 @@ subjective_all_sets = [
     'fofo_test_prompts', 'fofo_test_prompts_cn',
 ]
 
-base_prompt = """
+base_prompt_en = """
 I would like you to create a leaderboard that evaluates the correctness of the format of answers from various large language models. To accomplish this, you will need to analyze the text prompts given to the models and their corresponding answers. Specifically, please ensure that your evaluation outputs are properly formatted as a json string. I will provide both the prompts and the responses for this purpose.
 
 Here is the prompt:
@@ -45,9 +45,41 @@ Please evaluate the formatting of the model's responses by checking if they comp
 Please note that your response should be a properly formatted JSON string and should not contain any additional content. We will load it directly as a JSON string in Python.
 """
 
+
+base_prompt_cn = """
+我希望你创建一个排行榜，用于评估来自各种大型语言模型的回答格式的正确性。为了完成这个任务，你将需要分析给模型的文本提示以及它们对应的回答。具体来说，请确保你的评估输出正确地格式化为JSON字符串。我将为此提供提示和回答。
+以下是提示内容：
+{
+    "instruction": "{question}",
+}
+以下是模型的输出结果：
+[
+    {
+        "model": "model",
+        "answer": "{prediction}"
+    },
+]
+请通过检查模型回答是否符合提示中声明的格式规范来评估模型回答的格式。进行彻底的格式检查，并提供格式正确或错误的详细解释。你的反馈应包括模型的名称，接着是格式正确性的状态，用'1'表示正确，'0'表示错误。将你的推理以每个评估模型的单个字符串中的 bullet 点形式呈现。换句话说，你应该生成以下输出：
+```json
+[
+    {
+        'model': <模型名称>,
+        'format_correctness': <正确性>,
+        'reasons': <格式正确性的原因>
+    }
+]
+```
+请注意，你的回答应是一个正确格式化的JSON字符串，不应包含任何额外的内容。我们将在Python中直接将其作为JSON字符串加载。
+"""
+
+
 fofo_datasets = []
 
 for _name in subjective_all_sets:
+    if '_cn' in _name:
+        base_prompt = base_prompt_cn
+    else:
+        base_prompt = base_prompt_en
     subjective_infer_cfg = dict(
             prompt_template=dict(
                 type=PromptTemplate,

configs/datasets/subjective/followbench/followbench_llmeval.py

@@ -0,0 +1,63 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import LMEvaluator
+from opencompass.datasets import FollowBenchDataset
+from opencompass.summarizers import FollowBenchSummarizer
+
+subjective_reader_cfg = dict(
+    input_columns=['instruction', 'judge_prompt',],
+    output_column='judge',
+    )
+
+subjective_all_sets = [
+    'followbench_llmeval_cn', 'followbench_llmeval_en',
+]
+data_path ='data/subjective/followbench/converted_data'
+
+followbench_llmeval_dataset = []
+
+for _name in subjective_all_sets:
+    subjective_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt='{instruction}'
+                    ),
+                ]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+    subjective_eval_cfg = dict(
+        evaluator=dict(
+            type=LMEvaluator,
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt = '{judge_prompt}'
+                    ),
+                ]),
+            ),
+        ),
+        pred_role='BOT',
+    )
+
+    followbench_llmeval_dataset.append(
+        dict(
+            abbr=f'{_name}',
+            type=FollowBenchDataset,
+            path=data_path,
+            name=_name,
+            mode='singlescore',
+            cate='llm',
+            reader_cfg=subjective_reader_cfg,
+            infer_cfg=subjective_infer_cfg,
+            eval_cfg=subjective_eval_cfg,
+            summarizer = dict(type=FollowBenchSummarizer,)
+        ))

configs/datasets/subjective/wildbench/wildbench_pair_judge.py

@@ -3,7 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
 from opencompass.openicl.icl_evaluator import LMEvaluator
 from opencompass.datasets import WildBenchDataset
-
+from opencompass.summarizers import WildBenchPairSummarizer
 
 subjective_reader_cfg = dict(
     input_columns=['dialogue', 'prompt'],
@@ -61,5 +61,6 @@ wildbench_datasets.append(
                 {'abbr': 'llama-2-70b-chat-vllm', 'path':'./data/subjective/WildBench/llama2-70b'}],
         mode='m2n', # m个模型 与 n个模型进行对战
         infer_order='random',
-        base_models = [llama_2_70b, gpt4, claude]
+        base_models = [llama_2_70b, gpt4, claude],
+        summarizer = dict(type=WildBenchPairSummarizer),
     ))

configs/summarizers/groups/scicode.py

@@ -0,0 +1,23 @@
+scicode_summary_groups = [
+    {
+        'name': 'SciCode',
+        'subsets': [
+            ['SciCode', 'accuracy'],
+            ['SciCode', 'sub_accuracy'],
+        ]
+    },
+    {
+        'name': 'SciCode_with_background',
+        'subsets': [
+            ['SciCode_with_background', 'accuracy'],
+            ['SciCode_with_background', 'sub_accuracy'],
+        ]
+    },
+    {
+        'name': 'SciCode_wo_background',
+        'subsets': [
+            ['SciCode_wo_background', 'accuracy'],
+            ['SciCode_wo_background', 'sub_accuracy'],
+        ]
+    }
+]

configs/summarizers/scicode.py

@@ -0,0 +1,15 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .groups.scicode import scicode_summary_groups
+
+summarizer = dict(
+    dataset_abbrs=[
+        ['SciCode_with_background', 'accuracy'],
+        ['SciCode_with_background', 'sub_accuracy'],
+        ['SciCode_wo_background', 'accuracy'],
+        ['SciCode_wo_background', 'sub_accuracy'],
+    ],
+    summary_groups=sum(
+        [v for k, v in locals().items() if k.endswith('_summary_groups')], [])
+)

opencompass/__init__.py

@@ -1 +1 @@
-__version__ = '0.3.1'
+__version__ = '0.3.2.post1'

opencompass/configs/datasets/longbench/longbench2wikimqa/longbench_2wikimqa_gen_6b3efc.py

@@ -7,7 +7,7 @@ LongBench_2wikimqa_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_2wikimqa_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_2wikimqa_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_2wikimqa_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_2wikimqa_datasets = [
     dict(
         type=LongBench2wikimqaDataset,
         abbr='LongBench_2wikimqa',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='2wikimqa',
         reader_cfg=LongBench_2wikimqa_reader_cfg,
         infer_cfg=LongBench_2wikimqa_infer_cfg,
-        eval_cfg=LongBench_2wikimqa_eval_cfg)
+        eval_cfg=LongBench_2wikimqa_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchdureader/longbench_dureader_gen_c6c7e4.py

@@ -7,7 +7,7 @@ LongBench_dureader_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_dureader_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_dureader_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=128)
+    inferencer=dict(type=GenInferencer, max_out_len=128),
 )
 
 LongBench_dureader_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
+    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_dureader_datasets = [
     dict(
         type=LongBenchdureaderDataset,
         abbr='LongBench_dureader',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='dureader',
         reader_cfg=LongBench_dureader_reader_cfg,
         infer_cfg=LongBench_dureader_infer_cfg,
-        eval_cfg=LongBench_dureader_eval_cfg)
+        eval_cfg=LongBench_dureader_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchgov_report/longbench_gov_report_gen_54c5b0.py

@@ -7,7 +7,7 @@ LongBench_gov_report_reader_cfg = dict(
     input_columns=['context'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_gov_report_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_gov_report_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=512)
+    inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
 LongBench_gov_report_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator),
+    evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_gov_report_datasets = [
     dict(
         type=LongBenchgov_reportDataset,
         abbr='LongBench_gov_report',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='gov_report',
         reader_cfg=LongBench_gov_report_reader_cfg,
         infer_cfg=LongBench_gov_report_infer_cfg,
-        eval_cfg=LongBench_gov_report_eval_cfg)
+        eval_cfg=LongBench_gov_report_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchhotpotqa/longbench_hotpotqa_gen_6b3efc.py

@@ -7,7 +7,7 @@ LongBench_hotpotqa_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_hotpotqa_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_hotpotqa_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_hotpotqa_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_hotpotqa_datasets = [
     dict(
         type=LongBenchhotpotqaDataset,
         abbr='LongBench_hotpotqa',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='hotpotqa',
         reader_cfg=LongBench_hotpotqa_reader_cfg,
         infer_cfg=LongBench_hotpotqa_infer_cfg,
-        eval_cfg=LongBench_hotpotqa_eval_cfg)
+        eval_cfg=LongBench_hotpotqa_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchlcc/longbench_lcc_gen_6ba507.py

@@ -7,7 +7,7 @@ LongBench_lcc_reader_cfg = dict(
     input_columns=['context'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_lcc_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_lcc_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Please complete the code given below. \n{context}Next line of code:\n'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Please complete the code given below. \n{context}Next line of code:\n',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_lcc_eval_cfg = dict(
-    evaluator=dict(type=LongBenchCodeSimEvaluator),
+    evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_lcc_datasets = [
     dict(
         type=LongBenchlccDataset,
         abbr='LongBench_lcc',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='lcc',
         reader_cfg=LongBench_lcc_reader_cfg,
         infer_cfg=LongBench_lcc_infer_cfg,
-        eval_cfg=LongBench_lcc_eval_cfg)
+        eval_cfg=LongBench_lcc_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchlsht/longbench_lsht_gen_e8a339.py

@@ -1,13 +1,17 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchlshtDataset, lsht_postprocess
+from opencompass.datasets import (
+    LongBenchClassificationEvaluator,
+    LongBenchlshtDataset,
+    lsht_postprocess,
+)
 
 LongBench_lsht_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='all_labels',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_lsht_infer_cfg = dict(
@@ -15,10 +19,15 @@ LongBench_lsht_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_lsht_eval_cfg = dict(
@@ -31,9 +40,10 @@ LongBench_lsht_datasets = [
     dict(
         type=LongBenchlshtDataset,
         abbr='LongBench_lsht',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='lsht',
         reader_cfg=LongBench_lsht_reader_cfg,
         infer_cfg=LongBench_lsht_infer_cfg,
-        eval_cfg=LongBench_lsht_eval_cfg)
+        eval_cfg=LongBench_lsht_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchmulti_news/longbench_multi_news_gen_6f9da9.py

@@ -7,7 +7,7 @@ LongBench_multi_news_reader_cfg = dict(
     input_columns=['context'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_multi_news_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_multi_news_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=512)
+    inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
 LongBench_multi_news_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator),
+    evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_multi_news_datasets = [
     dict(
         type=LongBenchmulti_newsDataset,
         abbr='LongBench_multi_news',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='multi_news',
         reader_cfg=LongBench_multi_news_reader_cfg,
         infer_cfg=LongBench_multi_news_infer_cfg,
-        eval_cfg=LongBench_multi_news_eval_cfg)
+        eval_cfg=LongBench_multi_news_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchmultifieldqa_en/longbench_multifieldqa_en_gen_d3838e.py

@@ -7,7 +7,7 @@ LongBench_multifieldqa_en_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_multifieldqa_en_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_multifieldqa_en_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_multifieldqa_en_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_multifieldqa_en_datasets = [
     dict(
         type=LongBenchmultifieldqa_enDataset,
         abbr='LongBench_multifieldqa_en',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='multifieldqa_en',
         reader_cfg=LongBench_multifieldqa_en_reader_cfg,
         infer_cfg=LongBench_multifieldqa_en_infer_cfg,
-        eval_cfg=LongBench_multifieldqa_en_eval_cfg)
+        eval_cfg=LongBench_multifieldqa_en_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchmultifieldqa_zh/longbench_multifieldqa_zh_gen_e9a7ef.py

@@ -7,7 +7,7 @@ LongBench_multifieldqa_zh_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_multifieldqa_zh_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_multifieldqa_zh_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_multifieldqa_zh_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator, language='zh'),
+    evaluator=dict(type=LongBenchF1Evaluator, language='zh'), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_multifieldqa_zh_datasets = [
     dict(
         type=LongBenchmultifieldqa_zhDataset,
         abbr='LongBench_multifieldqa_zh',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='multifieldqa_zh',
         reader_cfg=LongBench_multifieldqa_zh_reader_cfg,
         infer_cfg=LongBench_multifieldqa_zh_infer_cfg,
-        eval_cfg=LongBench_multifieldqa_zh_eval_cfg)
+        eval_cfg=LongBench_multifieldqa_zh_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchmusique/longbench_musique_gen_6b3efc.py

@@ -7,7 +7,7 @@ LongBench_musique_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_musique_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_musique_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_musique_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_musique_datasets = [
     dict(
         type=LongBenchmusiqueDataset,
         abbr='LongBench_musique',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='musique',
         reader_cfg=LongBench_musique_reader_cfg,
         infer_cfg=LongBench_musique_infer_cfg,
-        eval_cfg=LongBench_musique_eval_cfg)
+        eval_cfg=LongBench_musique_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchnarrativeqa/longbench_narrativeqa_gen_a68305.py

@@ -7,7 +7,7 @@ LongBench_narrativeqa_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_narrativeqa_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_narrativeqa_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=128)
+    inferencer=dict(type=GenInferencer, max_out_len=128),
 )
 
 LongBench_narrativeqa_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_narrativeqa_datasets = [
     dict(
         type=LongBenchnarrativeqaDataset,
         abbr='LongBench_narrativeqa',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='narrativeqa',
         reader_cfg=LongBench_narrativeqa_reader_cfg,
         infer_cfg=LongBench_narrativeqa_infer_cfg,
-        eval_cfg=LongBench_narrativeqa_eval_cfg)
+        eval_cfg=LongBench_narrativeqa_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchpassage_count/longbench_passage_count_gen_dcdaab.py

@@ -7,7 +7,7 @@ LongBench_passage_count_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_passage_count_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_passage_count_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_passage_count_eval_cfg = dict(
-    evaluator=dict(type=LongBenchCountEvaluator),
+    evaluator=dict(type=LongBenchCountEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_passage_count_datasets = [
     dict(
         type=LongBenchpassage_countDataset,
         abbr='LongBench_passage_count',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='passage_count',
         reader_cfg=LongBench_passage_count_reader_cfg,
         infer_cfg=LongBench_passage_count_infer_cfg,
-        eval_cfg=LongBench_passage_count_eval_cfg)
+        eval_cfg=LongBench_passage_count_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchpassage_retrieval_en/longbench_passage_retrieval_en_gen_734db5.py

@@ -1,13 +1,16 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_enDataset
+from opencompass.datasets import (
+    LongBenchRetrievalEvaluator,
+    LongBenchpassage_retrieval_enDataset,
+)
 
 LongBench_passage_retrieval_en_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_passage_retrieval_en_infer_cfg = dict(
@@ -15,24 +18,29 @@ LongBench_passage_retrieval_en_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like \"Paragraph 1\", \"Paragraph 2\", etc.\n\nThe answer is: '),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_passage_retrieval_en_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRetrievalEvaluator),
+    evaluator=dict(type=LongBenchRetrievalEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_passage_retrieval_en_datasets = [
     dict(
         type=LongBenchpassage_retrieval_enDataset,
         abbr='LongBench_passage_retrieval_en',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='passage_retrieval_en',
         reader_cfg=LongBench_passage_retrieval_en_reader_cfg,
         infer_cfg=LongBench_passage_retrieval_en_infer_cfg,
-        eval_cfg=LongBench_passage_retrieval_en_eval_cfg)
+        eval_cfg=LongBench_passage_retrieval_en_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchpassage_retrieval_zh/longbench_passage_retrieval_zh_gen_01cca2.py

@@ -1,13 +1,16 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_zhDataset
+from opencompass.datasets import (
+    LongBenchRetrievalEvaluator,
+    LongBenchpassage_retrieval_zhDataset,
+)
 
 LongBench_passage_retrieval_zh_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_passage_retrieval_zh_infer_cfg = dict(
@@ -15,24 +18,29 @@ LongBench_passage_retrieval_zh_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是\"段落1\"，\"段落2\"等格式\n\n答案是：'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_passage_retrieval_zh_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'),
+    evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_passage_retrieval_zh_datasets = [
     dict(
         type=LongBenchpassage_retrieval_zhDataset,
         abbr='LongBench_passage_retrieval_zh',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='passage_retrieval_zh',
         reader_cfg=LongBench_passage_retrieval_zh_reader_cfg,
         infer_cfg=LongBench_passage_retrieval_zh_infer_cfg,
-        eval_cfg=LongBench_passage_retrieval_zh_eval_cfg)
+        eval_cfg=LongBench_passage_retrieval_zh_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchqasper/longbench_qasper_gen_6b3efc.py

@@ -7,7 +7,7 @@ LongBench_qasper_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_qasper_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_qasper_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_qasper_eval_cfg = dict(
-    evaluator=dict(type=LongBenchF1Evaluator),
+    evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_qasper_datasets = [
     dict(
         type=LongBenchqasperDataset,
         abbr='LongBench_qasper',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='qasper',
         reader_cfg=LongBench_qasper_reader_cfg,
         infer_cfg=LongBench_qasper_infer_cfg,
-        eval_cfg=LongBench_qasper_eval_cfg)
+        eval_cfg=LongBench_qasper_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchqmsum/longbench_qmsum_gen_d33331.py

@@ -7,7 +7,7 @@ LongBench_qmsum_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_qmsum_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_qmsum_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=512)
+    inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
 LongBench_qmsum_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator),
+    evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_qmsum_datasets = [
     dict(
         type=LongBenchqmsumDataset,
         abbr='LongBench_qmsum',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='qmsum',
         reader_cfg=LongBench_qmsum_reader_cfg,
         infer_cfg=LongBench_qmsum_infer_cfg,
-        eval_cfg=LongBench_qmsum_eval_cfg)
+        eval_cfg=LongBench_qmsum_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchrepobench/longbench_repobench_gen_6df953.py

@@ -7,7 +7,7 @@ LongBench_repobench_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_repobench_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_repobench_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Please complete the code given below. \n{context}{input}Next line of code:\n'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Please complete the code given below. \n{context}{input}Next line of code:\n',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_repobench_eval_cfg = dict(
-    evaluator=dict(type=LongBenchCodeSimEvaluator),
+    evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_repobench_datasets = [
     dict(
         type=LongBenchrepobenchDataset,
         abbr='LongBench_repobench-p',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='repobench-p',
         reader_cfg=LongBench_repobench_reader_cfg,
         infer_cfg=LongBench_repobench_infer_cfg,
-        eval_cfg=LongBench_repobench_eval_cfg)
+        eval_cfg=LongBench_repobench_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchsamsum/longbench_samsum_gen_f4416d.py

@@ -1,13 +1,17 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchRougeEvaluator, LongBenchsamsumDataset, samsum_postprocess
+from opencompass.datasets import (
+    LongBenchRougeEvaluator,
+    LongBenchsamsumDataset,
+    samsum_postprocess,
+)
 
 LongBench_samsum_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_samsum_infer_cfg = dict(
@@ -15,10 +19,15 @@ LongBench_samsum_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=128)
+    inferencer=dict(type=GenInferencer, max_out_len=128),
 )
 
 LongBench_samsum_eval_cfg = dict(
@@ -31,9 +40,10 @@ LongBench_samsum_datasets = [
     dict(
         type=LongBenchsamsumDataset,
         abbr='LongBench_samsum',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='samsum',
         reader_cfg=LongBench_samsum_reader_cfg,
         infer_cfg=LongBench_samsum_infer_cfg,
-        eval_cfg=LongBench_samsum_eval_cfg)
+        eval_cfg=LongBench_samsum_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchtrec/longbench_trec_gen_824187.py

@@ -1,13 +1,17 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchtrecDataset, trec_postprocess
+from opencompass.datasets import (
+    LongBenchClassificationEvaluator,
+    LongBenchtrecDataset,
+    trec_postprocess,
+)
 
 LongBench_trec_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='all_labels',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_trec_infer_cfg = dict(
@@ -15,10 +19,15 @@ LongBench_trec_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=64)
+    inferencer=dict(type=GenInferencer, max_out_len=64),
 )
 
 LongBench_trec_eval_cfg = dict(
@@ -31,9 +40,10 @@ LongBench_trec_datasets = [
     dict(
         type=LongBenchtrecDataset,
         abbr='LongBench_trec',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='trec',
         reader_cfg=LongBench_trec_reader_cfg,
         infer_cfg=LongBench_trec_infer_cfg,
-        eval_cfg=LongBench_trec_eval_cfg)
+        eval_cfg=LongBench_trec_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchtriviaqa/longbench_triviaqa_gen_d30cb9.py

@@ -1,13 +1,17 @@
 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import LongBenchF1Evaluator, LongBenchtriviaqaDataset, triviaqa_postprocess
+from opencompass.datasets import (
+    LongBenchF1Evaluator,
+    LongBenchtriviaqaDataset,
+    triviaqa_postprocess,
+)
 
 LongBench_triviaqa_reader_cfg = dict(
     input_columns=['context', 'input'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_triviaqa_infer_cfg = dict(
@@ -15,10 +19,15 @@ LongBench_triviaqa_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=32)
+    inferencer=dict(type=GenInferencer, max_out_len=32),
 )
 
 LongBench_triviaqa_eval_cfg = dict(
@@ -31,9 +40,10 @@ LongBench_triviaqa_datasets = [
     dict(
         type=LongBenchtriviaqaDataset,
         abbr='LongBench_triviaqa',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='triviaqa',
         reader_cfg=LongBench_triviaqa_reader_cfg,
         infer_cfg=LongBench_triviaqa_infer_cfg,
-        eval_cfg=LongBench_triviaqa_eval_cfg)
+        eval_cfg=LongBench_triviaqa_eval_cfg,
+    )
 ]

opencompass/configs/datasets/longbench/longbenchvcsum/longbench_vcsum_gen_f7a8ac.py

@@ -7,7 +7,7 @@ LongBench_vcsum_reader_cfg = dict(
     input_columns=['context'],
     output_column='answers',
     train_split='test',
-    test_split='test'
+    test_split='test',
 )
 
 LongBench_vcsum_infer_cfg = dict(
@@ -15,24 +15,29 @@ LongBench_vcsum_infer_cfg = dict(
         type=PromptTemplate,
         template=dict(
             round=[
-                dict(role='HUMAN', prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：'),
+                dict(
-            ], )),
+                    role='HUMAN',
+                    prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
+                ),
+            ],
+        ),
+    ),
     retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=GenInferencer, max_out_len=512)
+    inferencer=dict(type=GenInferencer, max_out_len=512),
 )
 
 LongBench_vcsum_eval_cfg = dict(
-    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
+    evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
-    pred_role='BOT'
 )
 
 LongBench_vcsum_datasets = [
     dict(
         type=LongBenchvcsumDataset,
         abbr='LongBench_vcsum',
-        path='THUDM/LongBench',
+        path='opencompass/Longbench',
         name='vcsum',
         reader_cfg=LongBench_vcsum_reader_cfg,
         infer_cfg=LongBench_vcsum_infer_cfg,
-        eval_cfg=LongBench_vcsum_eval_cfg)
+        eval_cfg=LongBench_vcsum_eval_cfg,
+    )
 ]

opencompass/configs/datasets/scicode/scicode_wbg_gen_085b98.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import ChatInferencer
+from opencompass.datasets import SciCodeDataset, SciCodeEvaluator
+
+
+SciCode_reader_cfg = dict(input_columns=['prompt'], output_column=None)
+
+SciCode_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template='',
+        ),
+
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=ChatInferencer, infer_mode='every', max_out_len=4096))
+
+SciCode_eval_cfg = dict(
+    evaluator=dict(
+        type=SciCodeEvaluator,
+        dataset_path='./data/scicode',
+        with_bg=True
+    )
+)
+
+SciCode_datasets = [
+    dict(
+        abbr='SciCode_with_background',
+        type=SciCodeDataset,
+        path='./data/scicode',
+        with_bg=True,
+        reader_cfg=SciCode_reader_cfg,
+        infer_cfg=SciCode_infer_cfg,
+        eval_cfg=SciCode_eval_cfg
+    )
+]

opencompass/configs/datasets/subjective/fofo/fofo_bilingual_judge.py

@@ -15,7 +15,7 @@ subjective_all_sets = [
     'fofo_test_prompts', 'fofo_test_prompts_cn',
 ]
 
-base_prompt = """
+base_prompt_en = """
 I would like you to create a leaderboard that evaluates the correctness of the format of answers from various large language models. To accomplish this, you will need to analyze the text prompts given to the models and their corresponding answers. Specifically, please ensure that your evaluation outputs are properly formatted as a json string. I will provide both the prompts and the responses for this purpose.
 
 Here is the prompt:
@@ -45,9 +45,41 @@ Please evaluate the formatting of the model's responses by checking if they comp
 Please note that your response should be a properly formatted JSON string and should not contain any additional content. We will load it directly as a JSON string in Python.
 """
 
+
+base_prompt_cn = """
+我希望你创建一个排行榜，用于评估来自各种大型语言模型的回答格式的正确性。为了完成这个任务，你将需要分析给模型的文本提示以及它们对应的回答。具体来说，请确保你的评估输出正确地格式化为JSON字符串。我将为此提供提示和回答。
+以下是提示内容：
+{
+    "instruction": "{question}",
+}
+以下是模型的输出结果：
+[
+    {
+        "model": "model",
+        "answer": "{prediction}"
+    },
+]
+请通过检查模型回答是否符合提示中声明的格式规范来评估模型回答的格式。进行彻底的格式检查，并提供格式正确或错误的详细解释。你的反馈应包括模型的名称，接着是格式正确性的状态，用'1'表示正确，'0'表示错误。将你的推理以每个评估模型的单个字符串中的 bullet 点形式呈现。换句话说，你应该生成以下输出：
+```json
+[
+    {
+        'model': <模型名称>,
+        'format_correctness': <正确性>,
+        'reasons': <格式正确性的原因>
+    }
+]
+```
+请注意，你的回答应是一个正确格式化的JSON字符串，不应包含任何额外的内容。我们将在Python中直接将其作为JSON字符串加载。
+"""
+
+
 fofo_datasets = []
 
 for _name in subjective_all_sets:
+    if '_cn' in _name:
+        base_prompt = base_prompt_cn
+    else:
+        base_prompt = base_prompt_en
     subjective_infer_cfg = dict(
             prompt_template=dict(
                 type=PromptTemplate,

opencompass/configs/datasets/subjective/followbench/followbench_llmeval.py

@@ -0,0 +1,63 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import LMEvaluator
+from opencompass.datasets import FollowBenchDataset
+from opencompass.summarizers import FollowBenchSummarizer
+
+subjective_reader_cfg = dict(
+    input_columns=['instruction', 'judge_prompt',],
+    output_column='judge',
+    )
+
+subjective_all_sets = [
+    'followbench_llmeval_cn', 'followbench_llmeval_en',
+]
+data_path ='data/subjective/followbench/converted_data'
+
+followbench_llmeval_dataset = []
+
+for _name in subjective_all_sets:
+    subjective_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt='{instruction}'
+                    ),
+                ]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+    subjective_eval_cfg = dict(
+        evaluator=dict(
+            type=LMEvaluator,
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt = '{judge_prompt}'
+                    ),
+                ]),
+            ),
+        ),
+        pred_role='BOT',
+    )
+
+    followbench_llmeval_dataset.append(
+        dict(
+            abbr=f'{_name}',
+            type=FollowBenchDataset,
+            path=data_path,
+            name=_name,
+            mode='singlescore',
+            cate='llm',
+            reader_cfg=subjective_reader_cfg,
+            infer_cfg=subjective_infer_cfg,
+            eval_cfg=subjective_eval_cfg,
+            summarizer = dict(type=FollowBenchSummarizer,)
+        ))

opencompass/configs/datasets/subjective/wildbench/wildbench_pair_judge.py

@@ -3,7 +3,7 @@ from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import ChatInferencer, GenInferencer
 from opencompass.openicl.icl_evaluator import LMEvaluator
 from opencompass.datasets import WildBenchDataset
-
+from opencompass.summarizers import WildBenchPairSummarizer
 
 subjective_reader_cfg = dict(
     input_columns=['dialogue', 'prompt'],
@@ -61,5 +61,6 @@ wildbench_datasets.append(
                 {'abbr': 'llama-2-70b-chat-vllm', 'path':'./data/subjective/WildBench/llama2-70b'}],
         mode='m2n', # m个模型 与 n个模型进行对战
         infer_order='random',
-        base_models = [llama_2_70b, gpt4, claude]
+        base_models = [llama_2_70b, gpt4, claude],
+        summarizer = dict(type=WildBenchPairSummarizer),
     ))

opencompass/configs/summarizers/groups/scicode.py

@@ -0,0 +1,23 @@
+scicode_summary_groups = [
+    {
+        'name': 'SciCode',
+        'subsets': [
+            ['SciCode', 'accuracy'],
+            ['SciCode', 'sub_accuracy'],
+        ]
+    },
+    {
+        'name': 'SciCode_with_background',
+        'subsets': [
+            ['SciCode_with_background', 'accuracy'],
+            ['SciCode_with_background', 'sub_accuracy'],
+        ]
+    },
+    {
+        'name': 'SciCode_wo_background',
+        'subsets': [
+            ['SciCode_wo_background', 'accuracy'],
+            ['SciCode_wo_background', 'sub_accuracy'],
+        ]
+    }
+]

opencompass/configs/summarizers/scicode.py

@@ -0,0 +1,15 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .groups.scicode import scicode_summary_groups
+
+summarizer = dict(
+    dataset_abbrs=[
+        ['SciCode_with_background', 'accuracy'],
+        ['SciCode_with_background', 'sub_accuracy'],
+        ['SciCode_wo_background', 'accuracy'],
+        ['SciCode_wo_background', 'sub_accuracy'],
+    ],
+    summary_groups=sum(
+        [v for k, v in locals().items() if k.endswith('_summary_groups')], [])
+)

opencompass/datasets/longbench/longbench_2wikim_qa.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBench2wikimqaDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_dureader.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchdureaderDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_gov_report.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchgov_reportDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_hotpot_qa.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchhotpotqaDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_lcc.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchlccDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_lsht.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchlshtDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_multi_news.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchmulti_newsDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_multifieldqa_en.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchmultifieldqa_enDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_multifieldqa_zh.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchmultifieldqa_zhDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_musique.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchmusiqueDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_narrative_qa.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchnarrativeqaDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_passage_count.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchpassage_countDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_passage_retrieval_en.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchpassage_retrieval_enDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_passage_retrieval_zh.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchpassage_retrieval_zhDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_qasper.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchqasperDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_qmsum.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchqmsumDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_repobench.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchrepobenchDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_samsum.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchsamsumDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_trec.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchtrecDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_trivia_qa.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchtriviaqaDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/longbench/longbench_vcsum.py

@@ -10,11 +10,12 @@ from ..base import BaseDataset
 class LongBenchvcsumDataset(BaseDataset):
 
     @staticmethod
-    def load(**kwargs):
+    def load(path: str, name: str):
-        if 'data_files' in kwargs:
+        path = get_data_path(path)
-            kwargs['data_files'] = get_data_path(kwargs['data_files'],
+        dataset = load_dataset(path=path,
-                                                 local_mode=True)
+                               name=name,
-        dataset = load_dataset(**kwargs)
+                               data_dir=path,
+                               trust_remote_code=True)
         split = 'test'
         raw_data = []
         for i in range(len(dataset[split])):

opencompass/datasets/subjective/__init__.py

@@ -9,6 +9,7 @@ from .compassbench_control_length_bias import \
 from .corev2 import Corev2Dataset  # noqa: F401, F403
 from .creationbench import CreationBenchDataset  # noqa: F401, F403
 from .fofo import FofoDataset  # noqa: F401, F403
+from .followbench import FollowBenchDataset  # noqa: F401, F403
 from .information_retrival import IRDataset  # noqa: F401, F403
 from .mtbench import MTBenchDataset  # noqa: F401, F403
 from .mtbench101 import MTBench101Dataset  # noqa: F401, F403

opencompass/datasets/subjective/followbench.py

@@ -0,0 +1,163 @@
+# flake8: noqa
+import json
+import os.path as osp
+import re
+
+from datasets import Dataset
+
+from opencompass.openicl.icl_evaluator import BaseEvaluator
+from opencompass.registry import LOAD_DATASET
+from opencompass.utils import get_data_path
+
+from ..base import BaseDataset
+
+
+def check_match(target, generation):
+    pattern_str = target.replace('{{',
+                                 '{').replace('}}',
+                                              '}').replace('{answer}', '.*')
+    pattern_str = re.escape(pattern_str).replace('\\.\\*', '.*')
+    match = re.fullmatch(pattern_str, generation)
+    return bool(match)
+
+
+@LOAD_DATASET.register_module()
+class FollowBenchDataset(BaseDataset):
+
+    def load(self, path: str, name: str, cate: str, *args, **kwargs):
+
+        path = get_data_path(path, local_mode=True)
+        filename = osp.join(path, f'{name}.json')
+        raw_data = []
+        raw_data = []
+        with open(filename, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            for item in data:
+                if cate == 'llm':
+                    raw_data.append({
+                        'instruction': item['instruction'],
+                        'judge_prompt': item['judge_prompt'],
+                        'judge': item
+                    })
+                elif cate == 'rule':
+                    raw_data.append({
+                        'instruction': item['instruction'],
+                        'judge': item
+                    })
+                else:
+                    raise NotImplementedError(
+                        f"Category '{cate}' is not implemented.")
+
+        dataset = Dataset.from_list(raw_data)
+        return dataset
+
+
+class FollowBenchEvaluator(BaseEvaluator):
+    """Evaluator for followbench rule-based eval."""
+
+    def __init__(self, num_workers=16) -> None:
+        self.num_workers = num_workers
+
+    def score_single(self, pred, refer):
+        generated_code = pred
+
+        # get current dir because we will enter a temp dir to
+        # execute generated code
+        cwd = os.getcwd()
+
+        def chdir_return(cwd, return_value):
+            os.chdir(cwd)
+            return return_value
+
+        # we create a tempdir to execute each generated program
+        with tempfile.TemporaryDirectory() as tempdir_name:
+
+            tempdir_name = Path(tempdir_name)
+            # copy all files and data dependencies from
+            shutil.copytree(refer['problem_path'],
+                            tempdir_name,
+                            dirs_exist_ok=True)
+            # generated outputs will be put into `result`
+            os.mkdir(tempdir_name / 'result')
+
+            program = refer['code_context'].replace('[insert]', generated_code)
+            with open(tempdir_name / 'program.py', 'w', encoding='UTF-8') as f:
+                f.write(program)
+
+            # enter into the tempdir to execute
+            os.chdir(tempdir_name)
+
+            execution_status = []
+            # a question may not have test case but we can still execute and
+            # see if there is error
+            test_cnt = max(1, int(refer['test_case_cnt']))
+            for i in range(1, test_cnt + 1):
+                # notice this command, e.g., you may need to
+                # replace `python` with `python3`
+                cmd_text = f'python program.py --test_case {i}'
+                time_limit = 60  # should not change the official time_limit
+                cmd = Command(cmd_text, )
+                exit_code = cmd.run(
+                    timeout=time_limit)  # 0 if there is no error
+                execution_status.append(exit_code)
+
+            # return if execution error
+            if sum(execution_status) > 0:
+                return chdir_return(cwd, False)
+
+            # loading testing code as a module
+            test_module = import_source_file(tempdir_name / 'test_code.py',
+                                             'test_code')
+            pass_flag = True
+
+            if int(refer['test_type']) == 3:
+                # stringTest parses the generated code into AST
+                # and check AST components
+                # if there is static error, stringTest may raise an exception
+                generated_code = generated_code.split('\n')
+                for line in generated_code:
+                    if 'print' in line and '#' not in line.split('print'):
+                        generated_code.remove(line)
+                generated_code = '\n'.join(generated_code)
+                try:
+                    pass_flag = test_module.stringTest(generated_code)
+                except Exception:
+                    # return False if stringTest error
+                    return chdir_return(cwd, False)
+
+            test_cnt = max(int(refer['test_case_cnt']), 1)
+            for i in range(1, test_cnt + 1):
+                try:
+                    ans = pickle.load(open(f'ans/ans{i}.pkl', 'rb'))
+                    # loading the generated output might still raise Exception
+                    # if the generated code is not correct
+                    result = pickle.load(open(f'result/result_{i}.pkl', 'rb'))
+                    pass_flag = test_module.test(result, ans) == 1
+                except Exception:
+                    # return False if stringTest error
+                    return chdir_return(cwd, False)
+
+        return chdir_return(cwd, pass_flag)
+
+    def score(self, predictions, references):
+        results = {'example': {'accuracy': [0, 0, 0, 0, 0], 'num': 0}}
+        for prediction, reference in zip(predictions, references):
+            if reference['category'] == 'example':
+                results['example']['num'] += 1
+                template = reference['target'].replace('{instruction}\n', '')
+                match_result = check_match(template, prediction)
+                print(match_result)
+                results['example']['accuracy'][reference['level'] -
+                                               1] += match_result
+        results['example']['accuracy'] = [
+            round(acc / (results['example']['num'] // 5), 2)
+            for acc in results['example']['accuracy']
+        ]
+        ######## Still not finished for rule-based evaluation
+
+        # Each process changes cwd, need to use multi-processing
+        with ProcessPoolExecutor(self.num_workers) as executor:
+            passed = sum(
+                list(executor.map(self.score_single, predictions, references)))
+
+        return {'accuracy': passed / total}

opencompass/summarizers/subjective/__init__.py

@@ -10,6 +10,7 @@ from .corev2 import Corev2Summarizer
 from .creationbench import CreationBenchSummarizer
 from .flames import FlamesSummarizer
 from .fofo import FofoSummarizer
+from .followbench import FollowBenchSummarizer
 from .mtbench import MTBenchSummarizer
 from .mtbench101 import MTBench101Summarizer
 from .multiround import MultiroundSummarizer

opencompass/summarizers/subjective/followbench.py

@@ -0,0 +1,149 @@
+# flake8: noqa: E501
+import csv
+import os
+import os.path as osp
+import re
+import statistics
+from collections import defaultdict
+from datetime import datetime
+
+import numpy as np
+from mmengine import ConfigDict
+
+try:
+    from prettytable import from_csv
+except ImportError:
+    from_csv = None
+
+from opencompass.utils import model_abbr_from_cfg
+
+from .subjective_post_process import post_process_autoj, post_process_judgelm
+from .utils import get_judgeanswer_and_reference_update, get_outdir
+
+
+def post_process_followbench(item):
+    generation, level = item['prediction'], item['gold']['level']
+    try:
+        satisfy = generation.strip('```').strip().split('\n')[-1]
+
+        if level == 1:
+            if 'YES' in satisfy:
+                return 1, 1
+            elif 'NO' in satisfy:
+                return 0, 0
+            else:
+                raise Exception('Invalid evaluation for level 1.')
+        else:
+            satisfy_list = re.search(r'\[.*\]', satisfy)
+            if satisfy_list:
+                satisfy_list = eval(satisfy_list.group())
+                if len(satisfy_list) == level:
+                    num_true = 0
+                    for i in satisfy_list:
+                        if i == 'YES' or i == 'True':
+                            num_true += 1
+                        elif i in [
+                                'NO', 'False', 'PARTIAL', 'MAYBE', 'UNKNOWN',
+                                'N/A'
+                        ]:
+                            num_true += 0
+                        else:
+                            raise Exception('Invalid element in the list.')
+                    return int(num_true == level), num_true / level
+                else:
+                    raise Exception('Invalid number of elements in the list.')
+            else:
+                raise Exception('Invalid list that cannot be parsed.')
+
+    except Exception as e:
+        return -1, -1
+
+
+def get_scores(judged_answers, references):
+    results = [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]
+    n_group = len(judged_answers) // 5
+    n_groups = [n_group] * 5
+
+    for judged_answer, reference in zip(judged_answers, references):
+        if judged_answer[0] == -1:
+            n_groups[reference['level'] - 1] -= 1
+        else:
+            results[0][reference['level'] - 1] += judged_answer[0]
+            results[1][reference['level'] - 1] += judged_answer[1]
+
+    for i in range(len(results)):
+        for j in range(len(results[i])):
+            if n_groups[j] != 0:
+                results[i][j] = results[i][j] / n_groups[j]
+            else:
+                results[i][j] = 0
+    temp_dict = {
+        'HSR_AVG': statistics.mean(results[0]),
+        'SSR_AVG': statistics.mean(results[1])
+    }
+    for idx, s in enumerate(results[0]):
+        temp_dict[f'HSR_L{idx+1}'] = s
+    for idx, s in enumerate(results[1]):
+        temp_dict[f'SSR_L{idx+1}'] = s
+
+    return temp_dict
+
+
+class FollowBenchSummarizer:
+    """Do the subjectivity analyze based on evaluation results.
+
+    Args:
+        config (ConfigDict): The configuration object of the evaluation task.
+            It's expected to be filled out at runtime.
+    """
+
+    def __init__(self, config: ConfigDict) -> None:
+        self.tasks = []
+        self.cfg = config
+        self.eval_model_cfgs = self.cfg['eval']['partitioner']['models']
+        self.eval_model_abbrs = [
+            model_abbr_from_cfg(model) for model in self.eval_model_cfgs
+        ]
+        self.judge_models = self.cfg.get('judge_models', None)
+
+        self.judge_function = post_process_followbench
+
+    def summarize(self,
+                  time_str: str = datetime.now().strftime('%Y%m%d_%H%M%S')):
+        """Summarize the subjectivity analysis based on evaluation results.
+
+        Args:
+            time_str (str): Timestamp for file naming.
+
+        Returns:
+            pd.DataFrame: The summary results.
+        """
+        all_scores = {}
+        for judge_model in self.judge_models:
+            score_by_judgemodel = {}
+            judge_abbr = model_abbr_from_cfg(judge_model)
+            dataset_cfgs = self.cfg['datasets']
+            dataset = dataset_cfgs[0]  # Alignbench just have only one subfile
+            output_dir, results_folder = get_outdir(self.cfg, time_str)
+
+            fout = osp.join(output_dir,
+                            'followbench-judged-by--' + judge_abbr + '.csv')
+
+            for eval_model_abbr in self.eval_model_abbrs:
+                subdir = eval_model_abbr + '_judged-by--' + judge_abbr
+                subdir_path = os.path.join(results_folder, subdir)
+                model = eval_model_abbr
+                if os.path.isdir(subdir_path):
+                    judged_answers, references = get_judgeanswer_and_reference_update(
+                        dataset, subdir_path, self.judge_function)
+                    if len(judged_answers) == 0:
+                        score_by_judgemodel[model] = None
+                        continue
+                    scores = get_scores(judged_answers, references)
+                    score_by_judgemodel[model] = scores
+                else:
+                    score_by_judgemodel[model] = None
+                    print(subdir_path + ' is not exist! please check!')
+
+            all_scores[judge_abbr] = score_by_judgemodel
+        return {'followbench': all_scores}

opencompass/summarizers/subjective/utils.py

@@ -73,3 +73,57 @@ def get_judgeanswer_and_reference(dataset, subdir_path, post_process):
         )
         print('*' * 100)
     return judged_answers, references
+
+
+def get_judgeanswer_and_reference_update(dataset, subdir_path, post_process):
+    """Extract judgements (scores) and references.
+
+    Args:
+        dataset (ConfigDict): Dataset config.
+        subdir_path (str): Model path in results dir.
+        post_process (function): The pre-defined extract function.
+    """
+    dataset_abbr = dataset_abbr_from_cfg(dataset)
+    filename = osp.join(subdir_path, dataset_abbr + '.json')
+    partial_filename = osp.join(subdir_path, dataset_abbr + '_0.json')
+    if osp.exists(osp.realpath(filename)):
+        result = mmengine.load(filename)
+    elif osp.exists(osp.realpath(partial_filename)):
+        filename = partial_filename
+        result = {}
+        i = 1
+        partial_dict_flag = 0
+        while osp.exists(osp.realpath(filename)):
+            res = mmengine.load(filename)
+            for k, v in res.items():
+                result[partial_dict_flag] = v
+                partial_dict_flag += 1
+            filename = osp.join(subdir_path,
+                                dataset_abbr + '_' + str(i) + '.json')
+            i += 1
+    else:
+        result = {}
+
+    if len(result) == 0:
+        print('*' * 100)
+        print('There are no results for ' + filename + ' or ' +
+              partial_filename)
+        print('*' * 100)
+
+    judged_answers = []
+    references = []
+    for k, v in result.items():
+        processed_judge = post_process(v)
+        if processed_judge is not None:
+            judged_answers.append(processed_judge)
+            references.append(v['gold'])
+        # else:
+        #     print(v['prediction'])
+        #     print('-' * 128)
+    if len(judged_answers) <= 0.95 * len(result):
+        print('*' * 100)
+        print(
+            f'For your {filename} judge. Among {len(result)} judgements, successfully extracted {len(judged_answers)} judgements, please check!'
+        )
+        print('*' * 100)
+    return judged_answers, references

opencompass/summarizers/subjective/wildbench.py

@@ -156,8 +156,8 @@ class WildBenchPairSummarizer(CompassArenaSummarizer):
         self.tasks = []
         self.cfg = config
 
-        self.base_models = self.cfg['eval']['partitioner']['base_models']
+        self.base_models = self.cfg['datasets'][0]['base_models']
-        self.compare_models = self.cfg['eval']['partitioner']['compare_models']
+        self.compare_models = self.cfg['eval']['partitioner']['models']
         self.judge_models = self.cfg.get('judge_models', None)
         self.meta_judge_model = self.cfg.eval.partitioner.get('meta_judge_model', None)
         self.judge_abbr = model_abbr_from_cfg(self.cfg['judge_models'][0])
@@ -247,8 +247,10 @@ class WildBenchPairSummarizer(CompassArenaSummarizer):
             pd.DataFrame: The summary results.
         """
         scores = self.get_score(time_str)
+        all_scores = {}
         output_dir, results_folder = get_outdir(self.cfg, time_str)
         for idx, judge_model in enumerate(self.judge_models):
+            score_by_judgemodel = {}
             judge_abbr = model_abbr_from_cfg(judge_model)
             for dataset in self.cfg['datasets']:
                 dataset_abbr = dataset_abbr_from_cfg(dataset)
@@ -258,7 +260,7 @@ class WildBenchPairSummarizer(CompassArenaSummarizer):
                 row_headers = [dataset_abbr, 'position_bias'] + row_headers
 
                 table = []
-                for row_header in row_headers:
+                for idx, row_header in enumerate(row_headers):
                     row = [row_header]
                     headers = ['']
                     for model_cfg in self.compare_models:
@@ -276,12 +278,13 @@ class WildBenchPairSummarizer(CompassArenaSummarizer):
                                 s = str(s)
                             row.append(s)
                         avg = avg/len(self.base_models)
+                        if idx == 0:
+                            score_by_judgemodel[model_abbr] = {'score': avg}
                         row.append(f'{avg:.2f}')
                         headers.append('Avg')
                     table.append(row)
 
                 txt = tabulate(table, headers=headers)
-                print(txt)
 
                 if idx == len(self.judge_models):
                     output_filename = osp.join(output_dir, 'summarized-by--' + judge_abbr + '-' + dataset_abbr + '-report.csv')
@@ -292,4 +295,5 @@ class WildBenchPairSummarizer(CompassArenaSummarizer):
                     f.write(','.join(headers) + '\n')
                     for line in table:
                         f.write(','.join(line) + '\n')
-                print(output_filename)
+            all_scores[judge_abbr] = score_by_judgemodel
+        return {'Wildbench': all_scores}

opencompass/utils/__init__.py

@@ -10,5 +10,6 @@ from .lark import *  # noqa
 from .logging import *  # noqa
 from .menu import *  # noqa
 from .model_postprocessors import *  # noqa
+from .postprocessors import *  # noqa
 from .prompt import *  # noqa
 from .text_postprocessors import *  # noqa

opencompass/utils/datasets_info.py

@@ -265,6 +265,12 @@ DATASETS_MAPPING = {
         "hf_id": "opencompass/xsum",
         "local": "./data/Xsum/dev.jsonl",
     },
+    # Longbench
+    "opencompass/Longbench": {
+        "ms_id": "",
+        "hf_id": "THUDM/LongBench",
+        "local": "./data/Longbench",
+    },
     # Needlebench
     "opencompass/needlebench": {
         "ms_id": "",
@@ -402,8 +408,16 @@ DATASETS_URL = {
         "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/mmlu_pro.zip",
         "md5": "e3200c7380f4cea5f13c768f2815fabb",
     },
+    "/Longbench": {
+        "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/Longbench.zip",
+        "md5": "ab0cb9e520ae5cfb899bf38b564249bb",
+    },
     "/needlebench": {
         "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/needlebench.zip",
         "md5": "b546da0397746eaff4d3ff0f20d6ede2",
-    }
+    },
+    "/teval": {
+        "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/teval.zip",
+        "md5": "7628ab5891a26bf96ca17becfd044867",
+    },
 }