[Update] Update dataset configuration with no max_out_len (#1754)

configs/datasets/subjective/wildbench/wildbench_pair_judge.py

@@ -20,7 +20,7 @@ subjective_infer_cfg = dict(
             template="""{dialogue}"""
         ),
         retriever=dict(type=ZeroRetriever),
-        inferencer=dict(type=ChatInferencer, max_seq_len=32768, max_out_len=4096, infer_mode='last'),
+        inferencer=dict(type=ChatInferencer, max_seq_len=32768, infer_mode='last'),
     )
 
 subjective_eval_cfg = dict(

configs/datasets/subjective/wildbench/wildbench_pair_judge_new.py

@@ -20,7 +20,7 @@ subjective_infer_cfg = dict(
             template="""{dialogue}"""
         ),
         retriever=dict(type=ZeroRetriever),
-        inferencer=dict(type=ChatInferencer, max_seq_len=32768, max_out_len=4096, infer_mode='last'),
+        inferencer=dict(type=ChatInferencer, max_seq_len=32768, infer_mode='last'),
     )
 
 subjective_eval_cfg = dict(

opencompass/configs/datasets/IFEval/IFEval_gen_353ae7.py

@@ -0,0 +1,33 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import IFEvalDataset, IFEvaluator
+
+ifeval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='reference')
+
+ifeval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+ifeval_eval_cfg = dict(
+    evaluator=dict(type=IFEvaluator),
+    pred_role='BOT',
+)
+
+ifeval_datasets = [
+    dict(
+        abbr='IFEval',
+        type=IFEvalDataset,
+        path='data/ifeval/input_data.jsonl',
+        reader_cfg=ifeval_reader_cfg,
+        infer_cfg=ifeval_infer_cfg,
+        eval_cfg=ifeval_eval_cfg)
+]

opencompass/configs/datasets/humaneval/humaneval_openai_sample_evals_gen_dcae0e.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+humaneval_datasets = [
+    dict(
+        abbr='openai_humaneval',
+        type=HumanevalDataset,
+        path='opencompass/humaneval',
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+]

opencompass/configs/datasets/humanevalx/humanevalx_gen_3d84a3.py

@@ -0,0 +1,41 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalXDataset, HumanevalXEvaluator
+
+humanevalx_reader_cfg = dict(
+    input_columns=['prompt'], output_column='declaration', train_split='test')
+
+humanevalx_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='{prompt}'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+humanevalx_eval_cfg_dict = {
+    lang : dict(
+        evaluator=dict(
+            type=HumanevalXEvaluator,
+            language=lang,
+            ip_address=
+            'localhost',  # replace to your code_eval_server ip_address, port
+            port=5001),  # refer to https://opencompass.readthedocs.io/en/latest/advanced_guides/code_eval_service.html to launch a server
+        pred_role='BOT')
+    for lang in ['python', 'cpp', 'go', 'java', 'js']   # do not support rust now
+}
+
+# Please download the needed `xx.jsonl.gz` from
+# https://github.com/THUDM/CodeGeeX2/tree/main/benchmark/humanevalx
+# and move them into `data/humanevalx/` folder
+humanevalx_datasets = [
+    dict(
+        type=HumanevalXDataset,
+        abbr=f'humanevalx-{lang}',
+        language=lang,
+        path='./data/humanevalx',
+        reader_cfg=humanevalx_reader_cfg,
+        infer_cfg=humanevalx_infer_cfg,
+        eval_cfg=humanevalx_eval_cfg_dict[lang])
+    for lang in ['python', 'cpp', 'go', 'java', 'js']
+]

opencompass/configs/datasets/math/math_0shot_gen_11c4b5.py

@@ -0,0 +1,35 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MATHDataset, MATHEvaluator, math_postprocess_v2, normalize_final_answer
+
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{problem}\nPlease reason step by step, and put your final answer within \\boxed{}.'),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+# postprocess v2
+math_eval_cfg = dict(
+    evaluator=dict(type=MATHEvaluator, version='v2'), pred_postprocessor=dict(type=math_postprocess_v2),
+)
+
+math_datasets = [
+    dict(
+        type=MATHDataset,
+        abbr='math',
+        path='opencompass/math',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg,
+    )
+]

opencompass/configs/datasets/subjective/wildbench/wildbench_pair_judge.py

@@ -20,7 +20,7 @@ subjective_infer_cfg = dict(
             template="""{dialogue}"""
         ),
         retriever=dict(type=ZeroRetriever),
-        inferencer=dict(type=ChatInferencer, max_seq_len=32768, max_out_len=4096, infer_mode='last'),
+        inferencer=dict(type=ChatInferencer, max_seq_len=32768, infer_mode='last'),
     )
 
 subjective_eval_cfg = dict(

opencompass/configs/datasets/subjective/wildbench/wildbench_pair_judge_new.py

@@ -20,7 +20,7 @@ subjective_infer_cfg = dict(
             template="""{dialogue}"""
         ),
         retriever=dict(type=ZeroRetriever),
-        inferencer=dict(type=ChatInferencer, max_seq_len=32768, max_out_len=4096, infer_mode='last'),
+        inferencer=dict(type=ChatInferencer, max_seq_len=32768, infer_mode='last'),
     )
 
 subjective_eval_cfg = dict(