From 41196c48aef7c2a1e4b2218e07305f7f66d1bc98 Mon Sep 17 00:00:00 2001
From: Jingming Zhuo <109033042+jingmingzhuo@users.noreply.github.com>
Date: Wed, 24 Apr 2024 17:40:50 +0800
Subject: [PATCH] Add humaneval prompt from simple_evals, openai (#1076)

* [Feature] Add IFEval

* add humaneval prompt from simple_evals, openai
---
 ...umaneval_openai_sample_evals_gen_159614.py | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 configs/datasets/humaneval/humaneval_openai_sample_evals_gen_159614.py

diff --git a/configs/datasets/humaneval/humaneval_openai_sample_evals_gen_159614.py b/configs/datasets/humaneval/humaneval_openai_sample_evals_gen_159614.py
new file mode 100644
index 00000000..2050eb74
--- /dev/null
+++ b/configs/datasets/humaneval/humaneval_openai_sample_evals_gen_159614.py
@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvaluator, humaneval_postprocess_v2
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+humaneval_datasets = [
+    dict(
+        abbr='openai_humaneval',
+        type=HumanevalDataset,
+        path='./data/humaneval/human-eval-v2-20210705.jsonl',
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+]