diff --git a/examples/eval_benchmax.py b/examples/eval_benchmax.py new file mode 100644 index 00000000..6eca0a3b --- /dev/null +++ b/examples/eval_benchmax.py @@ -0,0 +1,20 @@ +from mmengine.config import read_base + +with read_base(): + from opencompass.configs.datasets.xlivecodebench.xlivecodebench_gen import \ + LCB_datasets + from opencompass.configs.datasets.xgpqa.xgpqa_gen import \ + gpqa_datasets + from opencompass.configs.datasets.xIFEval.xIFeval_gen import \ + xifeval_datasets + from opencompass.configs.models.hf_llama.hf_llama3_8b_instruct import \ + models as hf_llama3_8b_instruct_models + +datasets = [ + *LCB_datasets, + *gpqa_datasets, + *xifeval_datasets +] +models = [ + *hf_llama3_8b_instruct_models +] diff --git a/opencompass/configs/datasets/xIFEval/IFEval.md b/opencompass/configs/datasets/xIFEval/IFEval.md new file mode 100644 index 00000000..96b67b48 --- /dev/null +++ b/opencompass/configs/datasets/xIFEval/IFEval.md @@ -0,0 +1,55 @@ +# IFEval +## Introduction +The following introduction comes from the abstract in [Instruction Following Evaluation for Large Language Models](arxiv.org/abs/2311.07911) + +``` +IFEval is a straightforward and easy-to-reproduce evaluation benchmark. It focuses on a set of "verifiable instructions" such as "write in more than 400 words" and "mention the keyword of AI at least 3 times". We identified 25 types of those verifiable instructions and constructed around 500 prompts, with each prompt containing one or more verifiable instructions. +``` + +## Official link + +### Paper + +[Instruction Following Evaluation for Large Language Models](arxiv.org/abs/2311.07911) + +### Repository + +[instruction_following_eval](github.com/google-research/google-research/tree/master/instruction_following_eval) + +## Examples +Input example I: +``` +Write a 300+ word summary of the wikipedia page \"https://en.wikipedia.org/wiki/Raymond_III,_Count_of_Tripoli\". Do not use any commas and highlight at least 3 sections that has titles in markdown format, for example *highlighted section part 1*, *highlighted section part 2*, *highlighted section part 3*. +``` +Output example I (from GPT-4): +``` +Raymond III was the Count of Tripoli from 1152 to 1187 and Prince of Galilee and Tiberias in the Kingdom of Jerusalem. He was born in 1140 to Raymond II of Tripoli and Hodierna of Jerusalem. His mother was the daughter of Baldwin II of Jerusalem. \n\n*Early Life and Succession*\n\nRaymond III was only a child when his father was murdered. His mother Hodierna was regent until Raymond came of age. In 1155 Raymond married Eschiva the daughter of Walter I of Beirut. They had three children: Raymond IV Bertrand and a daughter who married Guy of Lusignan. \n\n*Reign*\n\nRaymond III's reign was marked by a series of conflicts and alliances with the Muslim world. He was involved in the defense of the Kingdom of Jerusalem against Nur ad-Din and later Saladin. He was also a key figure in the internal politics of the kingdom. He was a supporter of the queen mother Amalric of Jerusalem and opposed the succession of Guy of Lusignan. \n\n*Later Life and Death*\n\nIn 1187 Raymond III was part of the disastrous Battle of Hattin where the Christian forces were defeated by Saladin. He was one of the few to escape the battlefield but died later that year. His son Raymond IV succeeded him as Count of Tripoli. \n\nRaymond III's life and reign were marked by the complex politics and warfare of the Crusader states. He was a key figure in the defense of the Kingdom of Jerusalem and his death marked a significant loss for the Christian forces in the region. His legacy is a testament to the turbulent times in which he lived and the challenges faced by the Crusader states in their relations with the Muslim world. +``` +Input example II: +``` +"I am planning a trip to Japan, and I would like thee to write an itinerary for my journey in a Shakespearean style. You are not allowed to use any commas in your response." +``` +Output example II (from GPT-4): +``` +Hark! Hearken to the tale of thy journey to the land of the rising sun, Japan. \n\nUpon the morrow of thy arrival, thou shalt find thyself in Tokyo, the city of lights and towers. In the morn, thou shalt explore the grandeur of the Imperial Palace, a sight to behold. As the sun reaches its zenith, thou shalt partake in a feast of sushi, a delicacy of the land. As the day wanes, thou shalt wander through the electric streets of Akihabara, a realm of technology and animation.\n\nOn the second day, thou shalt journey to the ancient city of Kyoto. In the soft light of dawn, thou shalt visit the golden pavilion of Kinkaku-ji, a vision of beauty and tranquility. As the day unfolds, thou shalt wander through the bamboo grove of Arashiyama, a place of peace and serenity. As the sun sets, thou shalt partake in a traditional tea ceremony, a ritual of grace and harmony.\n\nOn the third day, thou shalt travel to the city of Hiroshima. In the morning, thou shalt pay thy respects at the Peace Memorial Park, a place of remembrance and hope. As the day progresses, thou shalt journey to the island of Miyajima, home to the floating torii gate, a sight of wonder and awe. As the day ends, thou shalt feast on okonomiyaki, a dish of delight and satisfaction.\n\nOn the fourth day, thou shalt return to Tokyo. In the morning, thou shalt ascend the Tokyo Skytree, a tower of height and grandeur. As the day unfolds, thou shalt explore the bustling streets of Shibuya, a realm of fashion and culture. As the sun sets, thou shalt dine in the vibrant district of Roppongi, a place of gastronomy and nightlife.\n\nOn the fifth and final day, thou shalt bid farewell to the land of the rising sun. With memories of beauty and wonder, thou shalt embark on thy journey home, carrying with thee the spirit of Japan. \n\nThus ends the tale of thy journey, a tale of exploration and discovery. May thy travels be filled with joy and adventure, and may the land of the rising sun forever hold a place in thy heart. +``` +## Evaluation results + +``` +dataset version metric mode baichuan2-7b-chat-hf baichuan2-13b-chat-hf internlm2-chat-7b-hf internlm2-chat-20b-hf llama-2-7b-chat-hf llama-2-13b-chat-hf +--------- --------- ---------------------------- ------ ---------------------- ----------------------- ---------------------- ----------------------- -------------------- --------------------- +IFEval 3321a3 Prompt-level-strict-accuracy gen 36.04 35.49 38.26 33.09 33.46 33.64 +IFEval 3321a3 Inst-level-strict-accuracy gen 46.76 46.76 49.16 45.32 45.68 45.44 +IFEval 3321a3 Prompt-level-loose-accuracy gen 37.52 37.71 42.51 39.37 43.81 47.32 +IFEval 3321a3 Inst-level-loose-accuracy gen 48.44 49.16 53.72 51.08 55.64 58.03 +``` + +## Reference +``` +@article{zhou2023instruction, + title={Instruction-Following Evaluation for Large Language Models}, + author={Zhou, Jeffrey and Lu, Tianjian and Mishra, Swaroop and Brahma, Siddhartha and Basu, Sujoy and Luan, Yi and Zhou, Denny and Hou, Le}, + journal={arXiv preprint arXiv:2311.07911}, + year={2023} +} +``` diff --git a/opencompass/configs/datasets/xIFEval/IFEval_gen.py b/opencompass/configs/datasets/xIFEval/IFEval_gen.py new file mode 100644 index 00000000..9103c13b --- /dev/null +++ b/opencompass/configs/datasets/xIFEval/IFEval_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .IFEval_gen_3321a3 import ifeval_datasets # noqa: F401, F403 diff --git a/opencompass/configs/datasets/xIFEval/IFEval_gen_3321a3.py b/opencompass/configs/datasets/xIFEval/IFEval_gen_3321a3.py new file mode 100644 index 00000000..a3d641b0 --- /dev/null +++ b/opencompass/configs/datasets/xIFEval/IFEval_gen_3321a3.py @@ -0,0 +1,34 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import xIFEvalDataset, xIFEvaluator + +xifeval_reader_cfg = dict( + input_columns=['prompt'], output_column='reference') + +xifeval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=2048)) + +xifeval_eval_cfg = dict( + evaluator=dict(type=xIFEvaluator), + pred_role='BOT', +) +LANG = 'de' + +xifeval_datasets = [ + dict( + abbr='xIFEval', + type=xIFEvalDataset, + path=f'data/ifeval/input_data_google_{LANG}.jsonl', + reader_cfg=xifeval_reader_cfg, + infer_cfg=xifeval_infer_cfg, + eval_cfg=xifeval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xIFEval/README.md b/opencompass/configs/datasets/xIFEval/README.md new file mode 100644 index 00000000..3753262c --- /dev/null +++ b/opencompass/configs/datasets/xIFEval/README.md @@ -0,0 +1,31 @@ +# IFEval + +```bash +python3 run.py --models hf_internlm2_chat_7b --datasets IFEval_gen_3321a3 --debug +``` + +## Chat Models + +| model | Prompt-level-strict-accuracy | Inst-level-strict-accuracy | Prompt-level-loose-accuracy | Inst-level-loose-accuracy | +|:-----------------------------:|-------------------------------:|-----------------------------:|------------------------------:|----------------------------:| +| qwen1.5-0.5b-chat-hf | 13.12 | 23.26 | 15.71 | 26.38 | +| qwen1.5-1.8b-chat-hf | 16.08 | 26.26 | 18.30 | 29.02 | +| qwen1.5-4b-chat-hf | 25.51 | 35.97 | 28.84 | 39.81 | +| qwen1.5-7b-chat-hf | 38.82 | 50.00 | 42.70 | 53.48 | +| qwen1.5-14b-chat-hf | 42.51 | 54.20 | 49.17 | 59.95 | +| qwen1.5-32b-chat-hf | 49.54 | 60.43 | 53.97 | 64.39 | +| qwen1.5-72b-chat-hf | 51.02 | 61.99 | 57.12 | 67.27 | +| qwen1.5-110b-chat-hf | 55.08 | 65.59 | 61.18 | 70.86 | +| internlm2-chat-1.8b-hf | 18.30 | 28.78 | 21.44 | 32.01 | +| internlm2-chat-1.8b-sft-hf | 18.67 | 31.18 | 19.78 | 32.85 | +| internlm2-chat-7b-hf | 34.75 | 46.28 | 40.48 | 51.44 | +| internlm2-chat-7b-sft-hf | 39.19 | 50.12 | 42.33 | 52.76 | +| internlm2-chat-20b-hf | 36.41 | 48.68 | 40.67 | 53.24 | +| internlm2-chat-20b-sft-hf | 44.55 | 55.64 | 46.77 | 58.03 | +| llama-3-8b-instruct-hf | 68.02 | 76.74 | 75.42 | 82.85 | +| llama-3-70b-instruct-hf | 78.00 | 84.65 | 84.29 | 89.21 | +| llama-3-8b-instruct-lmdeploy | 69.13 | 77.46 | 77.26 | 83.93 | +| llama-3-70b-instruct-lmdeploy | 75.97 | 82.97 | 83.18 | 88.37 | +| mistral-7b-instruct-v0.1-hf | 40.30 | 50.96 | 41.96 | 53.48 | +| mistral-7b-instruct-v0.2-hf | 49.17 | 60.43 | 51.94 | 64.03 | +| mixtral-8x7b-instruct-v0.1-hf | 50.09 | 60.67 | 55.64 | 65.83 | diff --git a/opencompass/configs/datasets/xIFEval/xIFeval_gen.py b/opencompass/configs/datasets/xIFEval/xIFeval_gen.py new file mode 100644 index 00000000..8605b1e2 --- /dev/null +++ b/opencompass/configs/datasets/xIFEval/xIFeval_gen.py @@ -0,0 +1,37 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import xIFEvalDataset, xIFEvaluator + +xifeval_reader_cfg = dict( + input_columns=['prompt'], output_column='reference') + +xifeval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=4096)) + +xifeval_eval_cfg = dict( + evaluator=dict(type=xIFEvaluator), + pred_role='BOT', +) + +xifeval_datasets = [] + +LANGS = ['ar', 'bn', 'cs', 'de', 'es', 'fr', 'hu', 'ja', 'ko', 'ru', 'sr', 'sw', 'te', 'th', 'vi', 'zh'] +for LANG in LANGS: + path = f'data/xifeval/input_data_google_{LANG}.jsonl' + xifeval_datasets.append(dict( + type=xIFEvalDataset, + abbr=f'xIFEval_{LANG}', + path=path, + reader_cfg=xifeval_reader_cfg, + infer_cfg=xifeval_infer_cfg, + eval_cfg=xifeval_eval_cfg, + )) diff --git a/opencompass/configs/datasets/xgpqa/README.md b/opencompass/configs/datasets/xgpqa/README.md new file mode 100644 index 00000000..02560df5 --- /dev/null +++ b/opencompass/configs/datasets/xgpqa/README.md @@ -0,0 +1,69 @@ +# GPQA + +```bash +python3 run.py --models hf_internlm2_7b --datasets gpqa_ppl_6bf57a --debug +python3 run.py --models hf_internlm2_chat_7b --datasets gpqa_gen_4baadb --debug +``` + +## Base Models + +| model | GPQA_diamond | +|:------------------------:|---------------:| +| llama-7b-turbomind | 24.24 | +| llama-13b-turbomind | 25.25 | +| llama-30b-turbomind | 22.73 | +| llama-65b-turbomind | 21.72 | +| llama-2-7b-turbomind | 25.25 | +| llama-2-13b-turbomind | 23.74 | +| llama-2-70b-turbomind | 28.28 | +| llama-3-8b-turbomind | 31.82 | +| llama-3-70b-turbomind | 40.91 | +| internlm2-1.8b-turbomind | 24.24 | +| internlm2-7b-turbomind | 28.28 | +| internlm2-20b-turbomind | 31.31 | +| qwen-1.8b-turbomind | 28.79 | +| qwen-7b-turbomind | 24.75 | +| qwen-14b-turbomind | 27.78 | +| qwen-72b-turbomind | 31.31 | +| qwen1.5-0.5b-hf | 23.74 | +| qwen1.5-1.8b-hf | 28.79 | +| qwen1.5-4b-hf | 23.23 | +| qwen1.5-7b-hf | 20.71 | +| qwen1.5-14b-hf | 32.32 | +| qwen1.5-32b-hf | 30.81 | +| qwen1.5-72b-hf | 31.82 | +| qwen1.5-moe-a2-7b-hf | 28.79 | +| mistral-7b-v0.1-hf | 24.75 | +| mistral-7b-v0.2-hf | 23.74 | +| mixtral-8x7b-v0.1-hf | 28.79 | +| mixtral-8x22b-v0.1-hf | 36.36 | +| yi-6b-hf | 28.28 | +| yi-34b-hf | 35.86 | +| deepseek-7b-base-hf | 20.71 | +| deepseek-67b-base-hf | 25.25 | + +## Chat Models + +| model | GPQA_diamond | +|:-----------------------------:|---------------:| +| qwen1.5-0.5b-chat-hf | 19.70 | +| qwen1.5-1.8b-chat-hf | 29.80 | +| qwen1.5-4b-chat-hf | 25.25 | +| qwen1.5-7b-chat-hf | 31.82 | +| qwen1.5-14b-chat-hf | 30.30 | +| qwen1.5-32b-chat-hf | 31.31 | +| qwen1.5-72b-chat-hf | 32.83 | +| qwen1.5-110b-chat-hf | 35.86 | +| internlm2-chat-1.8b-hf | 25.76 | +| internlm2-chat-1.8b-sft-hf | 26.26 | +| internlm2-chat-7b-hf | 28.28 | +| internlm2-chat-7b-sft-hf | 27.27 | +| internlm2-chat-20b-hf | 30.30 | +| internlm2-chat-20b-sft-hf | 29.29 | +| llama-3-8b-instruct-hf | 25.76 | +| llama-3-70b-instruct-hf | 37.88 | +| llama-3-8b-instruct-lmdeploy | 25.76 | +| llama-3-70b-instruct-lmdeploy | 37.88 | +| mistral-7b-instruct-v0.1-hf | 30.30 | +| mistral-7b-instruct-v0.2-hf | 25.25 | +| mixtral-8x7b-instruct-v0.1-hf | 30.30 | diff --git a/opencompass/configs/datasets/xgpqa/gpqa_0shot_nocot_gen_772ea0.py b/opencompass/configs/datasets/xgpqa/gpqa_0shot_nocot_gen_772ea0.py new file mode 100644 index 00000000..4783dae4 --- /dev/null +++ b/opencompass/configs/datasets/xgpqa/gpqa_0shot_nocot_gen_772ea0.py @@ -0,0 +1,52 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import GPQADataset, GPQA_Simple_Eval_postprocess, GPQAEvaluator + +# openai_simple_eval prompt +align_prompt = """ +Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of ABCD. + +{question} + +A) {A} +B) {B} +C) {C} +D) {D} +""".strip() + +gpqa_reader_cfg = dict( + input_columns=['question', 'A', 'B', 'C', 'D'], + output_column='answer') + +gpqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt=align_prompt), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +gpqa_eval_cfg = dict(evaluator=dict(type=GPQAEvaluator), + pred_postprocessor=dict(type=GPQA_Simple_Eval_postprocess)) + +gpqa_datasets = [] +gpqa_subsets = { + # 'extended': 'gpqa_extended.csv', + # 'main': 'gpqa_main.csv', + 'diamond': 'gpqa_diamond.csv' +} + +for split in list(gpqa_subsets.keys()): + gpqa_datasets.append( + dict( + abbr='GPQA_' + split, + type=GPQADataset, + path='./data/gpqa/', + name=gpqa_subsets[split], + reader_cfg=gpqa_reader_cfg, + infer_cfg=gpqa_infer_cfg, + eval_cfg=gpqa_eval_cfg) + ) diff --git a/opencompass/configs/datasets/xgpqa/gpqa_few_shot_ppl_4b5a83.py b/opencompass/configs/datasets/xgpqa/gpqa_few_shot_ppl_4b5a83.py new file mode 100644 index 00000000..10eb1dee --- /dev/null +++ b/opencompass/configs/datasets/xgpqa/gpqa_few_shot_ppl_4b5a83.py @@ -0,0 +1,53 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import FixKRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.openicl.icl_evaluator import AccwithDetailsEvaluator +from opencompass.datasets import GPQADataset, GPQAEvaluator +from opencompass.utils import first_option_postprocess + +gpqa_reader_cfg = dict( + input_columns=['question', 'A', 'B', 'C', 'D'], + output_column='answer') + +hint = f'For the multiple choice question below, please provide the correct answer option directly.' +question_and_options = 'Question: {question}\n(A){A}\n(B){B}\n(C){C}\n(D){D}\n' +gpqa_infer_cfg = dict( + ice_template=dict( + type=PromptTemplate, + template={ + opt: f'{question_and_options}\nAnswer: {opt}' for opt in ['A', 'B', 'C', 'D']}, + ), + prompt_template=dict( + type=PromptTemplate, + template={ + opt: f'{hint}\n{question_and_options}\nAnswer: {opt}' for opt in ['A', 'B', 'C', 'D'] + }, + ice_token='' + ), + retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]), + inferencer=dict(type=PPLInferencer)) + +gpqa_eval_cfg = dict(evaluator=dict(type=AccwithDetailsEvaluator)) + +gpqa_datasets = [] +LANG = ['de'] +gpqa_subsets = { + lang: f'gpqa_main_{lang}.csv' for lang in LANG +} +# gpqa_subsets = { +# # 'extended': 'gpqa_extended.csv', +# # 'main': 'gpqa_main.csv', +# 'diamond': 'gpqa_main_{}.csv' +# } + +for split in list(gpqa_subsets.keys()): + gpqa_datasets.append( + dict( + abbr='GPQA_' + split, + type=GPQADataset, + path='./data/gpqa/', + name=gpqa_subsets[split], + reader_cfg=gpqa_reader_cfg, + infer_cfg=gpqa_infer_cfg, + eval_cfg=gpqa_eval_cfg) + ) diff --git a/opencompass/configs/datasets/xgpqa/gpqa_gen_015262.py b/opencompass/configs/datasets/xgpqa/gpqa_gen_015262.py new file mode 100644 index 00000000..227c0d49 --- /dev/null +++ b/opencompass/configs/datasets/xgpqa/gpqa_gen_015262.py @@ -0,0 +1,46 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import GPQADataset, GPQAEvaluator +from opencompass.utils import first_option_postprocess + +gpqa_reader_cfg = dict( + input_columns=['question', 'A', 'B', 'C', 'D'], + output_column='answer') + +gpqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='What is the correct answer to this question: {question}\nChoices:\n' + '(A){A}\n' + '(B){B}\n' + '(C){C}\n' + '(D){D}\n' + 'Format your response as follows: "The correct answer is (insert answer here)"'), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +gpqa_eval_cfg = dict(evaluator=dict(type=GPQAEvaluator), + pred_postprocessor=dict(type=first_option_postprocess, options='ABCD')) + +gpqa_datasets = [] +gpqa_subsets = { + 'extended': 'gpqa_extended.csv', + 'main': 'gpqa_main.csv', + 'diamond': 'gpqa_diamond.csv' +} + +for split in list(gpqa_subsets.keys()): + gpqa_datasets.append( + dict( + abbr='GPQA_' + split, + type=GPQADataset, + path='./data/gpqa/', + name=gpqa_subsets[split], + reader_cfg=gpqa_reader_cfg, + infer_cfg=gpqa_infer_cfg, + eval_cfg=gpqa_eval_cfg) + ) diff --git a/opencompass/configs/datasets/xgpqa/gpqa_gen_4baadb.py b/opencompass/configs/datasets/xgpqa/gpqa_gen_4baadb.py new file mode 100644 index 00000000..f4d39ed5 --- /dev/null +++ b/opencompass/configs/datasets/xgpqa/gpqa_gen_4baadb.py @@ -0,0 +1,46 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import GPQADataset, GPQAEvaluator +from opencompass.utils import first_option_postprocess + +gpqa_reader_cfg = dict( + input_columns=['question', 'A', 'B', 'C', 'D'], + output_column='answer') + +gpqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='What is the correct answer to this question: {question}\nChoices:\n' + '(A){A}\n' + '(B){B}\n' + '(C){C}\n' + '(D){D}\n' + 'Format your response as follows: "The correct answer is (insert answer here)"'), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +gpqa_eval_cfg = dict(evaluator=dict(type=GPQAEvaluator), + pred_postprocessor=dict(type=first_option_postprocess, options='ABCD')) + +gpqa_datasets = [] +gpqa_subsets = { + # 'extended': 'gpqa_extended.csv', + # 'main': 'gpqa_main.csv', + 'diamond': 'gpqa_diamond.csv' +} + +for split in list(gpqa_subsets.keys()): + gpqa_datasets.append( + dict( + abbr='GPQA_' + split, + type=GPQADataset, + path='./data/gpqa/', + name=gpqa_subsets[split], + reader_cfg=gpqa_reader_cfg, + infer_cfg=gpqa_infer_cfg, + eval_cfg=gpqa_eval_cfg) + ) diff --git a/opencompass/configs/datasets/xgpqa/gpqa_openai_simple_evals_gen_5aeece.py b/opencompass/configs/datasets/xgpqa/gpqa_openai_simple_evals_gen_5aeece.py new file mode 100644 index 00000000..90441cf5 --- /dev/null +++ b/opencompass/configs/datasets/xgpqa/gpqa_openai_simple_evals_gen_5aeece.py @@ -0,0 +1,57 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import GPQADataset, GPQA_Simple_Eval_postprocess, GPQAEvaluator + +# openai_simple_eval prompt +align_prompt = """ +Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. + +{question} + +A) {A} +B) {B} +C) {C} +D) {D} +""".strip() + +gpqa_reader_cfg = dict( + input_columns=['question', 'A', 'B', 'C', 'D'], + output_column='answer') + +gpqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt=align_prompt), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +gpqa_eval_cfg = dict(evaluator=dict(type=GPQAEvaluator), + pred_postprocessor=dict(type=GPQA_Simple_Eval_postprocess)) + +gpqa_datasets = [] +# gpqa_subsets = { +# # 'extended': 'gpqa_extended.csv', +# # 'main': 'gpqa_main.csv', +# 'diamond': 'gpqa_diamond.csv' +# } +gpqa_datasets = [] +LANGS = ['en','ar', 'bn', 'cs', 'es', 'fr', 'hu', 'ja', 'ko', 'ru', 'sr', 'sw', 'te', 'th', 'vi', 'zh'] +gpqa_subsets = { + lang: f'gpqa_main_{lang}.csv' for lang in LANGS +} + +for split in list(gpqa_subsets.keys()): + gpqa_datasets.append( + dict( + abbr='GPQA_' + split, + type=GPQADataset, + path='./data/gpqa/', # need to change + name=gpqa_subsets[split], + reader_cfg=gpqa_reader_cfg, + infer_cfg=gpqa_infer_cfg, + eval_cfg=gpqa_eval_cfg) + ) diff --git a/opencompass/configs/datasets/xgpqa/gpqa_ppl_6bf57a.py b/opencompass/configs/datasets/xgpqa/gpqa_ppl_6bf57a.py new file mode 100644 index 00000000..8b112009 --- /dev/null +++ b/opencompass/configs/datasets/xgpqa/gpqa_ppl_6bf57a.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer +from opencompass.datasets import GPQADataset, GPQAEvaluator +from opencompass.utils import first_option_postprocess + +gpqa_reader_cfg = dict( + input_columns=['question', 'A', 'B', 'C', 'D'], + output_column='answer') + +gpqa_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template={ + opt: f'Question: {{question}}\n(A){{A}}\n(B){{B}}\n(C){{C}}\n(D){{D}}\nAnswer: {opt}' for opt in ['A', 'B', 'C', 'D'] + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer)) + +gpqa_eval_cfg = dict(evaluator=dict(type=GPQAEvaluator), + pred_postprocessor=dict(type=first_option_postprocess, options='ABCD')) + +gpqa_datasets = [] +gpqa_subsets = { + # 'extended': 'gpqa_extended.csv', + # 'main': 'gpqa_main.csv', + 'diamond': 'gpqa_diamond.csv' +} + +for split in list(gpqa_subsets.keys()): + gpqa_datasets.append( + dict( + abbr='GPQA_' + split, + type=GPQADataset, + path='./data/gpqa/', + name=gpqa_subsets[split], + reader_cfg=gpqa_reader_cfg, + infer_cfg=gpqa_infer_cfg, + eval_cfg=gpqa_eval_cfg) + ) diff --git a/opencompass/configs/datasets/xgpqa/xgpqa_gen.py b/opencompass/configs/datasets/xgpqa/xgpqa_gen.py new file mode 100644 index 00000000..23353835 --- /dev/null +++ b/opencompass/configs/datasets/xgpqa/xgpqa_gen.py @@ -0,0 +1,7 @@ +from mmengine.config import read_base + +# with read_base(): +# from .gpqa_openai_simple_evals_gen_5aeece import gpqa_datasets + +with read_base(): + from .gpqa_openai_simple_evals_gen_5aeece import gpqa_datasets diff --git a/opencompass/configs/datasets/xhumaneval/README.md b/opencompass/configs/datasets/xhumaneval/README.md new file mode 100644 index 00000000..ce004785 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/README.md @@ -0,0 +1,69 @@ +# HumanEval + +```bash +python3 run.py --models hf_internlm2_7b --datasets deprecated_humaneval_gen_d2537e --debug +python3 run.py --models hf_internlm2_chat_7b --datasets humaneval_gen_8e312c --debug +``` + +## Base Models + +| model | pass@1 | +|:------------------------:|---------:| +| llama-7b-turbomind | 12.80 | +| llama-13b-turbomind | 15.24 | +| llama-30b-turbomind | 9.15 | +| llama-65b-turbomind | 7.32 | +| llama-2-7b-turbomind | 14.02 | +| llama-2-13b-turbomind | 15.24 | +| llama-2-70b-turbomind | 15.24 | +| llama-3-8b-turbomind | 28.05 | +| llama-3-70b-turbomind | 28.05 | +| internlm2-1.8b-turbomind | 30.49 | +| internlm2-7b-turbomind | 48.17 | +| internlm2-20b-turbomind | 51.83 | +| qwen-1.8b-turbomind | 16.46 | +| qwen-7b-turbomind | 23.78 | +| qwen-14b-turbomind | 23.78 | +| qwen-72b-turbomind | 66.46 | +| qwen1.5-0.5b-hf | 8.54 | +| qwen1.5-1.8b-hf | 23.17 | +| qwen1.5-4b-hf | 41.46 | +| qwen1.5-7b-hf | 53.05 | +| qwen1.5-14b-hf | 57.32 | +| qwen1.5-32b-hf | 70.12 | +| qwen1.5-72b-hf | 65.85 | +| qwen1.5-moe-a2-7b-hf | 45.73 | +| mistral-7b-v0.1-hf | 14.02 | +| mistral-7b-v0.2-hf | 9.15 | +| mixtral-8x7b-v0.1-hf | 24.39 | +| mixtral-8x22b-v0.1-hf | 16.46 | +| yi-6b-hf | 14.63 | +| yi-34b-hf | 17.07 | +| deepseek-7b-base-hf | 18.29 | +| deepseek-67b-base-hf | 23.17 | + +## Chat Models + +| model | pass@1 | +|:-----------------------------:|---------:| +| qwen1.5-0.5b-chat-hf | 9.15 | +| qwen1.5-1.8b-chat-hf | 15.85 | +| qwen1.5-4b-chat-hf | 30.49 | +| qwen1.5-7b-chat-hf | 40.85 | +| qwen1.5-14b-chat-hf | 50.00 | +| qwen1.5-32b-chat-hf | 57.93 | +| qwen1.5-72b-chat-hf | 60.37 | +| qwen1.5-110b-chat-hf | 65.24 | +| internlm2-chat-1.8b-hf | 33.54 | +| internlm2-chat-1.8b-sft-hf | 34.15 | +| internlm2-chat-7b-hf | 56.71 | +| internlm2-chat-7b-sft-hf | 61.59 | +| internlm2-chat-20b-hf | 67.68 | +| internlm2-chat-20b-sft-hf | 67.68 | +| llama-3-8b-instruct-hf | 55.49 | +| llama-3-70b-instruct-hf | 70.73 | +| llama-3-8b-instruct-lmdeploy | 57.93 | +| llama-3-70b-instruct-lmdeploy | 70.73 | +| mistral-7b-instruct-v0.1-hf | 32.32 | +| mistral-7b-instruct-v0.2-hf | 29.27 | +| mixtral-8x7b-instruct-v0.1-hf | 34.15 | diff --git a/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_4a6eef.py b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_4a6eef.py new file mode 100644 index 00000000..67dd58a5 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_4a6eef.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nCreate a Python script for this problem:\n{prompt}\n\n### Response:\n'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_6d1cc2.py b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_6d1cc2.py new file mode 100644 index 00000000..830d391f --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_6d1cc2.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nComplete the following python function.:\n{prompt}\n\n### Response:\n'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_a82cae.py b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_a82cae.py new file mode 100644 index 00000000..dc5d10f5 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_a82cae.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_d2537e.py b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_d2537e.py new file mode 100644 index 00000000..69231fdc --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_d2537e.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='Complete the following python code:\n{prompt}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_fd5822.py b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_fd5822.py new file mode 100644 index 00000000..ea56afd6 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_fd5822.py @@ -0,0 +1,31 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='{prompt}'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_ff7054.py b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_ff7054.py new file mode 100644 index 00000000..a1be3ba9 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/deprecated_humaneval_gen_ff7054.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt='Complete the following python code:'), + ], + round=[ + dict(role='HUMAN', prompt='{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/README.md b/opencompass/configs/datasets/xhumaneval/humaneval/README.md new file mode 100644 index 00000000..ce004785 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/README.md @@ -0,0 +1,69 @@ +# HumanEval + +```bash +python3 run.py --models hf_internlm2_7b --datasets deprecated_humaneval_gen_d2537e --debug +python3 run.py --models hf_internlm2_chat_7b --datasets humaneval_gen_8e312c --debug +``` + +## Base Models + +| model | pass@1 | +|:------------------------:|---------:| +| llama-7b-turbomind | 12.80 | +| llama-13b-turbomind | 15.24 | +| llama-30b-turbomind | 9.15 | +| llama-65b-turbomind | 7.32 | +| llama-2-7b-turbomind | 14.02 | +| llama-2-13b-turbomind | 15.24 | +| llama-2-70b-turbomind | 15.24 | +| llama-3-8b-turbomind | 28.05 | +| llama-3-70b-turbomind | 28.05 | +| internlm2-1.8b-turbomind | 30.49 | +| internlm2-7b-turbomind | 48.17 | +| internlm2-20b-turbomind | 51.83 | +| qwen-1.8b-turbomind | 16.46 | +| qwen-7b-turbomind | 23.78 | +| qwen-14b-turbomind | 23.78 | +| qwen-72b-turbomind | 66.46 | +| qwen1.5-0.5b-hf | 8.54 | +| qwen1.5-1.8b-hf | 23.17 | +| qwen1.5-4b-hf | 41.46 | +| qwen1.5-7b-hf | 53.05 | +| qwen1.5-14b-hf | 57.32 | +| qwen1.5-32b-hf | 70.12 | +| qwen1.5-72b-hf | 65.85 | +| qwen1.5-moe-a2-7b-hf | 45.73 | +| mistral-7b-v0.1-hf | 14.02 | +| mistral-7b-v0.2-hf | 9.15 | +| mixtral-8x7b-v0.1-hf | 24.39 | +| mixtral-8x22b-v0.1-hf | 16.46 | +| yi-6b-hf | 14.63 | +| yi-34b-hf | 17.07 | +| deepseek-7b-base-hf | 18.29 | +| deepseek-67b-base-hf | 23.17 | + +## Chat Models + +| model | pass@1 | +|:-----------------------------:|---------:| +| qwen1.5-0.5b-chat-hf | 9.15 | +| qwen1.5-1.8b-chat-hf | 15.85 | +| qwen1.5-4b-chat-hf | 30.49 | +| qwen1.5-7b-chat-hf | 40.85 | +| qwen1.5-14b-chat-hf | 50.00 | +| qwen1.5-32b-chat-hf | 57.93 | +| qwen1.5-72b-chat-hf | 60.37 | +| qwen1.5-110b-chat-hf | 65.24 | +| internlm2-chat-1.8b-hf | 33.54 | +| internlm2-chat-1.8b-sft-hf | 34.15 | +| internlm2-chat-7b-hf | 56.71 | +| internlm2-chat-7b-sft-hf | 61.59 | +| internlm2-chat-20b-hf | 67.68 | +| internlm2-chat-20b-sft-hf | 67.68 | +| llama-3-8b-instruct-hf | 55.49 | +| llama-3-70b-instruct-hf | 70.73 | +| llama-3-8b-instruct-lmdeploy | 57.93 | +| llama-3-70b-instruct-lmdeploy | 70.73 | +| mistral-7b-instruct-v0.1-hf | 32.32 | +| mistral-7b-instruct-v0.2-hf | 29.27 | +| mixtral-8x7b-instruct-v0.1-hf | 34.15 | diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_4a6eef.py b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_4a6eef.py new file mode 100644 index 00000000..67dd58a5 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_4a6eef.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nCreate a Python script for this problem:\n{prompt}\n\n### Response:\n'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_6d1cc2.py b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_6d1cc2.py new file mode 100644 index 00000000..830d391f --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_6d1cc2.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nComplete the following python function.:\n{prompt}\n\n### Response:\n'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_a82cae.py b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_a82cae.py new file mode 100644 index 00000000..dc5d10f5 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_a82cae.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_d2537e.py b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_d2537e.py new file mode 100644 index 00000000..69231fdc --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_d2537e.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='Complete the following python code:\n{prompt}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_fd5822.py b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_fd5822.py new file mode 100644 index 00000000..ea56afd6 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_fd5822.py @@ -0,0 +1,31 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='{prompt}'), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_ff7054.py b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_ff7054.py new file mode 100644 index 00000000..a1be3ba9 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/deprecated_humaneval_gen_ff7054.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt='Complete the following python code:'), + ], + round=[ + dict(role='HUMAN', prompt='{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen.py b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen.py new file mode 100644 index 00000000..74019908 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .humaneval_gen_8e312c import humaneval_datasets # noqa: F401, F403 diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen_66a7f4.py b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen_66a7f4.py new file mode 100644 index 00000000..b4109925 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen_66a7f4.py @@ -0,0 +1,35 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict(input_columns=['prompt'], output_column='task_id', train_split='test') + +HUMANEVAL_TEMPLATE = dict( + round=[ + dict(role='HUMAN', prompt='You are an intelligent programming assistant to produce Python algorithmic solutions.\nCan you complete the following Python function?\n```python\n{prompt}\n```'), + ] +) + +humaneval_infer_cfg = dict( + prompt_template=dict(type=PromptTemplate, template=HUMANEVAL_TEMPLATE), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024), +) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + k=[1, 10, 100], + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg, + ) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen_8e312c.py new file mode 100644 index 00000000..a8c6e587 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_gen_8e312c.py @@ -0,0 +1,37 @@ +# THIS SHALL ALSO BE DEPRECATED +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_openai_sample_evals_gen_159614.py b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_openai_sample_evals_gen_159614.py new file mode 100644 index 00000000..d364f938 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_openai_sample_evals_gen_159614.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_passk_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_passk_gen_8e312c.py new file mode 100644 index 00000000..6224696f --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_passk_gen_8e312c.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval_passk', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_repeat10_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_repeat10_gen_8e312c.py new file mode 100644 index 00000000..adcabde9 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/humaneval_repeat10_gen_8e312c.py @@ -0,0 +1,37 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval_repeat10', + type=HumanevalDataset, + path='opencompass/humaneval', + num_repeats=10, + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/internal_humaneval_gen_ce6b06.py b/opencompass/configs/datasets/xhumaneval/humaneval/internal_humaneval_gen_ce6b06.py new file mode 100644 index 00000000..53505e52 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/internal_humaneval_gen_ce6b06.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_internal_v2_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='# Complete the following python code:\n{prompt}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_internal_v2_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval/internal_humaneval_gen_d2537e.py b/opencompass/configs/datasets/xhumaneval/humaneval/internal_humaneval_gen_d2537e.py new file mode 100644 index 00000000..cb8e6223 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval/internal_humaneval_gen_d2537e.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_internal_v1_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='Complete the following python code:\n{prompt}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_internal_v1_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval_gen_66a7f4.py b/opencompass/configs/datasets/xhumaneval/humaneval_gen_66a7f4.py new file mode 100644 index 00000000..b4109925 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval_gen_66a7f4.py @@ -0,0 +1,35 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict(input_columns=['prompt'], output_column='task_id', train_split='test') + +HUMANEVAL_TEMPLATE = dict( + round=[ + dict(role='HUMAN', prompt='You are an intelligent programming assistant to produce Python algorithmic solutions.\nCan you complete the following Python function?\n```python\n{prompt}\n```'), + ] +) + +humaneval_infer_cfg = dict( + prompt_template=dict(type=PromptTemplate, template=HUMANEVAL_TEMPLATE), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024), +) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + k=[1, 10, 100], + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg, + ) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval/humaneval_gen_8e312c.py new file mode 100644 index 00000000..ed76790c --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval_gen_8e312c.py @@ -0,0 +1,49 @@ +# THIS SHALL ALSO BE DEPRECATED +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import xHumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2, HumanEvalPlusEvaluator + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +# humaneval_datasets = [ +# dict( +# abbr='openai_humaneval', +# type=xHumanevalDataset, +# path='opencompass/humaneval', +# reader_cfg=humaneval_reader_cfg, +# infer_cfg=humaneval_infer_cfg, +# eval_cfg=humaneval_eval_cfg) +# ] +LANGS = ['ar'] +humaneval_datasets = [] +for lang in LANGS: + humaneval_datasets.append( + dict( + abbr=f'humaneval_{lang}', + type=xHumanevalDataset, + path=f'data/xhumaneval_plus/humaneval_plus_gpt4o_{lang}.jsonl', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) + ) diff --git a/opencompass/configs/datasets/xhumaneval/humaneval_openai_sample_evals_gen_159614.py b/opencompass/configs/datasets/xhumaneval/humaneval_openai_sample_evals_gen_159614.py new file mode 100644 index 00000000..d364f938 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval_openai_sample_evals_gen_159614.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval_passk_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval/humaneval_passk_gen_8e312c.py new file mode 100644 index 00000000..6224696f --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval_passk_gen_8e312c.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval_passk', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/humaneval_repeat10_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval/humaneval_repeat10_gen_8e312c.py new file mode 100644 index 00000000..adcabde9 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/humaneval_repeat10_gen_8e312c.py @@ -0,0 +1,37 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval_repeat10', + type=HumanevalDataset, + path='opencompass/humaneval', + num_repeats=10, + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/internal_humaneval_gen_ce6b06.py b/opencompass/configs/datasets/xhumaneval/internal_humaneval_gen_ce6b06.py new file mode 100644 index 00000000..53505e52 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/internal_humaneval_gen_ce6b06.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_internal_v2_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='# Complete the following python code:\n{prompt}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_internal_v2_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/internal_humaneval_gen_d2537e.py b/opencompass/configs/datasets/xhumaneval/internal_humaneval_gen_d2537e.py new file mode 100644 index 00000000..cb8e6223 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/internal_humaneval_gen_d2537e.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_internal_v1_postprocess + +humaneval_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template='Complete the following python code:\n{prompt}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_internal_v1_postprocess), +) + +humaneval_datasets = [ + dict( + abbr='openai_humaneval', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_reader_cfg, + infer_cfg=humaneval_infer_cfg, + eval_cfg=humaneval_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval/xhumaneval_gen.py b/opencompass/configs/datasets/xhumaneval/xhumaneval_gen.py new file mode 100644 index 00000000..74019908 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval/xhumaneval_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .humaneval_gen_8e312c import humaneval_datasets # noqa: F401, F403 diff --git a/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen.py b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen.py new file mode 100644 index 00000000..9176bdee --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .humaneval_plus_gen_8e312c import humaneval_plus_datasets # noqa: F401, F403 diff --git a/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen_66a7f4.py b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen_66a7f4.py new file mode 100644 index 00000000..357ef91c --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen_66a7f4.py @@ -0,0 +1,35 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalPlusEvaluator, humaneval_postprocess_v2 + +humaneval_plus_reader_cfg = dict(input_columns=['prompt'], output_column='task_id', train_split='test') + +HUMANEVAL_TEMPLATE = dict( + round=[ + dict(role='HUMAN', prompt='You are an intelligent programming assistant to produce Python algorithmic solutions.\nCan you complete the following Python function?\n```python\n{prompt}\n```'), + ] +) + +humaneval_plus_infer_cfg = dict( + prompt_template=dict(type=PromptTemplate, template=HUMANEVAL_TEMPLATE), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024), +) + +humaneval_plus_eval_cfg = dict( + evaluator=dict(type=HumanEvalPlusEvaluator), + k=[1, 10, 100], + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_plus_datasets = [ + dict( + abbr='humaneval_plus', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_plus_reader_cfg, + infer_cfg=humaneval_plus_infer_cfg, + eval_cfg=humaneval_plus_eval_cfg, + ) +] diff --git a/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen_8e312c.py new file mode 100644 index 00000000..b37bf37f --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_gen_8e312c.py @@ -0,0 +1,57 @@ +# THIS SHALL ALSO BE DEPRECATED +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import xHumanevalDataset, HumanEvalPlusEvaluator, humaneval_postprocess_v2 + +humaneval_plus_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_plus_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=2048)) + + + + +# print('humaneval_plus_datasets ru') +# humaneval_plus_datasets = [ +# dict( +# abbr='xhumaneval_plus_ru', +# type=xHumanevalDataset, +# path='xhumaneval_plus', +# name='humaneval_plus_gpt4o_ru.jsonl', +# local_mode=True, +# reader_cfg=humaneval_plus_reader_cfg, +# infer_cfg=humaneval_plus_infer_cfg, +# eval_cfg=humaneval_plus_eval_cfg) +# ] + +LANGS = ['de'] + +humaneval_plus_datasets = [] +for lang in LANGS: + humaneval_plus_eval_cfg = dict( + evaluator=dict(type=HumanEvalPlusEvaluator), + pred_role='BOT', + k=[1, 10], # the parameter only for humaneval + lang=lang, + pred_postprocessor=dict(type=humaneval_postprocess_v2), + ) + humaneval_plus_datasets.append( + dict( + abbr=f'xhumaneval_plus_{lang}', + type=xHumanevalDataset, + path=f'data/xhumaneval_plus/humaneval_{lang}.jsonl', + reader_cfg=humaneval_plus_reader_cfg, + infer_cfg=humaneval_plus_infer_cfg, + eval_cfg=humaneval_plus_eval_cfg) + ) diff --git a/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_openai_simple_evals_gen_159614.py b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_openai_simple_evals_gen_159614.py new file mode 100644 index 00000000..8e91abdc --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_openai_simple_evals_gen_159614.py @@ -0,0 +1,38 @@ +# THIS SHALL ALSO BE DEPRECATED +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalPlusEvaluator, humaneval_postprocess_v2 + +humaneval_plus_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_plus_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{prompt}' + ), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_plus_eval_cfg = dict( + evaluator=dict(type=HumanEvalPlusEvaluator), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_plus_datasets = [ + dict( + abbr='humaneval_plus', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_plus_reader_cfg, + infer_cfg=humaneval_plus_infer_cfg, + eval_cfg=humaneval_plus_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_passk_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_passk_gen_8e312c.py new file mode 100644 index 00000000..d602d73b --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_passk_gen_8e312c.py @@ -0,0 +1,36 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_plus_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_plus_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_plus_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator, metric='EvalPlus'), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_plus_datasets = [ + dict( + abbr='humaneval_plus_passk', + type=HumanevalDataset, + path='opencompass/humaneval', + reader_cfg=humaneval_plus_reader_cfg, + infer_cfg=humaneval_plus_infer_cfg, + eval_cfg=humaneval_plus_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_repeat10_gen_8e312c.py b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_repeat10_gen_8e312c.py new file mode 100644 index 00000000..bbbafd90 --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval_plus/humaneval_plus_repeat10_gen_8e312c.py @@ -0,0 +1,37 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2 + +humaneval_plus_reader_cfg = dict( + input_columns=['prompt'], output_column='task_id', train_split='test') + +# TODO: allow empty output-column +humaneval_plus_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', + prompt='Complete the following python code:\n{prompt}'), + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512)) + +humaneval_plus_eval_cfg = dict( + evaluator=dict(type=HumanEvalEvaluator, metric='EvalPlus'), + pred_role='BOT', + k=[1, 10, 100], # the parameter only for humaneval + pred_postprocessor=dict(type=humaneval_postprocess_v2), +) + +humaneval_plus_datasets = [ + dict( + abbr='humaneval_plus_repeat10', + type=HumanevalDataset, + path='opencompass/humaneval', + num_repeats=10, + reader_cfg=humaneval_plus_reader_cfg, + infer_cfg=humaneval_plus_infer_cfg, + eval_cfg=humaneval_plus_eval_cfg) +] diff --git a/opencompass/configs/datasets/xhumaneval_plus/xhumaneval_plus_gen.py b/opencompass/configs/datasets/xhumaneval_plus/xhumaneval_plus_gen.py new file mode 100644 index 00000000..9176bdee --- /dev/null +++ b/opencompass/configs/datasets/xhumaneval_plus/xhumaneval_plus_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .humaneval_plus_gen_8e312c import humaneval_plus_datasets # noqa: F401, F403 diff --git a/opencompass/configs/datasets/xlivecodebench/README.md b/opencompass/configs/datasets/xlivecodebench/README.md new file mode 100644 index 00000000..834c3040 --- /dev/null +++ b/opencompass/configs/datasets/xlivecodebench/README.md @@ -0,0 +1,42 @@ +# LiveCodeBench + +## Dataset + +LiveCodeBench provides holistic and contamination-free evaluation of coding capabilities of LLMs. Particularly, LiveCodeBench continuously collects new problems over time from contests across three competition platforms -- LeetCode, AtCoder, and CodeForces. Next, LiveCodeBench also focuses on a broader range of code-related capabilities, such as self-repair, code execution, and test output prediction, beyond just code generation. Currently, LiveCodeBench hosts four hundred high-quality coding problems that were published between May 2023 and March 2024. + +- Origin Project: https://livecodebench.github.io/leaderboard.html + +## Setting + +| Model Type | Code Generation | Test Output Prediction | Code Execution | +|------------|--------|--------|--------| +| Base Model | ❌ | ❌ | ❌ | +| Chat Model | ✅ | ✅ | ✅ | + + + +## Baseline Performance + + +| Model Type | Code Generation(pass@1) | Test Output Prediction(pass@1) | Code Execution(pass@1) | +|------------|--------|--------|--------| +| Qwen2.5-7B-Instruct(HF) | 39.25 | 48.64 | 41.96 | +| Meta-Llama-3.1-8B-Instruct(HF) | 20.25 | 24.66 | 17.12 | + + +## Citation + +```bibtex +@article{jain2024livecodebench, + author = {Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica}, + title = {LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code}, + year = {2024}, + journal = {arXiv preprint}, +} +@misc{2023opencompass, + title={OpenCompass: A Universal Evaluation Platform for Foundation Models}, + author={OpenCompass Contributors}, + howpublished = {\url{https://github.com/open-compass/opencompass}}, + year={2023} +} +``` diff --git a/opencompass/configs/datasets/xlivecodebench/livecodebench_gen_6966bc.py b/opencompass/configs/datasets/xlivecodebench/livecodebench_gen_6966bc.py new file mode 100644 index 00000000..552e4e7d --- /dev/null +++ b/opencompass/configs/datasets/xlivecodebench/livecodebench_gen_6966bc.py @@ -0,0 +1,166 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets.xlivecodebench import ( + LCBCodeExecutionDataset, + LCBTestOutputPredictionDataset, + LCBCodeExecutionEvaluator, + LCBTestOutputEvaluator +) +from opencompass.datasets.xlivecodebench import xLCBCodeGenerationDataset, xLCBCodeGenerationEvaluator +from opencompass.datasets.livecodebench import TestOutputPromptConstants + + +lcb_code_generation_reader_cfg = dict( + input_columns=[ + 'question_content', + 'format_prompt', + ], + # output_column='evaluation_sample', + output_column='question_id', +) + +SYSTEM_MESSAGE_GENERIC = f'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +prompt_template = '### Question:\n{question_content}\n\n{format_prompt}' + \ + '### Answer: (use the provided format with backticks)\n\n' + + +# Code Generation Tasks +lcb_code_generation_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role='HUMAN', + prompt=prompt_template + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_generation_eval_cfg = dict( + evaluator=dict( + type=xLCBCodeGenerationEvaluator, + path='data/code_generation_lite/lcb_gpt4o_ar.jsonl', + num_process_evaluate=4, + timeout=6, + ), + pred_role='BOT', +) + +LCB_datasets = [] +LANGS = ['ar', 'bn', 'cs', 'de', 'es', 'fr', 'hu', 'ja', 'ko', 'ru', 'sr', 'sw', 'te', 'th', 'vi', 'zh'] +for LANG in LANGS: + LCBCodeGeneration_dataset = dict( + type=xLCBCodeGenerationDataset, + abbr=f'lcb_code_generation_{LANG}', + path=f'data/code_generation_lite/lcb_gpt4o_{LANG}.jsonl', + local_mode=True, + reader_cfg=lcb_code_generation_reader_cfg, + infer_cfg=lcb_code_generation_infer_cfg, + eval_cfg=lcb_code_generation_eval_cfg + ) + # Code Execution Dataset + lcb_code_execution_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', + ) + + lcb_code_execution_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + fallback_role='HUMAN', + prompt='You are an expert at Python programming, code execution, test case generation, and fuzzing.' + ), + ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) + ) + + lcb_code_execution_eval_cfg = dict( + evaluator=dict( + type=LCBCodeExecutionEvaluator, + ), + pred_role='BOT', + ) + + LCBCodeExecution_dataset = dict( + type=LCBCodeExecutionDataset, + abbr=f'lcb_code_execution_{LANG}', + path='opencompass/execution-v2', + reader_cfg=lcb_code_execution_reader_cfg, + infer_cfg=lcb_code_execution_infer_cfg, + eval_cfg=lcb_code_execution_eval_cfg, + ) + + # TestOuputput Dataset + lcb_test_output_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', + ) + + system_prompt = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + + lcb_test_output_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + # begin=[ + # dict( + # role='SYSTEM', + # prompt=system_prompt + # ), + # ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) + ) + + lcb_test_output_eval_cfg = dict( + evaluator=dict( + type=LCBTestOutputEvaluator, + ), + pred_role='BOT', + ) + + LCBTestOutput_dataset = dict( + type=LCBTestOutputPredictionDataset, + abbr=f'lcb_test_output_{LANG}', + path='opencompass/test_generation', + reader_cfg=lcb_test_output_reader_cfg, + infer_cfg=lcb_test_output_infer_cfg, + eval_cfg=lcb_test_output_eval_cfg, + ) + LCB_datasets += [ + LCBCodeGeneration_dataset, + LCBCodeExecution_dataset, + LCBTestOutput_dataset, + ] diff --git a/opencompass/configs/datasets/xlivecodebench/livecodebench_gen_b2b0fd.py b/opencompass/configs/datasets/xlivecodebench/livecodebench_gen_b2b0fd.py new file mode 100644 index 00000000..c2591e75 --- /dev/null +++ b/opencompass/configs/datasets/xlivecodebench/livecodebench_gen_b2b0fd.py @@ -0,0 +1,163 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import ( + LCBCodeGenerationDataset, + LCBCodeExecutionDataset, + LCBTestOutputPredictionDataset, + LCBCodeGenerationEvaluator, + LCBCodeExecutionEvaluator, + LCBTestOutputEvaluator +) +from opencompass.datasets.livecodebench import TestOutputPromptConstants + + +lcb_code_generation_reader_cfg = dict( + input_columns=[ + 'question_content', + 'format_prompt', + ], + # output_column='evaluation_sample', + output_column='question_id', +) + +SYSTEM_MESSAGE_GENERIC = f'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +prompt_template = '### Question:\n{question_content}\n\n{format_prompt}' + \ + '### Answer: (use the provided format with backticks)\n\n' + + +# Code Generation Tasks +lcb_code_generation_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role='HUMAN', + prompt=prompt_template + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_generation_eval_cfg = dict( + evaluator=dict( + type=LCBCodeGenerationEvaluator, + num_process_evaluate=4, + timeout=6, + ), + pred_role='BOT', +) + +LCBCodeGeneration_dataset = dict( + type=LCBCodeGenerationDataset, + abbr='lcb_code_generation', + path='opencompass/code_generation_lite', + reader_cfg=lcb_code_generation_reader_cfg, + infer_cfg=lcb_code_generation_infer_cfg, + eval_cfg=lcb_code_generation_eval_cfg +) + +# Code Execution Dataset +lcb_code_execution_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', +) + +lcb_code_execution_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + prompt='You are an expert at Python programming, code execution, test case generation, and fuzzing.' + ), + ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_execution_eval_cfg = dict( + evaluator=dict( + type=LCBCodeExecutionEvaluator, + ), + pred_role='BOT', +) + +LCBCodeExecution_dataset = dict( + type=LCBCodeExecutionDataset, + abbr='lcb_code_execution', + path='opencompass/execution-v2', + reader_cfg=lcb_code_execution_reader_cfg, + infer_cfg=lcb_code_execution_infer_cfg, + eval_cfg=lcb_code_execution_eval_cfg, +) + +# TestOuputput Dataset +lcb_test_output_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', +) + +system_prompt = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +lcb_test_output_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + # begin=[ + # dict( + # role='SYSTEM', + # prompt=system_prompt + # ), + # ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_test_output_eval_cfg = dict( + evaluator=dict( + type=LCBTestOutputEvaluator, + ), + pred_role='BOT', +) + +LCBTestOutput_dataset = dict( + type=LCBTestOutputPredictionDataset, + abbr='lcb_test_output', + path='opencompass/test_generation', + reader_cfg=lcb_test_output_reader_cfg, + infer_cfg=lcb_test_output_infer_cfg, + eval_cfg=lcb_test_output_eval_cfg, +) + +LCB_datasets = [ + LCBCodeGeneration_dataset, + LCBCodeExecution_dataset, + LCBTestOutput_dataset, +] diff --git a/opencompass/configs/datasets/xlivecodebench/livecodebench_o1_gen_f0ed6c.py b/opencompass/configs/datasets/xlivecodebench/livecodebench_o1_gen_f0ed6c.py new file mode 100644 index 00000000..b74e2c76 --- /dev/null +++ b/opencompass/configs/datasets/xlivecodebench/livecodebench_o1_gen_f0ed6c.py @@ -0,0 +1,165 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import ( + LCBCodeGenerationDataset, + LCBCodeExecutionDataset, + LCBTestOutputPredictionDataset, + LCBCodeGenerationEvaluator, + LCBCodeExecutionEvaluator, + LCBTestOutputEvaluator +) +from opencompass.datasets.livecodebench import TestOutputPromptConstants + + +lcb_code_generation_reader_cfg = dict( + input_columns=[ + 'question_content', + 'format_prompt', + ], + # output_column='evaluation_sample', + output_column='question_id', +) + +SYSTEM_MESSAGE_GENERIC = f'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +prompt_template = '### Question:\n{question_content}\n\n{format_prompt}' + \ + '### Answer: (use the provided format with backticks)\n\n' + + +# Code Generation Tasks +lcb_code_generation_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role='HUMAN', + prompt=prompt_template + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_generation_eval_cfg = dict( + evaluator=dict( + type=LCBCodeGenerationEvaluator, + num_process_evaluate=4, + timeout=6, + release_version='release_v4', + ), + pred_role='BOT', +) + +LCBCodeGeneration_dataset = dict( + type=LCBCodeGenerationDataset, + abbr='lcb_code_generation_v4', + path='opencompass/code_generation_lite', + reader_cfg=lcb_code_generation_reader_cfg, + infer_cfg=lcb_code_generation_infer_cfg, + eval_cfg=lcb_code_generation_eval_cfg, + release_version='release_v4', +) + +# Code Execution Dataset +lcb_code_execution_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', +) + +lcb_code_execution_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + prompt='You are an expert at Python programming, code execution, test case generation, and fuzzing.' + ), + ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_execution_eval_cfg = dict( + evaluator=dict( + type=LCBCodeExecutionEvaluator, + ), + pred_role='BOT', +) + +LCBCodeExecution_dataset = dict( + type=LCBCodeExecutionDataset, + abbr='lcb_code_execution', + path='opencompass/execution-v2', + reader_cfg=lcb_code_execution_reader_cfg, + infer_cfg=lcb_code_execution_infer_cfg, + eval_cfg=lcb_code_execution_eval_cfg, +) + +# TestOuputput Dataset +lcb_test_output_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', +) + +system_prompt = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +lcb_test_output_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + # begin=[ + # dict( + # role='SYSTEM', + # prompt=system_prompt + # ), + # ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_test_output_eval_cfg = dict( + evaluator=dict( + type=LCBTestOutputEvaluator, + ), + pred_role='BOT', +) + +LCBTestOutput_dataset = dict( + type=LCBTestOutputPredictionDataset, + abbr='lcb_test_output', + path='opencompass/test_generation', + reader_cfg=lcb_test_output_reader_cfg, + infer_cfg=lcb_test_output_infer_cfg, + eval_cfg=lcb_test_output_eval_cfg, +) + +LCB_datasets = [ + LCBCodeGeneration_dataset, + # LCBCodeExecution_dataset, + # LCBTestOutput_dataset, +] diff --git a/opencompass/configs/datasets/xlivecodebench/livecodebench_split_v4_o1_gen_f0ed6c.py b/opencompass/configs/datasets/xlivecodebench/livecodebench_split_v4_o1_gen_f0ed6c.py new file mode 100644 index 00000000..cd65c3ae --- /dev/null +++ b/opencompass/configs/datasets/xlivecodebench/livecodebench_split_v4_o1_gen_f0ed6c.py @@ -0,0 +1,165 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import ( + LCBCodeGenerationDataset, + LCBCodeExecutionDataset, + LCBTestOutputPredictionDataset, + LCBCodeGenerationEvaluator, + LCBCodeExecutionEvaluator, + LCBTestOutputEvaluator +) +from opencompass.datasets.livecodebench import TestOutputPromptConstants + + +lcb_code_generation_reader_cfg = dict( + input_columns=[ + 'question_content', + 'format_prompt', + ], + # output_column='evaluation_sample', + output_column='question_id', +) + +SYSTEM_MESSAGE_GENERIC = f'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +prompt_template = '### Question:\n{question_content}\n\n{format_prompt}' + \ + '### Answer: (use the provided format with backticks)\n\n' + + +# Code Generation Tasks +lcb_code_generation_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role='HUMAN', + prompt=prompt_template + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_generation_eval_cfg = dict( + evaluator=dict( + type=LCBCodeGenerationEvaluator, + num_process_evaluate=4, + timeout=6, + release_version='release_split_v4', + ), + pred_role='BOT', +) + +LCBCodeGeneration_dataset = dict( + type=LCBCodeGenerationDataset, + abbr='lcb_code_generation_split_v4', + path='opencompass/code_generation_lite', + reader_cfg=lcb_code_generation_reader_cfg, + infer_cfg=lcb_code_generation_infer_cfg, + eval_cfg=lcb_code_generation_eval_cfg, + release_version='release_split_v4', +) + +# Code Execution Dataset +lcb_code_execution_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', +) + +lcb_code_execution_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + prompt='You are an expert at Python programming, code execution, test case generation, and fuzzing.' + ), + ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_execution_eval_cfg = dict( + evaluator=dict( + type=LCBCodeExecutionEvaluator, + ), + pred_role='BOT', +) + +LCBCodeExecution_dataset = dict( + type=LCBCodeExecutionDataset, + abbr='lcb_code_execution', + path='opencompass/execution-v2', + reader_cfg=lcb_code_execution_reader_cfg, + infer_cfg=lcb_code_execution_infer_cfg, + eval_cfg=lcb_code_execution_eval_cfg, +) + +# TestOuputput Dataset +lcb_test_output_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', +) + +system_prompt = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +lcb_test_output_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + # begin=[ + # dict( + # role='SYSTEM', + # prompt=system_prompt + # ), + # ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_test_output_eval_cfg = dict( + evaluator=dict( + type=LCBTestOutputEvaluator, + ), + pred_role='BOT', +) + +LCBTestOutput_dataset = dict( + type=LCBTestOutputPredictionDataset, + abbr='lcb_test_output', + path='opencompass/test_generation', + reader_cfg=lcb_test_output_reader_cfg, + infer_cfg=lcb_test_output_infer_cfg, + eval_cfg=lcb_test_output_eval_cfg, +) + +LCB_datasets = [ + LCBCodeGeneration_dataset, + # LCBCodeExecution_dataset, + # LCBTestOutput_dataset, +] diff --git a/opencompass/configs/datasets/xlivecodebench/livecodebench_v1_o1_gen_f0ed6c.py b/opencompass/configs/datasets/xlivecodebench/livecodebench_v1_o1_gen_f0ed6c.py new file mode 100644 index 00000000..f2a0d77e --- /dev/null +++ b/opencompass/configs/datasets/xlivecodebench/livecodebench_v1_o1_gen_f0ed6c.py @@ -0,0 +1,164 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets import ( + LCBCodeGenerationDataset, + LCBCodeExecutionDataset, + LCBTestOutputPredictionDataset, + LCBCodeGenerationEvaluator, + LCBCodeExecutionEvaluator, + LCBTestOutputEvaluator +) +from opencompass.datasets.livecodebench import TestOutputPromptConstants + + +lcb_code_generation_reader_cfg = dict( + input_columns=[ + 'question_content', + 'format_prompt', + ], + # output_column='evaluation_sample', + output_column='question_id', +) + +SYSTEM_MESSAGE_GENERIC = f'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +prompt_template = '### Question:\n{question_content}\n\n{format_prompt}' + \ + '### Answer: (use the provided format with backticks)\n\n' + + +# Code Generation Tasks +lcb_code_generation_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict( + role='HUMAN', + prompt=prompt_template + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_generation_eval_cfg = dict( + evaluator=dict( + type=LCBCodeGenerationEvaluator, + num_process_evaluate=4, + timeout=6, + ), + pred_role='BOT', +) + +LCBCodeGeneration_dataset = dict( + type=LCBCodeGenerationDataset, + abbr='lcb_code_generation_v1', + path='opencompass/code_generation_lite', + reader_cfg=lcb_code_generation_reader_cfg, + infer_cfg=lcb_code_generation_infer_cfg, + eval_cfg=lcb_code_generation_eval_cfg, + release_version='release_v1', +) + +# Code Execution Dataset +lcb_code_execution_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', +) + +lcb_code_execution_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict( + role='SYSTEM', + prompt='You are an expert at Python programming, code execution, test case generation, and fuzzing.' + ), + ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_code_execution_eval_cfg = dict( + evaluator=dict( + type=LCBCodeExecutionEvaluator, + ), + pred_role='BOT', +) + +LCBCodeExecution_dataset = dict( + type=LCBCodeExecutionDataset, + abbr='lcb_code_execution', + path='opencompass/execution-v2', + reader_cfg=lcb_code_execution_reader_cfg, + infer_cfg=lcb_code_execution_infer_cfg, + eval_cfg=lcb_code_execution_eval_cfg, +) + +# TestOuputput Dataset +lcb_test_output_reader_cfg = dict( + input_columns=[ + 'prompt', + ], + output_column='evaluation_sample', +) + +system_prompt = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' + +lcb_test_output_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + # begin=[ + # dict( + # role='SYSTEM', + # prompt=system_prompt + # ), + # ], + round=[ + dict( + role='HUMAN', + prompt='{prompt}' + ) + ] + ) + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024) +) + +lcb_test_output_eval_cfg = dict( + evaluator=dict( + type=LCBTestOutputEvaluator, + ), + pred_role='BOT', +) + +LCBTestOutput_dataset = dict( + type=LCBTestOutputPredictionDataset, + abbr='lcb_test_output', + path='opencompass/test_generation', + reader_cfg=lcb_test_output_reader_cfg, + infer_cfg=lcb_test_output_infer_cfg, + eval_cfg=lcb_test_output_eval_cfg, +) + +LCB_datasets = [ + LCBCodeGeneration_dataset, + # LCBCodeExecution_dataset, + # LCBTestOutput_dataset, +] diff --git a/opencompass/configs/datasets/xlivecodebench/xlivecodebench_gen.py b/opencompass/configs/datasets/xlivecodebench/xlivecodebench_gen.py new file mode 100644 index 00000000..f663df06 --- /dev/null +++ b/opencompass/configs/datasets/xlivecodebench/xlivecodebench_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .livecodebench_gen_6966bc import LCB_datasets # noqa: F401, F403 diff --git a/opencompass/configs/datasets/xmgsm/README.md b/opencompass/configs/datasets/xmgsm/README.md new file mode 100644 index 00000000..31a9c90d --- /dev/null +++ b/opencompass/configs/datasets/xmgsm/README.md @@ -0,0 +1,67 @@ +# MGSM + +## Introduction + +The following introduction comes from the abstract in [Language models are multilingual chain-of-thought reasoners](https://arxiv.org/abs/2210.03057) + +``` +We introduce the Multilingual Grade School Math (MGSM) benchmark, by manually translating 250 grade-school math problems from the GSM8K dataset into ten typologically diverse languages. +``` + +## Official link + +### Paper + +[Language models are multilingual chain-of-thought reasoners](https://arxiv.org/abs/2210.03057) + +### Repository + +[MGSM](https://github.com/google-research/url-nlp) + +## Examples + +Input example I: + +``` +Solve this math problem. Give the reasoning steps before giving the final answer on the last line by itself in the format of "Answer:". Do not add anything other than the integer answer after "Answer:". + +Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market? +``` + +Output example I (from GPT-4): + +``` +Answer: 18 +``` + +## Evaluation results + +``` +dataset version metric mode llama-3-8b-instruct-hf +-------------- --------- ------------- ------ ------------------------ +mgsm_bn b65151 accuracy gen 14.4 +mgsm_de 2cc8ae accuracy gen 60 +mgsm_en 5de71e accuracy gen 76 +mgsm_es d6b459 accuracy gen 61.6 +mgsm_fr 813e3c accuracy gen 54.4 +mgsm_ja 04424f accuracy gen 42.8 +mgsm_ru 400469 accuracy gen 62.8 +mgsm_sw 9e41ed accuracy gen 0.8 +mgsm_te 346d97 accuracy gen 0 +mgsm_th e70bee accuracy gen 44 +mgsm_zh d5cf30 accuracy gen 28.4 +mgsm_latin - naive_average gen 50.56 +mgsm_non_latin - naive_average gen 32.07 +mgsm - naive_average gen 40.47 +``` + +## Reference + +``` +@article{shi2022language, + title={Language models are multilingual chain-of-thought reasoners}, + author={Shi, Freda and Suzgun, Mirac and Freitag, Markus and Wang, Xuezhi and Srivats, Suraj and Vosoughi, Soroush and Chung, Hyung Won and Tay, Yi and Ruder, Sebastian and Zhou, Denny and others}, + journal={arXiv preprint arXiv:2210.03057}, + year={2022} +} +``` diff --git a/opencompass/configs/datasets/xmgsm/mgsm_gen.py b/opencompass/configs/datasets/xmgsm/mgsm_gen.py new file mode 100644 index 00000000..8feea66c --- /dev/null +++ b/opencompass/configs/datasets/xmgsm/mgsm_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .mgsm_gen_d967bc import mgsm_datasets diff --git a/opencompass/configs/datasets/xmgsm/mgsm_gen_d967bc.py b/opencompass/configs/datasets/xmgsm/mgsm_gen_d967bc.py new file mode 100644 index 00000000..f56e3444 --- /dev/null +++ b/opencompass/configs/datasets/xmgsm/mgsm_gen_d967bc.py @@ -0,0 +1,62 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import JiebaRougeEvaluator +from opencompass.datasets import xMGSMSDataset, xMGSM_Evaluator, mgsm_postprocess + + +ALL_LANGUAGES = ['bn', 'de', 'en', 'es', 'fr', 'ja', 'ru', 'sw', 'te', 'th', 'zh', 'ar', 'sr', 'hu', 'cs', 'vi', 'ko'] + +LANG_TO_INSTRUCTIONS = { + 'en': """Solve this math problem. Give the reasoning steps before giving the final answer on the last line by itself in the format of "Answer:". Do not add anything other than the integer answer after "Answer:".\n\n{question}""", + 'bn': """এই গণিতের সমস্যাটি সমাধান করুন। চূড়ান্ত উত্তর দেওয়ার আগে যুক্তিসম্পন্ন পদক্ষেপ প্রদান করুন। চূড়ান্ত উত্তরটি একক সংখ্যা হিসাবে "উত্তর:" এর পরে শেষ লাইনে দিন। "উত্তর:" এর পরে অন্য কিছু যুক্ত করবেন না।.\n\n{question}""", + 'de': """Löse dieses Mathematikproblem. Gib die Schritte zur Begründung an, bevor du die endgültige Antwort in der letzten Zeile alleine im Format "Antwort:" gibst. Füge nichts anderes als die ganzzahlige Antwort nach "Antwort:" hinzu.\n\n{question}""", + 'es': """Resuelve este problema matemático. Proporciona los pasos de razonamiento antes de dar la respuesta final en la última línea por sí misma en el formato de "Respuesta:". No añadas nada más que la respuesta entera después de "Respuesta:".\n\n{question}""", + 'fr': """Résolvez ce problème de mathématiques. Donnez les étapes de raisonnement avant de fournir la réponse finale sur la dernière ligne elle-même dans le format de "Réponse:". N'ajoutez rien d'autre que la réponse entière après "Réponse:".\n\n{question}""", + 'ja': """の数学の問題を解いてください。最終的な答えを出す前に、解答の推論過程を記述してください。そして最後の行には "答え:" の形式で答えを記述し、その後には整数の答え以外何も追加しないでください。\n\n{question}""", + 'ru': """Решите эту математическую задачу. Объясните шаги рассуждения перед тем, как дать окончательный ответ в последней строке сам по себе в формате "Ответ:". Не добавляйте ничего, кроме целочисленного ответа после "Ответ:".\n\n{question}""", + 'sw': """Suluhisha tatizo hili la hesabu. Toa hatua za mantiki kabla ya kutoa jibu la mwisho kwenye mstari wa mwisho peke yake katika muundo wa "Jibu:". Usiongeze chochote kingine isipokuwa jibu la integer baada ya "Jibu:".\n\n{question}""", + 'te': """ఈ గణిత సమస్యను పరిష్కరించండి. చివరి సమాధానాన్ని ఇవ్వదానికి ముందు తర్కాత్మక అదుగులను ఇవ్వండి. చివరి పంక్తిలో మాత్రమే 'సమాధానం:' అనే ఆకారంలో చివరి సమాధానాద్ని ఇవ్వండి సమాధానం: తర్వాత పూర్ణాంక సమాధానానికి తప్పించి ఎదేనా చేర్చవద్దు.\n\n{question}""", + 'th': """แก้ปัญหาคณิตศาสตร์นี้ ให้ให้ขั้นตอนการใช้เหตุผลก่อนที่จะให้คำตอบสุดท้ายในบรรทัดสุดท้ายโดยอยู่ในรูปแบบ "คำตอบ:" ไม่ควรเพิ่มอะไรนอกจากคำตอบที่เป็นจำนวนเต็มหลังจาก "คำตอบ:"\n\n{question}""", + 'zh': """解决这个数学问题。在最后一行给出答案前，请提供推理步骤。最后一行应该以 "答案: " 的形式独立给出答案。在 "答案：" 后不要添加除整数答案之外的任何内容。\n\n{question}""", + 'ar': """حل هذه المسألة الرياضية. قبل أن تعطي الإجابة النهائية، يرجى إعطاء الخطوات التفكيرية. يجب أن تكون الإجابة النهائية في السطر الأخير بالشكل "الإجابة:". لا تضيف أي شيء آخر غير الإجابة العددية النهائية بعد "الإجابة:".\n\n{question}""", + 'sr': """Решење овог математичког проблема. Пре него што дате коначни одговор, пошаљите логички кораци. Коначни одговор мора бити на последњој линији у формату "Одговор:". Не додавајте ништа друго осим коначног целобројног одговора после "Одговор:".\n\n{question}""", + 'hu': """Megoldás ezt a matematikai problémát. Mielőtt megadná a végső választ, adjon meg logikai lépéseket. A végső válasz a végső sorban kell legyen a "Válasz:" formátumban. Ne adjon hozzá semmit mást, csak a végső egész szám választ a "Válasz:" után.\n\n{question}""", + 'cs': """Řešení tohoto matematického problému. Předtím, než dáte finální odpověď, přidejte logické kroky. Finální odpověď musí být na posledním řádku ve formátu "Odpověď:". Nezahrněte nic jiného než celočíselnou finální odpověď po "Odpověď:".\n\n{question}""", + 'vi': """Giải bài toán toán học này. Trước khi bạn cung cấp câu trả lời cuối cùng, hãy cung cấp các bước suy luận. Câu trả lời cuối cùng phải ở trên dòng cuối cùng trong định dạng "Câu trả lời:". Không thêm bất kỳ thứ gì khác ngoài câu trả lời số nguyên sau "Câu trả lời:".\n\n{question}""", + 'ko': """이 수학 문제를 풀어주세요. 마지막 줄에 최종 답을 제시하기 전에 논리적 단계를 제공해주세요. 마지막 줄에는 "답:" 형식으로 최종 답을 제시해주세요. "답:" 뒤에 정수 답만 추가해주세요.\n\n{question}""", +} + +mgsm_datasets = [] +for lang in ALL_LANGUAGES: + mgsm_reader_cfg = dict(input_columns=['question'], output_column='answer') + + mgsm_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt=LANG_TO_INSTRUCTIONS[lang]), + ] + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512), + ) + + mgsm_eval_cfg = dict( + evaluator=dict(type=xMGSM_Evaluator), + pred_role='BOT', + pred_postprocessor=dict(type=mgsm_postprocess, lang=lang), + ) + + mgsm_datasets.append( + dict( + type=xMGSMSDataset, + abbr=f'mgsm_{lang}', + path=f'data/mgsm/mgsm_test_{lang}_google.jsonl', + reader_cfg=mgsm_reader_cfg, + infer_cfg=mgsm_infer_cfg, + eval_cfg=mgsm_eval_cfg, + ) + ) diff --git a/opencompass/configs/datasets/xmgsm/mgsm_gen_d967bc_8shot.py b/opencompass/configs/datasets/xmgsm/mgsm_gen_d967bc_8shot.py new file mode 100644 index 00000000..0c153c4d --- /dev/null +++ b/opencompass/configs/datasets/xmgsm/mgsm_gen_d967bc_8shot.py @@ -0,0 +1,90 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import JiebaRougeEvaluator +from opencompass.datasets import xMGSMSDataset, xMGSM_Evaluator, mgsm_postprocess +import os + +ALL_LANGUAGES = ['bn', 'de', 'en', 'es', 'fr', 'ja', 'ru', 'sw', 'te', 'th', 'zh', 'ar', 'sr', 'hu', 'cs', 'vi', 'ko'] + +LANG_TO_INSTRUCTIONS = { + 'en': """Solve this math problem. Give the reasoning steps before giving the final answer on the last line by itself in the format of "Answer:". Do not add anything other than the integer answer after "Answer:".\n\n{question}""", + 'bn': """এই গণিতের সমস্যাটি সমাধান করুন। চূড়ান্ত উত্তর দেওয়ার আগে যুক্তিসম্পন্ন পদক্ষেপ প্রদান করুন। চূড়ান্ত উত্তরটি একক সংখ্যা হিসাবে "উত্তর:" এর পরে শেষ লাইনে দিন। "উত্তর:" এর পরে অন্য কিছু যুক্ত করবেন না।.\n\n{question}""", + 'de': """Löse dieses Mathematikproblem. Gib die Schritte zur Begründung an, bevor du die endgültige Antwort in der letzten Zeile alleine im Format "Antwort:" gibst. Füge nichts anderes als die ganzzahlige Antwort nach "Antwort:" hinzu.\n\n{question}""", + 'es': """Resuelve este problema matemático. Proporciona los pasos de razonamiento antes de dar la respuesta final en la última línea por sí misma en el formato de "Respuesta:". No añadas nada más que la respuesta entera después de "Respuesta:".\n\n{question}""", + 'fr': """Résolvez ce problème de mathématiques. Donnez les étapes de raisonnement avant de fournir la réponse finale sur la dernière ligne elle-même dans le format de "Réponse:". N'ajoutez rien d'autre que la réponse entière après "Réponse:".\n\n{question}""", + 'ja': """の数学の問題を解いてください。最終的な答えを出す前に、解答の推論過程を記述してください。そして最後の行には "答え:" の形式で答えを記述し、その後には整数の答え以外何も追加しないでください。\n\n{question}""", + 'ru': """Решите эту математическую задачу. Объясните шаги рассуждения перед тем, как дать окончательный ответ в последней строке сам по себе в формате "Ответ:". Не добавляйте ничего, кроме целочисленного ответа после "Ответ:".\n\n{question}""", + 'sw': """Suluhisha tatizo hili la hesabu. Toa hatua za mantiki kabla ya kutoa jibu la mwisho kwenye mstari wa mwisho peke yake katika muundo wa "Jibu:". Usiongeze chochote kingine isipokuwa jibu la integer baada ya "Jibu:".\n\n{question}""", + 'te': """ఈ గణిత సమస్యను పరిష్కరించండి. చివరి సమాధానాన్ని ఇవ్వదానికి ముందు తర్కాత్మక అదుగులను ఇవ్వండి. చివరి పంక్తిలో మాత్రమే 'సమాధానం:' అనే ఆకారంలో చివరి సమాధానాద్ని ఇవ్వండి సమాధానం: తర్వాత పూర్ణాంక సమాధానానికి తప్పించి ఎదేనా చేర్చవద్దు.\n\n{question}""", + 'th': """แก้ปัญหาคณิตศาสตร์นี้ ให้ให้ขั้นตอนการใช้เหตุผลก่อนที่จะให้คำตอบสุดท้ายในบรรทัดสุดท้ายโดยอยู่ในรูปแบบ "คำตอบ:" ไม่ควรเพิ่มอะไรนอกจากคำตอบที่เป็นจำนวนเต็มหลังจาก "คำตอบ:"\n\n{question}""", + 'zh': """解决这个数学问题。在最后一行给出答案前，请提供推理步骤。最后一行应该以 "答案: " 的形式独立给出答案。在 "答案：" 后不要添加除整数答案之外的任何内容。\n\n{question}""", + 'ar': """حل هذه المسألة الرياضية. قبل أن تعطي الإجابة النهائية، يرجى إعطاء الخطوات التفكيرية. يجب أن تكون الإجابة النهائية في السطر الأخير بالشكل "الإجابة:". لا تضيف أي شيء آخر غير الإجابة العددية النهائية بعد "الإجابة:".\n\n{question}""", + 'sr': """Решење овог математичког проблема. Пре него што дате коначни одговор, пошаљите логички кораци. Коначни одговор мора бити на последњој линији у формату "Одговор:". Не додавајте ништа друго осим коначног целобројног одговора после "Одговор:".\n\n{question}""", + 'hu': """Megoldás ezt a matematikai problémát. Mielőtt megadná a végső választ, adjon meg logikai lépéseket. A végső válasz a végső sorban kell legyen a "Válasz:" formátumban. Ne adjon hozzá semmit mást, csak a végső egész szám választ a "Válasz:" után.\n\n{question}""", + 'cs': """Řešení tohoto matematického problému. Předtím, než dáte finální odpověď, přidejte logické kroky. Finální odpověď musí být na posledním řádku ve formátu "Odpověď:". Nezahrněte nic jiného než celočíselnou finální odpověď po "Odpověď:".\n\n{question}""", + 'vi': """Giải bài toán toán học này. Trước khi bạn cung cấp câu trả lời cuối cùng, hãy cung cấp các bước suy luận. Câu trả lời cuối cùng phải ở trên dòng cuối cùng trong định dạng "Câu trả lời:". Không thêm bất kỳ thứ gì khác ngoài câu trả lời số nguyên sau "Câu trả lời:".\n\n{question}""", + 'ko': """이 수학 문제를 풀어주세요. 마지막 줄에 최종 답을 제시하기 전에 논리적 단계를 제공해주세요. 마지막 줄에는 "답:" 형식으로 최종 답을 제시해주세요. "답:" 뒤에 정수 답만 추가해주세요.\n\n{question}""", +} +import json +def load_jsonl(path): + with open(path, 'r', encoding='utf-8') as f: + return [json.loads(line) for line in f] +USER_HOME = os.path.expanduser("~") +DEFAULT_DATA_FOLDER = os.path.join(USER_HOME, '.cache/opencompass/') +mgsm_datasets = [] +for lang in ALL_LANGUAGES: + mgsm_reader_cfg = dict(input_columns=['question'], output_column='answer') + cache_dir = DEFAULT_DATA_FOLDER + + # For absolute path customized by the users + local_path = os.path.join(cache_dir, f'data/mgsm/mgsm_train_{lang}_google.jsonl') + data = load_jsonl(local_path) + questions = [item['question'] for item in data] + answers = [item['answer'] for item in data] + + mgsm_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt=questions[0]), + dict(role='BOT', prompt=answers[0]), + dict(role='HUMAN', prompt=questions[1]), + dict(role='BOT', prompt=answers[1]), + dict(role='HUMAN', prompt=questions[2]), + dict(role='BOT', prompt=answers[2]), + dict(role='HUMAN', prompt=questions[3]), + dict(role='BOT', prompt=answers[3]), + dict(role='HUMAN', prompt=questions[4]), + dict(role='BOT', prompt=answers[4]), + dict(role='HUMAN', prompt=questions[5]), + dict(role='BOT', prompt=answers[5]), + dict(role='HUMAN', prompt=questions[6]), + dict(role='BOT', prompt=answers[6]), + dict(role='HUMAN', prompt=questions[7]), + dict(role='BOT', prompt=answers[7]), + dict(role='HUMAN', prompt=LANG_TO_INSTRUCTIONS[lang]), + ] + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512), + ) + + mgsm_eval_cfg = dict( + evaluator=dict(type=xMGSM_Evaluator), + pred_role='BOT', + pred_postprocessor=dict(type=mgsm_postprocess, lang=lang), + ) + + mgsm_datasets.append( + dict( + type=xMGSMSDataset, + abbr=f'mgsm_{lang}_8shot', + path=f'data/mgsm/mgsm_test_{lang}_google.jsonl', + reader_cfg=mgsm_reader_cfg, + infer_cfg=mgsm_infer_cfg, + eval_cfg=mgsm_eval_cfg, + ) + ) diff --git a/opencompass/configs/datasets/xmgsm/xmgsm_ge_8shot.py b/opencompass/configs/datasets/xmgsm/xmgsm_ge_8shot.py new file mode 100644 index 00000000..735b9e87 --- /dev/null +++ b/opencompass/configs/datasets/xmgsm/xmgsm_ge_8shot.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .mgsm_gen_d967bc_8shot import mgsm_datasets diff --git a/opencompass/configs/datasets/xmgsm/xmgsm_gen.py b/opencompass/configs/datasets/xmgsm/xmgsm_gen.py new file mode 100644 index 00000000..8feea66c --- /dev/null +++ b/opencompass/configs/datasets/xmgsm/xmgsm_gen.py @@ -0,0 +1,4 @@ +from mmengine.config import read_base + +with read_base(): + from .mgsm_gen_d967bc import mgsm_datasets diff --git a/opencompass/configs/datasets/xruler/README.md b/opencompass/configs/datasets/xruler/README.md new file mode 100644 index 00000000..76c42e48 --- /dev/null +++ b/opencompass/configs/datasets/xruler/README.md @@ -0,0 +1,14 @@ +# Ruler +OpenCompass now supports the brand new long-context language model evaluation benchmark — [RULER](https://arxiv.org/pdf/2404.06654). RULER provides an evaluation of long-context including retrieval, multi-hop tracing, aggregation, and question answering through flexible configurations. + +OpenCompass have providied two types of evaluation demo for using different tokenizers. + +For using the same tokenizer (typicall GPT-4), you can follow the demo (configs/eval_ruler_fix_tokenizer.py) where most of the settings are already defined. + + +For evaluation using each model's own tokenizer, you have to build the settings when you run the demo (we do not know which model you are trying to evaluate!) you can create a new evaluation script following the example (configs/eval_ruler.py) and change the context window sizes or add models according to your settings. + +```bash +python run.py configs/eval_ruler_fix_tokenizer.py # For evaluation with GPT-4 tokenizer +python run.py configs/eval_ruler.py # For evaluation with model's tokenizer +``` diff --git a/opencompass/configs/datasets/xruler/ruler_128k_gen.py b/opencompass/configs/datasets/xruler/ruler_128k_gen.py new file mode 100644 index 00000000..4f302ad2 --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_128k_gen.py @@ -0,0 +1,28 @@ +from mmengine.config import read_base + +with read_base(): + from .ruler_cwe_gen import cwe_datasets as cwe # CWE + from .ruler_fwe_gen import fwe_datasets as fwe # FWE + from .ruler_niah_gen import niah_datasets as niah # Niah + from .ruler_qa_gen import qa_datasets as qa # QA + from .ruler_vt_gen import vt_datasets as vt # VT + + +import_ds = sum((cwe, fwe, niah, qa, vt), []) + +# Evaluation config +NUM_SAMPLES = 100 # Change to the number of samples you need +# Change the context lengths to be tested +max_seq_lens = [1024 * 128] +abbr_suffixs = ['128k'] + +ruler_datasets = [] + +# Different seq length +for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): + for dataset in import_ds: + tmp_dataset = dataset.deepcopy() + tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix + tmp_dataset['num_samples'] = NUM_SAMPLES + tmp_dataset['max_seq_length'] = max_seq_len + ruler_datasets.append(tmp_dataset) diff --git a/opencompass/configs/datasets/xruler/ruler_16k_gen.py b/opencompass/configs/datasets/xruler/ruler_16k_gen.py new file mode 100644 index 00000000..faab3cca --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_16k_gen.py @@ -0,0 +1,29 @@ + +from mmengine.config import read_base + +with read_base(): + from .ruler_cwe_gen import cwe_datasets as cwe # CWE + from .ruler_fwe_gen import fwe_datasets as fwe # FWE + from .ruler_niah_gen import niah_datasets as niah # Niah + from .ruler_qa_gen import qa_datasets as qa # QA + from .ruler_vt_gen import vt_datasets as vt # VT + + +import_ds = sum((cwe, fwe, niah, qa, vt), []) + +# Evaluation config +NUM_SAMPLES = 100 # Change to the number of samples you need +# Change the context lengths to be tested +max_seq_lens = [1024 * 16] +abbr_suffixs = ['16k'] + +ruler_datasets = [] + +# Different seq length +for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): + for dataset in import_ds: + tmp_dataset = dataset.deepcopy() + tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix + tmp_dataset['num_samples'] = NUM_SAMPLES + tmp_dataset['max_seq_length'] = max_seq_len + ruler_datasets.append(tmp_dataset) diff --git a/opencompass/configs/datasets/xruler/ruler_1m_gen.py b/opencompass/configs/datasets/xruler/ruler_1m_gen.py new file mode 100644 index 00000000..0b20375c --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_1m_gen.py @@ -0,0 +1,29 @@ + +from mmengine.config import read_base + +with read_base(): + from .ruler_cwe_gen import cwe_datasets as cwe # CWE + from .ruler_fwe_gen import fwe_datasets as fwe # FWE + from .ruler_niah_gen import niah_datasets as niah # Niah + from .ruler_qa_gen import qa_datasets as qa # QA + from .ruler_vt_gen import vt_datasets as vt # VT + + +import_ds = sum((cwe, fwe, niah, qa, vt), []) + +# Evaluation config +NUM_SAMPLES = 100 # Change to the number of samples you need +# Change the context lengths to be tested +max_seq_lens = [1024 * 1024] +abbr_suffixs = ['1m'] + +ruler_datasets = [] + +# Different seq length +for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): + for dataset in import_ds: + tmp_dataset = dataset.deepcopy() + tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix + tmp_dataset['num_samples'] = NUM_SAMPLES + tmp_dataset['max_seq_length'] = max_seq_len + ruler_datasets.append(tmp_dataset) diff --git a/opencompass/configs/datasets/xruler/ruler_32k_gen.py b/opencompass/configs/datasets/xruler/ruler_32k_gen.py new file mode 100644 index 00000000..ab02cb4a --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_32k_gen.py @@ -0,0 +1,29 @@ + +from mmengine.config import read_base + +with read_base(): + from .ruler_cwe_gen import cwe_datasets as cwe # CWE + from .ruler_fwe_gen import fwe_datasets as fwe # FWE + from .ruler_niah_gen import niah_datasets as niah # Niah + from .ruler_qa_gen import qa_datasets as qa # QA + from .ruler_vt_gen import vt_datasets as vt # VT + + +import_ds = sum((cwe, fwe, niah, qa, vt), []) + +# Evaluation config +NUM_SAMPLES = 100 # Change to the number of samples you need +# Change the context lengths to be tested +max_seq_lens = [1024 * 32] +abbr_suffixs = ['32k'] + +ruler_datasets = [] + +# Different seq length +for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): + for dataset in import_ds: + tmp_dataset = dataset.deepcopy() + tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix + tmp_dataset['num_samples'] = NUM_SAMPLES + tmp_dataset['max_seq_length'] = max_seq_len + ruler_datasets.append(tmp_dataset) diff --git a/opencompass/configs/datasets/xruler/ruler_4k_gen.py b/opencompass/configs/datasets/xruler/ruler_4k_gen.py new file mode 100644 index 00000000..f0031507 --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_4k_gen.py @@ -0,0 +1,28 @@ +from mmengine.config import read_base + +with read_base(): + from .ruler_cwe_gen import cwe_datasets as cwe # CWE + from .ruler_fwe_gen import fwe_datasets as fwe # FWE + from .ruler_niah_gen import niah_datasets as niah # Niah + from .ruler_qa_gen import qa_datasets as qa # QA + from .ruler_vt_gen import vt_datasets as vt # VT + + +import_ds = sum((cwe, fwe, niah, qa, vt), []) + +# Evaluation config +NUM_SAMPLES = 100 # Change to the number of samples you need +# Change the context lengths to be tested +max_seq_lens = [1024 * 4] +abbr_suffixs = ['4k'] + +ruler_datasets = [] + +# Different seq length +for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): + for dataset in import_ds: + tmp_dataset = dataset.deepcopy() + tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix + tmp_dataset['num_samples'] = NUM_SAMPLES + tmp_dataset['max_seq_length'] = max_seq_len + ruler_datasets.append(tmp_dataset) diff --git a/opencompass/configs/datasets/xruler/ruler_64k_gen.py b/opencompass/configs/datasets/xruler/ruler_64k_gen.py new file mode 100644 index 00000000..709260d6 --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_64k_gen.py @@ -0,0 +1,28 @@ +from mmengine.config import read_base + +with read_base(): + from .ruler_cwe_gen import cwe_datasets as cwe # CWE + from .ruler_fwe_gen import fwe_datasets as fwe # FWE + from .ruler_niah_gen import niah_datasets as niah # Niah + from .ruler_qa_gen import qa_datasets as qa # QA + from .ruler_vt_gen import vt_datasets as vt # VT + + +import_ds = sum((cwe, fwe, niah, qa, vt), []) + +# Evaluation config +NUM_SAMPLES = 100 # Change to the number of samples you need +# Change the context lengths to be tested +max_seq_lens = [1024 * 64] +abbr_suffixs: list[str] = ['64k'] + +ruler_datasets = [] + +# Different seq length +for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): + for dataset in import_ds: + tmp_dataset = dataset.deepcopy() + tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix + tmp_dataset['num_samples'] = NUM_SAMPLES + tmp_dataset['max_seq_length'] = max_seq_len + ruler_datasets.append(tmp_dataset) diff --git a/opencompass/configs/datasets/xruler/ruler_8k_gen.py b/opencompass/configs/datasets/xruler/ruler_8k_gen.py new file mode 100644 index 00000000..8c9a4ad8 --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_8k_gen.py @@ -0,0 +1,29 @@ + +from mmengine.config import read_base + +with read_base(): + from .ruler_cwe_gen import cwe_datasets as cwe # CWE + from .ruler_fwe_gen import fwe_datasets as fwe # FWE + from .ruler_niah_gen import niah_datasets as niah # Niah + from .ruler_qa_gen import qa_datasets as qa # QA + from .ruler_vt_gen import vt_datasets as vt # VT + + +import_ds = sum((cwe, fwe, niah, qa, vt), []) + +# Evaluation config +NUM_SAMPLES = 100 # Change to the number of samples you need +# Change the context lengths to be tested +max_seq_lens = [1024 * 8] +abbr_suffixs = ['8k'] + +ruler_datasets = [] + +# Different seq length +for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): + for dataset in import_ds: + tmp_dataset = dataset.deepcopy() + tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix + tmp_dataset['num_samples'] = NUM_SAMPLES + tmp_dataset['max_seq_length'] = max_seq_len + ruler_datasets.append(tmp_dataset) diff --git a/opencompass/configs/datasets/xruler/ruler_combined_gen.py b/opencompass/configs/datasets/xruler/ruler_combined_gen.py new file mode 100644 index 00000000..077c4f10 --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_combined_gen.py @@ -0,0 +1,12 @@ +from mmengine.config import read_base + +with read_base(): + from .ruler_1m_gen import ruler_datasets as ruler_1m_ds + from .ruler_4k_gen import ruler_datasets as ruler_4k_ds + from .ruler_8k_gen import ruler_datasets as ruler_8k_ds + from .ruler_16k_gen import ruler_datasets as ruler_16k_ds + from .ruler_32k_gen import ruler_datasets as ruler_32k_ds + from .ruler_64k_gen import ruler_datasets as ruler_64k_ds + from .ruler_128k_gen import ruler_datasets as ruler_128k_ds + +ruler_combined_datasets = sum((v for k, v in locals().items() if k.endswith('_ds')), []) diff --git a/opencompass/configs/datasets/xruler/ruler_cwe_gen.py b/opencompass/configs/datasets/xruler/ruler_cwe_gen.py new file mode 100644 index 00000000..2bc8fb4a --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_cwe_gen.py @@ -0,0 +1,34 @@ +from opencompass.datasets.ruler.ruler_cwe import RulerCweDataset +from opencompass.datasets.ruler.ruler_cwe import RulerCweEvaluator +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer + +# CWE Dataset +cwe_datasets = [ + { + 'abbr': 'ruler_cwe', + 'type': RulerCweDataset, + 'freq_cw': 30, + 'freq_ucw': 3, + 'num_cw': 10, + 'tokens_to_generate': 120, + 'reader_cfg': dict(input_columns=['prompt'], output_column='answer'), + 'infer_cfg': dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='{prompt}'), + dict(role='BOT', prompt='{answer}\n'), + ] + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + 'eval_cfg': dict( + evaluator=dict(type=RulerCweEvaluator), + ), + } +] diff --git a/opencompass/configs/datasets/xruler/ruler_fwe_gen.py b/opencompass/configs/datasets/xruler/ruler_fwe_gen.py new file mode 100644 index 00000000..c5585c30 --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_fwe_gen.py @@ -0,0 +1,33 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets.ruler.ruler_fwe import RulerFweDataset +from opencompass.datasets.ruler.ruler_fwe import RulerFweEvaluator + +# FWE Dataset +fwe_datasets = [ + { + 'abbr': 'ruler_fwe', + 'type': RulerFweDataset, + 'tokens_to_generate': 50, + 'alpha': 2.0, + 'coded_wordlen': 6, + 'reader_cfg': dict(input_columns=['prompt'], output_column='answer'), + 'infer_cfg': dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='{prompt}'), + dict(role='BOT', prompt='{answer}\n'), + ] + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + 'eval_cfg': dict( + evaluator=dict(type=RulerFweEvaluator), + ), + } +] diff --git a/opencompass/configs/datasets/xruler/ruler_niah_gen.py b/opencompass/configs/datasets/xruler/ruler_niah_gen.py new file mode 100644 index 00000000..916649a2 --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_niah_gen.py @@ -0,0 +1,122 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets.xruler.xruler_niah import xRulerNiahDataset +from opencompass.datasets.xruler.xruler_niah import xRulerNiahEvaluator + + +# Ruler Dataset settings +niah_configurations = [ + { + 'abbr': 'single_1', + 'type_haystack': 'repeat', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'single_2', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'single_3', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'uuids', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'multikey_1', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 4, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'multikey_2', + 'type_haystack': 'needle', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'multikey_3', + 'type_haystack': 'needle', + 'type_needle_k': 'uuids', + 'type_needle_v': 'uuids', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'multivalue', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 4, + 'num_needle_q': 1, + }, + { + 'abbr': 'multiquery', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 4, + }, +] + +niah_datasets = [] +LANGS = ['en', 'zh'] +# NIAH Dataset +base_path = './data/xruler/un_test' +for lang in LANGS: + file_path = f'un_test.{lang}.jsonl' + for index, config in enumerate(niah_configurations): + dataset_dict = { + 'abbr': f'ruler_niah_{config["abbr"]}', + 'type': xRulerNiahDataset, + 'base_path': base_path, + 'file_path': file_path, + # 'tokenizer_model': model_path, + 'tokens_to_generate': 128, + 'type_haystack': config['type_haystack'], + 'type_needle_k': config['type_needle_k'], + 'type_needle_v': config['type_needle_v'], + 'num_needle_k': config['num_needle_k'], + 'num_needle_v': config['num_needle_v'], + 'num_needle_q': config['num_needle_q'], + 'reader_cfg': dict(input_columns=['prompt'], output_column='answer'), + 'infer_cfg': dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='{prompt}'), + dict(role='BOT', prompt='{answer}\n'), + ] + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + 'eval_cfg': dict( + evaluator=dict(type=xRulerNiahEvaluator), + ), + } + niah_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/xruler/ruler_vt_gen.py b/opencompass/configs/datasets/xruler/ruler_vt_gen.py new file mode 100644 index 00000000..42dadc43 --- /dev/null +++ b/opencompass/configs/datasets/xruler/ruler_vt_gen.py @@ -0,0 +1,32 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets.ruler.ruler_vt import RulerVtDataset +from opencompass.datasets.ruler.ruler_vt import RulerVtEvaluator + +# VT Dataset +vt_datasets = [ + { + 'abbr': 'ruler_vt', + 'type': RulerVtDataset, + 'num_chains': 1, + 'num_hops': 4, + 'reader_cfg': dict(input_columns=['prompt'], output_column='answer'), + 'infer_cfg': dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='{prompt}'), + dict(role='BOT', prompt='{answer}\n'), + ] + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + 'eval_cfg': dict( + evaluator=dict(type=RulerVtEvaluator), + ), + } +] diff --git a/opencompass/configs/datasets/xruler/xruler_niah_gen.py b/opencompass/configs/datasets/xruler/xruler_niah_gen.py new file mode 100644 index 00000000..3f9a5b5d --- /dev/null +++ b/opencompass/configs/datasets/xruler/xruler_niah_gen.py @@ -0,0 +1,122 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets.xruler.xruler_niah import xRulerNiahDataset +from opencompass.datasets.xruler.xruler_niah import xRulerNiahEvaluator + + +# Ruler Dataset settings +niah_configurations = [ + { + 'abbr': 'single_1', + 'type_haystack': 'repeat', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'single_2', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'single_3', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'uuids', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'multikey_1', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 4, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'multikey_2', + 'type_haystack': 'needle', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'multikey_3', + 'type_haystack': 'needle', + 'type_needle_k': 'uuids', + 'type_needle_v': 'uuids', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 1, + }, + { + 'abbr': 'multivalue', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 4, + 'num_needle_q': 1, + }, + { + 'abbr': 'multiquery', + 'type_haystack': 'essay', + 'type_needle_k': 'words', + 'type_needle_v': 'numbers', + 'num_needle_k': 1, + 'num_needle_v': 1, + 'num_needle_q': 4, + }, +] + +niah_datasets = [] +LANGS = ['en', 'zh', 'ar', 'bn', 'cs', 'de', 'es', 'fr', 'hu'] +# NIAH Dataset +base_path = './data/xruler/un_test' +for lang in LANGS: + file_path = f'un_test.{lang}.json' + for index, config in enumerate(niah_configurations): + dataset_dict = { + 'abbr': f'ruler_niah_{config["abbr"]}_{lang}', + 'type': xRulerNiahDataset, + 'base_path': base_path, + 'file_path': file_path, + # 'tokenizer_model': model_path, + 'tokens_to_generate': 128, + 'type_haystack': config['type_haystack'], + 'type_needle_k': config['type_needle_k'], + 'type_needle_v': config['type_needle_v'], + 'num_needle_k': config['num_needle_k'], + 'num_needle_v': config['num_needle_v'], + 'num_needle_q': config['num_needle_q'], + 'reader_cfg': dict(input_columns=['prompt'], output_column='answer'), + 'infer_cfg': dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='{prompt}'), + dict(role='BOT', prompt='{answer}\n'), + ] + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + 'eval_cfg': dict( + evaluator=dict(type=xRulerNiahEvaluator), + ), + } + niah_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/xruler/xruler_qa_gen.py b/opencompass/configs/datasets/xruler/xruler_qa_gen.py new file mode 100644 index 00000000..2608bd72 --- /dev/null +++ b/opencompass/configs/datasets/xruler/xruler_qa_gen.py @@ -0,0 +1,40 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.datasets.xruler.ruler_qa import xRulerQaDataset +from opencompass.datasets.xruler.ruler_qa import xRulerQaEvaluator +LANGS = ['ar', 'bn', 'cs'] +qa_configurations = [ + {'lang': lang, 'dataset': 'squad', 'path': f'./data/xruler/xquad/xquad_{lang}.json'} for lang in LANGS + # {'dataset': 'hotpotqa', 'path': './data/ruler/hotpotqa.json'}, +] + +qa_datasets = [] +for index, config in enumerate(qa_configurations): + dataset_dict = { + 'abbr': f'xruler_qa_{config["dataset"]}_{config["lang"]}', + 'dataset': config['dataset'], + 'path': config['path'], + 'lang': config['lang'], + 'type': xRulerQaDataset, + 'tokens_to_generate': 50, + 'max_seq_length': 1024 * 8, + 'reader_cfg': dict(input_columns=['prompt'], output_column='answer'), + 'infer_cfg': dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt='{prompt}'), + dict(role='BOT', prompt='{answer}\n'), + ] + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + 'eval_cfg': dict( + evaluator=dict(type=xRulerQaEvaluator), + ), + } + qa_datasets.append(dataset_dict) diff --git a/opencompass/datasets/__init__.py b/opencompass/datasets/__init__.py index b1753221..bdc6df82 100644 --- a/opencompass/datasets/__init__.py +++ b/opencompass/datasets/__init__.py @@ -170,3 +170,7 @@ from .xcopa import * # noqa: F401, F403 from .xiezhi import XiezhiDataset, XiezhiRetriever # noqa: F401, F403 from .xlsum import * # noqa: F401, F403 from .xsum import * # noqa: F401, F403 +from .xhumaneval import xHumanevalDataset # noqa: F401, F403 +from .xIFEval.ifeval import xIFEvalDataset, xIFEvaluator # noqa: F401, F403 +from .xlivecodebench import xLCBCodeGenerationDataset, xLCBCodeGenerationEvaluator +from .xmgsm import * # noqa: F401, F403 diff --git a/opencompass/datasets/evalplus/__init__.py b/opencompass/datasets/evalplus/__init__.py new file mode 100644 index 00000000..6f5bd062 --- /dev/null +++ b/opencompass/datasets/evalplus/__init__.py @@ -0,0 +1,4 @@ +try: + from evalplus._version import __version__, __version_tuple__ +except ImportError: + __version__ = "local-dev" diff --git a/opencompass/datasets/evalplus/codegen.py b/opencompass/datasets/evalplus/codegen.py new file mode 100644 index 00000000..1229dcee --- /dev/null +++ b/opencompass/datasets/evalplus/codegen.py @@ -0,0 +1,270 @@ +import json +import os +from typing import Dict, List, Optional + +from .data import get_evalperf_data, get_human_eval_plus, get_mbpp_plus +from .provider import DecoderBase, make_model +from .sanitize import sanitize +from .utils import progress + +def codegen( + target_path: str, + model: DecoderBase, + dataset: Dict, + greedy=False, + n_samples=1, + id_range=None, + resume=True, +): + task2nexist = {} + if resume and target_path.endswith(".jsonl") and os.path.isfile(target_path): + with open(target_path, "r") as f: + for line in f: + if not line.strip(): + continue + task_id = json.loads(line)["task_id"] + task2nexist[task_id] = task2nexist.get(task_id, 0) + 1 + + if target_path.endswith(".jsonl"): + raw_target_path = target_path.replace(".jsonl", ".raw.jsonl") + else: + raw_target_path = target_path + ".raw" + os.makedirs(target_path, exist_ok=True) + + print(f"Sanitized code outputs will be saved to {target_path}") + print(f"Raw outputs will be saved to {raw_target_path}") + + backend_type: str = type(model).__name__ + with progress(backend_type) as p: + for task_id, task in p.track(dataset.items()): + if id_range is not None: + id_num = int(task_id.split("/")[1]) + low, high = id_range + if id_num < low or id_num >= high: + p.console.print(f"Skipping {task_id} as it is not in {id_range}") + continue + + if not target_path.endswith(".jsonl"): + p_name = task_id.replace("/", "_") + os.makedirs(os.path.join(target_path, p_name), exist_ok=True) + task2nexist[task_id] = len( + [ + f + for f in os.listdir(os.path.join(target_path, p_name)) + if f.endswith(".py") + ] + ) + + n_more_samples = n_samples + log = f"Codegen: {task_id} @ {model}" + if resume and task2nexist.get(task_id, 0) > 0: + log += f" (resuming from {task2nexist[task_id]})" + n_more_samples -= task2nexist[task_id] + + p.console.print(log) + + sidx = n_samples - n_more_samples + while sidx < n_samples: + prompt = task["prompt"].strip() + "\n" + outputs = model.codegen( + prompt, + do_sample=not greedy, + num_samples=n_samples - sidx, + ) + assert outputs, "No outputs from model!" + for impl in outputs: + solution = prompt + impl if model.is_direct_completion() else impl + sanitized_solution = sanitize( + solution, entrypoint=task["entry_point"] + ) + if target_path.endswith(".jsonl"): + # Writing the sanitized version + with open(target_path, "a") as f: + f.write( + json.dumps( + {"task_id": task_id, "solution": sanitized_solution} + ) + + "\n" + ) + + # Writing the raw version + with open(raw_target_path, "a") as f: + f.write( + json.dumps({"task_id": task_id, "solution": solution}) + + "\n" + ) + else: + # Writing the sanitized version + with open( + os.path.join(target_path, p_name, f"{sidx}.py"), + "w", + encoding="utf-8", + ) as f: + f.write(sanitized_solution) + + # Writing the raw version + with open( + os.path.join(raw_target_path, p_name, f"{sidx}.py"), + "w", + encoding="utf-8", + ) as f: + f.write(solution) + sidx += 1 + + +def run_codegen( + model: str, + dataset: str, + root: str = "evalplus_results", + bs: Optional[int] = None, + n_samples: int = 1, + temperature: float = 0.0, + resume: bool = True, + greedy: bool = False, + id_range: List = None, + version: str = "default", + backend: str = "vllm", + force_base_prompt: bool = False, + base_url: str = None, + tp: int = 1, + evalperf_type: str = None, # For EvalPerf + jsonl_fmt: bool = True, + attn_implementation: str = "eager", + device_map: Optional[str] = None, + trust_remote_code: bool = False, + enable_prefix_caching: bool = False, + enable_chunked_prefill: bool = False, + dtype: str = "bfloat16", + gptqmodel_backend: str = "auto", + lang: str = "en" # For GPTQModel +): + assert dataset in ["humaneval", "mbpp", "evalperf"], f"Invalid dataset {dataset}" + assert evalperf_type is None or evalperf_type in [ + "instruct", + "perf-instruct", + "perf-CoT", + ] + + # Make dir for codes generated by each model + identifier = model.strip("./").replace("/", "--") + f"_{backend}_temp_{temperature}" + if evalperf_type: + identifier += f"-{evalperf_type}" + + target_path = os.path.join(root, dataset, identifier) if lang == "en" else os.path.join(root, dataset, f"{lang}_{identifier}") + if jsonl_fmt: + target_path += ".jsonl" + else: + os.makedirs(target_path, exist_ok=True) + + if dataset == "humaneval": + dataset_dict = get_human_eval_plus(version=version, lang=lang) + elif dataset == "mbpp": + dataset_dict = get_mbpp_plus(version=version) + elif dataset == "evalperf": + original_dataset = {**get_human_eval_plus(), **get_mbpp_plus()} + dataset_dict = {k: original_dataset[k] for k in get_evalperf_data()} + assert id_range is None, "id_range not supported for evalperf" + else: + raise ValueError(f"Invalid dataset {dataset}") + + all_tasks_complete = False + if jsonl_fmt and os.path.isfile(target_path): + task_counts = {} + with open(target_path, "r") as f: + for line in f: + if not line.strip(): + continue + data = json.loads(line) + task_id = data["task_id"] + task_counts[task_id] = task_counts.get(task_id, 0) + 1 + + all_tasks_complete = all( + task_counts.get(task_id, 0) >= n_samples + for task_id in dataset_dict.keys() + ) + + if all_tasks_complete: + print("All samples are already cached. Skipping codegen.") + return target_path + + if greedy and (temperature != 0 or bs != 1 or n_samples != 1): + temperature = 0.0 + bs = 1 + n_samples = 1 + print("Greedy decoding ON (--greedy): setting bs=1, n_samples=1, temperature=0") + + if id_range is not None: + assert len(id_range) == 2, "id_range must be a list of length 2" + assert id_range[0] < id_range[1], "id_range must be increasing" + id_range = tuple(id_range) + + if bs is None: + bs = min(n_samples, 32) + print(f"Setting batch size to {bs}") + + # Make project dir + os.makedirs(root, exist_ok=True) + # Make dataset dir + os.makedirs(os.path.join(root, dataset), exist_ok=True) + + # Model instructions + instruction_prefix = "Please provide a self-contained Python script that solves the following problem in a markdown code block:" + response_prefix = "Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:" + + if evalperf_type == "perf-instruct": + instruction_prefix = "Please provide an efficient and self-contained Python script that solves the following problem in a markdown code block:" + response_prefix = "Below is a Python script with a self-contained function that efficiently solves the problem and passes corresponding tests:" + elif evalperf_type == "perf-CoT": + instruction_prefix = "Think step by step: please provide an efficient and self-contained Python script that solves the following problem in a markdown code block:" + response_prefix = "Below is a Python script with a self-contained function that efficiently solves the problem and passes corresponding tests:" + elif evalperf_type is not None and evalperf_type != "instruct": + raise ValueError(f"Invalid evalperf_type: {evalperf_type}") + + # Model creation + model_runner = make_model( + model=model, + backend=backend, + batch_size=bs, + temperature=temperature, + force_base_prompt=force_base_prompt, + dataset=dataset, + base_url=base_url, + tp=tp, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + device_map=device_map, + attn_implementation=attn_implementation, + trust_remote_code=trust_remote_code, + enable_prefix_caching=enable_prefix_caching, + enable_chunked_prefill=enable_chunked_prefill, + dtype=dtype, + gptqmodel_backend=gptqmodel_backend, + ) + + codegen( + target_path=target_path, + dataset=dataset_dict, + greedy=greedy, + model=model_runner, + n_samples=n_samples, + resume=resume, + id_range=id_range, + ) + + # force shutdown the model runner + del model_runner + import gc + + gc.collect() + + return target_path + + +def main(): + from fire import Fire + + Fire(run_codegen) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/config.py b/opencompass/datasets/evalplus/config.py new file mode 100644 index 00000000..ad5c87ac --- /dev/null +++ b/opencompass/datasets/evalplus/config.py @@ -0,0 +1,16 @@ +## EvalPlus +DEFAULT_GT_TIME_LIMIT_FACTOR = 4.0 +DEFAULT_MIN_TIME_LIMIT = 1.0 + +## EvalPerf + +### General +PERF_PROFILE_ROUNDS = 1 +PERF_RAM_GB_PER_PROC = 12 + +### Evaluation Phase +PERF_EVAL_TIMEOUT_SECOND = 45 + +### Curation Phase +PERF_CURATE_TIMEOUT_SECOND = 20 +PREF_CURATE_MIN_INSTRUCTION = 10000 diff --git a/opencompass/datasets/evalplus/eval/__init__.py b/opencompass/datasets/evalplus/eval/__init__.py new file mode 100644 index 00000000..419ff984 --- /dev/null +++ b/opencompass/datasets/evalplus/eval/__init__.py @@ -0,0 +1,316 @@ +# The MIT License +# +# Copyright (c) OpenAI (https://openai.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import itertools +import multiprocessing +import os +import time +from multiprocessing import Array, Value +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import psutil + +from opencompass.datasets.evalplus.config import * +from opencompass.datasets.evalplus.eval._special_oracle import ( + MBPP_OUTPUT_NOT_NONE_TASKS, + MBPP_OUTPUT_SET_EQ_TASKS, + _digit_distance_nums, + _poly, + _surface_Area, +) +from opencompass.datasets.evalplus.eval.utils import ( + create_tempdir, + reliability_guard, + swallow_io, + time_limit, +) + + +def compatible_eval_result(results: Dict) -> Dict: + # compatibility + for task_results in results["eval"].values(): + # update the "files" field to "nfiles" + if "files" in task_results and "nfiles" not in task_results: + task_results["nfiles"] = len(task_results.pop("files")) + return results + + +# unbiased estimator from https://github.com/openai/human-eval +def estimate_pass_at_k( + num_samples: Union[int, List[int], np.ndarray], + num_correct: Union[List[int], np.ndarray], + k: int, +) -> np.ndarray: + """ + Estimates pass@k of each problem and returns them in an array. + """ + + def estimator(n: int, c: int, k: int) -> float: + """ + Calculates 1 - comb(n - c, k) / comb(n, k). + """ + if n - c < k: + return 1.0 + return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) + + if isinstance(num_samples, int): + num_samples_it = itertools.repeat(num_samples, len(num_correct)) + else: + assert len(num_samples) == len(num_correct) + num_samples_it = iter(num_samples) + + return np.array( + [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)] + ) + + +PASS = "pass" +FAIL = "fail" +TIMEOUT = "timeout" + +_SUCCESS = 0 +_FAILED = 1 +_TIMEOUT = 2 +_UNKNOWN = 3 + +_mapping = {_SUCCESS: PASS, _FAILED: FAIL, _TIMEOUT: TIMEOUT, _UNKNOWN: None} + + +def query_maximum_memory_bytes() -> Optional[int]: + # Disable functionalities that can make destructive changes to the test. + # allow only 4GB memory usage + maximum_memory_bytes = os.getenv( + "EVALPLUS_MAX_MEMORY_BYTES", 4 * 1024 * 1024 * 1024 + ) + maximum_memory_bytes = min(int(maximum_memory_bytes), psutil.virtual_memory().total) + if maximum_memory_bytes == -1: + return None + return maximum_memory_bytes + + +def is_floats(x) -> bool: + # check if it is float; List[float]; Tuple[float] + if isinstance(x, float): + return True + if isinstance(x, (list, tuple)) and x: + return all(isinstance(i, float) for i in x) + if isinstance(x, np.ndarray): + return x.dtype == np.float64 or x.dtype == np.float32 + return False + + +def unsafe_execute( + dataset: str, + entry_point: str, + code: str, + inputs, + expected: List, + time_limits, + atol, + fast_check, + stat, # Value + details, # Array + progress, # Value +): + with create_tempdir(): + # These system calls are needed when cleaning up tempdir. + import os + import shutil + + rmtree = shutil.rmtree + rmdir = os.rmdir + chdir = os.chdir + reliability_guard(maximum_memory_bytes=query_maximum_memory_bytes()) + exec_globals = {} + try: + with swallow_io(): + exec(code, exec_globals) + fn = exec_globals[entry_point] + + for i, inp in enumerate(inputs): + try: + with time_limit(time_limits[i]): + with swallow_io(): + out = fn(*inp) + + exp = expected[i] + exact_match = out == exp + + # ================================================ # + # ============== special oracles ================= # + if dataset == "mbpp": + if "are_equivalent" == entry_point: # Mbpp/164 special oracle + exact_match = exact_match or True + elif "sum_div" == entry_point: # Mbpp/295 special oracle + exact_match = exact_match or out == 0 + elif "surface_Area" == entry_point: # Mbpp/581 special oracle + exact_match = ( + exact_match or abs(out - _surface_Area(*inp)) <= atol + ) + elif ( + "digit_distance_nums" == entry_point + ): # Mbpp/558 special oracle + exact_match = exact_match or out == _digit_distance_nums( + *inp + ) + elif entry_point in MBPP_OUTPUT_SET_EQ_TASKS: + exact_match = set(out) == set(exp) + elif entry_point in MBPP_OUTPUT_NOT_NONE_TASKS: + # exp is True if not None + # False if None + if isinstance(out, bool): + exact_match = out == exp + else: + exact_match = exp == (out is not None) + + if dataset == "humaneval": + if "find_zero" == entry_point: + assert abs(_poly(*inp, out)) <= atol + details[i] = True + progress.value += 1 + continue + # ============== special oracles ================= # + # ================================================ # + + if atol == 0 and is_floats(exp): + atol = 1e-6 # enforce atol for float comparison + if not exact_match and atol != 0: + # explicitly set rtol=1e-07 + # to match `np.testing.assert_allclose`'s default values + assert type(out) == type(exp) + if isinstance(exp, (list, tuple)): + assert len(out) == len(exp) + assert np.allclose(out, exp, rtol=1e-07, atol=atol) + else: + assert exact_match + except BaseException: + details[i] = False + progress.value += 1 + if fast_check: + raise + continue + + details[i] = True + progress.value += 1 + + stat.value = _SUCCESS + except BaseException: + stat.value = _FAILED + # Needed for cleaning up. + shutil.rmtree = rmtree + os.rmdir = rmdir + os.chdir = chdir + + +def untrusted_check( + dataset: str, + code: str, + inputs: List[Any], + entry_point: str, + expected, + atol, + ref_time: List[float], + fast_check: bool = False, + min_time_limit: float = DEFAULT_MIN_TIME_LIMIT, + gt_time_limit_factor: float = DEFAULT_GT_TIME_LIMIT_FACTOR, +) -> Tuple[str, np.ndarray]: + time_limits = [max(min_time_limit, gt_time_limit_factor * t) for t in ref_time] + timeout = min(os.getenv("EVALPLUS_TIMEOUT_PER_TASK", 60), sum(time_limits)) + 1 + if not fast_check: + timeout += 1 # extra time for data collection + + # shared memory objects + progress = Value("i", 0) + stat = Value("i", _UNKNOWN) + details = Array("b", [False for _ in range(len(inputs))]) + + p = multiprocessing.Process( + target=unsafe_execute, + args=( + dataset, + entry_point, + code, + inputs, + expected, + time_limits, + atol, + fast_check, + # return values + stat, + details, + progress, + ), + ) + p.start() + p.join(timeout=timeout + 1) + if p.is_alive(): + p.terminate() + time.sleep(0.1) + if p.is_alive(): + p.kill() + time.sleep(0.1) + + stat = _mapping[stat.value] + details = details[: progress.value] + + if not stat: + stat = TIMEOUT + + if stat == PASS: + if len(details) != len(inputs) or not all(details): + stat = FAIL + + return stat, details + + +def evaluate_files( + dataset: str, + files: List[str], + inputs: List, + expected: List, + entry_point: str, + atol: float, + ref_time: List[float], + fast_check: bool = False, + min_time_limit: float = DEFAULT_MIN_TIME_LIMIT, + gt_time_limit_factor: float = DEFAULT_GT_TIME_LIMIT_FACTOR, +) -> List[Tuple[str, List[bool]]]: + ret = [] + # sort files by the id in name (i.e., "../n.py") + files = sorted(files, key=lambda x: int(x.split("/")[-1].split(".")[0])) + for file in files: + code = open(file, "r").read() + stat, det = untrusted_check( + dataset, + code, + inputs, + entry_point, + expected=expected, + atol=atol, + ref_time=ref_time, + fast_check=fast_check, + min_time_limit=min_time_limit, + gt_time_limit_factor=gt_time_limit_factor, + ) + ret.append((stat, det.tolist())) + return ret diff --git a/opencompass/datasets/evalplus/eval/_special_oracle.py b/opencompass/datasets/evalplus/eval/_special_oracle.py new file mode 100644 index 00000000..1b13410b --- /dev/null +++ b/opencompass/datasets/evalplus/eval/_special_oracle.py @@ -0,0 +1,55 @@ +"""Special oracle handlings for problems where direct differential testing is not applicable.""" + +import math + +# For tasks whose output are not serializable, we only check the output is not None, which +# is also consistent with the original dataset. +MBPP_OUTPUT_NOT_NONE_TASKS = ["check_str", "text_match_three", "text_starta_endb"] + +# Tasks that needs to perform set comparison over two lists +MBPP_OUTPUT_SET_EQ_TASKS = [ + "similar_elements", # Mbpp/2 + "find_char_long", # Mbpp/7 + "common_in_nested_lists", # Mbpp/111 + "extract_singly", # Mbpp/140 + "larg_nnum", # Mbpp/232 + "intersection_array", # Mbpp/249 + "find_dissimilar", # Mbpp/579 + "Diff", # Mbpp/769 +] + + +# oracle for Mbpp/581 +def _surface_Area(base_edge, height): + """ + Recognizes the "height" as the perpendicular distance from the base to the apex of the pyramid + """ + slant_height = math.sqrt((base_edge / 2) ** 2 + height**2) + base_area = base_edge**2 + lateral_area = 4 * (base_edge * slant_height) / 2 + total_surface_area = base_area + lateral_area + return round(total_surface_area) + + +# oracle for Mbpp/558 +def _digit_distance_nums(num1, num2): + """ + Preprocesses the two numbers to have the same length by padding with zeros + """ + str_num1, str_num2 = str(num1), str(num2) + max_length = max(len(str_num1), len(str_num2)) + str_num1, str_num2 = str_num1.zfill(max_length), str_num2.zfill(max_length) + total_difference = 0 + for digit1, digit2 in zip(str_num1, str_num2): + difference = abs(int(digit1) - int(digit2)) + total_difference += difference + return total_difference + + +# oracle for HumaneEval/032 +def _poly(xs: list, x: float): + """ + Evaluates polynomial with coefficients xs at point x. + return xs[0] + xs[1] * x + xs[1] * x^2 + .... xs[n] * x^n + """ + return sum([coeff * math.pow(x, i) for i, coeff in enumerate(xs)]) diff --git a/opencompass/datasets/evalplus/eval/utils.py b/opencompass/datasets/evalplus/eval/utils.py new file mode 100644 index 00000000..4170bcbd --- /dev/null +++ b/opencompass/datasets/evalplus/eval/utils.py @@ -0,0 +1,187 @@ +# The MIT License +# +# Copyright (c) OpenAI (https://openai.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import contextlib +import faulthandler +import io +import os +import platform +import signal +import tempfile +from typing import Optional + + +@contextlib.contextmanager +def swallow_io(): + stream = WriteOnlyStringIO() + with contextlib.redirect_stdout(stream): + with contextlib.redirect_stderr(stream): + with redirect_stdin(stream): + yield + + +@contextlib.contextmanager +def time_limit(seconds: float): + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.setitimer(signal.ITIMER_REAL, seconds) + signal.signal(signal.SIGALRM, signal_handler) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +@contextlib.contextmanager +def create_tempdir(): + with tempfile.TemporaryDirectory() as dirname: + with chdir(dirname): + yield dirname + + +@contextlib.contextmanager +def chdir(root): + if root == ".": + yield + return + cwd = os.getcwd() + os.chdir(root) + try: + yield + except BaseException as exc: + raise exc + finally: + os.chdir(cwd) + + +class TimeoutException(Exception): + pass + + +class WriteOnlyStringIO(io.StringIO): + """StringIO that throws an exception when it's read from""" + + def read(self, *args, **kwargs): + raise IOError + + def readline(self, *args, **kwargs): + raise IOError + + def readlines(self, *args, **kwargs): + raise IOError + + def readable(self, *args, **kwargs): + """Returns True if the IO object can be read.""" + return False + + +class redirect_stdin(contextlib._RedirectStream): # type: ignore + _stream = "stdin" + + +def reliability_guard(maximum_memory_bytes: Optional[int] = None): + """ + This disables various destructive functions and prevents the generated code + from interfering with the test (e.g. fork bomb, killing other processes, + removing filesystem files, etc.) + + WARNING + This function is NOT a security sandbox. Untrusted code, including, model- + generated code, should not be blindly executed outside of one. See the + Codex paper for more information about OpenAI's code sandbox, and proceed + with caution. + """ + + if maximum_memory_bytes is not None: + import resource + + resource.setrlimit( + resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes) + ) + resource.setrlimit( + resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes) + ) + if not platform.uname().system == "Darwin": + resource.setrlimit( + resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes) + ) + + faulthandler.disable() + + import builtins + + builtins.exit = None + builtins.quit = None + + import os + + os.environ["OMP_NUM_THREADS"] = "1" + + os.kill = None + os.system = None + os.putenv = None + os.remove = None + os.removedirs = None + os.rmdir = None + os.fchdir = None + os.setuid = None + os.fork = None + os.forkpty = None + os.killpg = None + os.rename = None + os.renames = None + os.truncate = None + os.replace = None + os.unlink = None + os.fchmod = None + os.fchown = None + os.chmod = None + os.chown = None + os.chroot = None + os.fchdir = None + os.lchflags = None + os.lchmod = None + os.lchown = None + os.getcwd = None + os.chdir = None + builtins.open = None + + import shutil + + shutil.rmtree = None + shutil.move = None + shutil.chown = None + + import subprocess + + subprocess.Popen = None # type: ignore + + __builtins__["help"] = None + + import sys + + sys.modules["ipdb"] = None + sys.modules["joblib"] = None + sys.modules["resource"] = None + sys.modules["psutil"] = None + sys.modules["tkinter"] = None diff --git a/opencompass/datasets/evalplus/evalperf.py b/opencompass/datasets/evalplus/evalperf.py new file mode 100644 index 00000000..0a2f32b4 --- /dev/null +++ b/opencompass/datasets/evalplus/evalperf.py @@ -0,0 +1,557 @@ +"""Compute the Differential Performance Scores (DPS) and DPS_{norm} of given samples from a model. + +Check our COLM paper for more details: https://www.arxiv.org/abs/2408.06450 + +^Updates from the COLM paper: +* Condition to activate efficiency evaluation for a task: + * Paper: as long as you have at least one correct solution, and we select up to 10 correct solutions for efficiency sampling + * Here: you need to have at least `min_correct` correct solutions, and we evaluate the efficiency of all correct solutions + * Updating rationale: to make the evaluation more statistically robust + +@inproceedings{liu2024evaluating, + title = {Evaluating Language Models for Efficient Code Generation}, + author = {Liu, Jiawei and Xie, Songrun and Wang, Junhao and Wei, Yuxiang and Ding, Yifeng and Zhang, Lingming}, + booktitle = {First Conference on Language Modeling}, + year = {2024}, + url = {https://openreview.net/forum?id=IBCBMeAhmC}, +} +""" + +import json +import multiprocessing +import os +import socket +import time +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import closing +from datetime import datetime +from statistics import mean +from typing import Dict, List, Optional, Tuple + +import rich +from rich.rule import Rule +from rich.syntax import Syntax +from rich.table import Table + +from opencompass.datasets.evalplus.codegen import run_codegen +from opencompass.datasets.evalplus.config import * +from opencompass.datasets.evalplus.config import PERF_EVAL_TIMEOUT_SECOND +from opencompass.datasets.evalplus.data import ( + get_evalperf_data, + get_human_eval_plus, + get_human_eval_plus_hash, + get_mbpp_plus, + get_mbpp_plus_hash, +) +from opencompass.datasets.evalplus.data.mbpp import mbpp_deserialize_inputs +from opencompass.datasets.evalplus.data.utils import stream_jsonl +from opencompass.datasets.evalplus.eval import PASS, untrusted_check +from opencompass.datasets.evalplus.eval._special_oracle import MBPP_OUTPUT_NOT_NONE_TASKS +from opencompass.datasets.evalplus.evaluate import get_groundtruth +from opencompass.datasets.evalplus.perf.profile import ( + are_profiles_broken, + default_parallelism, + profile, + simple_test_profiler, +) +from opencompass.datasets.evalplus.utils import progress + +def rule(msg: str): + rich.print(Rule(msg)) + + +def not_none(l: list) -> list: + return [x for x in l if x is not None] + + +def get_free_port(): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +def correctness_check( + solution: str, dataset: str, task: Dict, expected_output: List +) -> Tuple: + assert isinstance(solution, str) + result = untrusted_check( + dataset, + solution, + task["base_input"] + list(task["plus_input"]), + task["entry_point"], + expected_output["base"] + expected_output["plus"], + task["atol"], + expected_output["base_time"] + expected_output["plus_time"], + fast_check=True, + min_time_limit=DEFAULT_MIN_TIME_LIMIT, + gt_time_limit_factor=DEFAULT_GT_TIME_LIMIT_FACTOR, + ) + return result, solution + + +def get_evalplus_data(): + problems_he = get_human_eval_plus(noextreme=True) + dataset_hash = get_human_eval_plus_hash(noextreme=True) + expected_output_human = get_groundtruth(problems_he, dataset_hash, []) + problems_mbpp = get_mbpp_plus(noextreme=True) + dataset_hash = get_mbpp_plus_hash(noextreme=True) + expected_output_mbpp = get_groundtruth( + problems_mbpp, + dataset_hash, + MBPP_OUTPUT_NOT_NONE_TASKS, + ) + problems = {**problems_he, **problems_mbpp} + expected_output = {**expected_output_human, **expected_output_mbpp} + return problems, expected_output + + +def table_print(table_name: str, kv: Dict): + table = Table( + title=table_name, + show_header=True, + header_style="bold", + ) + for col_name in kv: + table.add_column(col_name) + + table.add_row(*[str(v) for v in kv.values()]) + rich.print(table) + + +def correctness_worker(task_id: str, samples: list, ctask: Dict, expected_output: Dict): + assert isinstance( + samples, list + ), f"{task_id}: samples is not a list but {type(samples)}" + + results = [] + + for solution in samples: + result, solution = correctness_check( + solution, task_id.split("/")[0].lower(), ctask, expected_output + ) + results.append( + { + "solution": solution, + "pass": result[0] == PASS, + "profiled": False, + "matching_cluster_idx": None, + "dps": None, + "dps_norm": None, + } + ) + + return task_id, results + + +def perf_worker( + task_id: str, + ptask: Dict, # EvalPerf data + ret_dict: Dict, + lazy_evaluation: bool, + max_profile: int, +): + rich.print(f"{task_id}: Started") + start_time = time.time() + + ######################### Profiling Setup ######################### + n_reference = len(ptask["reference"]) + entry_point = ptask["entry_point"] + pe_input = ( + mbpp_deserialize_inputs(task_id, ptask["pe_input"])[0] + if task_id.startswith("Mbpp/") + else ptask["pe_input"][0] + ) + #################################################################### + + #################################################################### + ############### Lazily profile reference solutions ################# + #################################################################### + cache_ref_num_inst = [None] * n_reference + + def get_avg_ref_profile(idx, check_order=True) -> Optional[Tuple]: + nonlocal cache_ref_num_inst + + assert ( + idx < n_reference - 1 + and cache_ref_num_inst[idx + 1] is not None + or idx == n_reference - 1 + ), f"Calling get_avg_ref_profile({idx}) before get_avg_ref_profile({idx+1}) is called, is not allowed! {n_reference = }" + + if cache_ref_num_inst[idx] is not None: + return cache_ref_num_inst[idx], ptask["scores"][idx] + + evaluation_time = PERF_EVAL_TIMEOUT_SECOND + ref_solution = ptask["reference"][idx] + for _ in range(2): # at most retry twice + profiles = profile( + ref_solution, + entry_point, + [pe_input], + timeout_second_per_test=evaluation_time, + ) + + # Bad thing#1: timeout / failure happens + if are_profiles_broken(profiles): + print(f"{task_id}: [WARNING] Error in ref: {profiles}") + rich.print(Syntax(ref_solution, "python")) + print(f"{task_id}: Retrying w/ +10s timeout...") + evaluation_time += 10 + else: + break + + avg_profile = mean(profiles) + # Bad thing#2: if the current #instruction is faster than that of i+1 + if idx < n_reference - 1 and avg_profile < cache_ref_num_inst[idx + 1]: + print(f"{task_id}: [WARNING] #{idx} ref faster than #{idx + 1}") + print(f"ref {idx}: #inst {avg_profile}\tscore {ptask['scores'][idx]:.1f}") + print( + f"ref {idx+1}: #inst {cache_ref_num_inst[idx+1]}\tscore {ptask['scores'][idx+1]:.1f}" + ) + rich.print(Syntax(ref_solution, "python")) + if check_order: + return None + + cache_ref_num_inst[idx] = avg_profile + ret_dict["ref"][idx]["_num_cpu_instructions"] = avg_profile + return cache_ref_num_inst[idx], ptask["scores"][idx] + + #################################################################### + ############################## END ################################# + #################################################################### + + if not lazy_evaluation: # compute everything ahead of time + for i in range(n_reference - 1, -1, -1): + if get_avg_ref_profile(i) is None: + break + + assert ( + None not in cache_ref_num_inst + ), f"{task_id}: Failed to profile certain reference: {cache_ref_num_inst = }" + + profile_cache = {} + + cur_profiled = 0 + for result in ret_dict["results"]: + if cur_profiled >= max_profile: + rich.print(f"{task_id}: Reached max_profile limit {max_profile}, stopped") + break + if not result["pass"]: + continue + + solution = result["solution"] + + if solution in profile_cache: # reuse cache + sample_profiles = profile_cache[solution] + else: + sample_profiles = profile( + solution, + entry_point, + [pe_input], + timeout_second_per_test=PERF_EVAL_TIMEOUT_SECOND, + ) + profile_cache[solution] = sample_profiles # store cache + + score = 0 + norm_score = 0 + result["matching_cluster_idx"] = -1 # -1 means even slower than the slowest ref + # if the solution results in a timeout, score is 0 + if are_profiles_broken(sample_profiles): + print( + f"{task_id}: Tested solution error'ed out: {sample_profiles} ... regarded as 0 score" + ) + rich.print(Syntax(solution, "python")) + else: + avg_sample_profile = result["_num_cpu_instructions"] = mean(sample_profiles) + # Get profiles from fast to slow (back to front): + for j in range(n_reference - 1, -1, -1): + avg_ref_profile, ref_score = get_avg_ref_profile(j, check_order=False) + if avg_sample_profile <= avg_ref_profile: + result["matching_cluster_idx"] = j + score = ref_score + norm_score = 100 * (j + 1) / n_reference + break + + result["dps"] = score + result["dps_norm"] = norm_score + result["profiled"] = True + cur_profiled += 1 + + ret_dict["dps"] = mean(not_none([r["dps"] for r in ret_dict["results"]])) + ret_dict["dps_norm"] = mean(not_none([r["dps_norm"] for r in ret_dict["results"]])) + ret_dict["n_profiled"] = cur_profiled + + table_print( + f"[bold green]{task_id} Completed[/]", + { + "Duration": f"{time.time() - start_time:.1f}s", + "DPS": f"[green]{ret_dict['dps']:.1f}[/]", + "DPS_norm": f"[green]{ret_dict['dps_norm']:.1f}[/]", + "# Profiled": f"{cur_profiled} / {len(ret_dict['results'])}", + "Pass@1": f"{ret_dict['pass@1']:.1f}%", + }, + ) + + return ret_dict + + +# TODO(@ganler): OPTIMIZATION: reuse the samples from the generations of other datasets +def script( + samples: Optional[str] = None, + min_correct: int = 10, + max_profile: Optional[int] = None, + n_samples: int = 100, + temperature: float = 1.0, + parallel: Optional[int] = None, + lazy_evaluation: bool = True, + i_just_wanna_run: bool = False, + **model_kwargs, +): + max_profile = max_profile or min(min_correct * 2, n_samples) + assert min_correct <= max_profile <= n_samples + simple_test_profiler() # test linux perf setup + + if model_kwargs: + # To suppress the warning of tokenizers + os.environ["TOKENIZERS_PARALLELISM"] = os.environ.get( + "TOKENIZERS_PARALLELISM", "false" + ) + # overwrite parameters + samples = run_codegen( + dataset="evalperf", + n_samples=n_samples, + temperature=temperature, + **model_kwargs, + ) + + assert samples is not None, "Please provide the path to the samples" + + # Data loading + problems, expected_output = get_evalplus_data() + ptasks = get_evalperf_data() + + # Parallelism + max_workers = parallel or max(1, default_parallelism(divisor=4)) + assert 0 < max_workers < multiprocessing.cpu_count(), "Invalid max CPU workers" + + if os.path.isdir(samples): + result_path = os.path.join(samples, "evalperf_results.json") + else: + assert samples.endswith(".jsonl") + result_path = samples.replace(".jsonl", "_evalperf_results.json") + brief_result_path = result_path.replace( + "evalperf_results.json", "evalperf_results.brief.json" + ) + + # resume results + eval_results = {} + if not i_just_wanna_run and os.path.exists(result_path): + resumed_result = json.load(open(result_path, "r")) + if ( + resumed_result["n_samples"] == n_samples + and resumed_result["temperature"] == temperature + and resumed_result["min_correct"] == min_correct + and resumed_result["max_profile"] == max_profile + ): + eval_results = resumed_result["eval"] + for etask in eval_results: + ptasks.pop(etask, None) + + rich.print(f"Resumed {len(eval_results)} results from {result_path}") + + # Load model's samples: task_id -> a list of samples + sample_iter = stream_jsonl(samples) + samples = defaultdict(list) + for task in sample_iter: + samples[task["task_id"].replace("_", "/")].append(task["solution"]) + samples = {k: v[:n_samples] for k, v in samples.items()} + + # assert each task has n_samples + for task_id, s in samples.items(): + assert len(s) == n_samples, f"{task_id} has {len(s)} samples != {n_samples}" + + # Initialize eval_results + for task_id, ptask in ptasks.items(): + eval_results[task_id] = { + "task_id": task_id, + "results": [], + "ref": [ + {"solution": s, "score": r, "_num_cpu_instructions": None} + for s, r in zip(ptask["reference"], ptask["scores"]) + ], + "dps": None, + "dps_norm": None, + "pass@1": None, + "n_profiled": None, + } + + rule("Correctness Checking...") + with progress("Correctness") as p: + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [ + executor.submit( + correctness_worker, + task_id, + samples[task_id], + problems[task_id], + expected_output[task_id], + ) + for task_id in ptasks + ] + + for future in p.track(as_completed(futures), total=len(futures)): + task_id, results = future.result() + eval_results[task_id]["results"] = results + eval_results[task_id]["pass@1"] = ( + 100 * len([r for r in results if r["pass"]]) / n_samples + ) + + rule("EvalPerf Configurations") + if lazy_evaluation: + rich.print( + "[bold yellow]Lazy evaluation is enabled[/]: " + "Fast evaluation without enumeratively checking reference order consistency." + ) + + table_print( + "Configurations", + { + "Max CPU": max_workers, + "#Tasks": len(ptasks), + "#Samples per task": n_samples, + "Min correct": min_correct, + "Max profile": max_profile, + "Result path": result_path, + }, + ) + + rich.print(f"IDs of tasks to evaluate: {list(ptasks.keys())}") + rule("Evaluation Start") + undone = [] + with progress("Profiling") as p: + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for task_id, ptask in ptasks.items(): + n_pass = len([r for r in eval_results[task_id]["results"] if r["pass"]]) + if n_pass < min_correct: + rich.print( + f"{task_id}: [bold yellow]{n_pass} < {min_correct} correct solutions, skipped[/]" + ) + continue + futures.append( + executor.submit( + perf_worker, + task_id, + ptask, + eval_results[task_id], + lazy_evaluation, + max_profile, + ) + ) + undone.append(task_id) + rich.print(f"{task_id}: Queued") + + for future in p.track(as_completed(futures), total=len(futures)): + result = future.result() + eval_results[result["task_id"]] = result + undone.remove(result["task_id"]) + if undone and len(undone) < max_workers: + print(f"Still running: {undone}") + + rule("Evaluation Summary") + dps = mean(not_none([res["dps"] for res in eval_results.values()])) + dps_norm = mean(not_none([res["dps_norm"] for res in eval_results.values()])) + pass_1 = mean(not_none([res["pass@1"] for res in eval_results.values()])) + n_evalperfed = len(not_none([res["dps"] for res in eval_results.values()])) + + table_print( + "EvalPerf Summary", + { + "DPS": f"{dps:.1f}", + "DPS_norm": f"{dps_norm:.1f}", + "Pass@1": f"{pass_1:.1f}%", + "#EvalPerf-ed tasks": f"{n_evalperfed} / {len(eval_results)}", + "min_correct": min_correct, + "n_samples": n_samples, + "temperature": temperature, + }, + ) + + # Save full results + with open(result_path, "w") as f: + f.write( + json.dumps( + { + "date": datetime.now().strftime("%Y-%m-%d %H:%M"), + "n_samples": n_samples, + "temperature": temperature, + "min_correct": min_correct, + "max_profile": max_profile, + "eval": eval_results, + } + ) + ) + rich.print(f"Full results have been saved to {result_path}") + + # Save brief results + with open(brief_result_path, "w") as f: + f.write( + json.dumps( + { + "date": datetime.now().strftime("%Y-%m-%d %H:%M"), + "config": { + "n_samples": n_samples, + "temperature": temperature, + "min_correct": min_correct, + "max_profile": max_profile, + }, + "summary": { + "dps": dps, + "dps_norm": dps_norm, + "pass@1": pass_1, + }, + "eval": { + task_id: { + "dps": res["dps"], + "dps_norm": res["dps_norm"], + "pass@1": res["pass@1"], + "profiled": [ + { + "solution": r["solution"], + "matching_cluster_idx": r["matching_cluster_idx"], + } + for r in res["results"] + if r["profiled"] + ], + } + for task_id, res in eval_results.items() + }, + } + ) + ) + + rich.print(f"Brief results have been saved to {brief_result_path}") + + rule("To visualize win-rates and pair-wise DPS, run:") + rich.print( + Syntax( + f"""\ +git clone git@github.com:evalplus/evalplus.github.io.git +git --git-dir=evalplus.github.io/.git pull +cp {brief_result_path} evalplus.github.io/results/evalperf +python evalplus.github.io/results/evalperf/stats.py +python -m http.server -d evalplus.github.io {get_free_port()}""", + "bash", + ) + ) + + +def main(): + from fire import Fire + + Fire(script) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/evaluate.py b/opencompass/datasets/evalplus/evaluate.py new file mode 100644 index 00000000..d1389cbf --- /dev/null +++ b/opencompass/datasets/evalplus/evaluate.py @@ -0,0 +1,373 @@ +import json +import multiprocessing +import os +import pickle +import threading +import time +from collections import Counter, defaultdict +from concurrent.futures import ProcessPoolExecutor, as_completed +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple +from warnings import warn + +import numpy as np +from termcolor import cprint +from tqdm import tqdm + +from opencompass.datasets.evalplus.codegen import run_codegen +from opencompass.datasets.evalplus.config import * +from opencompass.datasets.evalplus.data import ( + get_human_eval_plus, + get_human_eval_plus_hash, + get_mbpp_plus, + get_mbpp_plus_hash, + load_solutions, +) +from opencompass.datasets.evalplus.data.mbpp import mbpp_serialize_inputs +from opencompass.datasets.evalplus.data.utils import CACHE_DIR +from opencompass.datasets.evalplus.eval import ( + PASS, + compatible_eval_result, + estimate_pass_at_k, + untrusted_check, +) +from opencompass.datasets.evalplus.eval._special_oracle import MBPP_OUTPUT_NOT_NONE_TASKS +from opencompass.datasets.evalplus.gen.util import trusted_exec + +# 1st item: the status +# 2nd item (optional): the detailed pass/fail boolean for each input +Result = Tuple[str, List[bool]] + +def get_groundtruth(problems, hashcode, tasks_only_output_not_none): + cache_file = os.path.join(CACHE_DIR, f"{hashcode}.pkl") + if os.path.exists(cache_file): + print(f"Load from ground-truth from {cache_file}") + with open(cache_file, "rb") as f: + return pickle.load(f) + + os.makedirs(CACHE_DIR, exist_ok=True) + print("Computing expected output...") + tbegin = time.time() + expected_output = {} + for task_id, problem in problems.items(): + oracle = {} + oracle["base"], oracle["base_time"] = trusted_exec( + problem["prompt"] + problem["canonical_solution"], + problem["base_input"], + problem["entry_point"], + record_time=True, + output_not_none=problem["entry_point"] in tasks_only_output_not_none, + ) + + oracle["plus"], oracle["plus_time"] = trusted_exec( + problem["prompt"] + problem["canonical_solution"], + problem["plus_input"], + problem["entry_point"], + record_time=True, + output_not_none=problem["entry_point"] in tasks_only_output_not_none, + ) + expected_output[task_id] = oracle + #print('oracle', oracle) + print(f"Expected outputs computed in {time.time() - tbegin:.2f}s") + + with open(cache_file, "wb") as f: + pickle.dump(expected_output, f) + # print('get_groundtruth', expected_output) + return expected_output + +def check_correctness( + dataset: str, + completion_id: int, + problem: Dict[str, Any], + solution: str, + expected_output: Dict[str, List], + base_only=False, + fast_check=False, + identifier=None, + min_time_limit: float = DEFAULT_MIN_TIME_LIMIT, + gt_time_limit_factor: float = DEFAULT_GT_TIME_LIMIT_FACTOR, +) -> Dict[str, Result]: # {...}, "base" | "plus" -> (status, details) + ret = { + "completion_id": completion_id, + "task_id": problem["task_id"], + "_identifier": identifier, + "solution": solution, + } + ret["base"] = untrusted_check( + dataset, + solution, + problem["base_input"], + problem["entry_point"], + expected=expected_output["base"], + atol=problem["atol"], + ref_time=expected_output["base_time"], + fast_check=fast_check, + min_time_limit=min_time_limit, + gt_time_limit_factor=gt_time_limit_factor, + ) + + if not base_only: + ret["plus"] = untrusted_check( + dataset, + solution, + problem["plus_input"], + problem["entry_point"], + expected=expected_output["plus"], + atol=problem["atol"], + ref_time=expected_output["plus_time"], + fast_check=fast_check, + min_time_limit=min_time_limit, + gt_time_limit_factor=gt_time_limit_factor, + ) + + return ret + + +def evaluate( + dataset: str, + samples: Optional[str] = None, + base_only: bool = False, + parallel: Optional[int] = None, + i_just_wanna_run: bool = False, + test_details: bool = False, + min_time_limit: float = DEFAULT_MIN_TIME_LIMIT, + gt_time_limit_factor: float = DEFAULT_GT_TIME_LIMIT_FACTOR, + mini: bool = False, + noextreme: bool = False, + version: str = "default", + output_file: Optional[str] = None, + lang: str = "ar", + **model_kwargs, +): + if model_kwargs: + print('model_kwargs', model_kwargs) + # To suppress the warning of tokenizers + os.environ["TOKENIZERS_PARALLELISM"] = os.environ.get( + "TOKENIZERS_PARALLELISM", "false" + ) + samples = run_codegen( + dataset=dataset, + **model_kwargs, + ) + assert samples is not None, "No samples provided" + + n_workers = parallel or max(1, multiprocessing.cpu_count() // 2) + + if os.path.isdir(samples): + result_path = os.path.join(samples, "eval_results.json") + else: + assert samples.endswith(".jsonl") + result_path = samples.replace(".jsonl", "_eval_results.json") + + if output_file is not None: + result_path = output_file + + if os.path.isfile(result_path) and not i_just_wanna_run: + print(f"Load from previous results from {result_path}") + with open(result_path, "r") as f: + results = json.load(f) + + results = compatible_eval_result(results) + else: + if dataset == "humaneval": + print('get_human_eval_plus') + problems = get_human_eval_plus( + mini=mini, noextreme=noextreme, version=version, lang=lang + ) + dataset_hash = get_human_eval_plus_hash( + mini=mini, noextreme=noextreme, version=version + ) + expected_output = get_groundtruth(problems, dataset_hash, []) + elif dataset == "mbpp": + problems = get_mbpp_plus(mini=mini, noextreme=noextreme, version=version) + dataset_hash = get_mbpp_plus_hash( + mini=mini, noextreme=noextreme, version=version + ) + expected_output = get_groundtruth( + problems, + dataset_hash, + MBPP_OUTPUT_NOT_NONE_TASKS, + ) + + results = { + "date": datetime.now().strftime("%Y-%m-%d %H:%M"), + "hash": dataset_hash, + "eval": {}, + } + + with ProcessPoolExecutor(max_workers=n_workers) as executor: + futures = [] + completion_id = Counter() + n_samples = 0 + eval_results = defaultdict(list) # task_id -> + remainings = set() + + print("Reading samples...") + print('samples', samples) + for sample in tqdm(load_solutions(samples)): + task_id = sample["task_id"] + if task_id not in problems: + warn( + f"Task {task_id} is found in the samples but not found in the dataset" + ) + continue + solution = ( + sample["solution"] + if "solution" in sample + else problems[task_id]["prompt"] + sample["completion"] + ) + remainings.add(sample["_identifier"]) + args = ( + dataset, + completion_id[task_id], + problems[task_id], + solution, + expected_output[task_id], + base_only, + not test_details, # fast_check + sample["_identifier"], + min_time_limit, + gt_time_limit_factor, + ) + futures.append(executor.submit(check_correctness, *args)) + completion_id[task_id] += 1 + n_samples += 1 + + assert n_samples == len(remainings), "Missing problems in unfinished" + assert len(completion_id) == len(problems), "Missing problems in samples" + + def stucking_checker(): + while remainings: + last_size = len(remainings) + time.sleep(20) + if last_size != len(remainings) or len(remainings) == 0: + continue + # Potential stucking + warn("No samples had finished testing in the last 20s") + warn(f"{len(remainings)} samples to be tested: {remainings}") + + threading.Thread(target=stucking_checker).start() + + for future in tqdm(as_completed(futures), total=n_samples): + result = future.result() + print('result 253', result) + remainings.remove(result["_identifier"]) + eval_results[result["task_id"]].append(result) + + # sort the results for each problem by completion_id + for task_id, task_results in eval_results.items(): + task_results.sort(key=lambda x: x["completion_id"]) + results["eval"][task_id] = [] + for res in task_results: + + def get_failed_tests(stat, details, inputs) -> List[Any]: + if stat == PASS or not details: + return [] + + if test_details: + return [ + inputs[i] for i in range(len(details)) if not details[i] + ] + + # else => simply return the only and the last fail test + return [inputs[len(details) - 1]] + + base_stat, base_details = res["base"] + base_fail_tests = get_failed_tests( + base_stat, base_details, problems[task_id]["base_input"] + ) + + # initialize plus tests + plus_stat = None + plus_fail_tests = [] + + # with plus tests + if not base_only: + plus_stat, plus_details = res["plus"] + plus_fail_tests = get_failed_tests( + plus_stat, plus_details, problems[task_id]["plus_input"] + ) + + if dataset == "mbpp": + base_fail_tests = mbpp_serialize_inputs(task_id, base_fail_tests) + plus_fail_tests = mbpp_serialize_inputs(task_id, plus_fail_tests) + + results["eval"][task_id].append( + { + "task_id": task_id, + "solution": res["solution"], + "base_status": base_stat, + "plus_status": plus_stat, + "base_fail_tests": base_fail_tests, + "plus_fail_tests": plus_fail_tests, + } + ) + + # Calculate pass@k. + total = np.array([len(r) for r in results["eval"].values()]) + base_correct = [] + new_correct = [] + + for res in results["eval"].values(): + bc = sum([r["base_status"] == PASS for r in res]) + base_correct.append(bc) + if not base_only: + new_correct.append( + sum( + [ + res[i]["base_status"] == res[i]["plus_status"] == PASS + for i in range(len(res)) + ] + ) + ) + base_correct = np.array(base_correct) + + pass_at_k = { + f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean() + for k in [1, 10, 100] + if total.min() >= k + } + cprint(f"{dataset} (base tests)", "red") + for k, v in pass_at_k.items(): + cprint(f"{k}:\t{v:.3f}", "red") + results["pass_at_k"] = {"base": pass_at_k} + + if new_correct: + cprint(f"{dataset}+ (base + extra tests)", "green") + pass_at_k = { + f"pass@{k}": estimate_pass_at_k(total, np.array(new_correct), k).mean() + for k in [1, 10, 100] + if (total >= k).all() + } + for k, v in pass_at_k.items(): + cprint(f"{k}:\t{v:.3f}", "green") + results["pass_at_k"]["plus"] = pass_at_k + + # save results + if os.path.isfile(result_path) and i_just_wanna_run: + decision = "" + while decision.lower() not in ["y", "n"]: + print(f"{result_path} already exists. Press [Y/N] to overwrite or exit...") + decision = input() + + if decision.lower() == "y": + # mv the file to a backup + new_path = result_path + ".bak" + while os.path.isfile(new_path): + new_path += ".bak" + os.rename(result_path, new_path) + print(f"Backup {result_path} to {new_path}") + + if not os.path.isfile(result_path): + with open(result_path, "w") as f: + json.dump(results, f) + + +def main(): + from fire import Fire + + Fire(evaluate) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/gen/__init__.py b/opencompass/datasets/evalplus/gen/__init__.py new file mode 100644 index 00000000..765cc2c5 --- /dev/null +++ b/opencompass/datasets/evalplus/gen/__init__.py @@ -0,0 +1,21 @@ +import copy +from typing import Any, List + + +class BaseGen(object): + def __init__(self, inputs: List[Any], entry_point: str, contract: str): + """Initializing a input mutator. + + Args: + inputs (List[Any]): The set of initial inputs (i.e., seeds) + entry_point (str): The function name to invoke with the input + contract (str): The contract to verify input validity + """ + self.contract = contract + self.entry_point = entry_point + self.seed_pool: List[Any] = copy.deepcopy(inputs) + self.new_inputs = [] + self.seed_hash = set([hash(str(x)) for x in self.seed_pool]) + + def generate(self, num: int) -> List[Any]: + raise NotImplementedError diff --git a/opencompass/datasets/evalplus/gen/chatgpt_gen.py b/opencompass/datasets/evalplus/gen/chatgpt_gen.py new file mode 100644 index 00000000..0b4118eb --- /dev/null +++ b/opencompass/datasets/evalplus/gen/chatgpt_gen.py @@ -0,0 +1,78 @@ +import ast +import random +from typing import List + +import openai +from openai.types.chat import ChatCompletion + +from evalplus.data.utils import to_raw +from evalplus.gen import BaseGen +from evalplus.gen.util import trusted_check_exec +from evalplus.gen.util.openai_request import make_auto_request + + +class ChatGPTGen(BaseGen): + def __init__(self, inputs: List, signature: str, contract_code: str, gd_code: str): + super().__init__(inputs, signature, contract_code) + self.gd_code = gd_code + self.prompt_messages = [ + "Please generate complex inputs to test the function.", + "Please generate corner case inputs to test the function.", + "Please generate difficult inputs to test the function.", + ] + self.iteration = 20 + self.client = openai.Client() + + def seed_selection(self) -> List: + # get 5 for now. + return random.sample(self.seed_pool, k=min(len(self.seed_pool), 5)) + + @staticmethod + def _parse_ret(ret: ChatCompletion) -> List: + rets = [] + output = ret.choices[0].message.content + if "```" in output: + for x in output.split("```")[1].splitlines(): + if x.strip() == "": + continue + try: + # remove comments + input = ast.literal_eval(f"[{x.split('#')[0].strip()}]") + except: # something wrong. + continue + rets.append(input) + return rets + + def chatgpt_generate(self, selected_inputs: List) -> List: + # append the groundtruth function + # actually it can be any function (maybe we can generate inputs for each llm generated code individually) + message = f"Here is a function that we want to test:\n```\n{self.gd_code}\n```" + str_inputs = "\n".join( + [ + ", ".join([f"'{to_raw(i)}'" if type(i) == str else str(i) for i in x]) + for x in selected_inputs + ] + ) + message += f"\nThese are some example inputs used to test the function:\n```\n{str_inputs}\n```" + message += f"\n{random.choice(self.prompt_messages)}" + ret = make_auto_request( + self.client, + message=message, + model="gpt-3.5-turbo", + max_tokens=256, + response_format={"type": "text"}, + ) + return self._parse_ret(ret) + + def generate(self, num: int): + while len(self.new_inputs) < num and self.iteration >= 0: + seeds = self.seed_selection() + new_inputs = self.chatgpt_generate(seeds) + for new_input in new_inputs: + if hash(str(new_input)) not in self.seed_hash: + if trusted_check_exec(self.contract, [new_input], self.entry_point): + self.seed_pool.append(new_input) + self.seed_hash.add(hash(str(new_input))) + self.new_inputs.append(new_input) + self.iteration -= 1 + return self.new_inputs[:num] diff --git a/opencompass/datasets/evalplus/gen/mut_gen.py b/opencompass/datasets/evalplus/gen/mut_gen.py new file mode 100644 index 00000000..93eb5c51 --- /dev/null +++ b/opencompass/datasets/evalplus/gen/mut_gen.py @@ -0,0 +1,30 @@ +import random +from abc import abstractmethod +from typing import Any, List + +from evalplus.gen import BaseGen +from evalplus.gen.util import trusted_check_exec + + +class MutateGen(BaseGen): + def __init__(self, inputs: List, signature: str, contract_code: str): + super().__init__(inputs, signature, contract_code) + + def seed_selection(self): + # random for now. + return random.choice(self.seed_pool) + + @abstractmethod + def mutate(self, seed_input: Any) -> Any: + pass + + def generate(self, num: int) -> List[Any]: + while len(self.new_inputs) < num: + seed = self.seed_selection() + new_input = self.mutate(seed) + if hash(str(new_input)) not in self.seed_hash: + if trusted_check_exec(self.contract, [new_input], self.entry_point): + self.seed_pool.append(new_input) + self.seed_hash.add(hash(str(new_input))) + self.new_inputs.append(new_input) + return self.new_inputs[:num] diff --git a/opencompass/datasets/evalplus/gen/type_mut.py b/opencompass/datasets/evalplus/gen/type_mut.py new file mode 100644 index 00000000..d978ed91 --- /dev/null +++ b/opencompass/datasets/evalplus/gen/type_mut.py @@ -0,0 +1,340 @@ +import copy +import random +import string +import time +from typing import Any, Dict, List, Set, Tuple + +from multipledispatch import dispatch + +from evalplus.gen.mut_gen import MutateGen +from evalplus.gen.util import trusted_check_exec + +MAX_MULTI_STEP_SIZE = 5 +MUTATE_BOUND_SIZE = 8 + +NoneType = type(None) + + +# decorator to use ingredients +class use_ingredient: + def __init__(self, prob: float): + assert 0 <= prob <= 0.95 + self.prob = prob + + def __call__(obj, func): + def wrapper(self, seed_input): + if random.random() < obj.prob and self.ingredients[type(seed_input)]: + return random.choice(list(self.ingredients[type(seed_input)])) + else: + return func(self, seed_input) + + return wrapper + + +class TypedMutGen(MutateGen): + def __init__(self, inputs: List, signature: str, contract_code: str): + super().__init__(inputs, signature, contract_code) + self.timeout = 60 * 60 # 1 hour + self.ingredients = { + int: set(), + float: set(), + str: set(), + complex: set(), + } + for x in inputs: + self.fetch_ingredient(x) + + def seed_selection(self): + # random for now. + return random.choice(self.seed_pool) + + def mutate(self, seed_input: Any) -> List: + new_input = copy.deepcopy(seed_input) + + patience = MUTATE_BOUND_SIZE + while new_input == seed_input or patience == 0: + new_input = self.typed_mutate(new_input) + patience -= 1 + + return new_input + + ######################### + # Type-aware generation # + ######################### + @dispatch(NoneType) + def typed_gen(self, _): + return None + + @dispatch(int) + def typed_gen(self, _): + @use_ingredient(0.5) + def _impl(*_): + return random.randint(-100, 100) + + return _impl(self, _) + + @dispatch(float) + def typed_gen(self, _): + @use_ingredient(0.5) + def _impl(*_): + return random.uniform(-100, 100) + + return _impl(self, _) + + @dispatch(bool) + def typed_gen(self, _): + return random.choice([True, False]) + + @dispatch(str) + def typed_gen(self, _): + @use_ingredient(0.5) + def _impl(*_): + return "".join( + random.choice(string.ascii_letters) + for _ in range(random.randint(0, 10)) + ) + + return _impl(self, _) + + def any_gen(self): + # weighted choose + choice = random.choices( + [ + True, + 1, + 1.1, + "str", + [], # list + tuple(), # tuple + dict(), # dict + None, # None + ], + [0.2, 0.2, 0.2, 0.2, 0.05, 0.05, 0.05, 0.05], + )[0] + return self.typed_gen(choice) + + @dispatch(list) + def typed_gen(self, _): + ret = [] + size = random.randint(0, 10) + if random.randint(0, 4) == 0: # heterogeneous + for _ in range(size): + ret.append(self.any_gen()) + else: # homogeneous + t = random.choice([bool(), int(), float(), str()]) + for _ in range(size): + ret.append(self.typed_gen(t)) + return ret + + @dispatch(tuple) + def typed_gen(self, _): + return tuple(self.typed_gen([])) + + # NOTE: disable set for now as Steven is too weak in Python (/s) + # @dispatch(set) + # def typed_gen(self, _): + # return set(self.typed_gen([])) + + @dispatch(dict) + def typed_gen(self, _): + ret = dict() + values = self.typed_gen([]) + # NOTE: Assumption: nobody uses dict with heterogeneous keys + # NOTE: Assumption: nobody uses dict with boolean keys + key_type = random.choice([int(), float(), str()]) + for v in values: + ret[self.typed_gen(key_type)] = self.typed_gen(v) + return ret + + ######################## + # Type-aware mutation # + ######################## + # Simple primitives + @dispatch(int) + def typed_mutate(self, seed_input: int): + @use_ingredient(0.5) + def _impl(_, seed_input: int): + return seed_input + random.randint(-1, 1) + + return _impl(self, seed_input) + + @dispatch(float) + def typed_mutate(self, seed_input: float): + @use_ingredient(0.5) + def _impl(_, seed_input: float): + if random.randint(0, 1): + return seed_input + random.uniform(-1, 1) + return seed_input * (1 + random.uniform(-0.5, 0.5)) + + return _impl(self, seed_input) + + @dispatch(complex) + def typed_mutate(self, seed_input: complex): + @use_ingredient(0.5) + def _impl(_, seed_input: complex): + imag = seed_input.imag + random.uniform(-1, 1) + return complex(0, imag) + + return _impl(self, seed_input) + + @dispatch(bool) + def typed_mutate(self, seed_input: bool): + return random.choice([True, False]) + + @dispatch(NoneType) + def typed_mutate(self, seed_input: NoneType): + return None + + # List-like + @dispatch(list) + def typed_mutate(self, seed_input: List): + if len(seed_input) == 0: + return self.typed_gen([]) + + choice = random.randint(0, 3) + idx = random.randint(0, len(seed_input) - 1) + if choice == 0: # remove one element + seed_input.pop(random.randint(0, len(seed_input) - 1)) + elif choice == 1 and len(seed_input) > 0: # add one mutated element + seed_input.insert( + random.randint(0, len(seed_input) - 1), + self.typed_mutate(seed_input[idx]), + ) + elif choice == 2 and len(seed_input) > 0: # repeat one element + seed_input.append(seed_input[idx]) + else: # inplace element change + seed_input[idx] = self.typed_mutate(seed_input[idx]) + return seed_input + + @dispatch(tuple) + def typed_mutate(self, seed_input: Tuple): + return tuple(self.typed_mutate(list(seed_input))) + + # String + @dispatch(str) + def typed_mutate(self, seed_input: str): + @use_ingredient(0.4) + def _impl(_, seed_input: str): + choice = random.randint(0, 2) if seed_input else 0 + if choice == 0 and self.ingredients[str]: # insert an ingredient + idx = random.randint(0, len(seed_input)) + return ( + seed_input[:idx] + + random.choice(list(self.ingredients[str])) + + seed_input[idx:] + ) + # other choices assume len(seed_input) > 0 + elif choice == 1: # replace a substring with empty or mutated string + start = random.randint(0, len(seed_input) - 1) + end = random.randint(start + 1, len(seed_input)) + mid = ( + "" + if random.randint(0, 1) + else self.typed_mutate(seed_input[start:end]) + ) + return seed_input[:start] + mid + seed_input[end:] + elif choice == 2: # repeat one element + idx = random.randint(0, len(seed_input) - 1) + return ( + seed_input[:idx] + + seed_input[random.randint(0, len(seed_input) - 1)] + + seed_input[idx:] + ) + + # random char + return self.typed_gen(str()) + + return _impl(self, seed_input) + + # Set + @dispatch(set) + def typed_mutate(self, seed_input: Set): + return set(self.typed_mutate(list(seed_input))) + + # Dict + @dispatch(dict) + def typed_mutate(self, seed_input: Dict): + if len(seed_input) == 0: + return self.typed_gen(dict()) + + choice = random.randint(0, 2) + if choice == 0: # remove a kv + del seed_input[random.choice(list(seed_input.keys()))] + elif choice == 1: # add a kv + k = self.typed_mutate(random.choice(list(seed_input.keys()))) + v = self.typed_mutate(random.choice(list(seed_input.values()))) + seed_input[k] = v + elif choice == 2: # inplace value change + k0, v0 = random.choice(list(seed_input.items())) + seed_input[k0] = self.typed_mutate(v0) + return seed_input + + ############################################ + # Fetching ingredients to self.ingredients # + ############################################ + def fetch_ingredient(self, seed_input): + self.typed_fetch(seed_input) + + @dispatch(int) + def typed_fetch(self, seed_input: int): + self.ingredients[int].add(seed_input) + + @dispatch(float) + def typed_fetch(self, seed_input: float): + self.ingredients[float].add(seed_input) + + @dispatch(complex) + def typed_fetch(self, seed_input: complex): + self.ingredients[complex].add(seed_input) + + @dispatch(str) + def typed_fetch(self, seed_input: str): + self.ingredients[str].add(seed_input) + for token in seed_input.strip().split(): + self.ingredients[str].add(token) + + # List-like + def _fetch_list_like(self, seed_input): + for x in seed_input: + if self.typed_fetch.dispatch(type(x)): + self.fetch_ingredient(x) + + @dispatch(list) + def typed_fetch(self, seed_input: List): + self._fetch_list_like(seed_input) + + @dispatch(tuple) + def typed_fetch(self, seed_input: Tuple): + self._fetch_list_like(seed_input) + + # NOTE: disable set for now as Steven is too weak in Python (/s) + # @dispatch(set) + # def typed_fetch(self, seed_input: Set): + # self._fetch_list_like(seed_input) + + # Dict + @dispatch(dict) + def typed_fetch(self, seed_input: Dict): + self._fetch_list_like(seed_input.keys()) + self._fetch_list_like(seed_input.values()) + + def generate(self, num: int): + start = time.time() + num_generated = 1 + while len(self.new_inputs) < num and time.time() - start < self.timeout: + if num_generated % 1000 == 0: + print( + f"generated {num_generated} already with {len(self.new_inputs)} new inputs ... " + ) + new_input = self.seed_selection() + # Multi-step instead of single-step + for _ in range(random.randint(1, MAX_MULTI_STEP_SIZE)): + new_input = self.mutate(new_input) + num_generated += 1 + if hash(str(new_input)) not in self.seed_hash: + if trusted_check_exec(self.contract, [new_input], self.entry_point): + self.typed_fetch(new_input) + self.seed_pool.append(new_input) + self.new_inputs.append(new_input) + self.seed_hash.add(hash(str(new_input))) + return self.new_inputs[:num] diff --git a/opencompass/datasets/evalplus/gen/util/__init__.py b/opencompass/datasets/evalplus/gen/util/__init__.py new file mode 100644 index 00000000..9bd03bf3 --- /dev/null +++ b/opencompass/datasets/evalplus/gen/util/__init__.py @@ -0,0 +1,40 @@ +import time +from copy import deepcopy + +from opencompass.datasets.evalplus.eval.utils import time_limit + + +def trusted_exec(code, inputs, entry_point, record_time=False, output_not_none=False): + """Execute trusted code in place.""" + exec_globals = {} + exec(code, exec_globals) + fn = exec_globals[entry_point] + + rtime = [] + ret = [] + for inp in inputs: + inp = deepcopy(inp) + if record_time: + start = time.time() + ret.append(fn(*inp)) + rtime.append(time.time() - start) + else: + ret.append(fn(*inp)) + + if output_not_none: + ret = [i is not None for i in ret] + + if record_time: + return ret, rtime + else: + return ret + + +def trusted_check_exec(code, inputs, entry_point): + """Check trusted_exec success.""" + try: + with time_limit(seconds=1.0): + trusted_exec(code, inputs, entry_point) + except Exception: + return False + return True diff --git a/opencompass/datasets/evalplus/inputgen.py b/opencompass/datasets/evalplus/inputgen.py new file mode 100644 index 00000000..82c344dc --- /dev/null +++ b/opencompass/datasets/evalplus/inputgen.py @@ -0,0 +1,107 @@ +"""Generate a .jsonl file where each line is a json object +representing a programming problem with a task ID ("task_id") +and a list of enhanced inputs ("inputs") for that task. +""" + +import argparse +import json +import os + +from opencompass.datasets.evalplus.data.mbpp import mbpp_serialize_inputs +from opencompass.datasets.evalplus.gen.chatgpt_gen import ChatGPTGen +from opencompass.datasets.evalplus.gen.type_mut import TypedMutGen + +class SetEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return list(obj) + return json.JSONEncoder.default(self, obj) + + +# Used for MBPP as MBPP's prompt is not a formal function signature +def insert_contract_into_code(entry_point, code, contract): + lines = code.split("\n") + index = lines.index( + next(line for line in lines if line.startswith(f"def {entry_point}")) + ) + lines.insert(index + 1, contract) + return "\n".join(lines) + + +def input_generation(args, problems): + with open(args.output, "w") as file: + for problem in problems.values(): + new_input = {} + task_id = problem["task_id"] + print(f"generating inputs for {task_id} ...") + # by default we do not include constraints in the prompt (code) + code = problem["prompt"] + problem["canonical_solution"] + # but we use c_code to include contract which checks input validity at execution time + if args.dataset == "humaneval": + c_code = ( + problem["prompt"] + + problem["contract"] + + problem["canonical_solution"] + ) + elif args.dataset == "mbpp": + c_code = problem["prompt"] + insert_contract_into_code( + entry_point=problem["entry_point"], + code=problem["canonical_solution"], + contract=problem["contract"], + ) + + # first generate chatgpt + input_gen = ChatGPTGen( + problem["base_input"], problem["entry_point"], c_code, code + ).generate(args.chatgpt_len) + # generate mutation next + + if input_gen is None or len(input_gen) == 0: + new_input["task_id"] = task_id + new_input["inputs"] = {} + file.write(json.dumps(new_input, cls=SetEncoder) + "\n") + continue + + input_gen.extend( + TypedMutGen(input_gen, problem["entry_point"], c_code).generate( + args.mut_len + ) + ) + print(f"generated {len(input_gen)} inputs") + new_input["task_id"] = task_id + if args.dataset == "mbpp": + new_input["inputs"] = mbpp_serialize_inputs(task_id, input_gen) + new_input["inputs"] = input_gen + file.write(json.dumps(new_input, cls=SetEncoder) + "\n") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset", required=True, type=str, choices=["humaneval", "mbpp"] + ) + parser.add_argument("--chatgpt_len", required=True, type=int) + parser.add_argument("--mut_len", required=True, type=int) + parser.add_argument("--output", type=str, help="Output .jsonl path") + args = parser.parse_args() + + problems = None + if args.dataset == "humaneval": + from evalplus.data import get_human_eval_plus + + # Allow it to be incomplete + problems = get_human_eval_plus(err_incomplete=False) + args.output = args.output or "HumanEvalPlusInputs.jsonl" + + if args.dataset == "mbpp": + from evalplus.data import get_mbpp_plus + + problems = get_mbpp_plus(err_incomplete=False) + args.output = args.output or "MbppPlusInput.jsonl" + + assert not os.path.isfile(args.output), f"{args.output} already exists!" + input_generation(args, problems) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/lecacy_sanitize.py b/opencompass/datasets/evalplus/lecacy_sanitize.py new file mode 100644 index 00000000..f8c50db0 --- /dev/null +++ b/opencompass/datasets/evalplus/lecacy_sanitize.py @@ -0,0 +1,200 @@ +"""Legacy version of post-processing LLM-generated Python code. +This sanitizer is implemented using regex and string manipulation. +You might want to use the latest tree-sitter-based sanitizer (evalplus.sanitize) instead. +""" + +import os +import pathlib +import re +from typing import List, Optional + +from tqdm import tqdm + +from opencompass.datasets.evalplus.data import ( + get_human_eval_plus, + get_mbpp_plus, + load_solutions, + write_directory, + write_jsonl, +) +from opencompass.datasets.evalplus.syncheck import syntax_check + +def remove_unindented_lines( + code: str, protect_before: str, execeptions: List[str], trim_tails: List[str] +) -> str: + lines = code.splitlines() + cut_idx = [] + cut_enabled = False + for i, line in enumerate(lines): + if not cut_enabled and line.startswith(protect_before): + cut_enabled = True + continue + if line.strip() == "": + continue + if any(line.startswith(e) for e in execeptions): + continue + + lspace = len(line) - len(line.lstrip()) + if lspace == 0: + cut_idx.append(i) + + if any(line.rstrip().startswith(t) for t in trim_tails): + # cut off everything behind + cut_idx.extend(list(range(i, len(lines)))) + break + + return "\n".join([line for i, line in enumerate(lines) if i not in cut_idx]) + + +def to_four_space_indents(old_code): + new_code = "" + for line in old_code.splitlines(): + lspace = len(line) - len(line.lstrip()) + if lspace == 3: + new_code += " " + new_code += line + "\n" + return new_code + + +def sanitize( + old_code: str, + entry_point: str, + rm_prefix_lines: Optional[str] = None, + eofs: List = None, +): + new_code = old_code + if rm_prefix_lines is not None: + new_code = "\n".join( + [ + line + for line in old_code.splitlines() + if not line.startswith(rm_prefix_lines) + ] + ) + + new_code = "\n" + new_code + def_left = "def " + entry_point + + # basic handling of chat output + new_code = new_code.replace("\n```python\n", "\n```\n") + for chunk in new_code.split("\n```\n"): + if def_left in chunk: + new_code = chunk + break + + chunks = [chunk for chunk in re.split(f"{def_left}\\s*\\(", new_code)] + # TODO: having return does not mean this is complete + bodies = [chunk for chunk in chunks[1:] if " return " in chunk.split("\ndef")[0]] + def_left = def_left + "(" + new_code = def_left + def_left.join(bodies) if len(bodies) > 0 else "" # fn + impl + new_code = to_four_space_indents(new_code) + + for eof in eofs or []: + new_code = new_code.split(eof)[0] + + # remove lines starting from the first unindented line after def_left + new_code = remove_unindented_lines( + new_code, + protect_before=def_left, + execeptions=["def ", "import ", "from "], + trim_tails=['"""', "if", "print"], + ) + new_code = chunks[0] + new_code + + # cut all functions that are not syntactically correct && not the entry point + parts = new_code.split("\ndef ") + includes = [parts[0]] + for fn in new_code.split("\ndef ")[1:]: + if ( + fn.strip().startswith(entry_point + " ") + or fn.strip().startswith(entry_point + "(") + or syntax_check("\ndef " + fn) + ): + includes.append(fn) + new_code = "\ndef ".join(includes) + return new_code.strip() + + +def script( + samples: str, + eofs: List[str] = [], + inplace: bool = False, + rm_prefix_lines: str = None, + debug_task: str = None, + mbpp_version: str = "default", +): + # task_id -> entry_point + entry_point = {} + dataset = {**get_human_eval_plus(), **get_mbpp_plus(version=mbpp_version)} + + for task_id, problem in dataset.items(): + entry_point[task_id] = problem["entry_point"] + + # make a new folder with "-sanitized" suffix + is_folder = os.path.isdir(samples) + target_path = pathlib.Path(samples) + if not inplace: + if is_folder: + new_name = target_path.name + "-sanitized" + else: + new_name = target_path.name.replace(".jsonl", "-sanitized.jsonl") + target_path = target_path.parent / new_name + target_path = str(target_path) + + nsan = 0 + ntotal = 0 + + new_solutions = [] + + for solution in tqdm(load_solutions(samples)): + task_id = solution["task_id"] + dbg_identifier = solution["_identifier"] + if debug_task is not None and task_id != debug_task: + continue + + ntotal += 1 + if "solution" in solution: + old_code = solution["solution"] + else: + assert "completion" in solution + old_code = dataset[task_id]["prompt"] + "\n" + solution["completion"] + + old_code = old_code.strip() + + new_code = sanitize( + old_code=old_code, + entry_point=entry_point[task_id], + rm_prefix_lines=rm_prefix_lines, + eofs=eofs, + ).strip() + + # if changed, print the message + if new_code != old_code: + msg = "Sanitized: " + dbg_identifier + if is_folder: + msg += " -> " + dbg_identifier.replace(samples, target_path) + print(msg) + nsan += 1 + + new_solutions.append({"task_id": task_id, "solution": new_code}) + + if is_folder: + write_directory(target_path, new_solutions) + else: + write_jsonl(target_path, new_solutions) + + if nsan > 0: + print(f"Sanitized {nsan} out of {ntotal} files.") + else: + print(f"All files seems valid -- no files are sanitized.") + print(f"Check the sanitized files at {target_path}") + + +def main(): + from fire import Fire + + Fire(script) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/perf/__init__.py b/opencompass/datasets/evalplus/perf/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/evalplus/perf/profile.py b/opencompass/datasets/evalplus/perf/profile.py new file mode 100644 index 00000000..7daa391a --- /dev/null +++ b/opencompass/datasets/evalplus/perf/profile.py @@ -0,0 +1,183 @@ +import time +from concurrent.futures import ProcessPoolExecutor +from multiprocessing import Process, Value, cpu_count +from platform import system +from time import perf_counter +from traceback import format_exc +from typing import Any, Callable, List, Optional + +import psutil +from cirron import Collector + +from evalplus.config import PERF_PROFILE_ROUNDS, PERF_RAM_GB_PER_PROC +from evalplus.eval.utils import ( + TimeoutException, + create_tempdir, + reliability_guard, + swallow_io, + time_limit, +) + + +def get_max_ram_gb(): + total_ram = psutil.virtual_memory().total + return total_ram / (1024**3) + + +def default_parallelism(divisor=4): + return max(1, max(cpu_count(), get_max_ram_gb() // PERF_RAM_GB_PER_PROC) // divisor) + + +def simple_test_profiler(): + # assert linux + assert system() == "Linux", "EvalPerf requires Linux's perf_event_open" + try: + with Collector(): + pass + except Exception as e: + print("It seems your system does not support instruction counting.") + print("Try this on Linux:") + print(" sudo sh -c 'echo 0 > /proc/sys/kernel/perf_event_paranoid' ") + print("Also check more info at: https://github.com/s7nfo/Cirron") + print("Re-raising the original exception...") + raise e + + +def are_profiles_broken(profiles) -> bool: + return not all(isinstance(profile, (float, int)) for profile in profiles) + + +def physical_runtime_profiler(function, test_inputs) -> float: + start = perf_counter() + for test_input in test_inputs: + function(*test_input) + return perf_counter() - start + + +def num_instruction_profiler(function, test_inputs) -> int: + with Collector() as c: + for test_input in test_inputs: + function(*test_input) + return int(c.counters.instruction_count) + + +_STAT_NONE = 0 +_STAT_START = 1 +_STAT_SUCC = 2 +_STAT_ERROR = 3 + + +def get_instruction_count_shared_mem( + profiler: Callable, + func_code: str, + entry_point: str, + test_inputs: List[Any], + timeout_second_per_test: float, + memory_bound_gb: int, + warmup_inputs: Optional[List[Any]], + # shared memory + compute_cost, # Value("d", 0.0), + progress, # Value("i", 0), +) -> Optional[float]: + + error = None + + with create_tempdir(): + # These system calls are needed when cleaning up tempdir. + import os + import shutil + + rmtree = shutil.rmtree + rmdir = os.rmdir + chdir = os.chdir + + # Disable functionalities that can make destructive changes to the test. + maximum_memory_bytes = memory_bound_gb * 1024 * 1024 * 1024 + reliability_guard(maximum_memory_bytes=maximum_memory_bytes) + exec_globals = {} + + # run (eval) the func def + exec(func_code, exec_globals) + fn = exec_globals[entry_point] + + # warmup the function + if warmup_inputs: + for _ in range(3): + fn(*warmup_inputs) + + progress.value = _STAT_START + try: # run the function + with time_limit(timeout_second_per_test): + with swallow_io(): + compute_cost.value = profiler(fn, test_inputs) + progress.value = _STAT_SUCC + except TimeoutException: + print("[Warning] Profiling hits TimeoutException") + except MemoryError: + print("[Warning] Profiling hits MemoryError") + except: + print("[CRITICAL] ! Unknown exception during profiling !") + error = format_exc() + print(error) + + if progress.value != _STAT_SUCC: + progress.value = _STAT_ERROR + + # Needed for cleaning up. + shutil.rmtree = rmtree + os.rmdir = rmdir + os.chdir = chdir + + +def profile( + func_code: str, + entry_point: str, + test_inputs: List[Any], + timeout_second_per_test: float, + memory_bound_gb: int = PERF_RAM_GB_PER_PROC, + profile_rounds: int = PERF_PROFILE_ROUNDS, + profiler: Callable = num_instruction_profiler, + warmup_inputs: Optional[List[Any]] = None, # multiple inputs +) -> List[int | float | str]: + """Profile the func_code against certain input tests. + The function code is assumed to be correct and if a string is returned, it is an error message. + """ + timeout = timeout_second_per_test * len(test_inputs) * profile_rounds + + def _run(): + compute_cost = Value("d", 0.0) + progress = Value("i", _STAT_NONE) + + p = Process( + target=get_instruction_count_shared_mem, + args=( + profiler, + func_code, + entry_point, + test_inputs, + timeout_second_per_test, + memory_bound_gb, + warmup_inputs, + # shared memory + compute_cost, + progress, + ), + ) + p.start() + p.join(timeout=timeout + 1) + if p.is_alive(): + p.terminate() + time.sleep(0.1) + + if p.is_alive(): + p.kill() + time.sleep(0.1) + + if progress.value == _STAT_SUCC: + return compute_cost.value + elif progress.value == _STAT_NONE: + return "PROFILING DID NOT START" + elif progress.value == _STAT_ERROR: + return "SOLUTION ERROR ENCOUNTERED WHILE PROFILING" + + return [_run() for _ in range(profile_rounds)] diff --git a/opencompass/datasets/evalplus/perf/sampling.py b/opencompass/datasets/evalplus/perf/sampling.py new file mode 100644 index 00000000..771a12a4 --- /dev/null +++ b/opencompass/datasets/evalplus/perf/sampling.py @@ -0,0 +1,295 @@ +import json +import os +import re +from concurrent.futures import ProcessPoolExecutor +from copy import deepcopy +from traceback import format_exc +from typing import Any, List, Optional, Tuple + +from pympler.asizeof import asizeof +from rich.syntax import Syntax +from termcolor import colored + +from evalplus.config import PERF_CURATE_TIMEOUT_SECOND, PERF_RAM_GB_PER_PROC +from evalplus.data import get_human_eval_plus, get_mbpp_plus +from evalplus.data.mbpp import mbpp_serialize_inputs +from evalplus.eval.utils import TimeoutException, reliability_guard, time_limit +from evalplus.sanitize import syntax_check +from evalplus.utils import progress + + +# this is more of a hack... rather than a "verified" implementation +def insert_contract(entry_point: str, code: str, contract: str): + # why is this so complicated? because the contract might be mis-indented... + def get_first_indent_size(source, body_char_start_idx): + assert source.strip() + indent_size = 0 + while source[body_char_start_idx - indent_size - 1] == " ": + indent_size += 1 + return indent_size + + code = code.replace("\t", " " * 4) + contract = contract.replace("\t", " " * 4) + + lines = [line for line in code.split("\n") if line.strip()] + fn_def_line = [line for line in lines if line.startswith(f"def {entry_point}")][0] + def_line_idx = lines.index(fn_def_line) + body_start_idx = code.index(code.split(fn_def_line)[1].lstrip()) + + source_indent: int = get_first_indent_size(code, body_start_idx) + contract_indent: int = get_first_indent_size( + contract, len(contract) - len(contract.lstrip()) + ) + return "\n".join( + lines[: def_line_idx + 1] + + [ + " " * max(0, source_indent - contract_indent) + cline + for cline in contract.split("\n") + if cline + ] + + [ + " " * max(0, contract_indent - source_indent) + sline + for sline in lines[def_line_idx + 1 :] + if sline + ] + ) + + +def post_process(text: str) -> Optional[str]: + """Post-process the LLM generated text to make it valid.""" + if "\n```" not in text: + return None + + # split ```python3 or ```python + text = re.split(r"\n```python3?\n", text)[1] + text = text.split("\n```")[0].strip() + + # perform syntax check + if not syntax_check(text): + print(colored("⚠️ Syntax check failed for the code below:", "red")) + print(text[:256], "..." if len(text) > 256 else "") + return None + + return text + + +# returns: +# 1. generated and validated (by the contract) inputs +# 2. whether the generator stops in a well-defined manner +# -- if False, we might want to try another generator +def sample_one_input( + ref_code_with_contract: str, + entry_point: str, + generator_code: str, + timeout_second: float = PERF_CURATE_TIMEOUT_SECOND + 1, +) -> Tuple[List[Any], bool]: + # These system calls are needed when cleaning up tempdir. + import os + import shutil + + rmtree = shutil.rmtree + rmdir = os.rmdir + chdir = os.chdir + # Disable functionalities that can make destructive changes to the test. + # :imit memory usages. + maximum_memory_bytes = PERF_RAM_GB_PER_PROC * 1024 * 1024 * 1024 + reliability_guard(maximum_memory_bytes=maximum_memory_bytes) + exec_globals = {} + + # eval the func def with contract + exec(ref_code_with_contract, exec_globals) + fn = exec_globals[entry_point] + + # eval the generator + generator_code = "from typing import *\n" + generator_code + try: + exec(generator_code, exec_globals) + generator = exec_globals["perf_input_gen"] + except Exception: + print(colored(f"⚠️ [GEN EVAL] Exception ~ {entry_point}:", "red")) + print(colored(format_exc(), "red")) + return [], False + + well_defined_exit = True + return_inputs = [] + + for fac in range(1, 27): + scale = 2**fac + print(f"[INPUT GEN] scale=2**{fac}") + try: + with time_limit(timeout_second): + test_input = generator(scale) + if not isinstance(test_input, tuple): + test_input = (test_input,) + # integers should stay in the range of 64-bit + if any( + isinstance(arg, int) and not (-(2**63) <= arg < 2**63) + for arg in test_input + ): + print(colored(f"[INPUT GEN] Int overflow against 64bit", "yellow")) + break + # hack list integer + if isinstance(test_input[0], list) and any( + not (-(2**63) <= v < 2**63) + for v in test_input[0] + if isinstance(v, int) + ): + print(colored(f"[INPUT GEN] Int overflow against 64bit", "yellow")) + break + # stop here if the input is of 64M. + INPUT_LIMIT_MB = 64 + if asizeof(test_input) > 1024 * 1024 * INPUT_LIMIT_MB: + print(colored(f"[INPUT GEN] Size > {INPUT_LIMIT_MB}MB", "yellow")) + break + except TimeoutException: + print(colored(f"[INPUT GEN] TimeoutException at scale=2**{fac}", "yellow")) + break + except MemoryError: + print(colored(f"[INPUT GEN] MemoryError at scale=2**{fac}", "yellow")) + break + except Exception: + print(colored(f"⚠️ [INPUT GEN] Exception at scale=2**{fac}", "red")) + print(colored(format_exc(), "red")) + well_defined_exit = False + break + + try: + with time_limit(timeout_second): + # deepcopy in case fn modifies the input + fn(*deepcopy(test_input)) + return_inputs = [test_input] # only keep on input + except TimeoutException: + print(colored(f"[Testing] Timeout at scale=2**{fac}", "yellow")) + break + except MemoryError: + print(colored(f"[Testing] MemoryError at scale=2**{fac}", "yellow")) + break + except Exception: + print(colored(f"⚠️ [Testing] Exception ~ {entry_point}", "red")) + print(colored(format_exc(), "red")) + well_defined_exit = False + break + + # Needed for cleaning up. + shutil.rmtree = rmtree + os.rmdir = rmdir + os.chdir = chdir + + return return_inputs, well_defined_exit + + +def main(input: str, output: str): + """In the synthesizer file, each line includes a set of input generators for a task. + The goal of this script is to use these generators to sample inputs for each task. + The generated inputs are expected to be valid. + """ + assert output.endswith(".jsonl"), "output must be a .jsonl file" + + id2task = {} + for task_id, item in get_human_eval_plus().items(): + id2task[task_id] = item + + for task_id, item in get_mbpp_plus().items(): + id2task[task_id] = item + + # loading the synthesizers + with open(input, "r") as f: + synthesizers = [json.loads(l) for l in f] + + n_total = 0 + n_parsed = 0 + n_dedup = 0 + + for item in synthesizers: + item["synthesizers"] = [post_process(s) for s in item["synthesizers"]] + n_total += len(item["synthesizers"]) + item["synthesizers"] = [s for s in item["synthesizers"] if s is not None] + n_parsed += len(item["synthesizers"]) + + dedup_set = set() + for s in item["synthesizers"]: + dedup_set.add( + "\n".join( + [l for l in s.splitlines() if l.strip() and not l.startswith("#")] + ) + ) + item["synthesizers"] = list(dedup_set) + n_dedup += len(item["synthesizers"]) + + print( + colored( + f"#Total {n_total} with {n_parsed} parsed => {100 * (1 - n_parsed / n_total) :.1f}% syntax err", + "green", + ) + ) + + print( + colored( + f"#Parsed {n_parsed} with {n_dedup} dedup => {100 * (1 - n_dedup / n_parsed) :.1f}% duplicate", + "green", + ) + ) + + # resume mode check finished tasks + finished_tasks = set() + if os.path.isfile(output): + with open(output, "r") as f: + for l in f: + item = json.loads(l) + finished_tasks.add(item["task_id"]) + + print("Resumed finished tasks:", finished_tasks) + with open(output, "ab+") as f: + with progress() as p: + for item in p.track(synthesizers): + task_id = item["task_id"] + entry_point = id2task[task_id]["entry_point"] + if task_id in finished_tasks: + p.console.print(f"{task_id}: {entry_point} ~ Resumed") + continue + + ref_code_with_contract = insert_contract( + entry_point, item["ref_code"], id2task[task_id]["contract"] + ) + p.console.print(f"{task_id}: PE input generation...") + p.console.print(Syntax(ref_code_with_contract.strip(), "python")) + + results = [] + for i, generator_code in enumerate(item["synthesizers"]): + p.console.print( + f"Using generator {i+1}/{len(item['synthesizers'])}:" + ) + p.console.print(Syntax(generator_code, "python")) + args = ( + ref_code_with_contract, + entry_point, + generator_code, + ) + with ProcessPoolExecutor(max_workers=1) as executor: + tmp_results, status = executor.submit( + sample_one_input, *args + ).result() + + results.extend(tmp_results) + + # if the func returns in a well-defined manner, we can stop here. + if status: + break + + p.console.print("Serializing and storing results...") + + if "Mbpp/" in task_id: + results = mbpp_serialize_inputs(task_id, results) + + to_write = {"task_id": item["task_id"], "inputs": results} + to_write = (json.dumps(to_write) + "\n").encode("utf-8") + + # task_id => list of inputs + f.write(to_write) + f.flush() + + +if __name__ == "__main__": + import fire + + fire.Fire(main) diff --git a/opencompass/datasets/evalplus/perf/sas.py b/opencompass/datasets/evalplus/perf/sas.py new file mode 100644 index 00000000..f50e8ab5 --- /dev/null +++ b/opencompass/datasets/evalplus/perf/sas.py @@ -0,0 +1,168 @@ +"""This file implements the "Synthesizing an Synthesizer" idea using OpenAI API. +Specifically, for each HumanEval+ and MBPP+ task, we generate n test input synthesizers +by querying a vLLM server (https://docs.vllm.ai/en/latest/). +""" + +import json +from typing import Optional + +import openai +from tqdm import tqdm + +from evalplus.data import get_human_eval_plus, get_mbpp_plus + + +def fewshot_cot( + task_id, + client: openai.OpenAI, + entry_point: str, + code: str, + model: str, + n: int = 1, + max_tokens: int = 2048, +): + responses = client.completions.create( + model=model, + prompt=f'''\ +You are an AI programming assistant, proficient in analyzing and generating Python code. \ +You are going to produce a self-contained Python function to generate a large input for a given function, \ +to test its performance at scale. +### Instruction: +Generate a `perf_input_gen(scale: int)` function to produce a "large" input to exercise the performance of the `add` function: +```python3 +def add(x: int, y: int): + """Add two numbers x and y + >>> add(2, 3) + 5 + >>> add(5, 7) + 12 + """ + return x + y +``` +### Response: +Analysis: +1. Input format: two integers `x` and `y` +2. Is this task O(1) solvable? Yes +### Instruction: +Generate a `perf_input_gen(scale: int)` function to produce a "large" input to exercise the performance of the `prime_num` function: +```python3 +""" +Write a function to check if a number is prime or not. +assert prime_num(2) == True +""" +import math +def prime_num(num): + if num < 2: return False + for i in range(2, math.isqrt(num)): + if num % i == 0: + return False + return True +``` +### Response: +Analysis: +1. Input format: An integer `n` +2. Is this task O(1) solvable? No +3. Time complexity: O(n) +4. Space complexity: O(1) +5. What kind of input can exercise its performance? Large prime numbers +```python3 +# Can reuse the `prime_num` function +# `scale` is a rough estimate of the input size -- larger `scale` means larger input +# use case: prime_num(*perf_input_gen(scale)) +import random +def perf_input_gen(scale: int): + for i in range(scale, 2, -1): + if prime_num(i): + return (i,) + return (2,) +``` +### Instruction: +Generate a `perf_input_gen(scale: int)` function to produce a "large" input to exercise the performance of the `{entry_point}` function: +```python3 +{code} +``` +### Response: +Analysis: +1. Input format: ''', + n=n, + stop=["\n```\n", "\n2. Is this task O(1) solvable? Yes"], + max_tokens=max_tokens, + temperature=0.2, + ) + + # warn if any response is out of context + for r in responses.choices: + if r.finish_reason == "length": + print(f"Warning: response is too long for {task_id}") + + return [r.text for r in responses.choices] + + +def main( + output: str, # output file + n: int = 16, # sample size and batch size + model: Optional[str] = "TheBloke/deepseek-coder-33B-instruct-AWQ", + port: str = 8088, +): + assert output.endswith(".jsonl"), "output must be a .jsonl file" + + base_url = f"http://localhost:{port}/v1" + print(f"Trying to query vLLM model: {model} at {base_url}") + print(f"Note: To use SaS, you need to first set up a vLLM server for {model}") + print(f"For example:") + print( + f"""python -m vllm.entrypoints.openai.api_server \\ +--model "{model}" \\ +--port {port} \\ +--tensor-parallel-size 2 \\ +--max-num-seqs 16 \\ +--gpu-memory-utilization 1.0""" + ) + + # "task_id" -> { "task_id", "entry_point", "ref_code", } + tasks = {} + for task_id, item in get_human_eval_plus().items(): + tasks[task_id] = { + "task_id": task_id, + "entry_point": item["entry_point"], + "ref_code": item["prompt"] + item["canonical_solution"], + } + + for task_id, item in get_mbpp_plus().items(): + tasks[task_id] = { + "task_id": task_id, + "entry_point": item["entry_point"], + "ref_code": item["prompt"] + item["canonical_solution"], + } + + # Using vLLM as a backend, please make sure that a vLLM server is available first. + # vLLM document: https://docs.vllm.ai/en/latest/ + client = openai.OpenAI(api_key="none", base_url=base_url) + + with open(output, "w") as f: + for task_id, item in tqdm(tasks.items(), total=len(tasks)): + responses = fewshot_cot( + task_id=task_id, + client=client, + entry_point=item["entry_point"], + code=item["ref_code"], + model=model, + n=n, + ) + f.write( + json.dumps( + { + "task_id": task_id, + "ref_code": item["ref_code"], + "synthesizers": responses, + } + ) + + "\n" + ) + f.flush() + + +if __name__ == "__main__": + import fire + + fire.Fire(main) diff --git a/opencompass/datasets/evalplus/perf/select_pe_inputs.py b/opencompass/datasets/evalplus/perf/select_pe_inputs.py new file mode 100644 index 00000000..acc41334 --- /dev/null +++ b/opencompass/datasets/evalplus/perf/select_pe_inputs.py @@ -0,0 +1,126 @@ +"""Select the most performance-exercising inputs from pe_inputs obtained from `sampling.py`. +""" + +import json +from statistics import median + +from tqdm import tqdm + +from evalplus.config import PERF_CURATE_TIMEOUT_SECOND +from evalplus.data import get_human_eval_plus, get_mbpp_plus +from evalplus.data.mbpp import mbpp_deserialize_inputs, mbpp_serialize_inputs +from evalplus.perf.profile import are_profiles_broken, profile + + +def script(solutions: str, output_profiled_solutions: str, pe_inputs: str = None): + assert solutions.endswith(".jsonl") + assert pe_inputs is None or pe_inputs.endswith(".jsonl") + assert output_profiled_solutions.endswith(".jsonl") + + evalplus = get_human_eval_plus(noextreme=True) + mbppplus = get_mbpp_plus(noextreme=True) + tasks = {**evalplus, **mbppplus} + + # assume each line's format is: { + # "task_id": task's id, + # "inputs": a list of inputs, + inputs_dict = None + + if pe_inputs is not None: + print("Loading performance-exercising inputs...") + with open(pe_inputs, "r") as f: + inputs_dict = { + task["task_id"]: task["inputs"] for l in f for task in [json.loads(l)] + } + + # Notably, the solutions are already validated and cleaned. + with open(solutions, "r") as f: + solutions = {} + for l in f: + solution = json.loads(l) + solutions[solution["task_id"]] = solution["solution"] + + for task_id, task in tqdm(tasks.items()): + if inputs_dict: + inputs = ( + mbpp_deserialize_inputs(task_id, inputs_dict[task_id]) + if "Mbpp/" in task_id + else inputs_dict[task_id] + ) + else: + inputs = task["base_input"] + list(task["plus_input"]) + + input_costs = [] + + if task_id.startswith("HumanEval"): + canonical_solution = task["prompt"] + task["canonical_solution"] + else: + canonical_solution = task["canonical_solution"] + + for inp in inputs: + costs = profile( + canonical_solution, + task["entry_point"], + [inp], + timeout_second_per_test=PERF_CURATE_TIMEOUT_SECOND, + ) + if are_profiles_broken(costs): + continue + input_costs.append((median(costs), inp)) + input_costs.sort(reverse=True, key=lambda x: x[0]) + + for _, pe_input in input_costs: + solution_costs = [] + + for solution in solutions[task_id]: + costs = profile( + solution, + task["entry_point"], + [pe_input], + timeout_second_per_test=PERF_CURATE_TIMEOUT_SECOND, + ) + if not are_profiles_broken(costs): + solution_costs.append(costs) + continue + + # stop once we find the first also the most performance-exercising input + break + + # This means no timeouts happen for the input, so we use it. + if len(solution_costs) == len(solutions[task_id]): + break + + # If no satisfied input found, we don't save any profiled data. + if len(input_costs) == 0 or len(solution_costs) != len(solutions[task_id]): + print(f"Skipping {task_id}...") + pe_input = None + solution_costs = None + else: + pe_input = ( + mbpp_serialize_inputs(task_id, [pe_input]) + if task_id.startswith("Mbpp/") + else [pe_input] + ) + + with open(output_profiled_solutions, "a") as f: + f.write( + json.dumps( + { + "task_id": task_id, + "pe_input": pe_input, + "solutions": solutions[task_id], + "counter_profile": solution_costs, + } + ) + + "\n" + ) + + +def main(): + from fire import Fire + + Fire(script) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/perf/select_pe_tasks.py b/opencompass/datasets/evalplus/perf/select_pe_tasks.py new file mode 100644 index 00000000..637ce919 --- /dev/null +++ b/opencompass/datasets/evalplus/perf/select_pe_tasks.py @@ -0,0 +1,240 @@ +"""Analyze the variance of PE and their time cost. +Filter those with high variance and low time cost. +""" + +import json +import math +from datetime import datetime +from typing import List + +import numpy as np +from rich.console import Console +from rich.syntax import Syntax +from termcolor import colored + +from evalplus.config import PREF_CURATE_MIN_INSTRUCTION + + +def cv(time_costs: List[float]) -> float: + """ + We use Coefficient of Variation (CV) to as the variance of PE. + CV = 100 * standard deviation / mean + """ + if len(time_costs) == 0: + raise ValueError("time_costs is empty.") + return 100 * np.std(time_costs) / np.mean(time_costs) + + +def filter_by_profile_size(task2profile: dict, threshold: int = 10): + to_remove = [] + for task_id, profile in task2profile.items(): + if ( + profile is None + or len(profile) < threshold + or any(None in p for p in profile) + ): + print(colored(f"⚠️ {task_id} skipped: #profile < {threshold}", "red")) + to_remove.append(task_id) + for task_id in to_remove: + del task2profile[task_id] + return task2profile + + +def filter_by_compute_cost( + task2profile: dict, thresh: float = PREF_CURATE_MIN_INSTRUCTION +): + """Filter out tasks that can be solved using less than threshold #instruction.""" + to_remove = [] + for task_id, profile in task2profile.items(): + if ( + min(np.mean(p) for p in profile) < thresh + ): # filter if some solution is too fast + print( + colored( + f"⚠️ {task_id} skipped: some solution is faster than {thresh} #instruction", + "red", + ) + ) + to_remove.append(task_id) + for task_id in to_remove: + del task2profile[task_id] + return task2profile + + +def filter_by_cv(task2profile: dict, thresh: float, percentile: int = 95): + to_remove = [] + for task_id, profile in task2profile.items(): + mean_var = np.percentile([cv(p) for p in profile], percentile) + if mean_var > thresh: + print( + colored( + f"⚠️ {task_id} skipped: P{percentile} CV = {mean_var:.1f}% > {thresh}%", + "red", + ) + ) + to_remove.append(task_id) + for task_id in to_remove: + del task2profile[task_id] + return task2profile + + +# smaller time, larger threshold +def thresh_fn(base_thresh, x, weight=0.002): + return base_thresh + math.sqrt(weight / x) + + +def adaptive_seg1d(arr1d, base_thresh=0.10): + # sort from large to small + arr1d = np.sort(arr1d)[::-1] + # relative distance + relative_distance = -np.diff(arr1d) / arr1d[:-1] + + splitter_idx = [] + for i, rel in enumerate(relative_distance): + if rel > thresh_fn(base_thresh, arr1d[i], weight=PREF_CURATE_MIN_INSTRUCTION): + splitter_idx.append(i + 1) + + # [9, 8, 7, |-> 3, 2 1] + # splitter_idx points to the slowest in each cluster + return np.split(arr1d, splitter_idx) + + +def filter_by_clustering(task2profile: dict, base_threshold=0.2, min_clusters=3): + to_remove = [] + for task_id, profile in task2profile.items(): + if len(adaptive_seg1d(np.mean(profile, axis=1), base_threshold)) < min_clusters: + print( + colored( + f"⚠️ {task_id} skipped: #Cluster = 0 with {base_threshold=}%", + "red", + ) + ) + to_remove.append(task_id) + for task_id in to_remove: + del task2profile[task_id] + return task2profile + + +def brief_list_repr(lst, head_count=4, tail_count=4): + if len(lst) <= head_count + tail_count: + return f"{lst}" + else: + head = ", ".join(str(x) for x in lst[:head_count]) + tail = ", ".join(str(x) for x in lst[-tail_count:]) + return f"[{head}, ..., {tail}]" + + +def script( + profiled_solutions: str, + output_dataset: str = f"evalperf-{datetime.now():%Y%m%d}.jsonl", + debug_tasks: List[str] = [], + min_clusters=4, +): + assert profiled_solutions.endswith(".jsonl") + assert output_dataset.endswith(".jsonl") + + # read jsonl + with open(profiled_solutions, "r") as f: + profiled_solutions = [json.loads(l) for l in f if l.strip()] + + console = Console() + + task2profile = {d["task_id"]: d["counter_profile"] for d in profiled_solutions} + print(f"Loaded {len(task2profile)} tasks.") + + # * Criteria 1: Profile cannot be empty + task2profile = filter_by_profile_size(task2profile) + print(f"{len(task2profile)} tasks with profile.") + + # * Criteria 2: Solutions should run more than MIN_SLOWEST_INSTRUCTION_COUNT + task2profile = filter_by_compute_cost(task2profile) + print( + f"{len(task2profile)} tasks with slowest mean time > {PREF_CURATE_MIN_INSTRUCTION}s." + ) + + # * Criteria 3: P99-CV should be less than 5% + final_thresh = 5 + percentile = 99 + task2profile = filter_by_cv( + task2profile, thresh=final_thresh, percentile=percentile + ) + print(f"{len(task2profile)} tasks with CV <= {final_thresh}%.") + + # * Criteria 4: Cluster should be more than 1 + task2profile = filter_by_clustering( + task2profile, base_threshold=0.2, min_clusters=min_clusters + ) + print(f"{len(task2profile)} tasks with #Cluster >= {min_clusters}.") + + # export dataset + task2solution = {d["task_id"]: d for d in profiled_solutions} + # each item is {"task_id": "xxx", "solutions": [...], "percentile": [...]} + export_dataset = [] + total_clusters = 0 + for task_id, profile in task2profile.items(): + print(colored(f"-========== {task_id} ==========-", "green")) + if task_id in debug_tasks: + print(colored(f"Debugging {task_id}", "red")) + mean_runtime = [np.mean(p) for p in profile] + clusters = adaptive_seg1d(mean_runtime) # descend + print(colored(f"#seg = {len(clusters)}", "green")) + + accumulative_ratio = [] + ref_idx = [] + for i, cluster in enumerate(clusters): + prior_ar = 0 if i == 0 else accumulative_ratio[-1] + ratio = 100 * len(cluster) / len(mean_runtime) + acc_ratio = prior_ar + ratio + brief_list_str = brief_list_repr([round(1000 * v) for v in cluster]) + print( + f"#{i} |{len(cluster):<3}| ({acc_ratio:<4.1f}) @cv {cv(cluster):.1f}: {brief_list_str}" + ) + accumulative_ratio.append(acc_ratio) + ref_idx.append(np.where(mean_runtime == cluster[0])[0][0]) + + if task_id in debug_tasks: + # print solutions + solution_text = task2solution[task_id]["solutions"][ref_idx[-1]] + # remove empty lines + solution_text = "\n".join( + line for line in solution_text.split("\n") if line.strip() + ) + console.print(Syntax(solution_text, "python")) + print(colored("-" * 32, "green")) + + total_clusters += len(clusters) + + # add reference solution and check consistency + for i in range(len(ref_idx)): + if i == 0: + continue + # prior runtime must be larger than current + assert mean_runtime[ref_idx[i - 1]] > mean_runtime[ref_idx[i]] + + reference = [task2solution[task_id]["solutions"][idx] for idx in ref_idx] + + assert len(reference) == len(clusters) + assert len(accumulative_ratio) == len(reference) + item = { + "task_id": task_id, + "reference": reference, + "pe_input": task2solution[task_id]["pe_input"], + "scores": accumulative_ratio, + } + export_dataset.append(item) + + print(f"Total clusters: {total_clusters}") + + with open(output_dataset, "w") as f: + for item in export_dataset: + f.write(json.dumps(item) + "\n") + + +def main(): + from fire import Fire + + Fire(script) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/provider/__init__.py b/opencompass/datasets/evalplus/provider/__init__.py new file mode 100644 index 00000000..c7e1b395 --- /dev/null +++ b/opencompass/datasets/evalplus/provider/__init__.py @@ -0,0 +1,121 @@ +from opencompass.datasets.evalplus.provider.base import DecoderBase + + +def make_model( + model: str, + backend: str, + dataset: str, + batch_size: int = 1, + temperature: float = 0.0, + force_base_prompt: bool = False, + # instruction model only + instruction_prefix=None, + response_prefix=None, + # non-server only + dtype="bfloat16", + trust_remote_code=False, + # vllm only + tp=1, + enable_prefix_caching=False, + enable_chunked_prefill=False, + # openai only + base_url=None, + # hf only + attn_implementation="eager", + device_map=None, + # gptqmodel only + gptqmodel_backend: str = 'auto', + gguf_file: str = None, +) -> DecoderBase: + if backend == "vllm": + from evalplus.provider.vllm import VllmDecoder + + return VllmDecoder( + name=model, + batch_size=batch_size, + temperature=temperature, + dataset=dataset, + force_base_prompt=force_base_prompt, + tensor_parallel_size=tp, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + trust_remote_code=trust_remote_code, + enable_prefix_caching=enable_prefix_caching, + enable_chunked_prefill=enable_chunked_prefill, + dtype=dtype, + gguf_file=gguf_file + ) + elif backend == "hf": + from evalplus.provider.hf import HuggingFaceDecoder + + return HuggingFaceDecoder( + name=model, + batch_size=batch_size, + temperature=temperature, + dataset=dataset, + force_base_prompt=force_base_prompt, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + attn_implementation=attn_implementation, + device_map=device_map, + trust_remote_code=trust_remote_code, + dtype=dtype, + gguf_file=gguf_file, + ) + elif backend == "openai": + from evalplus.provider.openai import OpenAIChatDecoder + + assert not force_base_prompt, f"{backend} backend does not serve base model" + return OpenAIChatDecoder( + name=model, + batch_size=batch_size, + temperature=temperature, + base_url=base_url, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) + elif backend == "anthropic": + from evalplus.provider.anthropic import AnthropicDecoder + + assert not force_base_prompt, f"{backend} backend does not serve base model" + return AnthropicDecoder( + name=model, + batch_size=batch_size, + temperature=temperature, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) + elif backend == "google": + from evalplus.provider.google import GeminiDecoder + + assert not force_base_prompt, f"{backend} backend does not serve base model" + return GeminiDecoder( + name=model, + batch_size=batch_size, + temperature=temperature, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) + elif backend == "bedrock": + from evalplus.provider.bedrock import BedrockDecoder + + assert not force_base_prompt, f"{backend} backend does not serve base model" + return BedrockDecoder( + name=model, + batch_size=batch_size, + temperature=temperature, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + ) + elif backend == "gptqmodel": + from evalplus.provider.gptqmodel import GPTQModelDecoder + + return GPTQModelDecoder( + name=model, + dataset=dataset, + force_base_prompt=force_base_prompt, + instruction_prefix=instruction_prefix, + response_prefix=response_prefix, + trust_remote_code=trust_remote_code, + gptqmodel_backend=gptqmodel_backend, + ) diff --git a/opencompass/datasets/evalplus/provider/anthropic.py b/opencompass/datasets/evalplus/provider/anthropic.py new file mode 100644 index 00000000..de130c15 --- /dev/null +++ b/opencompass/datasets/evalplus/provider/anthropic.py @@ -0,0 +1,46 @@ +import os +from typing import List + +import anthropic + +from opencompass.datasets.evalplus.provider.base import DecoderBase +# from opencompass.datasets.evalplus.provider.utility import anthropic_request + + +# class AnthropicDecoder(DecoderBase): +# def __init__(self, name: str, **kwargs) -> None: +# super().__init__(name, **kwargs) +# self.client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_KEY")) + +# def codegen( +# self, prompt: str, do_sample: bool = True, num_samples: int = 200 +# ) -> List[str]: +# if do_sample: +# assert self.temperature > 0, "Temperature must be positive for sampling" + +# batch_size = min(self.batch_size, num_samples) +# if not do_sample: +# assert batch_size == 1, "Sampling only supports batch size of 1" + +# outputs = [] +# for _ in range(batch_size): +# message = anthropic_request.make_auto_request( +# client=self.client, +# model=self.name, +# messages=[ +# { +# "role": "user", +# "content": self.instruction_prefix +# + f"\n```python\n{prompt.strip()}\n```\n", +# } +# ], +# max_tokens=self.max_new_tokens, +# temperature=self.temperature, +# stop_sequences=self.eos, +# ) +# outputs.append(message.content[0].text) + +# return outputs + +# def is_direct_completion(self) -> bool: +# return False diff --git a/opencompass/datasets/evalplus/provider/base.py b/opencompass/datasets/evalplus/provider/base.py new file mode 100644 index 00000000..57a849df --- /dev/null +++ b/opencompass/datasets/evalplus/provider/base.py @@ -0,0 +1,45 @@ +from abc import ABC, abstractmethod +from typing import List + +from opencompass.datasets.evalplus.provider.utility import EOS + + +class DecoderBase(ABC): + def __init__( + self, + name: str, + batch_size: int = 1, + temperature: float = 0.8, + max_new_tokens: int = 768, + dtype: str = "bfloat16", # default + trust_remote_code: bool = False, + instruction_prefix: str = None, + response_prefix: str = None, + ) -> None: + print("Initializing a decoder model: {} ...".format(name)) + self.name = name + self.batch_size = batch_size + self.temperature = temperature + self.eos = EOS + self.skip_special_tokens = False + self.max_new_tokens = max_new_tokens + self.dtype = dtype + self.trust_remote_code = trust_remote_code + self.instruction_prefix = instruction_prefix + self.response_prefix = response_prefix + + @abstractmethod + def codegen( + self, prompt: str, do_sample: bool = True, num_samples: int = 200 + ) -> List[str]: + pass + + @abstractmethod + def is_direct_completion(self) -> bool: + pass + + def __repr__(self) -> str: + return self.name + + def __str__(self) -> str: + return self.name diff --git a/opencompass/datasets/evalplus/provider/bedrock.py b/opencompass/datasets/evalplus/provider/bedrock.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/evalplus/provider/google.py b/opencompass/datasets/evalplus/provider/google.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/evalplus/provider/gptqmodel.py b/opencompass/datasets/evalplus/provider/gptqmodel.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/evalplus/provider/hf.py b/opencompass/datasets/evalplus/provider/hf.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/evalplus/provider/openai.py b/opencompass/datasets/evalplus/provider/openai.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/evalplus/provider/utility.py b/opencompass/datasets/evalplus/provider/utility.py new file mode 100644 index 00000000..673f11cf --- /dev/null +++ b/opencompass/datasets/evalplus/provider/utility.py @@ -0,0 +1,64 @@ +from concurrent.futures import ThreadPoolExecutor +from typing import List + +EOS = [ + "<|endoftext|>", + "<|endofmask|>", + "", + "\nif __name__", + "\ndef main(", + "\nprint(", +] + + +def extra_eos_for_direct_completion(dataset) -> List[str]: + if dataset.lower() == "humaneval": + return ["\ndef ", "\nclass ", "\nimport ", "\nfrom ", "\nassert "] + elif dataset.lower() == "mbpp": + return ['\n"""', "\nassert"] + raise ValueError(f"Unknown dataset: {dataset}") + + +# some random words which serves as the splitter +_MAGIC_SPLITTER_ = "-[[]]-this-is-really-our-highest-priority-[[]]-" + + +def make_raw_chat_prompt( + task_prompt: str, + instruction_prefix: str, + response_prefix: str, + tokenizer, +) -> str: + # directly return prompt if it does not have a tokenizer.chat_template + if tokenizer.chat_template is None: + return task_prompt + + assert instruction_prefix is not None, "Instruction prefix is required!" + assert response_prefix is not None, "Response prefix is required!" + + task_prompt = f"""\ +{instruction_prefix} +``` +{task_prompt.strip()} +``` +""" + response = f"""\ +{response_prefix} +```python +{_MAGIC_SPLITTER_} +``` +""" + task_prompt = tokenizer.apply_chat_template( + [ + {"role": "user", "content": task_prompt}, + {"role": "assistant", "content": response}, + ], + tokenize=False, + ).split(_MAGIC_SPLITTER_)[0] + return task_prompt + + +def concurrent_call(n, callback, /, *args, **kwargs): + with ThreadPoolExecutor(max_workers=n) as executor: + futures = [executor.submit(callback, *args, **kwargs) for _ in range(n)] + return [future.result() for future in futures] \ No newline at end of file diff --git a/opencompass/datasets/evalplus/provider/vllm.py b/opencompass/datasets/evalplus/provider/vllm.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/evalplus/sanitize.py b/opencompass/datasets/evalplus/sanitize.py new file mode 100644 index 00000000..2e4b8484 --- /dev/null +++ b/opencompass/datasets/evalplus/sanitize.py @@ -0,0 +1,253 @@ +"""Post-processing LLM-generated Python code implemented using tree-sitter.""" + +import os +import pathlib +from typing import Dict, Generator, List, Optional, Set, Tuple + +import tree_sitter_python +from tqdm import tqdm +from tree_sitter import Language, Node, Parser + +from opencompass.datasets.evalplus.data import ( + get_human_eval_plus, + get_mbpp_plus, + load_solutions, + write_directory, + write_jsonl, +) +from opencompass.datasets.evalplus.syncheck import syntax_check + +CLASS_TYPE = "class_definition" +FUNCTION_TYPE = "function_definition" +IMPORT_TYPE = ["import_statement", "import_from_statement"] +IDENTIFIER_TYPE = "identifier" +ATTRIBUTE_TYPE = "attribute" +RETURN_TYPE = "return_statement" +EXPRESSION_TYPE = "expression_statement" +ASSIGNMENT_TYPE = "assignment" + + +def code_extract(text: str) -> str: + lines = text.split("\n") + longest_line_pair = (0, 0) + longest_so_far = 0 + + for i in range(len(lines)): + for j in range(i + 1, len(lines)): + current_lines = "\n".join(lines[i : j + 1]) + if syntax_check(current_lines): + current_length = sum(1 for line in lines[i : j + 1] if line.strip()) + if current_length > longest_so_far: + longest_so_far = current_length + longest_line_pair = (i, j) + + return "\n".join(lines[longest_line_pair[0] : longest_line_pair[1] + 1]) + + +def get_deps(nodes: List[Tuple[str, Node]]) -> Dict[str, Set[str]]: + def dfs_get_deps(node: Node, deps: Set[str]) -> None: + for child in node.children: + if child.type == IDENTIFIER_TYPE: + deps.add(child.text.decode("utf8")) + else: + dfs_get_deps(child, deps) + + name2deps = {} + for name, node in nodes: + deps = set() + dfs_get_deps(node, deps) + name2deps[name] = deps + return name2deps + + +def get_function_dependency(entrypoint: str, call_graph: Dict[str, str]) -> Set[str]: + queue = [entrypoint] + visited = {entrypoint} + while queue: + current = queue.pop(0) + if current not in call_graph: + continue + for neighbour in call_graph[current]: + if not (neighbour in visited): + visited.add(neighbour) + queue.append(neighbour) + return visited + + +def get_definition_name(node: Node) -> str: + for child in node.children: + if child.type == IDENTIFIER_TYPE: + return child.text.decode("utf8") + + +def traverse_tree(node: Node) -> Generator[Node, None, None]: + cursor = node.walk() + depth = 0 + + visited_children = False + while True: + if not visited_children: + yield cursor.node + if not cursor.goto_first_child(): + depth += 1 + visited_children = True + elif cursor.goto_next_sibling(): + visited_children = False + elif not cursor.goto_parent() or depth == 0: + break + else: + depth -= 1 + + +def has_return_statement(node: Node) -> bool: + traverse_nodes = traverse_tree(node) + for node in traverse_nodes: + if node.type == RETURN_TYPE: + return True + return False + + +def extract_target_code_or_empty(code: str, entrypoint: Optional[str] = None) -> str: + code = code_extract(code) + code_bytes = bytes(code, "utf8") + parser = Parser(Language(tree_sitter_python.language())) + tree = parser.parse(code_bytes) + class_names = set() + function_names = set() + variable_names = set() + + root_node = tree.root_node + import_nodes = [] + definition_nodes = [] + + for child in root_node.children: + if child.type in IMPORT_TYPE: + import_nodes.append(child) + elif child.type == CLASS_TYPE: + name = get_definition_name(child) + if not ( + name in class_names or name in variable_names or name in function_names + ): + definition_nodes.append((name, child)) + class_names.add(name) + elif child.type == FUNCTION_TYPE: + name = get_definition_name(child) + if not ( + name in function_names or name in variable_names or name in class_names + ) and has_return_statement(child): + definition_nodes.append((name, child)) + function_names.add(get_definition_name(child)) + elif ( + child.type == EXPRESSION_TYPE and child.children[0].type == ASSIGNMENT_TYPE + ): + subchild = child.children[0] + name = get_definition_name(subchild) + if not ( + name in variable_names or name in function_names or name in class_names + ): + definition_nodes.append((name, subchild)) + variable_names.add(name) + + if entrypoint: + name2deps = get_deps(definition_nodes) + reacheable = get_function_dependency(entrypoint, name2deps) + + sanitized_output = b"" + + for node in import_nodes: + sanitized_output += code_bytes[node.start_byte : node.end_byte] + b"\n" + + for pair in definition_nodes: + name, node = pair + if entrypoint and not (name in reacheable): + continue + sanitized_output += code_bytes[node.start_byte : node.end_byte] + b"\n" + return sanitized_output[:-1].decode("utf8") + + +def sanitize(code: str, entrypoint: Optional[str] = None) -> str: + sanitized_code = extract_target_code_or_empty(code, entrypoint).strip() + if not sanitized_code: + return code_extract(code) + return sanitized_code + + +def script( + samples: str, inplace: bool = False, debug_task: str = None, mbpp_version="default" +): + # task_id -> entry_point + entry_point = {} + # merge two datasets + dataset = {**get_human_eval_plus(), **get_mbpp_plus(version=mbpp_version)} + + for task_id, problem in dataset.items(): + entry_point[task_id] = problem["entry_point"] + + # make a new folder with "-sanitized" suffix + is_folder = os.path.isdir(samples) + target_path = pathlib.Path(samples) + if not inplace: + if is_folder: + new_name = target_path.name + "-sanitized" + else: + new_name = target_path.name.replace(".jsonl", "-sanitized.jsonl") + target_path = target_path.parent / new_name + target_path = str(target_path) + + nsan = 0 + ntotal = 0 + + new_solutions = [] + + for solution in tqdm(load_solutions(samples)): + task_id = solution["task_id"] + if task_id not in dataset: + print( + f"Skiping {task_id} as it does not existing in the latest EvalPlus dataset." + ) + continue + + function_name = entry_point[task_id] if task_id in entry_point else None + dbg_identifier = solution["_identifier"] + if debug_task is not None and task_id != debug_task: + continue + + ntotal += 1 + if "solution" in solution: + old_code = solution["solution"] + else: + assert "completion" in solution + old_code = dataset[task_id]["prompt"] + "\n" + solution["completion"] + + new_code = sanitize(code=old_code, entrypoint=function_name) + + # if changed, print the message + if new_code != old_code: + msg = "Sanitized: " + dbg_identifier + if is_folder: + msg += " -> " + dbg_identifier.replace(samples, target_path) + print(msg) + nsan += 1 + + new_solutions.append({"task_id": task_id, "solution": new_code}) + + if is_folder: + write_directory(target_path, new_solutions) + else: + write_jsonl(target_path, new_solutions) + + if nsan > 0: + print(f"Sanitized {nsan} out of {ntotal} files.") + else: + print(f"All files seems valid -- no files are sanitized.") + print(f"Check the sanitized files at {target_path}") + + +def main(): + from fire import Fire + + Fire(script) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/setup.py b/opencompass/datasets/evalplus/setup.py new file mode 100644 index 00000000..76551df2 --- /dev/null +++ b/opencompass/datasets/evalplus/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup, find_packages + +setup( + name='evalplus', # 包名 (安装时使用: pip install mypackage) + version='0.1.0', # 版本号 + author='default', # 作者信息 + description='A sample Python package', # 简短描述 + packages=find_packages(), # 自动发现子包 + install_requires=[], # (可选) 依赖项列表 + python_requires='>=3.10', # (可选) 支持的Python版本 +) + diff --git a/opencompass/datasets/evalplus/syncheck.py b/opencompass/datasets/evalplus/syncheck.py new file mode 100644 index 00000000..1e05a7ae --- /dev/null +++ b/opencompass/datasets/evalplus/syncheck.py @@ -0,0 +1,118 @@ +"""This file checks two things: +1. Is the LLMs codegen completed for each benchmark? +2. Warn the code that are not compilable (it could be some impl issues). +""" + +import ast +import traceback + +from termcolor import colored + +from opencompass.datasets.evalplus.data import load_solutions + + +def syntax_check(code, verbose=False): + try: + ast.parse(code) + return True + except (SyntaxError, MemoryError): + if verbose: + traceback.print_exc() + return False + + +def script( + samples: str, dataset: str, nsample_check: int = None, verbose: bool = False +): + # List[Dict{"task_id", "solution"}] + solutions = load_solutions(samples) + + if dataset == "humaneval": + from evalplus.data import get_human_eval_plus + + dataset = get_human_eval_plus() + dataset_name = "HumanEval" + elif dataset == "mbpp": + from evalplus.data import get_mbpp_plus + + dataset = get_mbpp_plus() + dataset_name = "Mbpp" + + print(colored(f"Dataset: {dataset_name}", "blue")) + + id2solutions = {} + for solution in solutions: + task_id = solution["task_id"] + if task_id not in id2solutions: + id2solutions[task_id] = [] + if "solution" not in solution: + assert "completion" in solution, "solution or completion must exist!" + solution["solution"] = dataset[task_id]["prompt"] + solution["completion"] + id2solutions[task_id].append(solution) + + print(colored("==============================", "blue")) + print(colored(" ::: Checking completeness... ", "blue")) + print(colored(" ::::: All tasks complete? ", "blue")) + ndone = 0 + + task_ids = dataset.keys() + ntask = len(task_ids) + for task_id in task_ids: + if task_id not in id2solutions: + print(colored(f" ⚠️ {task_id} is missing!", "red")) + continue + nfiles = len(id2solutions[task_id]) + + if nsample_check is None or nfiles <= nsample_check: + ndone += 1 + continue + + print( + colored( + f" ⚠️ {task_id} only has {nfiles} samples! But {nsample_check} are expected.", + "red", + ) + ) + + # check if there is enough number of samples here. + if nsample_check is not None: + if ntask != ndone: + ntbd = ntask - ndone + print(colored(f" ::::: ⚠️ {ntbd}/{ntask} tasks incomplete!", "red")) + else: + print(colored(f" ::::: All {ntask} tasks complete!", "green")) + + print(colored("==============================", "blue")) + print(colored(" ::: Checking compilation... ", "blue")) + print(colored(" ::::: All code compilable? ", "blue")) + ncode = 0 + nwrong = 0 + for task_id in task_ids: + # task_id must exist + if task_id not in id2solutions: + continue + + for solution in id2solutions[task_id]: + ncode += 1 + code = solution["solution"] + dbg_identifier = solution["_identifier"] + if code.strip() == "": + print(colored(f" ⚠️ {dbg_identifier} is empty!", "red")) + nwrong += 1 + elif not syntax_check(code, verbose): + print(colored(f" ⚠️ {dbg_identifier} is not compilable!", "red")) + nwrong += 1 + if 0 != nwrong: + print(colored(f" ::::: ⚠️ {nwrong}/{ncode} code are not compilable!", "red")) + else: + print(colored(f" ::::: All {ncode} code are compilable!", "green")) + + +def main(): + from fire import Fire + + Fire(script) + + +if __name__ == "__main__": + main() diff --git a/opencompass/datasets/evalplus/utils.py b/opencompass/datasets/evalplus/utils.py new file mode 100644 index 00000000..5273c04c --- /dev/null +++ b/opencompass/datasets/evalplus/utils.py @@ -0,0 +1,17 @@ +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + TextColumn, + TimeElapsedColumn, +) + + +def progress(note: str = "processing"): + return Progress( + TextColumn(f"{note} •" + "[progress.percentage]{task.percentage:>3.0f}%"), + BarColumn(), + MofNCompleteColumn(), + TextColumn("•"), + TimeElapsedColumn(), + ) diff --git a/opencompass/datasets/xIFEval/__init__.py b/opencompass/datasets/xIFEval/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/opencompass/datasets/xIFEval/evaluation_main.py b/opencompass/datasets/xIFEval/evaluation_main.py new file mode 100644 index 00000000..66092540 --- /dev/null +++ b/opencompass/datasets/xIFEval/evaluation_main.py @@ -0,0 +1,256 @@ +# flake8: noqa +# yapf: disable + +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +from typing import Dict, List, Optional, Union + +from absl import flags + +import opencompass.datasets.xIFEval.instructions_registry as instructions_registry + +# _INPUT_DATA = flags.DEFINE_string('input_data', +# None, +# 'path to input data', +# required=True) + +# _INPUT_RESPONSE_DATA = flags.DEFINE_string('input_response_data', +# None, +# 'path to input response data', +# required=False) + +# _OUTPUT_DIR = flags.DEFINE_string( +# 'output_dir', +# None, +# 'Output directory for inference and eval results.', +# required=True, +# ) + + +@dataclasses.dataclass +class InputExample: + key: int + instruction_id_list: List[str] + prompt: str + kwargs: List[Dict[str, Optional[Union[str, int]]]] + lang: str + + +@dataclasses.dataclass +class OutputExample: + instruction_id_list: List[str] + prompt: str + response: str + follow_all_instructions: bool + follow_instruction_list: List[bool] + + +# def test_instruction_following_strict( +# inp, +# response, +# ): +# """Tests response to see if instrutions are followed.""" +# instruction_list = inp.instruction_id_list +# is_following_list = [] + +# for index, instruction_id in enumerate(instruction_list): +# instruction_cls = instructions_registry.INSTRUCTION_DICT[ +# instruction_id] +# instruction = instruction_cls(instruction_id) +# instruction.build_description(**inp.kwargs[index]) +# args = instruction.get_instruction_args() +# if args and 'prompt' in args: +# instruction.build_description(prompt=inp.prompt) + +# if response.strip() and instruction.check_following(response): +# is_following_list.append(True) +# else: +# is_following_list.append(False) + +# return OutputExample( +# instruction_id_list=inp.instruction_id_list, +# prompt=inp.prompt, +# response=response, +# follow_all_instructions=all(is_following_list), +# follow_instruction_list=is_following_list, +# ) + + +# def test_instruction_following_loose( +# inp, +# response, +# ): +# """Tests response for an upper bound for following instructions.""" +# r = response.split('\n') +# response_remove_first = '\n'.join(r[1:]).strip() +# response_remove_last = '\n'.join(r[:-1]).strip() +# response_remove_both = '\n'.join(r[1:-1]).strip() +# revised_response = response.replace('*', '') +# revised_response_remove_first = response_remove_first.replace('*', '') +# revised_response_remove_last = response_remove_last.replace('*', '') +# revised_response_remove_both = response_remove_both.replace('*', '') +# all_responses = [ +# response, +# revised_response, +# response_remove_first, +# response_remove_last, +# response_remove_both, +# revised_response_remove_first, +# revised_response_remove_last, +# revised_response_remove_both, +# ] +# instruction_list = inp.instruction_id_list +# is_following_list = [] + +# for index, instruction_id in enumerate(instruction_list): +# instruction_cls = instructions_registry.INSTRUCTION_DICT[ +# instruction_id] +# instruction = instruction_cls(instruction_id) + +# instruction.build_description(**inp.kwargs[index]) +# args = instruction.get_instruction_args() +# if args and 'prompt' in args: +# instruction.build_description(prompt=inp.prompt) + +# is_following = False +# for r in all_responses: +# if r.strip() and instruction.check_following(r): +# is_following = True +# break + +# is_following_list.append(is_following) + +# return OutputExample( +# instruction_id_list=inp.instruction_id_list, +# prompt=inp.prompt, +# response=response, +# follow_all_instructions=all(is_following_list), +# follow_instruction_list=is_following_list, +# ) + + +def test_instruction_following_strict( + inp, + response, +): + """Tests response to see if instructions are followed.""" + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id, inp.lang) + + # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method. + kwargs = {k: v for k, v in inp.kwargs[index].items() if v} + instruction.build_description(**kwargs) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + if response.strip() and instruction.check_following(response): + is_following_list.append(True) + else: + is_following_list.append(False) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def test_instruction_following_loose( + inp, + response, +): + """Tests response for an upper bound for following instructions.""" + r = response.split("\n") + response_remove_first = "\n".join(r[1:]).strip() + response_remove_last = "\n".join(r[:-1]).strip() + response_remove_both = "\n".join(r[1:-1]).strip() + revised_response = response.replace("*", "") + revised_response_remove_first = response_remove_first.replace("*", "") + revised_response_remove_last = response_remove_last.replace("*", "") + revised_response_remove_both = response_remove_both.replace("*", "") + all_responses = [ + response, + revised_response, + response_remove_first, + response_remove_last, + response_remove_both, + revised_response_remove_first, + revised_response_remove_last, + revised_response_remove_both, + ] + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id, inp.lang) + + # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method. + kwargs = {k: v for k, v in inp.kwargs[index].items() if v} + instruction.build_description(**kwargs) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + is_following = False + for r in all_responses: + if r.strip() and instruction.check_following(r): + is_following = True + break + + is_following_list.append(is_following) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def process_results(doc, results): + inp = InputExample( + key=doc["key"], + instruction_id_list=doc["instruction_id_list"], + prompt=doc["prompt"], + kwargs=doc["kwargs"], + lang=doc.get("lang", "en") + ) + response = results[0] + + out_strict = test_instruction_following_strict(inp, response) + out_loose = test_instruction_following_loose(inp, response) + + return { + "prompt_level_strict_acc": out_strict.follow_all_instructions, + "inst_level_strict_acc": out_strict.follow_instruction_list, + "prompt_level_loose_acc": out_loose.follow_all_instructions, + "inst_level_loose_acc": out_loose.follow_instruction_list, + } + + +def agg_inst_level_acc(items): + flat_items = [item for sublist in items for item in sublist] + inst_level_acc = sum(flat_items) / len(flat_items) + return inst_level_acc \ No newline at end of file diff --git a/opencompass/datasets/xIFEval/ifeval.py b/opencompass/datasets/xIFEval/ifeval.py new file mode 100644 index 00000000..3677ed86 --- /dev/null +++ b/opencompass/datasets/xIFEval/ifeval.py @@ -0,0 +1,99 @@ +import json + +from datasets import Dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path + +from ..base import BaseDataset +from .evaluation_main import (InputExample, test_instruction_following_loose, + test_instruction_following_strict) + + +@LOAD_DATASET.register_module() +class xIFEvalDataset(BaseDataset): + + @staticmethod + def load(path): + path = get_data_path(path, local_mode=True) + datasets = [] + with open(path, 'r', encoding='utf-8') as file: + for line in file: + tmp = json.loads(line.strip()) + dataset = dict(prompt=tmp['prompt'], reference=tmp) + datasets.append(dataset) + return Dataset.from_list(datasets) + + +class xIFEvaluator(BaseEvaluator): + + def score(self, predictions, references, origin_prompt): + prompt_strict_correct, prompt_strict_total = 0, 0 + inst_strict_correct, inst_strict_total = 0, 0 + prompt_loose_correct, prompt_loose_total = 0, 0 + inst_loose_correct, inst_loose_total = 0, 0 + details = {} + for index, (pred, refer) in enumerate(zip(predictions, references)): + input = InputExample( + key=refer['key'], + instruction_id_list=refer['instruction_id_list'], + prompt=refer['prompt'], + kwargs=refer['kwargs'], + lang=refer['lang'] + ) + for kwarg in input.kwargs: + for k in list(kwarg.keys()): + if kwarg[k] is None: + kwarg.pop(k, None) + + # strict + example = test_instruction_following_strict(input, pred) + follow_instruction_list = example.follow_instruction_list + instruction_id_list = example.instruction_id_list + prompt_strict_total += 1 + is_strict_correct = all(follow_instruction_list) + prompt_strict_correct += is_strict_correct + inst_strict_total += len(instruction_id_list) + inst_strict_correct += sum(follow_instruction_list) + + # loose + example = test_instruction_following_loose(input, pred) + follow_instruction_list = example.follow_instruction_list + instruction_id_list = example.instruction_id_list + prompt_loose_total += 1 + is_loose_correct = all(follow_instruction_list) + prompt_loose_correct += is_loose_correct + inst_loose_total += len(instruction_id_list) + inst_loose_correct += sum(follow_instruction_list) + + if is_strict_correct: + grade = 'strict' + elif is_loose_correct: + grade = 'loose' + else: + grade = 'none' + + details[str(index)] = { + 'prompt': origin_prompt[index], + 'pred': pred, + 'refer': refer, + 'is_strict_correct': is_strict_correct, + 'is_loose_correct': is_loose_correct, + 'is_correct': is_strict_correct, + 'grade': grade + } + + results = { + 'Prompt-level-strict-accuracy': + prompt_strict_correct / prompt_strict_total * 100, + 'Inst-level-strict-accuracy': + inst_strict_correct / inst_strict_total * 100, + 'Prompt-level-loose-accuracy': + prompt_loose_correct / prompt_loose_total * 100, + 'Inst-level-loose-accuracy': + inst_loose_correct / inst_loose_total * 100, + 'details': + details + } + return results diff --git a/opencompass/datasets/xIFEval/instructions.py b/opencompass/datasets/xIFEval/instructions.py new file mode 100644 index 00000000..bd0e835e --- /dev/null +++ b/opencompass/datasets/xIFEval/instructions.py @@ -0,0 +1,1727 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library of instructions.""" + +import collections +import json +import logging +import random +import re +import string +from typing import Dict, Optional, Sequence, Union + +import langdetect + +from opencompass.datasets.xIFEval import instructions_util + + +logger = logging.getLogger(__name__) + +_InstructionArgsDtype = Optional[Dict[str, Union[int, str, Sequence[str]]]] + +_LANGUAGES = instructions_util.LANGUAGE_CODES + +# The relational operation for comparison. +_COMPARISON_RELATION = ("less than", "at least") + +# The maximum number of sentences. +_MAX_NUM_SENTENCES = 20 + +# The number of placeholders. +_NUM_PLACEHOLDERS = 4 + +# The number of bullet lists. +_NUM_BULLETS = 5 + +# The options of constrained response. +_CONSTRAINED_RESPONSE_OPTIONS = { + "en": ( + "My answer is yes.", + "My answer is no.", + "My answer is maybe.", + ), + "zh-CN": ( + "我的回答是肯定的。", + "我的回答是否定的。", + "我的回答是可能。", + ), + "es": ( + "Mi respuesta es sí.", + "Mi respuesta es no.", + "Mi respuesta es tal vez.", + ), + "fr": ( + "Ma réponse est oui.", + "Ma réponse est non.", + "Ma réponse est peut-être.", + ), + "de": ( + "Meine Antwort ist ja.", + "Meine Antwort ist nein.", + "Meine Antwort ist vielleicht.", + ), + "hu": ( + "A válaszom igen.", + "A válaszom nem.", + "A válaszom talán.", + ), + "ru": ( + "Мой ответ — да.", + "Мой ответ — нет.", + "Мой ответ — может быть.", + ), + "ja": ( + "私の答えは「はい」です。", + "私の答えはノーです。", + "私の答えは「多分」です。", + ), + "th": ( + "คำตอบของฉันคือใช่", + "คำตอบของฉันคือไม่", + "คำตอบของฉันคือบางที", + ), + "sw": ( + "Jibu langu ni ndiyo.", + "Jibu langu ni hapana.", + "Jibu langu ni labda.", + ), + "bn": ( + "আমার উত্তর হ্যাঁ।", + "আমার উত্তর না।", + "আমার উত্তর হয়তো।", + ), + "te": ( + "నా సమాధానం అవును.", + "నా సమాధానం లేదు.", + "నా సమాధానం కావచ్చు.", + ), + "ar": ( + "جوابي هو نعم.", + "جوابي هو لا.", + "جوابي هو ربما.", + ), + "ko": ( + "제 대답은 '예'입니다.", + "내 대답은 아니오입니다.", + "내 대답은 아마도입니다.", + ), + "vi": ( + "Câu trả lời của tôi là có.", + "Câu trả lời của tôi là không.", + "Câu trả lời của tôi là có thể.", + ), + "cs": ( + "Moje odpověď je ano.", + "Moje odpověď je ne.", + "Moje odpověď je možná.", + ), + "sr": ( + "Мој одговор је да.", + "Мој одговор је не.", + "Мој одговор је можда.", + ), +} + +# The options of starter keywords. +_STARTER_OPTIONS = ( + "I would say", + "My answer is", + "I believe", + "In my opinion", + "I think", + "I reckon", + "I feel", + "From my perspective", + "As I see it", + "According to me", + "As far as I'm concerned", + "To my understanding", + "In my view", + "My take on it is", + "As per my perception", +) + +# The options of ending keywords. +# TODO(jeffreyzhou) add more ending options +_ENDING_OPTIONS = ("Any other questions?", "Is there anything else I can help with?") + +# The number of highlighted sections. +_NUM_HIGHLIGHTED_SECTIONS = 4 + +# The section splitter. +_SECTION_SPLITER = ("Section", "SECTION") + +# The number of sections. +_NUM_SECTIONS = 5 + +# The number of paragraphs. +_NUM_PARAGRAPHS = 5 + +# The postscript marker. +_POSTSCRIPT_MARKER = ("P.S.", "P.P.S") + +# The number of keywords. +_NUM_KEYWORDS = 2 + +# The occurrences of a single keyword. +_KEYWORD_FREQUENCY = 3 + +# The occurrences of a single letter. +_LETTER_FREQUENCY = 10 + +# The occurrences of words with all capital letters. +_ALL_CAPITAL_WORD_FREQUENCY = 20 + +# The number of words in the response. +_NUM_WORDS_LOWER_LIMIT = 100 +_NUM_WORDS_UPPER_LIMIT = 500 + + +class Instruction: + """An instruction template.""" + + def __init__(self, instruction_id, lang="en"): + self.id = instruction_id + self.lang = lang + + def build_description(self, **kwargs): + raise NotImplementedError("`build_description` not implemented.") + + def get_instruction_args(self): + raise NotImplementedError("`get_instruction_args` not implemented.") + + def get_instruction_args_keys(self): + raise NotImplementedError("`get_instruction_args_keys` not implemented.") + + def check_following(self, value): + raise NotImplementedError("`check_following` not implemented.") + + +class ResponseLanguageChecker(Instruction): + """Check the language of the entire response.""" + + def build_description(self, *, language=None): + """Build the instruction description. + + Args: + language: A string representing the expected language of the response. The + language has to comply to the 97 types defined in + `langid.py` (https://pypi.org/project/langid/1.1.5/), which follows + ISO 639-1 codes (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes); + for example, `en` for English, `zh` for Chinese, `fr` for French. + + Returns: + A string representing the instruction description. + """ + self._language = language + if self._language is None: + self._language = random.choice(list(_LANGUAGES.keys())) + # TODO(tianjianlu): opens the description generation to more choices. + self._description_pattern = ( + "Your ENTIRE response should be in {language} language, no other " + + "language is allowed." + ) + return self._description_pattern.format(language=_LANGUAGES[self._language]) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"language": self._language} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["language"] + + def check_following(self, value): + """Check if the language of the entire response follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the language of `value` follows instruction; otherwise False. + """ + assert isinstance(value, str) + + try: + return langdetect.detect(value) == self._language + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class NumberOfSentences(Instruction): + """Check the number of sentences.""" + + def build_description(self, *, num_sentences=None, relation=None): + """Build the instruction description. + + Args: + num_sentences: An integer specifying the number of sentences as a + threshold. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of sentences < the threshold; + if 'at least', the actual number of sentences >= the threshold. + + Returns: + A string representing the instruction description. + """ + # The number of sentences as a threshold for comparison. + self._num_sentences_threshold = num_sentences + if self._num_sentences_threshold is None or self._num_sentences_threshold < 0: + self._num_sentences_threshold = random.randint(1, _MAX_NUM_SENTENCES) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "Your response should contain {relation} {num_sentences} sentences." + ) + return self._description_pattern.format( + relation=self._comparison_relation, + num_sentences=self._num_sentences_threshold, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_sentences": self._num_sentences_threshold, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "relation"] + + def check_following(self, value): + """Check if the number of sentences follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the response follows the instruction. + + Raise: + ValueError if the string in `instruction_args` is not in + [`less_than`, `at_least`]. + """ + if self.lang == "zh-CN" or self.lang == "ja": + num_sentences = instructions_util.count_chinese_sentences(value) + elif self.lang == "th": + num_sentences = instructions_util.count_thai_sentences(value) + elif self.lang == "bn": + num_sentences = instructions_util.count_bengali_sentences(value) + elif self.lang == "ko": + num_sentences = instructions_util.count_korean_sentences(value) + else: + num_sentences = instructions_util.count_sentences(value) + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_sentences < self._num_sentences_threshold + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_sentences >= self._num_sentences_threshold + + +class PlaceholderChecker(Instruction): + """Check the placeholders in template writing.""" + + def build_description(self, *, num_placeholders=None): + """Build the instruction description. + + Args: + num_placeholders: An integer denoting the minimum number of + placeholders required in the response. + + Returns: + A string representing the instruction description. + """ + self._num_placeholders = num_placeholders + if self._num_placeholders is None or self._num_placeholders < 0: + self._num_placeholders = random.randint(1, _NUM_PLACEHOLDERS) + self._description_pattern = ( + "The response must contain at least {num_placeholders} placeholders " + + "represented by square brackets, such as [address]." + ) + return self._description_pattern.format(num_placeholders=self._num_placeholders) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_placeholders": self._num_placeholders} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_placeholders"] + + def check_following(self, value): + """Check if the number of placeholders follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the actual number of placeholders in the response is greater than + or equal to `num_placeholders`; otherwise, False. + """ + placeholders = re.findall(r"\[.*?\]", value) + num_placeholders = len(placeholders) + return num_placeholders >= self._num_placeholders + + +class BulletListChecker(Instruction): + """Checks the bullet list in the prompt.""" + + def build_description(self, *, num_bullets=None): + """Build the instruction description. + + Args: + num_bullets: An integer specifying the exact number of bullet lists + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._num_bullets = num_bullets + if self._num_bullets is None or self._num_bullets < 0: + self._num_bullets = random.randint(1, _NUM_BULLETS) + self._description_pattern = ( + "Your answer must contain exactly {num_bullets} bullet points. " + + "Use the markdown bullet points such as:\n" + + "* This is point 1. \n" + + "* This is point 2" + ) + return self._description_pattern.format(num_bullets=self._num_bullets) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_bullets": self._num_bullets} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_bullets"] + + def check_following(self, value): + r"""Check if the number of bullet lists meets the requirement. + + Args: + value: A string representing the response. The response is expected to + contain some bullet lists that start with `\*`. + + Returns: + True if the actual number of bullet lists in the response meets the + requirement. + """ + bullet_lists = re.findall(r"^\s*\*[^\*].*$", value, flags=re.MULTILINE) + bullet_lists_2 = re.findall(r"^\s*-.*$", value, flags=re.MULTILINE) + num_bullet_lists = len(bullet_lists) + len(bullet_lists_2) + return num_bullet_lists == self._num_bullets + + +class ConstrainedResponseChecker(Instruction): + """Checks the constrained response.""" + + def build_description(self): + """Build the instruction description.""" + # A sequence of string(s) representing the options of the expected response. + self._constrained_responses = _CONSTRAINED_RESPONSE_OPTIONS[self.lang] + self._description_pattern = ( + "Answer with one of the following options: {response_options}" + ) + return self._description_pattern.format( + response_options=self._constrained_responses + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response matches the constrained options. + + Args: + value: A string representing the response. + + Returns: + True if the actual response contains one of the options in the constrained + responses; otherwise False. + """ + value = value.strip() + for constrained_response in self._constrained_responses: + if constrained_response in value: + return True + return False + + +class ConstrainedStartChecker(Instruction): + """Checks the response start.""" + + def build_description(self, *, starter=None): + """Build the instruction description. + + Args: + starter: A string representing the keyword that the response should start + with. + + Returns: + A string representing the instruction description. + """ + self._starter = starter.strip() if isinstance(starter, str) else starter + if self._starter is None: + self._starter = random.choice(_STARTER_OPTIONS) + self._description_pattern = ( + "During the conversation, when it is your turn, " + + "please always start with {starter}" + ) + return self._description_pattern.format(starter=self._starter) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"starter": self._starter} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["starter"] + + def check_following(self, value): + """Checks if the response starts with the constrained keyword or phrase. + + Args: + value: A string representing the response. + + Returns: + True if the response starts with the given phrase or keyword that is + contained in `instruction_args`; otherwise, False. + """ + response_pattern = r"^\s*" + self._starter + r".*$" + response_with_constrained_start = re.search( + response_pattern, value, flags=re.MULTILINE + ) + return True if response_with_constrained_start else False + + +class HighlightSectionChecker(Instruction): + """Checks the highlighted section.""" + + def build_description(self, *, num_highlights=None): + """Build the instruction description. + + Args: + num_highlights: An integer specifying the minimum number of highlighted + sections. + + Returns: + A string representing the instruction description. + """ + self._num_highlights = num_highlights + if self._num_highlights is None or self._num_highlights < 0: + self._num_highlights = random.randint(1, _NUM_HIGHLIGHTED_SECTIONS) + + self._description_pattern = ( + "Highlight at least {num_highlights} sections in your answer with " + + "markdown, i.e. *highlighted section*." + ) + + return self._description_pattern.format(num_highlights=self._num_highlights) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_highlights": self._num_highlights} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_highlights"] + + def check_following(self, value): + """Checks if the number of highlighted sections meets the requirement. + + Args: + value: a string representing the response. The response is expected to + contain highlighted sections in the format of *highlighted*. + + Returns: + True if the actual number of highlighted sections in the format of + *highlighted sections* meets the minimum requirement; otherwise False. + """ + num_highlights = 0 + highlights = re.findall(r"\*[^\n\*]*\*", value) + double_highlights = re.findall(r"\*\*[^\n\*]*\*\*", value) + for highlight in highlights: + if highlight.strip("*").strip(): + num_highlights += 1 + for highlight in double_highlights: + if highlight.removeprefix("**").removesuffix("**").strip(): + num_highlights += 1 + + return num_highlights >= self._num_highlights + + +class SectionChecker(Instruction): + """Checks the sections.""" + + def build_description(self, *, section_spliter=None, num_sections=None): + """Build the instruction description. + + Args: + section_spliter: A string represents the section spliter keyword that + marks a new section, i.e., `Section` or `SECTION`. + num_sections: An integer specifying the number of sections. + + Returns: + A string representing the instruction description. + """ + self._section_spliter = ( + section_spliter.strip() + if isinstance(section_spliter, str) + else section_spliter + ) + if self._section_spliter is None: + self._section_spliter = random.choice(_SECTION_SPLITER) + + self._num_sections = num_sections + if self._num_sections is None or self._num_sections < 0: + self._num_sections = random.randint(1, _NUM_SECTIONS) + + self._description_pattern = ( + "Your response must have {num_sections} sections. Mark the beginning " + + "of each section with {section_spliter} X, such as:\n" + + "{section_spliter} 1\n" + + "[content of section 1]\n" + + "{section_spliter} 2\n" + + "[content of section 2]" + ) + + return self._description_pattern.format( + num_sections=self._num_sections, section_spliter=self._section_spliter + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "section_spliter": self._section_spliter, + "num_sections": self._num_sections, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["section_spliter", "num_sections"] + + def check_following(self, value): + """Checks the response contains multiple sections. + + Args: + value: A string representing the response. The response is expected + to contain multiple sections (number of sections is greater than 1). + A new section starts with `Section 1`, where the number denotes the + section index. + + Returns: + True if the number of sections in the response is greater than or equal to + the minimum number of sections; otherwise, False. + """ + section_splitter_patten = r"\s?" + self._section_spliter + r"\s?\d+\s?" + sections = re.split(section_splitter_patten, value) + num_sections = len(sections) - 1 + return num_sections >= self._num_sections + + +class ParagraphChecker(Instruction): + """Checks the paragraphs.""" + + def build_description(self, *, num_paragraphs=None): + """Build the instruction description. + + Args: + num_paragraphs: An integer specifying the number of paragraphs. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs are separated with the markdown divider: ***" + ) + + return self._description_pattern.format(num_paragraphs=self._num_paragraphs) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_paragraphs": self._num_paragraphs} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs"] + + def check_following(self, value): + """Checks the response contains required number of paragraphs. + + Args: + value: A string representing the response. The response may contain + paragraphs that are separated by the markdown divider: `***`. + + Returns: + True if the actual number of paragraphs is the same as required; + otherwise, False. + """ + paragraphs = re.split(r"\s?\*\*\*\s?", value) + num_paragraphs = len(paragraphs) + + for index, paragraph in enumerate(paragraphs): + if not paragraph.strip(): + if index == 0 or index == len(paragraphs) - 1: + num_paragraphs -= 1 + else: + return False + + return num_paragraphs == self._num_paragraphs + + +class PostscriptChecker(Instruction): + """Checks the postscript.""" + + def build_description(self, *, postscript_marker=None): + """Build the instruction description. + + Args: + postscript_marker: A string containing the keyword that marks the start + of the postscript section. + + Returns: + A string representing the instruction description. + """ + self._postscript_marker = ( + postscript_marker.strip() + if isinstance(postscript_marker, str) + else postscript_marker + ) + if self._postscript_marker is None: + self._postscript_marker = random.choice(_POSTSCRIPT_MARKER) + + self._description_pattern = ( + "At the end of your response, please explicitly add a postscript " + + "starting with {postscript}" + ) + + return self._description_pattern.format(postscript=self._postscript_marker) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"postscript_marker": self._postscript_marker} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["postscript_marker"] + + def check_following(self, value): + """Checks if the response follows the postscript format. + + Args: + value: a string representing the response. The response is expected to + contain a postscript section. + + Returns: + True if the response contains a postscript section starting with + the keyword containing in the `instruction_args`; otherwise False. + """ + value = value.lower() + if self._postscript_marker == "P.P.S": + postscript_pattern = r"\s*p\.\s?p\.\s?s.*$" + elif self._postscript_marker == "P.S.": + postscript_pattern = r"\s*p\.\s?s\..*$" + else: + postscript_pattern = r"\s*" + re.escape(self._postscript_marker.lower()) + r".*$" + postscript = re.findall(postscript_pattern, value, flags=re.MULTILINE) + return True if postscript else False + + +class RephraseChecker(Instruction): + """Checks the rephrase.""" + + def build_description(self, *, original_message): + """Build the instruction description. + + Args: + original_message: A string representing the original message. The + rephrased response should only change its words/sentences in between + its two asterisks, for example, *change me*. Both original and rephrased + messages should contain the changes in the form of *change me*. + + Returns: + A string representing the instruction description. + """ + if not self.is_change(original_message): + raise ValueError( + f"Message {original_message} does not contain changes " + "in the form of *change me*." + ) + + self._reference_without_change = original_message + self._description = ( + "Rephrasing: Your rephrased response should only" + + "change the words/sentences in between two asterisks" + + "such as *change me*." + ) + return self._description + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"original_message": self._reference_without_change} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_message"] + + def check_following(self, value): + r"""Checks if the rephrasing follows the instruction. + + Args: + value: A string representing the response, which is expected to rephras + the string of `instruction_args`. + + Returns: + True if `value` and `instruction_args` only differ by the words/sentences + in between two asterisks such as *change me*; otherwise, False. + """ + + if not self.is_change(value): + raise ValueError( + f"value {value} does not contain " "changes in the form of *change me*." + ) + + response_without_changes = self.strip_changes(value) + reference_without_changes = self.strip_changes(self._reference_without_change) + + return response_without_changes == reference_without_changes + + def is_change(self, response): + """Check if there is change in the response in the form of *change me*.""" + return re.search(r"\*.*\*", response) + + def strip_changes(self, response): + """Strips off the changes.""" + return re.sub(r"\*.*\*", "", response) + + +class KeywordChecker(Instruction): + """Check the exisitence of certain keywords.""" + + def build_description(self, *, keywords=None): + """Build the instruction description. + + Args: + keywords: A sequence of strings representing the keywords that are + expected in the response. + + Returns: + A string representing the instruction description. + """ + + if not keywords: + self._keywords = instructions_util.generate_keywords( + num_keywords=_NUM_KEYWORDS + ) + else: + self._keywords = keywords + self._keywords = sorted(self._keywords) + + self._description_pattern = "Include keywords {keywords} in the response." + + return self._description_pattern.format(keywords=self._keywords) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"keywords": self._keywords} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keywords"] + + def check_following(self, value): + """Check if the response contain the expected keywords.""" + for keyword in self._keywords: + if not re.search(keyword, value, flags=re.IGNORECASE): + return False + return True + + +class KeywordFrequencyChecker(Instruction): + """Check the keyword frequency.""" + + def build_description(self, *, keyword=None, frequency=None, relation=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + frequency: An integer specifying the number of times `keyword` is expected + to appear in the response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of occurrences < frequency; + if 'at least', the actual number of occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if not keyword: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + + self._frequency = frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _KEYWORD_FREQUENCY) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "In your response, the word {keyword} should appear {relation} " + + "{frequency} times." + ) + + return self._description_pattern.format( + keyword=self._keyword, + relation=self._comparison_relation, + frequency=self._frequency, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "keyword": self._keyword, + "frequency": self._frequency, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword", "frequency", "relation"] + + def check_following(self, value): + """Checks if the response contain the keyword with required frequency.""" + actual_occurrences = len(re.findall(self._keyword, value, flags=re.IGNORECASE)) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return actual_occurrences < self._frequency + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return actual_occurrences >= self._frequency + + +class NumberOfWords(Instruction): + """Checks the number of words.""" + + def build_description(self, *, num_words=None, relation=None): + """Build the instruction description. + + Args: + num_words: An integer specifying the number of words contained in the + response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of words < num_words; + if 'at least', the actual number of words >= num_words. + + Returns: + A string representing the instruction description. + """ + + self._num_words = num_words + if self._num_words is None or self._num_words < 0: + self._num_words = random.randint( + _NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT + ) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = "Answer with {relation} {num_words} words." + + return self._description_pattern.format( + relation=self._comparison_relation, num_words=self._num_words + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_words": self._num_words, "relation": self._comparison_relation} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_words", "relation"] + + def check_following(self, value): + """Checks if the response contains the expected number of words.""" + if self.lang == "zh-CN": + num_words = instructions_util.count_chinese_words(value) + elif self.lang == "ja": + num_words = instructions_util.count_japanese_words(value) + elif self.lang == "th": + num_words = instructions_util.count_words_by_spm(value) + elif self.lang == "ko": + num_words = instructions_util.count_korean_words(value) + elif self.lang == "bn" or self.lang == "te": + num_words = instructions_util.count_words_by_space(value) + else: + num_words = instructions_util.count_words(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_words < self._num_words + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_words >= self._num_words + + +class JsonFormat(Instruction): + """Check the Json format.""" + + def build_description(self): + self._description_pattern = ( + "Entire output should be wrapped in JSON format. You can use markdown" + " ticks such as ```." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + value = ( + value.strip() + .removeprefix("```json") + .removeprefix("```Json") + .removeprefix("```JSON") + .removeprefix("```") + .removesuffix("```") + .strip() + ) + try: + json.loads(value) + except ValueError: + return False + return True + + +class ParagraphFirstWordCheck(Instruction): + """Check the paragraph and the first word of the nth paragraph.""" + + def build_description( + self, num_paragraphs=None, nth_paragraph=None, first_word=None + ): + r"""Build the instruction description. + + Args: + num_paragraphs: An integer indicating the number of paragraphs expected + in the response. A paragraph is a subset of the string that is + expected to be separated by '\n\n'. + nth_paragraph: An integer indicating the paragraph number that we look at. + Note that n starts from 1. + first_word: A string that represent the first word of the bth paragraph. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._nth_paragraph = nth_paragraph + if ( + self._nth_paragraph is None + or self._nth_paragraph <= 0 + or self._nth_paragraph > self._num_paragraphs + ): + self._nth_paragraph = random.randint(1, self._num_paragraphs + 1) + + self._first_word = first_word + if self._first_word is None: + self._first_word = instructions_util.generate_keywords(num_keywords=1)[0] + self._first_word = self._first_word.lower() + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs and only paragraphs are separated with each other by two " + + "new lines as if it was '\\n\\n' in python. " + + "Paragraph {nth_paragraph} must start with word {first_word}." + ) + + return self._description_pattern.format( + num_paragraphs=self._num_paragraphs, + nth_paragraph=self._nth_paragraph, + first_word=self._first_word, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_paragraphs": self._num_paragraphs, + "nth_paragraph": self._nth_paragraph, + "first_word": self._first_word, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs", "nth_paragraph", "first_word"] + + def check_following(self, value): + """Checks for required number of paragraphs and correct first word. + + Args: + value: a string representing the response. The response may contain + paragraphs that are separated by two new lines and the first word of + the nth paragraph will have to match a specified word. + + Returns: + True if the number of paragraphs is the same as required and the first + word of the specified paragraph is the same as required. Otherwise, false. + """ + + paragraphs = re.split(r"\n\n", value) + num_paragraphs = len(paragraphs) + + for paragraph in paragraphs: + if not paragraph.strip(): + num_paragraphs -= 1 + + # check that index doesn't go out of bounds + if self._nth_paragraph <= num_paragraphs: + paragraph = paragraphs[self._nth_paragraph - 1].strip() + if not paragraph: + return False + else: + return False + + if self.lang == "zh-CN" or self.lang == "ja" or self.lang == "th": + if paragraph.lstrip(" \"'‘“「").startswith(self._first_word): + first_word = self._first_word + else: + first_word = "" + else: + first_word = "" + punctuation = {".", ",", "?", "!", "'", '"'} + + # get first word and remove punctuation + word = paragraph.split()[0].strip() + # TODO(jeffrey): make more complex? + word = word.lstrip("'") + word = word.lstrip('"') + if self.lang == "es": + word = word.lstrip('¿') + + for letter in word: + if letter in punctuation: + break + first_word += letter.lower() + + return num_paragraphs == self._num_paragraphs and first_word == self._first_word + + +# TODO(jeffrey) add relation - at least/at most? +class KeySentenceChecker(Instruction): + """Check the existence of certain key sentences.""" + + def build_description(self, key_sentences=None, num_sentences=None): + """Build the instruction description. + + Args: + key_sentences: A sequences of strings representing the key sentences that + are expected in the response. + num_sentences: The number of key sentences that are expected to be seen in + the response. + + Returns: + A string representing the instruction description. + """ + + if not key_sentences: + # TODO(jeffrey) make a generate sentences function? wonderwords package + self._key_sentences = set(["For now, this is fine."]) + else: + self._key_sentences = key_sentences + + if not num_sentences: + self._num_sentences = random.randint(1, len(self._key_sentences)) + else: + self._num_sentences = num_sentences + + self._description_pattern = ( + "Include {num_sentences} of the following sentences {key_sentences}" + ) + + return self._description_pattern.format( + num_sentences=self._num_sentences, key_sentences=self._key_sentences + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_sentences": self._num_sentences, + "key_sentences": list(self._key_sentences), + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "key_sentences"] + + def check_following(self, value): + """Checks if the response contains the expected key sentences.""" + count = 0 + sentences = instructions_util.split_into_sentences(value) + for sentence in self._key_sentences: + if sentence in sentences: + count += 1 + + return count == self._num_sentences + + +class ForbiddenWords(Instruction): + """Checks that specified words are not used in response.""" + + def build_description(self, forbidden_words=None): + """Build the instruction description. + + Args: + forbidden_words: A sequences of strings representing words that are not + allowed in the response. + + Returns: + A string representing the instruction description. + """ + + if not forbidden_words: + self._forbidden_words = instructions_util.generate_keywords( + num_keywords=_NUM_KEYWORDS + ) + else: + self._forbidden_words = list(set(forbidden_words)) + self._forbidden_words = sorted(self._forbidden_words) + self._description_pattern = ( + "Do not include keywords {forbidden_words} in the response." + ) + + return self._description_pattern.format(forbidden_words=self._forbidden_words) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"forbidden_words": self._forbidden_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["forbidden_words"] + + def check_following(self, value): + """Check if the response does not contain the expected keywords.""" + for word in self._forbidden_words: + if self.lang == "zh-CN" or self.lang == "ja" or self.lang == "th" or self.lang == "bn" or self.lang == "te": + if re.search(word, value, flags=re.IGNORECASE): + return False + else: + if re.search(r"\b" + word + r"\b", value, flags=re.IGNORECASE): + return False + return True + + +class RephraseParagraph(Instruction): + """Checks that the paragraph is rephrased.""" + + def build_description(self, *, original_paragraph, low, high): + """Builds the instruction description. + + Args: + original_paragraph: A string presenting the original paragraph. The + rephrases response should have betweeb low-high words in common. + low: An integer presenting the lower bound of similar words. + high: An integer representing the upper bound of similar words. + + Returns: + A string representing the instruction description. + """ + # TODO(jeffrey) make more encompassing + self._original_paragraph = original_paragraph + self._low = low + self._high = high + + self._description = ( + "Rephrase the following paragraph: " + + "{original_paragraph}\nYour response should have " + + "between {low} and {high} of the same words. " + + "Words are the same if and only if all of the " + + "letters, ignoring cases, are the same. For " + + "example, 'run' is the same as 'Run' but different " + + "to 'ran'." + ) + + return self._description.format( + original_paragraph=original_paragraph, low=self._low, high=self._high + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "original_paragraph": self._original_paragraph, + "low": self._low, + "high": self._high, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_paragraph", "low", "high"] + + def check_following(self, value): + val_words = re.findall(r"\w+", value.lower()) + original_words = re.findall(r"\w+", self._original_paragraph.lower()) + similar_words = 0 + + dict_val = collections.Counter(val_words) + dict_original = collections.Counter(original_words) + + for word in dict_original: + similar_words += min(dict_original[word], dict_val[word]) + + return similar_words >= self._low and similar_words <= self._high + + +class TwoResponsesChecker(Instruction): + """Check that two responses were given.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Give two different responses. Responses and only responses should" + " be separated by 6 asterisk symbols: ******." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has two different answers. + + Args: + value: A string representing the response. + + Returns: + True if two responses are detected and false otherwise. + """ + valid_responses = list() + responses = value.split("******") + for index, response in enumerate(responses): + if not response.strip(): + if index != 0 and index != len(responses) - 1: + return False + else: + valid_responses.append(response) + return ( + len(valid_responses) == 2 + and valid_responses[0].strip() != valid_responses[1].strip() + ) + + +class RepeatPromptThenAnswer(Instruction): + """Checks that Prompt is first repeated then answered.""" + + def build_description(self, *, prompt_to_repeat=None): + """Build the instruction description. + + Args: + prompt_to_repeat: The prompt that is meant to be repeated. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + self._description_pattern = ( + "First repeat the request word for word without change," + " then give your answer (1. do not say any words or characters" + " before repeating the request; 2. the request you need to repeat" + " does not include this sentence)" + ) + return self._description_pattern + + def get_instruction_args(self): + return {"prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["prompt_to_repeat"] + + def check_following(self, value): + if value.strip().lower().startswith(self._prompt_to_repeat.strip().lower()): + return True + return False + + +class EndChecker(Instruction): + """Checks that the prompt ends with a given phrase.""" + + def build_description(self, *, end_phrase=None): + """Build the instruction description. + + Args: + end_phrase: A string representing the phrase the response should end with. + + Returns: + A string representing the instruction description. + """ + self._end_phrase = ( + end_phrase.strip() if isinstance(end_phrase, str) else end_phrase + ) + if self._end_phrase is None: + self._end_phrase = random.choice(_ENDING_OPTIONS) + self._description_pattern = ( + "Finish your response with this exact phrase {ender}. " + "No other words should follow this phrase." + ) + return self._description_pattern.format(ender=self._end_phrase) + + def get_instruction_args(self): + return {"end_phrase": self._end_phrase} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["end_phrase"] + + def check_following(self, value): + """Checks if the response ends with the expected phrase.""" + value = value.strip().strip('"').lower() + self._end_phrase = self._end_phrase.strip().lower() + return value.endswith(self._end_phrase) + + +class TitleChecker(Instruction): + """Checks the response for a title.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your answer must contain a title, wrapped in double angular brackets," + " such as <>." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains a title.""" + pattern = r"<<[^\n]+>>" + re_pattern = re.compile(pattern) + titles = re.findall(re_pattern, value) + + for title in titles: + if title.lstrip("<").rstrip(">").strip(): + return True + return False + + +class LetterFrequencyChecker(Instruction): + """Checks letter frequency.""" + + def build_description(self, *, letter=None, let_frequency=None, let_relation=None): + """Build the instruction description. + + Args: + letter: A string representing a letter that is expected in the response. + let_frequency: An integer specifying the number of times `keyword` is + expected to appear in the response. + let_relation: A string in (`less than`, `at least`), defining the + relational operator for comparison. Two relational comparisons are + supported for now; if 'less than', the actual number of + occurrences < frequency; if 'at least', the actual number of + occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if ( + not letter + or len(letter) > 1 + or ord(letter.lower()) < 97 + or ord(letter.lower()) > 122 + ): + self._letter = random.choice(list(string.ascii_letters)) + else: + self._letter = letter.strip() + self._letter = self._letter.lower() + + self._frequency = let_frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _LETTER_FREQUENCY) + + if let_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif let_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {let_relation} is given." + ) + else: + self._comparison_relation = let_relation + + self._description_pattern = ( + "In your response, the letter {letter} should appear {let_relation}" + " {let_frequency} times." + ) + + return self._description_pattern.format( + letter=self._letter, + let_frequency=self._frequency, + let_relation=self._comparison_relation, + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "letter": self._letter, + "let_frequency": self._frequency, + "let_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["letter", "let_frequency", "let_relation"] + + def check_following(self, value): + """Checks that the response contains the letter at the right frequency.""" + value = value.lower() + letters = collections.Counter(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return letters[self._letter] < self._frequency + else: + return letters[self._letter] >= self._frequency + + +class CapitalLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all capital letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all capital letters." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all capital letters.""" + assert isinstance(value, str) + + try: + return value.isupper() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class LowercaseLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all lowercase letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all lowercase" + " letters. No capital letters are allowed." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all lowercase letters.""" + assert isinstance(value, str) + + try: + return value.islower() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class CommaChecker(Instruction): + """Checks the response for no commas.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "In your entire response, refrain from the use of any commas." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response does not contain commas.""" + return not re.search(r"[\,，]", value) + + +class CapitalWordFrequencyChecker(Instruction): + """Checks frequency of words with all capital letters.""" + + def build_description( + self, + capital_frequency=None, + capital_relation=None, + ): + """Build the instruction description. + + Args: + capital_frequency: An integer that represents the number of words that + should be in all capital letters. + capital_relation: A string that is 'at least' or 'at most' that refers to + the frequency. + + Returns: + A string representing the instruction description. + """ + self._frequency = capital_frequency + if self._frequency is None: + self._frequency = random.randint(1, _ALL_CAPITAL_WORD_FREQUENCY) + + self._comparison_relation = capital_relation + if capital_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif capital_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {capital_relation} is given." + ) + + self._description_pattern = ( + "In your response, words with all capital letters should appear" + " {relation} {frequency} times." + ) + + return self._description_pattern.format( + frequency=self._frequency, relation=self._comparison_relation + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "capital_frequency": self._frequency, + "capital_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["capital_frequency", "capital_relation"] + + def check_following(self, value): + """Checks the frequency of words with all capital letters.""" + # Hyphenated words will count as one word + words = instructions_util.nltk.word_tokenize(value) + capital_words = [word for word in words if word.isupper()] + + capital_words = len(capital_words) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return capital_words < self._frequency + else: + return capital_words >= self._frequency + + +class QuotationChecker(Instruction): + """Checks response is wrapped with double quotation marks.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Wrap your entire response with double quotation marks." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is wrapped with double quotation marks.""" + value = value.strip() + return len(value) > 1 and (value[0] in ['"', "“", "„", "«"]) and (value[-1] in ['"', "”", "“", "»"]) diff --git a/opencompass/datasets/xIFEval/instructions_registry.py b/opencompass/datasets/xIFEval/instructions_registry.py new file mode 100644 index 00000000..e40e936f --- /dev/null +++ b/opencompass/datasets/xIFEval/instructions_registry.py @@ -0,0 +1,167 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registry of all instructions.""" + +from opencompass.datasets.xIFEval import instructions + +_KEYWORD = "keywords:" + +_LANGUAGE = "language:" + +_LENGTH = "length_constraints:" + +_CONTENT = "detectable_content:" + +_FORMAT = "detectable_format:" + +_MULTITURN = "multi-turn:" + +_COMBINATION = "combination:" + +_STARTEND = "startend:" + +_CHANGE_CASES = "change_case:" + +_PUNCTUATION = "punctuation:" + +INSTRUCTION_DICT = { + _KEYWORD + "existence": instructions.KeywordChecker, + _KEYWORD + "frequency": instructions.KeywordFrequencyChecker, + # TODO(jeffreyzhou): make a proper set of sentences to choose from + # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, + _KEYWORD + "forbidden_words": instructions.ForbiddenWords, + _KEYWORD + "letter_frequency": instructions.LetterFrequencyChecker, + _LANGUAGE + "response_language": instructions.ResponseLanguageChecker, + _LENGTH + "number_sentences": instructions.NumberOfSentences, + _LENGTH + "number_paragraphs": instructions.ParagraphChecker, + _LENGTH + "number_words": instructions.NumberOfWords, + _LENGTH + "nth_paragraph_first_word": instructions.ParagraphFirstWordCheck, + _CONTENT + "number_placeholders": instructions.PlaceholderChecker, + _CONTENT + "postscript": instructions.PostscriptChecker, + _FORMAT + "number_bullet_lists": instructions.BulletListChecker, + # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace + # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, + _FORMAT + "constrained_response": instructions.ConstrainedResponseChecker, + _FORMAT + "number_highlighted_sections": (instructions.HighlightSectionChecker), + _FORMAT + "multiple_sections": instructions.SectionChecker, + # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. + # _FORMAT + "rephrase": instructions.RephraseChecker, + _FORMAT + "json_format": instructions.JsonFormat, + _FORMAT + "title": instructions.TitleChecker, + # TODO(tianjianlu): Re-enable with specific prompts. + # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, + _COMBINATION + "two_responses": instructions.TwoResponsesChecker, + _COMBINATION + "repeat_prompt": instructions.RepeatPromptThenAnswer, + _STARTEND + "end_checker": instructions.EndChecker, + _CHANGE_CASES + "capital_word_frequency": instructions.CapitalWordFrequencyChecker, + _CHANGE_CASES + "english_capital": instructions.CapitalLettersEnglishChecker, + _CHANGE_CASES + "english_lowercase": instructions.LowercaseLettersEnglishChecker, + _PUNCTUATION + "no_comma": instructions.CommaChecker, + _STARTEND + "quotation": instructions.QuotationChecker, +} + +INSTRUCTION_CONFLICTS = { + _KEYWORD + "existence": {_KEYWORD + "existence"}, + _KEYWORD + "frequency": {_KEYWORD + "frequency"}, + # TODO(jeffreyzhou): make a proper set of sentences to choose from + # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, + _KEYWORD + "forbidden_words": {_KEYWORD + "forbidden_words"}, + _KEYWORD + "letter_frequency": {_KEYWORD + "letter_frequency"}, + _LANGUAGE + "response_language": { + _LANGUAGE + "response_language", + _FORMAT + "multiple_sections", + _KEYWORD + "existence", + _KEYWORD + "frequency", + _KEYWORD + "forbidden_words", + _STARTEND + "end_checker", + _CHANGE_CASES + "english_capital", + _CHANGE_CASES + "english_lowercase", + }, + _LENGTH + "number_sentences": {_LENGTH + "number_sentences"}, + _LENGTH + "number_paragraphs": { + _LENGTH + "number_paragraphs", + _LENGTH + "nth_paragraph_first_word", + _LENGTH + "number_sentences", + _LENGTH + "nth_paragraph_first_word", + }, + _LENGTH + "number_words": {_LENGTH + "number_words"}, + _LENGTH + "nth_paragraph_first_word": { + _LENGTH + "nth_paragraph_first_word", + _LENGTH + "number_paragraphs", + }, + _CONTENT + "number_placeholders": {_CONTENT + "number_placeholders"}, + _CONTENT + "postscript": {_CONTENT + "postscript"}, + _FORMAT + "number_bullet_lists": {_FORMAT + "number_bullet_lists"}, + # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace + # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, + _FORMAT + "constrained_response": set(INSTRUCTION_DICT.keys()), + _FORMAT + "number_highlighted_sections": {_FORMAT + "number_highlighted_sections"}, + _FORMAT + "multiple_sections": { + _FORMAT + "multiple_sections", + _LANGUAGE + "response_language", + _FORMAT + "number_highlighted_sections", + }, + # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. + # _FORMAT + "rephrase": instructions.RephraseChecker, + _FORMAT + "json_format": set(INSTRUCTION_DICT.keys()).difference( + {_KEYWORD + "forbidden_words", _KEYWORD + "existence"} + ), + _FORMAT + "title": {_FORMAT + "title"}, + # TODO(tianjianlu): Re-enable with specific prompts. + # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, + _COMBINATION + "two_responses": set(INSTRUCTION_DICT.keys()).difference( + { + _KEYWORD + "forbidden_words", + _KEYWORD + "existence", + _LANGUAGE + "response_language", + _FORMAT + "title", + _PUNCTUATION + "no_comma", + } + ), + _COMBINATION + "repeat_prompt": set(INSTRUCTION_DICT.keys()).difference( + {_KEYWORD + "existence", _FORMAT + "title", _PUNCTUATION + "no_comma"} + ), + _STARTEND + "end_checker": {_STARTEND + "end_checker"}, + _CHANGE_CASES + "capital_word_frequency": { + _CHANGE_CASES + "capital_word_frequency", + _CHANGE_CASES + "english_lowercase", + _CHANGE_CASES + "english_capital", + }, + _CHANGE_CASES + "english_capital": {_CHANGE_CASES + "english_capital"}, + _CHANGE_CASES + "english_lowercase": { + _CHANGE_CASES + "english_lowercase", + _CHANGE_CASES + "english_capital", + }, + _PUNCTUATION + "no_comma": {_PUNCTUATION + "no_comma"}, + _STARTEND + "quotation": {_STARTEND + "quotation", _FORMAT + "title"}, +} + + +def conflict_make(conflicts): + """Makes sure if A conflicts with B, B will conflict with A. + + Args: + conflicts: Dictionary of potential conflicts where key is instruction id + and value is set of instruction ids that it conflicts with. + + Returns: + Revised version of the dictionary. All instructions conflict with + themselves. If A conflicts with B, B will conflict with A. + """ + for key in conflicts: + for k in conflicts[key]: + conflicts[k].add(key) + conflicts[key].add(key) + return conflicts diff --git a/opencompass/datasets/xIFEval/instructions_util.py b/opencompass/datasets/xIFEval/instructions_util.py new file mode 100644 index 00000000..a46c2a8e --- /dev/null +++ b/opencompass/datasets/xIFEval/instructions_util.py @@ -0,0 +1,1737 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility library of instructions.""" + +import functools +import random +import re + +import immutabledict +import nltk +import pkg_resources +from packaging import version + + +# Downloading 'punkt' with nltk<3.9 has a remote code vuln. +# see https://github.com/EleutherAI/lm-evaluation-harness/issues/2210 +# and https://github.com/nltk/nltk/issues/3266 +# for more information. +NLTK_MIN_VERSION = "3.9.1" + + +def download_nltk_resources(): + """Download 'punkt' if not already installed""" + nltk_version = pkg_resources.get_distribution("nltk").version + assert ( + version.parse(nltk_version) >= version.parse(NLTK_MIN_VERSION) + ), f"`nltk` version {nltk_version} is not >= {NLTK_MIN_VERSION}. Please update `nltk` before proceeding--older versions are vulnerable to a remote code execution vulnerability." + + try: + nltk.data.find("tokenizers/punkt_tab") + except LookupError: + nltk.download("punkt_tab") + + +download_nltk_resources() + +WORD_LIST = [ + "western", + "sentence", + "signal", + "dump", + "spot", + "opposite", + "bottom", + "potato", + "administration", + "working", + "welcome", + "morning", + "good", + "agency", + "primary", + "wish", + "responsibility", + "press", + "problem", + "president", + "steal", + "brush", + "read", + "type", + "beat", + "trainer", + "growth", + "lock", + "bone", + "case", + "equal", + "comfortable", + "region", + "replacement", + "performance", + "mate", + "walk", + "medicine", + "film", + "thing", + "rock", + "tap", + "total", + "competition", + "ease", + "south", + "establishment", + "gather", + "parking", + "world", + "plenty", + "breath", + "claim", + "alcohol", + "trade", + "dear", + "highlight", + "street", + "matter", + "decision", + "mess", + "agreement", + "studio", + "coach", + "assist", + "brain", + "wing", + "style", + "private", + "top", + "brown", + "leg", + "buy", + "procedure", + "method", + "speed", + "high", + "company", + "valuable", + "pie", + "analyst", + "session", + "pattern", + "district", + "pleasure", + "dinner", + "swimming", + "joke", + "order", + "plate", + "department", + "motor", + "cell", + "spend", + "cabinet", + "difference", + "power", + "examination", + "engine", + "horse", + "dimension", + "pay", + "toe", + "curve", + "literature", + "bother", + "fire", + "possibility", + "debate", + "activity", + "passage", + "hello", + "cycle", + "background", + "quiet", + "author", + "effect", + "actor", + "page", + "bicycle", + "error", + "throat", + "attack", + "character", + "phone", + "tea", + "increase", + "outcome", + "file", + "specific", + "inspector", + "internal", + "potential", + "staff", + "building", + "employer", + "shoe", + "hand", + "direction", + "garden", + "purchase", + "interview", + "study", + "recognition", + "member", + "spiritual", + "oven", + "sandwich", + "weird", + "passenger", + "particular", + "response", + "reaction", + "size", + "variation", + "a", + "cancel", + "candy", + "exit", + "guest", + "condition", + "fly", + "price", + "weakness", + "convert", + "hotel", + "great", + "mouth", + "mind", + "song", + "sugar", + "suspect", + "telephone", + "ear", + "roof", + "paint", + "refrigerator", + "organization", + "jury", + "reward", + "engineering", + "day", + "possession", + "crew", + "bar", + "road", + "description", + "celebration", + "score", + "mark", + "letter", + "shower", + "suggestion", + "sir", + "luck", + "national", + "progress", + "hall", + "stroke", + "theory", + "offer", + "story", + "tax", + "definition", + "history", + "ride", + "medium", + "opening", + "glass", + "elevator", + "stomach", + "question", + "ability", + "leading", + "village", + "computer", + "city", + "grand", + "confidence", + "candle", + "priest", + "recommendation", + "point", + "necessary", + "body", + "desk", + "secret", + "horror", + "noise", + "culture", + "warning", + "water", + "round", + "diet", + "flower", + "bus", + "tough", + "permission", + "week", + "prompt", + "connection", + "abuse", + "height", + "save", + "corner", + "border", + "stress", + "drive", + "stop", + "rip", + "meal", + "listen", + "confusion", + "girlfriend", + "living", + "relation", + "significance", + "plan", + "creative", + "atmosphere", + "blame", + "invite", + "housing", + "paper", + "drink", + "roll", + "silver", + "drunk", + "age", + "damage", + "smoke", + "environment", + "pack", + "savings", + "influence", + "tourist", + "rain", + "post", + "sign", + "grandmother", + "run", + "profit", + "push", + "clerk", + "final", + "wine", + "swim", + "pause", + "stuff", + "singer", + "funeral", + "average", + "source", + "scene", + "tradition", + "personal", + "snow", + "nobody", + "distance", + "sort", + "sensitive", + "animal", + "major", + "negotiation", + "click", + "mood", + "period", + "arrival", + "expression", + "holiday", + "repeat", + "dust", + "closet", + "gold", + "bad", + "sail", + "combination", + "clothes", + "emphasis", + "duty", + "black", + "step", + "school", + "jump", + "document", + "professional", + "lip", + "chemical", + "front", + "wake", + "while", + "inside", + "watch", + "row", + "subject", + "penalty", + "balance", + "possible", + "adult", + "aside", + "sample", + "appeal", + "wedding", + "depth", + "king", + "award", + "wife", + "blow", + "site", + "camp", + "music", + "safe", + "gift", + "fault", + "guess", + "act", + "shame", + "drama", + "capital", + "exam", + "stupid", + "record", + "sound", + "swing", + "novel", + "minimum", + "ratio", + "machine", + "shape", + "lead", + "operation", + "salary", + "cloud", + "affair", + "hit", + "chapter", + "stage", + "quantity", + "access", + "army", + "chain", + "traffic", + "kick", + "analysis", + "airport", + "time", + "vacation", + "philosophy", + "ball", + "chest", + "thanks", + "place", + "mountain", + "advertising", + "red", + "past", + "rent", + "return", + "tour", + "house", + "construction", + "net", + "native", + "war", + "figure", + "fee", + "spray", + "user", + "dirt", + "shot", + "task", + "stick", + "friend", + "software", + "promotion", + "interaction", + "surround", + "block", + "purpose", + "practice", + "conflict", + "routine", + "requirement", + "bonus", + "hole", + "state", + "junior", + "sweet", + "catch", + "tear", + "fold", + "wall", + "editor", + "life", + "position", + "pound", + "respect", + "bathroom", + "coat", + "script", + "job", + "teach", + "birth", + "view", + "resolve", + "theme", + "employee", + "doubt", + "market", + "education", + "serve", + "recover", + "tone", + "harm", + "miss", + "union", + "understanding", + "cow", + "river", + "association", + "concept", + "training", + "recipe", + "relationship", + "reserve", + "depression", + "proof", + "hair", + "revenue", + "independent", + "lift", + "assignment", + "temporary", + "amount", + "loss", + "edge", + "track", + "check", + "rope", + "estimate", + "pollution", + "stable", + "message", + "delivery", + "perspective", + "mirror", + "assistant", + "representative", + "witness", + "nature", + "judge", + "fruit", + "tip", + "devil", + "town", + "emergency", + "upper", + "drop", + "stay", + "human", + "neck", + "speaker", + "network", + "sing", + "resist", + "league", + "trip", + "signature", + "lawyer", + "importance", + "gas", + "choice", + "engineer", + "success", + "part", + "external", + "worker", + "simple", + "quarter", + "student", + "heart", + "pass", + "spite", + "shift", + "rough", + "lady", + "grass", + "community", + "garage", + "youth", + "standard", + "skirt", + "promise", + "blind", + "television", + "disease", + "commission", + "positive", + "energy", + "calm", + "presence", + "tune", + "basis", + "preference", + "head", + "common", + "cut", + "somewhere", + "presentation", + "current", + "thought", + "revolution", + "effort", + "master", + "implement", + "republic", + "floor", + "principle", + "stranger", + "shoulder", + "grade", + "button", + "tennis", + "police", + "collection", + "account", + "register", + "glove", + "divide", + "professor", + "chair", + "priority", + "combine", + "peace", + "extension", + "maybe", + "evening", + "frame", + "sister", + "wave", + "code", + "application", + "mouse", + "match", + "counter", + "bottle", + "half", + "cheek", + "resolution", + "back", + "knowledge", + "make", + "discussion", + "screw", + "length", + "accident", + "battle", + "dress", + "knee", + "log", + "package", + "it", + "turn", + "hearing", + "newspaper", + "layer", + "wealth", + "profile", + "imagination", + "answer", + "weekend", + "teacher", + "appearance", + "meet", + "bike", + "rise", + "belt", + "crash", + "bowl", + "equivalent", + "support", + "image", + "poem", + "risk", + "excitement", + "remote", + "secretary", + "public", + "produce", + "plane", + "display", + "money", + "sand", + "situation", + "punch", + "customer", + "title", + "shake", + "mortgage", + "option", + "number", + "pop", + "window", + "extent", + "nothing", + "experience", + "opinion", + "departure", + "dance", + "indication", + "boy", + "material", + "band", + "leader", + "sun", + "beautiful", + "muscle", + "farmer", + "variety", + "fat", + "handle", + "director", + "opportunity", + "calendar", + "outside", + "pace", + "bath", + "fish", + "consequence", + "put", + "owner", + "go", + "doctor", + "information", + "share", + "hurt", + "protection", + "career", + "finance", + "force", + "golf", + "garbage", + "aspect", + "kid", + "food", + "boot", + "milk", + "respond", + "objective", + "reality", + "raw", + "ring", + "mall", + "one", + "impact", + "area", + "news", + "international", + "series", + "impress", + "mother", + "shelter", + "strike", + "loan", + "month", + "seat", + "anything", + "entertainment", + "familiar", + "clue", + "year", + "glad", + "supermarket", + "natural", + "god", + "cost", + "conversation", + "tie", + "ruin", + "comfort", + "earth", + "storm", + "percentage", + "assistance", + "budget", + "strength", + "beginning", + "sleep", + "other", + "young", + "unit", + "fill", + "store", + "desire", + "hide", + "value", + "cup", + "maintenance", + "nurse", + "function", + "tower", + "role", + "class", + "camera", + "database", + "panic", + "nation", + "basket", + "ice", + "art", + "spirit", + "chart", + "exchange", + "feedback", + "statement", + "reputation", + "search", + "hunt", + "exercise", + "nasty", + "notice", + "male", + "yard", + "annual", + "collar", + "date", + "platform", + "plant", + "fortune", + "passion", + "friendship", + "spread", + "cancer", + "ticket", + "attitude", + "island", + "active", + "object", + "service", + "buyer", + "bite", + "card", + "face", + "steak", + "proposal", + "patient", + "heat", + "rule", + "resident", + "broad", + "politics", + "west", + "knife", + "expert", + "girl", + "design", + "salt", + "baseball", + "grab", + "inspection", + "cousin", + "couple", + "magazine", + "cook", + "dependent", + "security", + "chicken", + "version", + "currency", + "ladder", + "scheme", + "kitchen", + "employment", + "local", + "attention", + "manager", + "fact", + "cover", + "sad", + "guard", + "relative", + "county", + "rate", + "lunch", + "program", + "initiative", + "gear", + "bridge", + "breast", + "talk", + "dish", + "guarantee", + "beer", + "vehicle", + "reception", + "woman", + "substance", + "copy", + "lecture", + "advantage", + "park", + "cold", + "death", + "mix", + "hold", + "scale", + "tomorrow", + "blood", + "request", + "green", + "cookie", + "church", + "strip", + "forever", + "beyond", + "debt", + "tackle", + "wash", + "following", + "feel", + "maximum", + "sector", + "sea", + "property", + "economics", + "menu", + "bench", + "try", + "language", + "start", + "call", + "solid", + "address", + "income", + "foot", + "senior", + "honey", + "few", + "mixture", + "cash", + "grocery", + "link", + "map", + "form", + "factor", + "pot", + "model", + "writer", + "farm", + "winter", + "skill", + "anywhere", + "birthday", + "policy", + "release", + "husband", + "lab", + "hurry", + "mail", + "equipment", + "sink", + "pair", + "driver", + "consideration", + "leather", + "skin", + "blue", + "boat", + "sale", + "brick", + "two", + "feed", + "square", + "dot", + "rush", + "dream", + "location", + "afternoon", + "manufacturer", + "control", + "occasion", + "trouble", + "introduction", + "advice", + "bet", + "eat", + "kill", + "category", + "manner", + "office", + "estate", + "pride", + "awareness", + "slip", + "crack", + "client", + "nail", + "shoot", + "membership", + "soft", + "anybody", + "web", + "official", + "individual", + "pizza", + "interest", + "bag", + "spell", + "profession", + "queen", + "deal", + "resource", + "ship", + "guy", + "chocolate", + "joint", + "formal", + "upstairs", + "car", + "resort", + "abroad", + "dealer", + "associate", + "finger", + "surgery", + "comment", + "team", + "detail", + "crazy", + "path", + "tale", + "initial", + "arm", + "radio", + "demand", + "single", + "draw", + "yellow", + "contest", + "piece", + "quote", + "pull", + "commercial", + "shirt", + "contribution", + "cream", + "channel", + "suit", + "discipline", + "instruction", + "concert", + "speech", + "low", + "effective", + "hang", + "scratch", + "industry", + "breakfast", + "lay", + "join", + "metal", + "bedroom", + "minute", + "product", + "rest", + "temperature", + "many", + "give", + "argument", + "print", + "purple", + "laugh", + "health", + "credit", + "investment", + "sell", + "setting", + "lesson", + "egg", + "middle", + "marriage", + "level", + "evidence", + "phrase", + "love", + "self", + "benefit", + "guidance", + "affect", + "you", + "dad", + "anxiety", + "special", + "boyfriend", + "test", + "blank", + "payment", + "soup", + "obligation", + "reply", + "smile", + "deep", + "complaint", + "addition", + "review", + "box", + "towel", + "minor", + "fun", + "soil", + "issue", + "cigarette", + "internet", + "gain", + "tell", + "entry", + "spare", + "incident", + "family", + "refuse", + "branch", + "can", + "pen", + "grandfather", + "constant", + "tank", + "uncle", + "climate", + "ground", + "volume", + "communication", + "kind", + "poet", + "child", + "screen", + "mine", + "quit", + "gene", + "lack", + "charity", + "memory", + "tooth", + "fear", + "mention", + "marketing", + "reveal", + "reason", + "court", + "season", + "freedom", + "land", + "sport", + "audience", + "classroom", + "law", + "hook", + "win", + "carry", + "eye", + "smell", + "distribution", + "research", + "country", + "dare", + "hope", + "whereas", + "stretch", + "library", + "if", + "delay", + "college", + "plastic", + "book", + "present", + "use", + "worry", + "champion", + "goal", + "economy", + "march", + "election", + "reflection", + "midnight", + "slide", + "inflation", + "action", + "challenge", + "guitar", + "coast", + "apple", + "campaign", + "field", + "jacket", + "sense", + "way", + "visual", + "remove", + "weather", + "trash", + "cable", + "regret", + "buddy", + "beach", + "historian", + "courage", + "sympathy", + "truck", + "tension", + "permit", + "nose", + "bed", + "son", + "person", + "base", + "meat", + "usual", + "air", + "meeting", + "worth", + "game", + "independence", + "physical", + "brief", + "play", + "raise", + "board", + "she", + "key", + "writing", + "pick", + "command", + "party", + "yesterday", + "spring", + "candidate", + "physics", + "university", + "concern", + "development", + "change", + "string", + "target", + "instance", + "room", + "bitter", + "bird", + "football", + "normal", + "split", + "impression", + "wood", + "long", + "meaning", + "stock", + "cap", + "leadership", + "media", + "ambition", + "fishing", + "essay", + "salad", + "repair", + "today", + "designer", + "night", + "bank", + "drawing", + "inevitable", + "phase", + "vast", + "chip", + "anger", + "switch", + "cry", + "twist", + "personality", + "attempt", + "storage", + "being", + "preparation", + "bat", + "selection", + "white", + "technology", + "contract", + "side", + "section", + "station", + "till", + "structure", + "tongue", + "taste", + "truth", + "difficulty", + "group", + "limit", + "main", + "move", + "feeling", + "light", + "example", + "mission", + "might", + "wait", + "wheel", + "shop", + "host", + "classic", + "alternative", + "cause", + "agent", + "consist", + "table", + "airline", + "text", + "pool", + "craft", + "range", + "fuel", + "tool", + "partner", + "load", + "entrance", + "deposit", + "hate", + "article", + "video", + "summer", + "feature", + "extreme", + "mobile", + "hospital", + "flight", + "fall", + "pension", + "piano", + "fail", + "result", + "rub", + "gap", + "system", + "report", + "suck", + "ordinary", + "wind", + "nerve", + "ask", + "shine", + "note", + "line", + "mom", + "perception", + "brother", + "reference", + "bend", + "charge", + "treat", + "trick", + "term", + "homework", + "bake", + "bid", + "status", + "project", + "strategy", + "orange", + "let", + "enthusiasm", + "parent", + "concentrate", + "device", + "travel", + "poetry", + "business", + "society", + "kiss", + "end", + "vegetable", + "employ", + "schedule", + "hour", + "brave", + "focus", + "process", + "movie", + "illegal", + "general", + "coffee", + "ad", + "highway", + "chemistry", + "psychology", + "hire", + "bell", + "conference", + "relief", + "show", + "neat", + "funny", + "weight", + "quality", + "club", + "daughter", + "zone", + "touch", + "tonight", + "shock", + "burn", + "excuse", + "name", + "survey", + "landscape", + "advance", + "satisfaction", + "bread", + "disaster", + "item", + "hat", + "prior", + "shopping", + "visit", + "east", + "photo", + "home", + "idea", + "father", + "comparison", + "cat", + "pipe", + "winner", + "count", + "lake", + "fight", + "prize", + "foundation", + "dog", + "keep", + "ideal", + "fan", + "struggle", + "peak", + "safety", + "solution", + "hell", + "conclusion", + "population", + "strain", + "alarm", + "measurement", + "second", + "train", + "race", + "due", + "insurance", + "boss", + "tree", + "monitor", + "sick", + "course", + "drag", + "appointment", + "slice", + "still", + "care", + "patience", + "rich", + "escape", + "emotion", + "royal", + "female", + "childhood", + "government", + "picture", + "will", + "sock", + "big", + "gate", + "oil", + "cross", + "pin", + "improvement", + "championship", + "silly", + "help", + "sky", + "pitch", + "man", + "diamond", + "most", + "transition", + "work", + "science", + "committee", + "moment", + "fix", + "teaching", + "dig", + "specialist", + "complex", + "guide", + "people", + "dead", + "voice", + "original", + "break", + "topic", + "data", + "degree", + "reading", + "recording", + "bunch", + "reach", + "judgment", + "lie", + "regular", + "set", + "painting", + "mode", + "list", + "player", + "bear", + "north", + "wonder", + "carpet", + "heavy", + "officer", + "negative", + "clock", + "unique", + "baby", + "pain", + "assumption", + "disk", + "iron", + "bill", + "drawer", + "look", + "double", + "mistake", + "finish", + "future", + "brilliant", + "contact", + "math", + "rice", + "leave", + "restaurant", + "discount", + "sex", + "virus", + "bit", + "trust", + "event", + "wear", + "juice", + "failure", + "bug", + "context", + "mud", + "whole", + "wrap", + "intention", + "draft", + "pressure", + "cake", + "dark", + "explanation", + "space", + "angle", + "word", + "efficiency", + "management", + "habit", + "star", + "chance", + "finding", + "transportation", + "stand", + "criticism", + "flow", + "door", + "injury", + "insect", + "surprise", + "apartment", +] # pylint: disable=line-too-long + +# ISO 639-1 codes to language names. +LANGUAGE_CODES = immutabledict.immutabledict( + { + "en": "English", + "es": "Spanish", + "pt": "Portuguese", + "ar": "Arabic", + "hi": "Hindi", + "fr": "French", + "ru": "Russian", + "de": "German", + "ja": "japanese", + "it": "Italian", + "bn": "Bengali", + "uk": "Ukrainian", + "th": "Thai", + "ur": "Urdu", + "ta": "Tamil", + "te": "Telugu", + "bg": "Bulgarian", + "ko": "Korean", + "pl": "Polish", + "he": "Hebrew", + "fa": "Persian", + "vi": "Vietnamese", + "ne": "Nepali", + "sw": "Swahili", + "kn": "Kannada", + "mr": "Marathi", + "gu": "Gujarati", + "pa": "Punjabi", + "ml": "Malayalam", + "fi": "Finnish", + } +) + +_ALPHABETS = "([A-Za-z])" +_PREFIXES = "(Mr|St|Mrs|Ms|Dr)[.]" +_SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)" +_STARTERS = r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" +_ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" +_WEBSITES = "[.](com|net|org|io|gov|edu|me)" +_DIGITS = "([0-9])" +_MULTIPLE_DOTS = r"\.{2,}" + + +def split_into_sentences(text): + """Split the text into sentences. + + Args: + text: A string that consists of more than or equal to one sentences. + + Returns: + A list of strings where each string is a sentence. + """ + text = " " + text + " " + text = text.replace("\n", " ") + text = re.sub(_PREFIXES, "\\1", text) + text = re.sub(_WEBSITES, "\\1", text) + text = re.sub(_DIGITS + "[.]" + _DIGITS, "\\1\\2", text) + text = re.sub( + _MULTIPLE_DOTS, + lambda match: "" * len(match.group(0)) + "", + text, + ) + if "Ph.D" in text: + text = text.replace("Ph.D.", "PhD") + text = re.sub(r"\s" + _ALPHABETS + "[.] ", " \\1 ", text) + text = re.sub(_ACRONYMS + " " + _STARTERS, "\\1 \\2", text) + text = re.sub( + _ALPHABETS + "[.]" + _ALPHABETS + "[.]" + _ALPHABETS + "[.]", + "\\1\\2\\3", + text, + ) + text = re.sub(_ALPHABETS + "[.]" + _ALPHABETS + "[.]", "\\1\\2", text) + text = re.sub(" " + _SUFFIXES + "[.] " + _STARTERS, " \\1 \\2", text) + text = re.sub(" " + _SUFFIXES + "[.]", " \\1", text) + text = re.sub(" " + _ALPHABETS + "[.]", " \\1", text) + if "”" in text: + text = text.replace(".”", "”.") + if '"' in text: + text = text.replace('."', '".') + if "!" in text: + text = text.replace('!"', '"!') + if "?" in text: + text = text.replace('?"', '"?') + text = text.replace(".", ".") + text = text.replace("?", "?") + text = text.replace("!", "!") + text = text.replace("", ".") + sentences = text.split("") + sentences = [s.strip() for s in sentences] + if sentences and not sentences[-1]: + sentences = sentences[:-1] + return sentences + + +def count_words(text): + """Counts the number of words.""" + tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_words = len(tokens) + return num_words + +def count_chinese_words(text): + from sacrebleu.tokenizers.tokenizer_zh import TokenizerZh + tokenizer = TokenizerZh() + return len(tokenizer(text.strip()).split()) + +def count_japanese_words(text): + from sacrebleu.tokenizers.tokenizer_ja_mecab import TokenizerJaMecab + tokenizer = TokenizerJaMecab() + return len(tokenizer(text.strip()).split()) + +def count_korean_words(text): + from sacrebleu.tokenizers.tokenizer_ko_mecab import TokenizerKoMecab + tokenizer = TokenizerKoMecab() + return len(tokenizer(text.strip()).split()) + +def count_words_by_space(text): + return len(text.strip().split()) + +def count_words_by_spm(text): + from sacrebleu.tokenizers.tokenizer_spm import Flores101Tokenizer + tokenizer = Flores101Tokenizer() + return len(tokenizer(text.strip()).split()) + + +@functools.lru_cache(maxsize=None) +def _get_sentence_tokenizer(): + return nltk.data.load("nltk:tokenizers/punkt/english.pickle") + + +def count_sentences(text): + """Count the number of sentences.""" + tokenizer = _get_sentence_tokenizer() + tokenized_sentences = tokenizer.tokenize(text) + return len(tokenized_sentences) + +def count_chinese_sentences(text): + text = re.sub(r"([。！!？\?]|\.{6}|……)\s*([^”’\"」\s])", r"\1\n\2", text) + text = re.sub(r"(([。！!？\?]|\.{6}|……)\s*[”’\"」])\s*([^,，。！!？\?\s])", r"\1\n\3", text) + return len(text.strip().split("\n")) + +def count_thai_sentences(text): + from pythainlp import sent_tokenize + return len(sent_tokenize(text)) + +def count_bengali_sentences(text): + text = re.sub(r"([!\?।]|\.{6})\s*([^”’\"\s])", r"\1\n\2", text) + text = re.sub(r"(([!\?।]|\.{6})\s*[”’\"])\s*([^,!\?।\s])", r"\1\n\3", text) + return len(text.strip().split("\n")) + +def count_korean_sentences(text): + import kss + return len(kss.split_sentences(text)) + + +def generate_keywords(num_keywords): + """Randomly generates a few keywords.""" + return random.sample(WORD_LIST, k=num_keywords) diff --git a/opencompass/datasets/xhumaneval.py b/opencompass/datasets/xhumaneval.py new file mode 100644 index 00000000..7bc8588b --- /dev/null +++ b/opencompass/datasets/xhumaneval.py @@ -0,0 +1,73 @@ +# flake8: noqa: E501 +# yapf: disable +import copy +import json +import os.path as osp +import re +import tempfile +from os import environ +from typing import List +import os +from datasets import Dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path + +from .base import BaseDataset + +HUMANEVAL_IMPORT_ERROR = '''\ +Please install human_eval use following steps: +git clone git@github.com:open-compass/human-eval.git +cd human-eval && pip install -e .''' + +HUMANEVAL_PLUS_IMPORT_ERROR = '''\ +Please install evalplus use following steps: +git clone --recurse-submodules git@github.com:open-compass/human-eval.git +cd human-eval +pip install -e . +pip install -e evalplus''' + + +@LOAD_DATASET.register_module() +class xHumanevalDataset(BaseDataset): + + @staticmethod + def load(path: str, num_repeats: int = 1, local_mode: bool = False): + """Load humaneval dataset for pass k mode. + + Note that you can use num_repeats > 1 when your model does not support + `num_return_sequence` in generation, otherwise use the raw + humaneval dataset and set `num_return_sequence` in model config to + generate multiple responses for testing pass@k>1. + + It better to change your dataset abbr correspondingly if you want to + change num_repeats>1, otherwise the number in + `.cache/dataset_size.json` might be inconsistent. + + Args: + num_repeats(int): Number of repetition for this dataset to get + multiple responses in special cases. + """ + path = get_data_path(path, local_mode=True) + # path = os.path.join(path, name) + if environ.get('DATASET_SOURCE') == 'ModelScope': + from modelscope import MsDataset + dataset = MsDataset.load(path, subset_name='openai_humaneval', split='test') + dataset_list = [] + for example in dataset: + dataset_list.extend([example] * num_repeats) + dataset = Dataset.from_list(dataset_list) + else: + dataset = [] + with open(path, 'r', encoding='utf-8') as f: + for line in f: + item = json.loads(line.strip()) + item = {'prompt': item['prompt'], 'task_id': item['task_id'], 'test': item['test'], 'entry_point': item['entry_point'], 'canonical_solution': item['canonical_solution']} + dataset.extend( + [item for _ in range(num_repeats)]) + print(dataset[:10]) + dataset = Dataset.from_list(dataset) + return dataset + + diff --git a/opencompass/datasets/xlivecodebench/__init__.py b/opencompass/datasets/xlivecodebench/__init__.py new file mode 100644 index 00000000..555b546b --- /dev/null +++ b/opencompass/datasets/xlivecodebench/__init__.py @@ -0,0 +1,9 @@ +from .evaluator import LCBCodeExecutionEvaluator # noqa: F401, F403 +from .evaluator import xLCBCodeGenerationEvaluator # noqa: F401, F403 +from .evaluator import LCBTestOutputEvaluator # noqa: F401, F403 +from .livecodebench import CompassBenchCodeExecutionDataset # noqa: F401, F403 +from .livecodebench import LCBCodeExecutionDataset # noqa: F401, F403 +from .livecodebench import xLCBCodeGenerationDataset # noqa: F401, F403 +from .livecodebench import LCBTestOutputPredictionDataset # noqa: F401, F403 +from .prompts import TestOutputPromptConstants # noqa: F401, F403 +# from .xlivecodebench import xLCBCodeGenerationDataset # noqa: F401, F403 diff --git a/opencompass/datasets/xlivecodebench/evaluator.py b/opencompass/datasets/xlivecodebench/evaluator.py new file mode 100644 index 00000000..235062fb --- /dev/null +++ b/opencompass/datasets/xlivecodebench/evaluator.py @@ -0,0 +1,456 @@ +import ast +import json +import multiprocessing +from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor, as_completed + +import numpy as np +from tqdm import tqdm + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import ICL_EVALUATORS +from opencompass.utils import get_logger + +from .execute_utils import BASE_IMPORTS, codeexecute_check_correctness +from .extract_utils import (extract_code_execution, extract_code_generation, + extract_test_output_code) +from .livecodebench import xLCBCodeGenerationDataset +from .pass_k_utils import compute_metrics_from_results + + +def codegen_check_correctness(sample, generation, timeout, debug=True): + """Check correctness of code generation with a global timeout. + + The global timeout is to catch some extreme/rare cases not handled by the + timeouts inside `run_test` + """ + + def _temp_run(sample, generation, debug, result, metadata_list, timeout): + from .testing_util import run_test + res, metadata = run_test(sample, + test=generation, + debug=debug, + timeout=timeout) + result.append(res) + metadata_list.append(metadata) + + manager = multiprocessing.Manager() + result = manager.list() + metadata_list = manager.list() + p = multiprocessing.Process( + target=_temp_run, + args=(sample, generation, debug, result, metadata_list, timeout), + ) + p.start() + p.join(timeout=(timeout + 1) * + len(json.loads(sample['input_output'])['inputs']) + 5) + if p.is_alive(): + p.kill() + if not result: + in_outs = json.loads(sample['input_output']) + # consider that all tests failed + result = [[-1 for i in range(len(in_outs['inputs']))]] + if debug: + logger = get_logger() + logger.info('global timeout') + return result[0], metadata_list[0] + + +def evaluate_generations_by_problem(problem_generations: list, sample: list, + debug: bool, timeout: int): + """Evaluate each problem. + + Args: + problem_generations: + sample: + debug: + timeout + """ + # problem_generations: list[str] = args[0] + # sample = args[1] + # debug: bool = args[2] + # timeout: int = args[3] + logger = get_logger() + + res = [] + metadata = [] + for o_idx, o in enumerate(problem_generations): + curr_res = [-2] + try: + curr_res, curr_metadata = codegen_check_correctness( + sample, o, timeout=timeout, debug=debug) + if debug: + logger.info(f'\nSuccessful compilation of task {o_idx}!') + fixed = [] + for e in curr_res: + if isinstance(e, np.ndarray): + e = e.item(0) + if isinstance(e, np.bool_): + e = bool(e) + fixed.append(e) + curr_res = fixed + if not np.all(curr_res): + if debug: + logger.info( + f'Results were not True for all test cases' # noqa: F541, E501 + f' {curr_res=}\n') + except Exception as e: + if debug: + logger.info( + f'Compilation failed, test framework exception' # noqa: F541, E501 + f' = {repr(e)}{e}\n') + # break + curr_metadata = {} + finally: + assert isinstance(curr_res, list) + assert isinstance(curr_metadata, dict) + res.append(curr_res) + metadata.append(curr_metadata) + if debug: + for i, r in enumerate(problem_generations): + logger.info(f'Sample\n{r}\nResult\n{res[i]}') + logger.info('*' * 30 + '\n\n') + return res, metadata + + +def evaluate_generations( + samples_list: list, + generations_list: list[list[str]], + debug: bool = False, + num_process_evaluate: int = 16, + timeout=6, +): + """We take the list of code generations and try to compile them and the run + their corresponding unit tests which are retrieved from the APPS dataset. + + Args: + generations: list of code generations (same order as samples in APPS + dataset) + level: difficulty level used in the generation, can be "all", + "introductory", "interview" or "competition" + + Returns: + results: dictionary of results, key is the problem index, value is + a list of results for each generation + [-2] = compile error, [-1] = runtime error [False] = failed test + case [True] = passed test case + """ + + # generations are code generations in the same order of the dataset + + inputs = [[(generations_list[index], samples_list[index], debug, timeout), + index] for index in range(len(generations_list))] + + with tqdm(total=len(inputs)) as pbar: + with ProcessPoolExecutor( + max_workers=1 if debug else num_process_evaluate) as executor: + futures = { + executor.submit(evaluate_generations_by_problem, + problem_generations, sample, debug, timeout): + index + for (problem_generations, sample, debug, + timeout), index in inputs + } + + results = {} + metadata = {} + for future in as_completed(futures): + index = futures[future] + results[index], metadata[index] = future.result() + pbar.update(1) + + assert len(results) == len( + inputs), f'results = {len(results)} inputs = {len(inputs)} {results=}' + # results = {i: r for r, (_, i) in zip(results, inputs)} + + return results, metadata + + +def codegen_metrics( + samples_list, + generations_list, + k_list=[1, 5, 10, 20, 40, 50, 75, 100, 125, 150, 200, 500, 1000], + num_process_evaluate=16, + timeout=6, + debug=False, +): + logger = get_logger() + + samples_linear = [] + generations_linear = [] + remap_index = [] + results = defaultdict(list) + metadatas = defaultdict(list) + for idx, (sample, + generation_list) in enumerate(zip(samples_list, + generations_list)): + assert isinstance(generation_list, list), generations_list[0] + for generation in generation_list: + assert isinstance(generation, str), generations_list[0] + samples_linear.append(sample) + generations_linear.append([generation]) + remap_index.append(idx) + + logger.info(f'LCBCodeGeneration: Evaluating {len(samples_linear)}...') + + results_linear, metadatas_linear = evaluate_generations( + samples_linear, + generations_linear, + debug=debug, + num_process_evaluate=num_process_evaluate, + timeout=timeout, + ) + + for idx, sub_results in sorted(results_linear.items(), key=lambda x: x[0]): + results[remap_index[idx]].append(sub_results[0]) + + for idx, sub_metadatas in sorted(metadatas_linear.items(), + key=lambda x: x[0]): + metadatas[remap_index[idx]].append(sub_metadatas[0]) + + metrics = compute_metrics_from_results(results, k_list=k_list) + + final_metadata = [] + for key in sorted(list(metadatas.keys())): + final_metadata.append(metadatas[key]) + for i in range(len(final_metadata)): + if type(final_metadata[i]) is not list: + final_metadata[i] = [json.dumps(final_metadata[i])] + else: + final_metadata[i] = [json.dumps(x) for x in final_metadata[i]] + + assert len(final_metadata[i]) == len( + generations_list[0]), f'{len(final_metadata[i])=}' + + return [metrics, results, final_metadata] + + +@ICL_EVALUATORS.register_module() +class xLCBCodeGenerationEvaluator(BaseEvaluator): + + def __init__(self, + num_process_evaluate, + path, + timeout=6, + release_version='release_v1'): + super().__init__() + self.num_process_evaluate = num_process_evaluate + self.timeout = timeout + self.dataset = xLCBCodeGenerationDataset.load(path=path, + release_version=release_version, local_mode=True)['test'] + + def score(self, predictions, references): + predictions = [[extract_code_generation(item)] for item in predictions] + + evaluation_samples = dict() + for idx in range(len(self.dataset)): + evaluation_samples[self.dataset[idx][ + 'question_id']] = self.dataset[idx]['evaluation_sample'] + + references = [evaluation_samples[item] for item in references] + + references = [{'input_output': item} for item in references] + + BaseEvaluator.is_num_equal(predictions, references) + + results = { # noqa: F841 + 'pass': 0, + 'timeout': 0, + 'failed': 0, + 'wrong_answer': 0 + } # noqa: F401, F403 + + metrics, eval_results, final_metadata = codegen_metrics( + references, + predictions, + k_list=[1], + num_process_evaluate=self.num_process_evaluate, + timeout=self.timeout, + ) + + return metrics + + +def evaluate_score(args) -> list[bool]: + gs, (c, i, o) = args + + execution_results = [] + for g in gs: + if i in g: + pass + else: + code_to_execute = f'{BASE_IMPORTS}\n{c}\nassert {o} == {g}' + execution_results.append( + codeexecute_check_correctness(code_to_execute, 3)) + if len(execution_results) == 0: + execution_results = [False] * len(gs) + return execution_results + + +def code_execution_metrics( + samples, + generations, +): + + def pass_at_k(n, c, k): + if n - c < k: + return 1.0 + return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) + + # execute the code + references = [(doc['code'], doc['input'], doc['output']) + for doc in samples] + with ProcessPoolExecutor() as executor: + args_list = zip(generations, references) + results = executor.map(evaluate_score, args_list) + all_results = list(results) + + # serial version + # all_results = [] + # for i in range(len(generations)): + # generation = generations[i] + # result = evaluate_score([generation, references[i]]) + # all_results.append(result) + + # compute pass@1 + pass_at_1s = [] + for execution_result in all_results: + c, n = execution_result.count(True), len(execution_result) + pass_at_1s.append(pass_at_k(n, c, 1)) + metrics = {'pass@1': sum(pass_at_1s) / len(pass_at_1s) * 100} + + results = {} + for i, r in enumerate(all_results): + r_new = [] + for _r in r: + r_new.append([_r]) + results[i] = r_new + return [metrics, results] + + +@ICL_EVALUATORS.register_module() +class LCBCodeExecutionEvaluator(BaseEvaluator): + + def __init__(self): + super().__init__() + # self.num_process_evaluate = num_process_evaluate + # self.timeout = timeout + + def score(self, predictions, references): + predictions = [[extract_code_execution(item)] for item in predictions] + references = [json.loads(item) for item in references] + metrics, results = code_execution_metrics(references, predictions) + return metrics + + +def parse_assert_statement(statement): + """Parse a Python assert statement and extract the expected output from the + right side of the '==' operator as a string. + + :param statement: A string containing the assert statement. + :return: The expected output from the assert statement as a string. + """ + try: + parsed = ast.parse(statement, mode='exec') + except SyntaxError: + return 'Invalid syntax' + + if len(parsed.body) == 0: + return 'Empty statement' + + if not isinstance(parsed.body[0], ast.Assert): + return 'Not an assert statement' + + comparison = parsed.body[0].test + + if not isinstance(comparison, ast.Compare) or not isinstance( + comparison.ops[0], ast.Eq): + return 'Not an equality assertion' + + # Extract and return the right side of the '==' operator as a string + return ast.get_source_segment(statement, comparison.comparators[0]) + + +def check_testcase_output(testcase_str, expected_output): + + if len(testcase_str.splitlines()) > 1: + for line in testcase_str.splitlines(): + if line.startswith('#'): + continue + if 'assert' in line: + testcase_str = line + break + + testcase_str = testcase_str.strip() + + if 'assert' in testcase_str: + testcase_output_str = str(parse_assert_statement(testcase_str)) + + else: + testcase_output_str = testcase_str + + global_result = None + + try: + testcase_output_eval = eval(testcase_output_str) + except Exception as e: + print(e) + global_result = False + # print("Failed to eval testcase output", testcase_output_str) + # breakpoint() + + try: + expected_output_eval = json.loads(expected_output) + except Exception as e: + print(e) + global_result = False + print('Failed to eval expected testcase output', expected_output) + + if global_result is None: + global_result = testcase_output_eval == expected_output_eval + + return global_result + + +def test_output_metrics( + samples, + generations, + k_list=[1, 5], +): + num_samples = len(samples) + results = [] + for idx in tqdm(list(range(num_samples))): + idx_results = [] + sample = samples[idx] + extracted_generation_list = generations[idx] + for extracted_generation in extracted_generation_list: + global_result = check_testcase_output(extracted_generation, + sample['output']) + idx_results.append([global_result]) + results.append(idx_results) + + results = { + result_idx: results[result_idx] + for result_idx in range(len(results)) + } + + metrics = compute_metrics_from_results(results, k_list=k_list) + + return [metrics, results] + + +@ICL_EVALUATORS.register_module() +class LCBTestOutputEvaluator(BaseEvaluator): + + def __init__(self): + super().__init__() + + def score(self, predictions, references): + + predictions = [[extract_test_output_code(item)] + for item in predictions] + references = [json.loads(item) for item in references] + metrics, results = test_output_metrics(references, + predictions, + k_list=[1]) + return metrics diff --git a/opencompass/datasets/xlivecodebench/execute_utils.py b/opencompass/datasets/xlivecodebench/execute_utils.py new file mode 100644 index 00000000..c6af4e1c --- /dev/null +++ b/opencompass/datasets/xlivecodebench/execute_utils.py @@ -0,0 +1,271 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the +# current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code is adapted from OpenAI's release +# https://github.com/openai/human-eval/blob/master/human_eval/execution.py + +import contextlib +import faulthandler +import io +import multiprocessing +import os +import platform +import signal +import tempfile + +BASE_IMPORTS = """from itertools import accumulate, chain, combinations, count, permutations, product, groupby, islice, repeat +from copy import deepcopy +from string import ascii_lowercase +from math import floor, log2, log10, sqrt, comb, gcd, ceil, inf, isqrt +from collections import defaultdict, deque, Counter +from bisect import bisect, bisect_left, bisect_right, insort +from heapq import heappush, heappop, heapify, merge +from functools import reduce, cache, lru_cache +from random import randrange, shuffle +from operator import itemgetter, sub +from re import search as re_search # Assuming 're' refers to a regex search +from os.path import commonprefix +from typing import List, Tuple, Dict, Set, Optional, Union, Any, Callable, Iterable, Iterator, Generator +import copy +import string +import math +import collections +import bisect +import heapq +import functools +import random +import itertools +import operator +import re +import numpy as np +import pandas as pd +from math import log, prod # 'log' and 'prod' are functions in the math module +from collections import deque, defaultdict, Counter, OrderedDict +from itertools import accumulate, permutations, combinations, product, groupby, islice, chain, repeat, zip_longest, cycle +from functools import lru_cache, reduce, partial +# from sortedcontainers import SortedList, SortedDict, SortedSet +# import sortedcontainers +from operator import iand +import sys +""" # noqa: E501 + + +def codeexecute_check_correctness(check_program, timeout=3): + """Evaluates the functional correctness of a completion by running the test + suite provided in the problem. + + :param completion_id: an optional completion ID so we can match + the results later even if execution finishes asynchronously. + """ + manager = multiprocessing.Manager() + result = manager.list() + + p = multiprocessing.Process(target=unsafe_execute, + args=(check_program, result, timeout)) + p.start() + p.join(timeout=timeout + 1) + if p.is_alive(): + p.kill() + + if not result: + result.append('timed out') + + return result[0] == 'passed' + + +def unsafe_execute(check_program, result, timeout): + + with create_tempdir(): + + # These system calls are needed when cleaning up tempdir. + import os + import shutil + + rmtree = shutil.rmtree + rmdir = os.rmdir + chdir = os.chdir + + # Disable functionalities that can make destructive changes + # to the test. + reliability_guard() + + # Run program. + try: + exec_globals = {} + with swallow_io(): + with time_limit(timeout): + exec(check_program, exec_globals) + result.append('passed') + except TimeoutException: + result.append('timed out') + except BaseException as e: + result.append(f'failed: {e}') + + # Needed for cleaning up. + shutil.rmtree = rmtree + os.rmdir = rmdir + os.chdir = chdir + + +@contextlib.contextmanager +def time_limit(seconds): + + def signal_handler(signum, frame): + raise TimeoutException('Timed out!') + + signal.setitimer(signal.ITIMER_REAL, seconds) + signal.signal(signal.SIGALRM, signal_handler) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +@contextlib.contextmanager +def swallow_io(): + stream = WriteOnlyStringIO() + with contextlib.redirect_stdout(stream): + with contextlib.redirect_stderr(stream): + with redirect_stdin(stream): + yield + + +@contextlib.contextmanager +def create_tempdir(): + with tempfile.TemporaryDirectory() as dirname: + with chdir(dirname): + yield dirname + + +class TimeoutException(Exception): + pass + + +class WriteOnlyStringIO(io.StringIO): + """StringIO that throws an exception when it's read from.""" + + def read(self, *args, **kwargs): + raise OSError + + def readline(self, *args, **kwargs): + raise OSError + + def readlines(self, *args, **kwargs): + raise OSError + + def readable(self, *args, **kwargs): + """Returns True if the IO object can be read.""" + return False + + +class redirect_stdin(contextlib._RedirectStream): # type: ignore + _stream = 'stdin' + + +@contextlib.contextmanager +def chdir(root): + if root == '.': + yield + return + cwd = os.getcwd() + os.chdir(root) + try: + yield + except BaseException as exc: + raise exc + finally: + os.chdir(cwd) + + +def reliability_guard(maximum_memory_bytes=None): + """This disables various destructive functions and prevents the generated + code from interfering with the test (e.g. fork bomb, killing other + processes, removing filesystem files, etc.) + + WARNING This function is NOT a security sandbox. Untrusted code, including, + model- generated code, should not be blindly executed outside of one. See + the Codex paper for more information about OpenAI's code sandbox, and + proceed with caution. + """ + + if maximum_memory_bytes is not None: + import resource + + resource.setrlimit(resource.RLIMIT_AS, + (maximum_memory_bytes, maximum_memory_bytes)) + resource.setrlimit(resource.RLIMIT_DATA, + (maximum_memory_bytes, maximum_memory_bytes)) + if not platform.uname().system == 'Darwin': + resource.setrlimit(resource.RLIMIT_STACK, + (maximum_memory_bytes, maximum_memory_bytes)) + + faulthandler.disable() + + import builtins + + builtins.exit = None + builtins.quit = None + + import os + + os.environ['OMP_NUM_THREADS'] = '1' + + os.kill = None + os.system = None + os.putenv = None + os.remove = None + os.removedirs = None + os.rmdir = None + os.fchdir = None + os.setuid = None + os.fork = None + os.forkpty = None + os.killpg = None + os.rename = None + os.renames = None + os.truncate = None + os.replace = None + os.unlink = None + os.fchmod = None + os.fchown = None + os.chmod = None + os.chown = None + os.chroot = None + os.fchdir = None + os.lchflags = None + os.lchmod = None + os.lchown = None + os.getcwd = None + os.chdir = None + + import shutil + + shutil.rmtree = None + shutil.move = None + shutil.chown = None + + import subprocess + + subprocess.Popen = None # type: ignore + + __builtins__['help'] = None + + import sys + + sys.modules['ipdb'] = None + sys.modules['joblib'] = None + sys.modules['resource'] = None + sys.modules['psutil'] = None + sys.modules['tkinter'] = None diff --git a/opencompass/datasets/xlivecodebench/extract_utils.py b/opencompass/datasets/xlivecodebench/extract_utils.py new file mode 100644 index 00000000..d296961d --- /dev/null +++ b/opencompass/datasets/xlivecodebench/extract_utils.py @@ -0,0 +1,75 @@ +# Copyright LiveCodeBench @ 2024, + +import re + + +def extract_code_generation(model_output: str, model_type: str = 'chat'): + # modified from + outputlines = model_output.split('\n') + # TODO: handle codellama + + if model_type == 'base': + return model_output.strip() + elif model_type == 'chat': + indexlines = [i for i, line in enumerate(outputlines) if '```' in line] + else: + raise ValueError(f'Invalid mode type: {model_type}') + + if len(indexlines) < 2: + return '' + return '\n'.join(outputlines[indexlines[0] + 1:indexlines[1]]) + + +def extract_code_execution(model_output: str, cot: bool = False): + pattern = r'\[PYTHON\](.*?)\[\/PYTHON\]' + matches = re.findall(pattern, model_output, re.DOTALL) + if matches: + # fetch the last one + model_output = matches[-1] + + if '[PYTHON]' in model_output: + model_output + if cot: + if '[ANSWER]' in model_output: + model_output = model_output.split('[ANSWER]')[1].strip() + if '==' in model_output: + model_output = model_output.split('==')[1].strip() + if '[/ANSWER]' in model_output: + model_output = model_output.split('[/ANSWER]')[0].strip() + else: + model_output = model_output.split('\n')[0].strip() + return model_output.strip() + + +def extract_test_output_code(model_output: str): + outputlines = model_output.split('\n') + # find the last line startwith assert... + indexlines = [ + i for i, line in enumerate(outputlines) if line.startswith('assert') + ] + if indexlines: + return outputlines[indexlines[-1]] + + # TODO: handle codellama format + # if lmstyle and lmstyle == LMStyle.CodeLLaMaInstruct: + # indexlines = \ + # [i for i, line in enumerate(outputlines) if "PYTHON]" in line] + # else: + + # first try to extract ```python if not then try ``` + indexlines = [ + i for i, line in enumerate(outputlines) + if '```python' in line or '```Python' in line + ] + if indexlines: + start_index = indexlines[0] + else: + start_index = None + indexlines = [i for i, line in enumerate(outputlines) if '```' in line] + if start_index is not None: + indexlines = [i for i in indexlines if i > start_index] + indexlines = [start_index] + indexlines + + if len(indexlines) < 2: + return '' + return '\n'.join(outputlines[indexlines[0] + 1:indexlines[1]]) diff --git a/opencompass/datasets/xlivecodebench/livecodebench.py b/opencompass/datasets/xlivecodebench/livecodebench.py new file mode 100644 index 00000000..7aae5564 --- /dev/null +++ b/opencompass/datasets/xlivecodebench/livecodebench.py @@ -0,0 +1,262 @@ +# Copyright (c) 2024, LiveCodeBench and its contributors. +# Copyright (c) 2023, OpenCompass and its contributors. + +import base64 +import json +import pickle +import zlib +from dataclasses import dataclass +from enum import Enum + +from datasets import DatasetDict, load_dataset, load_from_disk, Dataset + +from opencompass.utils import get_data_path # noqa: F401, F403 + +from ..base import BaseDataset +from .prompts import SelfRepairPromptConstants # noqa: F401, F403 +from .prompts import TestOutputPromptConstants # noqa: F401, F403 +from .prompts import (CodeGenerationPromptConstants, + get_generic_question_template_answer_self_repair, + get_generic_question_template_test_completion, + make_code_execution_prompt) + + +class Platform(Enum): + LEETCODE = 'leetcode' + CODEFORCES = 'codeforces' + ATCODER = 'atcoder' + + +class Difficulty(Enum): + EASY = 'easy' + MEDIUM = 'medium' + HARD = 'hard' + + +class TestType(Enum): + STDIN = 'stdin' + FUNCTIONAL = 'functional' + + +@dataclass +class Test: + input: str + output: str + testtype: TestType + + def __post_init__(self): + self.testtype = TestType(self.testtype) + + +class xLCBCodeGenerationDataset(BaseDataset): + + @staticmethod + def load(path: str = 'opencompass/code_generation_lite', + local_mode: bool = False, + release_version: str = 'release_v1'): + + def transform(item): + # Define the dataitem mapping logic + + # starter_code + if item['starter_code']: + format_prompt = f'### Format: {CodeGenerationPromptConstants.FORMATTING_MESSAGE_WITH_STARTER_CODE}\n' # noqa: E501 + format_prompt += f"```python\n{item['starter_code']}\n```\n\n" + else: + format_prompt = f'### Format: {CodeGenerationPromptConstants.FORMATTING_WITHOUT_STARTER_CODE}\n' # noqa: E501 + format_prompt += '```python\n# YOUR CODE HERE\n```\n\n' + + item['format_prompt'] = format_prompt + + # load test cases + public_test_cases = item['public_test_cases'] + public_test_cases = json.loads(item['public_test_cases']) + + private_test_cases = item['private_test_cases'] + try: + private_test_cases = json.loads(item['private_test_cases']) + except Exception as e: # noqa: F841 + private_test_cases = json.loads( + pickle.loads( + zlib.decompress( + base64.b64decode(private_test_cases.encode( + 'utf-8')) # type: ignore + ))) # type: ignore + + # load metadata + metadata = json.loads(item['metadata']) + evaluation_sample = json.dumps({ + 'inputs': + [t['input'] for t in public_test_cases + private_test_cases], + 'outputs': + [t['output'] for t in public_test_cases + private_test_cases], + 'fn_name': + metadata.get('func_name', None), + }) + item['evaluation_sample'] = evaluation_sample + + return item + + path = get_data_path(path, local_mode=local_mode) + print('path', path) + def load_jsonl(path): + dataset_dict = [] + with open(path, 'r') as f: + for line in f: + dataset_dict.append(json.loads(line)) + return dataset_dict + + dataset_dict = load_jsonl(path) + # tranform list to da + #print(dataset_dict) + dataset = Dataset.from_list(dataset_dict) + # dataset = load_dataset( + # path, # 'livecodebench/code_generation_lite' + # split='test', + # version_tag=release_version, + # trust_remote_code=True) + + dataset = dataset.map(transform) + + return DatasetDict({'test': dataset, 'train': dataset}) + + +class LCBCodeExecutionDataset(BaseDataset): + + @staticmethod + def load( + path: str = 'opencompass/execution-v2', + local_mode: bool = False, + cot: bool = False, + # release_version: str = "release_v1" + ): + # path = get_data_path(path, local_mode=local_mode) + + def transform(item): + code, input = item['code'], item['input'] + prompt = make_code_execution_prompt(code, input, cot=cot) + + item['prompt'] = prompt + + evaluation_sample = json.dumps({ + 'code': item['code'], + 'input': item['input'], + 'output': item['output'] + }) + item['evaluation_sample'] = evaluation_sample + + return item + + path = get_data_path(path, local_mode=local_mode) + dataset = load_dataset(path, + split='test') # 'livecodebench/execution-v2' + dataset = dataset.map(transform) + + return DatasetDict({'test': dataset, 'train': dataset}) + + +class LCBTestOutputPredictionDataset(BaseDataset): + + @staticmethod + def load( + path: str = 'opencompass/test_generation', + local_mode: bool = False, + # release_version: str = "release_v1" + ): + # path = get_data_path(path, local_mode=local_mode) + + def transform(item): + question_content = item['question_content'] + starter_code = item['starter_code'] + test = json.loads(item['test']) + + testcase_input = test[0]['input'] + testcase_output = test[0]['output'] + + item['testcase_input'] = testcase_input + item['testcase_output'] = testcase_output + + item['prompt'] = get_generic_question_template_test_completion( + question_content=question_content, + starter_code=starter_code, + testcase_input=testcase_input) + + evaluation_sample = json.dumps({ + 'input': + item['question_content'], + 'output': + json.loads(item['test'])[0]['output'] + }) + item['evaluation_sample'] = evaluation_sample + + return item + + path = get_data_path(path, local_mode=local_mode) + # 'livecodebench/test_generation', + dataset = load_dataset(path, split='test', trust_remote_code=True) + dataset = dataset.map(transform) + + return DatasetDict({'test': dataset, 'train': dataset}) + + +class LCBSelfRepairDataset(BaseDataset): + + @staticmethod + def load(path: str = 'livecodebench/code_generation_lite', + local_mode: bool = False, + release_version: str = 'release_v1'): + + def transform(item): + # define the data item mapping logic + + question = item['question_content'] + code = item['code_list'][0] + metadata = item['metadata'] + + prompt = get_generic_question_template_answer_self_repair( + question=question, code=code, metadata=metadata) + item['prompt'] = prompt + + return + + dataset = load_dataset(path, + split='test', + version_tag=release_version, + trust_remote_code=True) + dataset = dataset.map(transform) + + return DatasetDict({'test': dataset, 'train': dataset}) + + +class CompassBenchCodeExecutionDataset(BaseDataset): + + @staticmethod + def load( + path: str = 'opencompass/execution-v2', + local_mode: bool = False, + cot: bool = False, + # release_version: str = "release_v1" + ): + # path = get_data_path(path, local_mode=local_mode) + + def transform(item): + code, input = item['code'], item['input'] + prompt = make_code_execution_prompt(code, input, cot=cot) + + item['prompt'] = prompt + + evaluation_sample = json.dumps({ + 'code': item['code'], + 'input': item['input'], + 'output': item['output'] + }) + item['evaluation_sample'] = evaluation_sample + + return item + + path = get_data_path(path, local_mode=local_mode) + dataset = load_from_disk(path) # 'livecodebench/execution-v2' + dataset = dataset['test'] + dataset = dataset.map(transform) + + return DatasetDict({'test': dataset, 'train': dataset}) diff --git a/opencompass/datasets/xlivecodebench/pass_k_utils.py b/opencompass/datasets/xlivecodebench/pass_k_utils.py new file mode 100644 index 00000000..ec895bbb --- /dev/null +++ b/opencompass/datasets/xlivecodebench/pass_k_utils.py @@ -0,0 +1,72 @@ +# Copyright LiveCodeBench @ 2024, + +import numpy as np + + +def estimate_pass_at_k(num_samples, num_correct, k): + """Estimates pass@k of each problem and returns them in an array.""" + + def estimator(n: int, c: int, k: int) -> float: + """Calculates 1 - comb(n - c, k) / comb(n, k).""" + if n - c < k: + return 1.0 * 100 + return 100 * (1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))) + + import itertools + + if isinstance(num_samples, int): + num_samples_it = itertools.repeat(num_samples, len(num_correct)) + else: + assert len(num_samples) == len(num_correct) + num_samples_it = iter(num_samples) + + return np.array([ + estimator(int(n), int(c), k) + for n, c in zip(num_samples_it, num_correct) + ]) + + +def compute_metrics_from_results(results, k_list=[1, 5]): + total = [] + correct = [] + task_ids = [] + for task_id, res in results.items(): + all_correct = [] + for generation in res: + gen = np.array(generation) + all_correct.append(np.all(gen > 0)) + task_ids.append(task_id) + total.append(len(all_correct)) + correct.append(sum(all_correct)) + total = np.array(total) + correct = np.array(correct) + ks = k_list + detail_pass_at_k = { + f'pass@{k}': estimate_pass_at_k(total, correct, k).tolist() + for k in ks if (total >= k).all() + } + pass_at_k = { + f'pass@{k}': estimate_pass_at_k(total, correct, k).mean() + for k in ks if (total >= k).all() + } + detail_metrics = { + k: dict(zip(task_ids, v)) + for k, v in detail_pass_at_k.items() + } + pass_at_k['detail'] = detail_metrics + return pass_at_k + + +def extract_instance_results(results): + instance_wise_grades = {} + for task_id, res in results.items(): + instance_wise_grades[task_id] = [] + for generation in res: + instance_wise_grades[task_id].append( + all([g > 0 for g in generation])) + + instance_wise_grades = [ + v for _, v in sorted(instance_wise_grades.items(), + key=lambda item: item[0]) + ] + return instance_wise_grades diff --git a/opencompass/datasets/xlivecodebench/prompts.py b/opencompass/datasets/xlivecodebench/prompts.py new file mode 100644 index 00000000..406ff1a3 --- /dev/null +++ b/opencompass/datasets/xlivecodebench/prompts.py @@ -0,0 +1,210 @@ +# Copyright LiveCodeBench @ 2024, + +import json + + +class CodeGenerationPromptConstants: + SYSTEM_MESSAGE_GENERIC = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.' # noqa: E501 + + SYSTEM_MESSAGE_GEMINI = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. Do NOT use system calls like `exit` in the generated program.' # noqa: E501 + + SYSTEM_MESSAGE_DEEPSEEK = 'You are an AI programming assistant, utilizing the DeepSeek Coder model, developed by DeepSeek Company, and you answer questions related to computer science.' # noqa: E501 + + SYSTEM_MESSAGE_MAGIC = 'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n@@ Instruction\n' # noqa: E501 + + SYSTEM_MESSAGE_WIZARD = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' # noqa: E501 + + SYSTEM_MESSAGE_PHIND = """You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. Put your fixed program within code delimiters, for example: +```python +# YOUR CODE HERE +```""" # noqa: E501 + + SYSTEM_MESSAGE_CODEQWEN = ( + '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user' # noqa: E501 + ) + + FORMATTING_MESSAGE_WITH_STARTER_CODE = 'You will use the following starter code to write the solution to the problem and enclose your code within delimiters.' # noqa: E501 + + FORMATTING_WITHOUT_STARTER_CODE = 'Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows.' # noqa: E501 + + PYTHON_FORMAT = '```python\n# YOUR CODE HERE\n```\n\n' + + +class TestOutputPromptConstants: + SYSTEM_MESSAGE_CHAT_GENERIC = 'You are a helpful programming assistant and an expert Python programmer. You are helping a user to write a test case to help to check the correctness of the function. The user has written a input for the testcase. You will calculate the output of the testcase and write the whole assertion statement in the markdown code block with the correct output.' # noqa: E501 + + SYSTEM_MESSAGE_COMPLETION_GENERIC = 'You are a helpful programming assistant and an expert Python programmer. You are helping a user to write a test case to help to check the correctness of the function.' # noqa: E501 + + SYSTEM_MESSAGE_INST_CLLAMA = 'You are a helpful programming assistant and an expert Python programmer. You are helping a user to write a test case to help to check the correctness of the function. The user has written a input for the testcase. You will calculate the output of the testcase and write out the complete assertion statement between [PYTHON] and [/PYTHON] tags.' # noqa: E501 + + SYSTEM_MESSAGE_WIZARD = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' # noqa: E501 + + SYSTEM_MESSAGE_PHIND = """You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. You must put the entire fixed program within code delimiters only for once., for example: +```python +# YOUR CODE HERE +```""" # noqa: E501 + + FORMATTING_MESSAGE = 'You will use the following starter code to write the solution to the problem and enclose your code within delimiters.' # noqa: E501 + + FORMATTING_WITHOUT_STARTER_MESSAGE = 'Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows.' # noqa: E501 + + +class SelfRepairPromptConstants: + SYSTEM_MESSAGE_GENERIC = 'You are a helpful programming assistant and an expert Python programmer. You are helping a user write a program to solve a problem. The user has written some code, but it has some errors and is not passing the tests. You will help the user by first giving a concise (at most 2-3 sentences) textual explanation of what is wrong with the code. After you have pointed out what is wrong with the code, you will then generate a fixed version of the program. You must put the entire fixed program within code delimiters only for once.' # noqa: E501 + + SYSTEM_MESSAGE_DEEPSEEK = 'You are an AI programming assistant, utilizing the DeepSeek Coder model, developed by DeepSeek Company, and you are helping a user correct a error program for code competition. The user has written some code, but it has some errors and is not passing the tests. You will help the user by first giving a concise (at most 2-3 sentences) textual explanation of what is wrong with the code. After you have pointed out what is wrong with the code, you will then generate a fixed version of the entire executable program. You must put the entire fixed executable program within code delimiters.' # noqa: E501 + + SYSTEM_MESSAGE_MAGIC = 'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n@@ Instruction\n' # noqa: E501 + + SYSTEM_MESSAGE_WIZARD = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' # noqa: E501 + + SYSTEM_MESSAGE_PHIND = """You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. You must put the entire fixed program within code delimiters only for once., for example: +```python +# YOUR CODE HERE +```""" # noqa: E501 + + FORMATTING_REPEAT = 'First reason about the code providing a textual explanation of what is wrong with the code and then generate a fixed of the program enclosed code delimiters.' # noqa: E501 + + FORMATTING_MESSAGE = 'You will use the following starter code to write the solution to the problem and enclose your code within delimiters.' # noqa: E501 + + FORMATTING_WITHOUT_STARTER_CODE = 'Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows.' # noqa: E501 + + +def make_code_execution_prompt(code, input, cot): + if cot: + # make_cot_output_prompt + return f"""You are given a Python function and an assertion containing an input to the function. Complete the assertion with a literal (no unsimplified expressions, no function calls) containing the output when executing the provided code on the given input, even if the function is incorrect or incomplete. Do NOT output any extra information. Execute the program step by step before arriving at an answer, and provide the full assertion with the correct output in [ANSWER] and [/ANSWER] tags, following the examples. + +[PYTHON] +def performOperation(s): + s = s + s + return "b" + s + "a" +assert performOperation(s = "hi") == ?? +[/PYTHON] +[THOUGHT] +Let's execute the code step by step: + +1. The function performOperation is defined, which takes a single argument s. +2. The function is called with the argument "hi", so within the function, s is initially "hi". +3. Inside the function, s is concatenated with itself, so s becomes "hihi". +4. The function then returns a new string that starts with "b", followed by the value of s (which is now "hihi"), and ends with "a". +5. The return value of the function is therefore "bhihia". +[/THOUGHT] +[ANSWER] +assert performOperation(s = "hi") == "bhihia" +[/ANSWER] + +[PYTHON] +{code} +assert {input} == ?? +[/PYTHON] +[THOUGHT] +""" # noqa: E501 + else: + # make_direct_output_prompt + return f"""You are given a Python function and an assertion containing an input to the function. Complete the assertion with a literal (no unsimplified expressions, no function calls) containing the output when executing the provided code on the given input, even if the function is incorrect or incomplete. Do NOT output any extra information. Provide the full assertion with the correct output in [ANSWER] and [/ANSWER] tags, following the examples. + +[PYTHON] +def repeatNumber(number : int) -> int: + return number +assert repeatNumber(number = 17) == ?? +[/PYTHON] +[ANSWER] +assert repeatNumber(number = 17) == 17 +[/ANSWER] + +[PYTHON] +def addCharacterA(string : str) -> str: + return string + "a" +assert addCharacterA(string = "x9j") == ?? +[/PYTHON] +[ANSWER] +assert addCharacterA(string = "x9j") == "x9ja" +[/ANSWER] + +[PYTHON] +{code} +assert {input} == ?? +[/PYTHON] +[ANSWER] +""" # noqa: E501 + + +def get_generic_question_template_test_completion(question_content, + starter_code, + testcase_input: str): + + def format_testcase_func_name_input(function_name, testcase): + """use the form of "assert func_name(input) == ".""" + # TODO should there be a space after the == ? + input_str = ', '.join(testcase.split('\n')) + return f'assert {function_name}({input_str}) == # TODO' + + def parse_function_name_from_starter_code(starter_code): + """ + starter_code : str + """ + import ast + + tree = ast.parse(starter_code) + fn = None + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + assert fn is None + fn = node.name + return fn + + prompt = f'Problem:\n{question_content}' + prompt += f'Function:\n```\n{starter_code}\n```\n' + + # parse function name from starter_code + func_name = parse_function_name_from_starter_code(starter_code) + prompt += 'Please complete the following test case:\n\n' + prompt += ( + f'```\n{format_testcase_func_name_input(func_name, testcase_input)}\n```\n' # noqa: E501 + ) + + return prompt + + +def get_generic_question_template_answer_self_repair(question: str, code, + metadata): + + def get_check_prompt(metadata): + # def get_check_prompt(question: str, result, metadata): + # # assumes i/o examples are already truncated! + # # less pressure on storing 10 MB json because on a single large + # # input-output pair + # result_by_test_case = result + # assert len(metadata) == 1, f"metadata = {metadata}" + # metadata = metadata[0] + metadata = json.loads(metadata) + if 'error_code' not in metadata: + return '' + if metadata['error_code'] == -1: + # time limit exceeded + message = f"The above code is incorrect and got the following compilation error.\n{metadata['error']}" # noqa: E501 + elif metadata['error_code'] == -2: + # wrong answer + message = f"The above code is incorrect and got a wrong answer.\nInput: {metadata['inputs']}\nGenerated Output: {metadata['output']}\nExpected: {metadata['expected']}" # noqa: E501 + elif metadata['error_code'] == -3: + # time limit exceeded + message = f"The above code is incorrect and got time limit exceeded.\n{metadata['error']}\nInput: {metadata['inputs']}\nExpected: {metadata['expected']}" # noqa: E501 + pass + elif metadata['error_code'] == -4: + # runtime error + message = f"The above code is incorrect and got a runtime error.\nInput: {metadata['inputs']}\nExpected: {metadata['expected']}\n{metadata['error']}" # noqa: E501 + else: + raise NotImplementedError( + f"metadata['error_code'] = {metadata['error_code']} not implemented || {metadata=}" # noqa: E501 + ) + return message + + prompt = f'### Question:\n{question}\n\n' + prompt += f'### Answer:\n```python\n{code}\n```\n\n' + # prompt += get_check_prompt(question, result, metadata) + "\n" + prompt += get_check_prompt(metadata) + '\n' + prompt += f'### Format: {SelfRepairPromptConstants.FORMATTING_WITHOUT_STARTER_CODE}\n' # noqa: E501 + prompt += '```python\n# YOUR CODE HERE\n```\n\n' + prompt += '### Answer: (use the provided format with backticks)\n\n' + return prompt diff --git a/opencompass/datasets/xlivecodebench/testing_util.py b/opencompass/datasets/xlivecodebench/testing_util.py new file mode 100644 index 00000000..6583b17b --- /dev/null +++ b/opencompass/datasets/xlivecodebench/testing_util.py @@ -0,0 +1,762 @@ +# Copyright LiveCodeBench @ 2024, + +import ast +import faulthandler +import json +import platform +# to run the solution files we're using a timing based approach +import signal +import sys +# used for debugging to time steps +from datetime import datetime +from enum import Enum +# for capturing the stdout +from io import StringIO +# used for testing the code that reads from input +from unittest.mock import mock_open, patch + +import numpy as np +from pyext import RuntimeModule + + +def truncatefn(s, length=300): + assert isinstance(s, str) + if len(s) <= length: + return s + + return s[:length // 2] + '...(truncated) ...' + s[-length // 2:] + + +class CODE_TYPE(Enum): + call_based = 0 + standard_input = 1 + + +# stuff for setting up signal timer +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + print('alarm went off') + # return + raise TimeoutException + + +signal.signal(signal.SIGALRM, timeout_handler) + +# timeout = 6 # seconds + + +# used to capture stdout as a list +# from https://stackoverflow.com/a/16571630/6416660 +# alternative use redirect_stdout() from contextlib +class Capturing(list): + + def __enter__(self): + self._stdout = sys.stdout + sys.stdout = self._stringio = StringIO() + # Make closing the StringIO a no-op + self._stringio.close = lambda x: 1 + return self + + def __exit__(self, *args): + self.append(self._stringio.getvalue()) + del self._stringio # free up some memory + sys.stdout = self._stdout + + +def only_int_check(val): + return isinstance(val, int) + + +def string_int_check(val): + return isinstance(val, str) and val.isdigit() + + +def combined_int_check(val): + return only_int_check(val) or string_int_check(val) + + +def run_test(sample, test=None, debug=False, timeout=6): + """if test(generated_code) is not None it'll try to run the code. + + otherwise it'll just return an input and output pair. + """ + # Disable functionalities that can make destructive changes to the test. + reliability_guard() + + if debug: + print(f'start = {datetime.now().time()}') + + try: + in_outs = json.loads(sample['input_output']) + except ValueError: + in_outs = None + if in_outs: + if in_outs.get('fn_name') is None: + which_type = CODE_TYPE.standard_input # Standard input + method_name = None + else: + which_type = CODE_TYPE.call_based # Call-based + method_name = in_outs['fn_name'] + + if debug: + print(f'loaded input_output = {datetime.now().time()}') + + if test is None: + assert False, 'should not happen: test code is none' + return in_outs, {'error': 'no test code provided'} + elif test is not None: + results = [] + sol = 'from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(6*10**5)\n' # noqa: E501 + if debug: + print(f'loading test code = {datetime.now().time()}') + + if which_type == CODE_TYPE.call_based: + + sol += test + if debug: + print(f'sol = {sol}') + signal.alarm(timeout) + try: + tmp_sol = RuntimeModule.from_string('tmp_sol', '', sol) + if 'class Solution' not in test: + tmp = tmp_sol + else: + tmp = tmp_sol.Solution() + signal.alarm(0) + except Exception as e: + signal.alarm(0) + if debug: + print(f'type 0 compilation error = {e}') + results.append(-2) + return results, { + 'error': repr(e), + 'error_code': -1, + 'error_message': 'Compilation Error', + } + signal.alarm(0) + + elif which_type == CODE_TYPE.standard_input: + # sol + # if code has if __name__ == "__main__": then remove it + try: + astree = ast.parse(test) + last_block = astree.body[-1] + if isinstance(last_block, ast.If): + condition = last_block.test + if ast.unparse( + condition).strip() == "__name__ == '__main__'": + test = (ast.unparse(astree.body[:-1]) + '\n' + + ast.unparse(last_block.body)) + except Exception as e: # noqa: + pass + + tmp_test = test.split('\n') + + new_test = [] + for x in tmp_test: + if (not x.startswith('from ')) and ( + not x.startswith('import ')): + new_test.append('\t' + x + '\n') + else: + new_test.append(x + '\n') + tmp_test = new_test + + new_test = '' + started = False + for i in tmp_test: + if i.startswith('\t') and not started: + new_test += 'stdin = sys.stdin\nstdout = sys.stdout\n' + new_test += 'def code():\n' + new_test += i + started = True + elif started and ((i.startswith('from ')) or + (i.startswith('import '))): + new_test += '\t' + i + else: + new_test += i + tmp_test = new_test + + sol += tmp_test + if debug: + print(f'sol = {sol}') + method_name = 'code' + signal.alarm(timeout) + try: + tmp_sol = RuntimeModule.from_string('tmp_sol', '', sol) + tmp = tmp_sol + signal.alarm(0) + except Exception as e: + signal.alarm(0) + if debug: + print(f'type 1 compilation error = {e}') + results.append(-2) + return results, { + 'error': repr(e), + 'error_code': -1, + 'error_message': 'Compilation Error', + } + signal.alarm(0) + if debug: + print(f'get method = {datetime.now().time()}') + + try: + method = getattr(tmp, + method_name) # get_attr second arg must be str + except Exception as e: + signal.alarm(0) + e = sys.exc_info() + print(f'unable to get function error = {e}') + results.append(-2) + return results, { + 'error': repr(e), + 'error_code': -1, + 'error_message': 'Unable to extract code', + } + + for index, inputs in enumerate(in_outs['inputs']): + raw_inputs = inputs + raw_outputs = in_outs['outputs'][index] + if which_type == CODE_TYPE.call_based: + inputs = [json.loads(line) for line in inputs.split('\n')] + in_outs['outputs'][index] = json.loads( + in_outs['outputs'][index]) + + truncate_line_size = 300 // (raw_inputs.count('\n') + 1) + raw_inputs = '\n'.join([ + truncatefn(line, truncate_line_size) + for line in raw_inputs.strip().split('\n') + ]) + raw_outputs = truncatefn(raw_outputs, 200) + else: + raw_inputs = truncatefn(raw_inputs) + raw_outputs = truncatefn(raw_outputs, 200) + # JSON forces dictionaries to have string keys; this undoes this + # (assuming a singleton list) + try: + if isinstance(inputs[0], dict): + inputs = [{int(k): v for k, v in inputs[0].items()}] + except Exception as e: # noqa: F841 + True + try: + if isinstance(in_outs['outputs'][index], dict): + in_outs['outputs'][index] = [{ + int(k): v + for k, v in in_outs['outputs'][index].items() + }] + except Exception as e: # noqa: F841 + True + try: + if isinstance(in_outs['outputs'][index][0], dict): + in_outs['outputs'][index] = [{ + int(k): v + for k, v in in_outs['outputs'][index][0].items() + }] + except Exception as e: # noqa: F841 + True + + if debug: + print( + f'time: {datetime.now().time()} testing index = {index} ' + f'inputs = {inputs}, {type(inputs)}. type = {which_type}') + if which_type == CODE_TYPE.call_based: # Call-based + signal.alarm(timeout) + faulthandler.enable() + try: + output = method(*inputs) + raw_true_output = output + + raw_true_output_copy = json.dumps(output) + raw_true_output_copy = truncatefn(raw_true_output_copy, + 200) + + # ground truth sequences are not tuples + if isinstance(output, tuple): + output = list(output) + + tmp_result = output == in_outs['outputs'][index] + if (isinstance(in_outs['outputs'][index], list) + and in_outs['outputs'][index]): + tmp_result = tmp_result or ( + output == in_outs['outputs'][index][0]) + + # ground truth sequences are not tuples + try: + if isinstance(output[0], tuple): + tmp_result = tmp_result or ([ + list(x) for x in output + ] == in_outs['outputs'][index][0]) + except Exception as e: # noqa: F841 + True + results.append(tmp_result) + if tmp_result is not True: + return results, { + 'output': raw_true_output_copy, + 'expected': raw_outputs, + 'inputs': raw_inputs, + 'error_code': -2, + 'error_message': 'Wrong Answer', + } + # reset the alarm + signal.alarm(0) + except Exception as e: + signal.alarm(0) + faulthandler.disable() + if debug: + print( + f'Standard input runtime error or time limit exceeded error = {e}' # noqa: E501 + ) + results.append(-1) + if 'timeoutexception' in repr(e).lower(): + return results, { + 'error': repr(e), + 'error_code': -3, + 'error_message': 'Time Limit Exceeded', + 'inputs': raw_inputs, + 'expected': raw_outputs, + } + else: + return results, { + 'error': repr(e), + 'error_code': -4, + 'error_message': 'Runtime Error', + 'inputs': raw_inputs, + 'expected': raw_outputs, + } + faulthandler.disable() + signal.alarm(0) + if debug: + print( + f"outputs = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" # noqa: E501 + ) + elif which_type == CODE_TYPE.standard_input: # Standard input + faulthandler.enable() + passed = False + + if isinstance(inputs, list): + inputs = '\n'.join(inputs) + if isinstance(in_outs['outputs'][index], list): + in_outs['outputs'][index] = '\n'.join( + in_outs['outputs'][index]) + + signal.alarm(timeout) + with Capturing() as output: + try: + call_method(method, inputs) + # reset the alarm + signal.alarm(0) + passed = True + except Exception as e: + # runtime error or took too long + signal.alarm(0) + print( + f'Call-based runtime error or time limit exceeded error = {repr(e)}{e}' # noqa: E501 + ) + results.append(-1) + if 'timeoutexception' in repr(e).lower(): + return results, { + 'error': repr(e), + 'error_code': -3, + 'error_message': 'Time Limit Exceeded', + 'inputs': raw_inputs, + 'expected': raw_outputs, + } + else: + return results, { + 'error': repr(e), + 'error_code': -4, + 'error_message': 'Runtime Error', + 'inputs': raw_inputs, + 'expected': raw_outputs, + } + signal.alarm(0) + raw_true_output = output[0] + raw_true_output_copy = truncatefn(raw_true_output, 200) + output = raw_true_output.splitlines() + if not passed: + if debug: + nl = '\n' + if not isinstance(inputs, list): + print( + f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" # noqa: E501 + ) + else: + print( + f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" # noqa: E501 + ) + continue + + if passed and debug: + print( + f"==> output = {output}, test outputs = {in_outs['outputs'][index]}" # noqa: E501 + ) + + if custom_compare_(output, in_outs['outputs'][index]): + tmp_result = True + results.append(tmp_result) + continue + + # ground truth sequences are expressed as lists not tuples + if isinstance(output, tuple): + output = list(output) + + tmp_result = False + try: + tmp_result = output == [in_outs['outputs'][index]] + if isinstance(in_outs['outputs'][index], list): + tmp_result = tmp_result or ( + output == in_outs['outputs'][index]) + if isinstance(output[0], str): + tmp_result = tmp_result or ([ + e.strip() for e in output + ] == in_outs['outputs'][index]) + except Exception as e: + if debug: + print(f'Failed check1 exception = {e}') + pass + + if tmp_result is True: + results.append(tmp_result) + continue + + # try one more time without \n + if isinstance(in_outs['outputs'][index], list): + for tmp_index, i in enumerate(in_outs['outputs'][index]): + in_outs['outputs'][index][tmp_index] = i.split('\n') + in_outs['outputs'][index][tmp_index] = [ + x.strip() + for x in in_outs['outputs'][index][tmp_index] if x + ] + else: + in_outs['outputs'][index] = in_outs['outputs'][ + index].split('\n') + in_outs['outputs'][index] = list( + filter(len, in_outs['outputs'][index])) + in_outs['outputs'][index] = list( + map(lambda x: x.strip(), in_outs['outputs'][index])) + + try: + tmp_result = output == [in_outs['outputs'][index]] + if isinstance(in_outs['outputs'][index], list): + tmp_result = tmp_result or ( + output == in_outs['outputs'][index]) + except Exception as e: + if debug: + print(f'Failed check2 exception = {e}') + pass + + if tmp_result is True: + results.append(tmp_result) + continue + + # try by converting the output into a split up list too + if isinstance(output, list): + output = list(filter(len, output)) + + if debug: + nl = '\n' + if not isinstance(inputs, list): + print( + f"@1 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]} {tmp_result=}" # noqa: E501 + ) + else: + print( + f"@1 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]} {tmp_result=}" # noqa: E501 + ) + + if tmp_result is True: + results.append(tmp_result) + continue + + if debug: + print(f'{tmp_result=} @a') + + try: + tmp_result = output == [in_outs['outputs'][index]] + if isinstance(in_outs['outputs'][index], list): + tmp_result = tmp_result or ( + output == in_outs['outputs'][index]) + except Exception as e: + if debug: + print(f'Failed check3 exception = {e}') + pass + + if debug: + print(f'{tmp_result=} @b') + + try: + all_ints = all( + combined_int_check(e1) and combined_int_check(e2) + for e1, e2 in zip(output, in_outs['outputs'][index])) + if not all_ints: + if debug: + print([ + combined_int_check(e1) + and combined_int_check(e2) for e1, e2 in zip( + output, in_outs['outputs'][index]) + ]) + output_float = [float(e) for e in output] + gt_float = [ + float(e) for e in in_outs['outputs'][index] + ] + tmp_result = tmp_result or ( + (len(output_float) == len(gt_float)) + and np.allclose(output_float, gt_float)) + except Exception as e: # noqa: F841 + pass + + if debug: + print(f'{tmp_result=} @c') + + try: + if isinstance(output[0], list): + all_ints = all( + combined_int_check(e1) and combined_int_check(e2) + for e1, e2 in zip(output[0], in_outs['outputs'] + [index])) + if not all_ints: + output_float = [float(e) for e in output[0]] + gt_float = [ + float(e) for e in in_outs['outputs'][index][0] + ] + tmp_result = tmp_result or ( + (len(output_float) == len(gt_float)) + and np.allclose(output_float, gt_float)) + except Exception as e: # noqa: F841 + pass + + if tmp_result is True: + results.append(tmp_result) + continue + + if debug: + print(f'{tmp_result=} @d') + # try by converting the stuff into split up list + if isinstance(in_outs['outputs'][index], list): + for tmp_index, i in enumerate(in_outs['outputs'][index]): + in_outs['outputs'][index][tmp_index] = set(i.split()) + else: + in_outs['outputs'][index] = set( + in_outs['outputs'][index].split()) + + if debug: + print(f'{tmp_result=} @e') + + try: + tmp_result = output == in_outs['outputs'][index] + except Exception as e: + if debug: + print(f'Failed check4 exception = {e}') + continue + + if tmp_result is True: + results.append(tmp_result) + continue + + if debug: + print(f'{tmp_result=} @f') + + # try by converting the output into a split up list too + if isinstance(output, list): + for tmp_index, i in enumerate(output): + output[tmp_index] = i.split() + output = list(filter(len, output)) + for tmp_index, i in enumerate(output): + output[tmp_index] = set(i) + else: + output = output.split() + output = list(filter(len, output)) + output = set(output) + + if debug: + print(f'{tmp_result=} @g') + # try: + # tmp_result = set(frozenset(s) for s in output) == set( + # frozenset(s) for s in in_outs["outputs"][index] + # ) + # except Exception as e: + # if debug: + # print(f"Failed check5 exception = {e}") + + # if they are all numbers, round so that similar numbers are + # treated as identical + # try: + # all_ints = all( + # combined_int_check(e1) and combined_int_check(e2) + # for e1, e2 in zip(output, in_outs["outputs"][index]) + # ) + # tmp_result = tmp_result or ( + # set( + # frozenset(round(float(t), 3) for t in s) for s in output) + # == set( + # frozenset(round(float(t), 3) for t in s) + # for s in in_outs["outputs"][index] + # ) + # ) + # except Exception as e: + # if debug: + # print(f"Failed check6 exception = {e}") + + if debug: + print(f'{tmp_result=} @h') + + if tmp_result is True and debug: + print('PASSED') + + results.append(tmp_result) + if tmp_result is not True: + return results, { + 'output': raw_true_output_copy, + 'expected': raw_outputs, + 'inputs': raw_inputs, + 'error_code': -2, + 'error_message': 'Wrong Answer', + } + + if debug: + nl = '\n' + if not isinstance(inputs, list): + print( + f"@2 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" # noqa: E501 + ) + else: + print( + f"@2 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}" # noqa: E501 + ) + + print(f'results = {results}') + + return results, {} + + +def custom_compare_(output, ground_truth): + + if isinstance(output, list): + output_1 = '\n'.join(output) + if stripped_string_compare(output_1, ground_truth): + return True + + if isinstance(output, list): + output_2 = [o.lstrip().rstrip() for o in output] + output_2 = '\n'.join(output_2) + if stripped_string_compare(output_2, ground_truth): + return True + + return False + + +def stripped_string_compare(s1, s2): + s1 = s1.lstrip().rstrip() + s2 = s2.lstrip().rstrip() + return s1 == s2 + + +def call_method(method, inputs): + + if isinstance(inputs, list): + inputs = '\n'.join(inputs) + + inputs_line_iterator = iter(inputs.split('\n')) + + # sys.setrecursionlimit(10000) + + # @patch('builtins.input', side_effect=inputs.split("\n")) + @patch('builtins.open', mock_open(read_data=inputs)) + @patch('sys.stdin', StringIO(inputs)) + @patch('sys.stdin.readline', lambda *args: next(inputs_line_iterator)) + @patch('sys.stdin.readlines', lambda *args: inputs.split('\n')) + @patch('sys.stdin.read', lambda *args: inputs) + # @patch('sys.stdout.write', print) + def _inner_call_method(_method): + try: + return _method() + except SystemExit as e: # noqa: F841 + pass + finally: + pass + + return _inner_call_method(method) + + +def reliability_guard(maximum_memory_bytes=None): + """This disables various destructive functions and prevents the generated + code from interfering with the test (e.g. fork bomb, killing other + processes, removing filesystem files, etc.) WARNING This function is NOT a + security sandbox. + + Untrusted code, including, model- generated code, should not be blindly + executed outside of one. See the Codex paper for more information about + OpenAI's code sandbox, and proceed with caution. + """ + + if maximum_memory_bytes is not None: + import resource + + resource.setrlimit(resource.RLIMIT_AS, + (maximum_memory_bytes, maximum_memory_bytes)) + resource.setrlimit(resource.RLIMIT_DATA, + (maximum_memory_bytes, maximum_memory_bytes)) + if not platform.uname().system == 'Darwin': + resource.setrlimit(resource.RLIMIT_STACK, + (maximum_memory_bytes, maximum_memory_bytes)) + + faulthandler.disable() + + import builtins + + builtins.exit = None + builtins.quit = None + + import os + + os.environ['OMP_NUM_THREADS'] = '1' + + os.kill = None + os.system = None + os.putenv = None + os.remove = None + os.removedirs = None + os.rmdir = None + os.fchdir = None + os.setuid = None + os.fork = None + os.forkpty = None + os.killpg = None + os.rename = None + os.renames = None + os.truncate = None + os.replace = None + os.unlink = None + os.fchmod = None + os.fchown = None + os.chmod = None + os.chown = None + os.chroot = None + os.fchdir = None + os.lchflags = None + os.lchmod = None + os.lchown = None + os.getcwd = None + os.chdir = None + + import shutil + + shutil.rmtree = None + shutil.move = None + shutil.chown = None + + import subprocess + + subprocess.Popen = None # type: ignore + + __builtins__['help'] = None + + import sys + + sys.modules['ipdb'] = None + sys.modules['joblib'] = None + sys.modules['resource'] = None + sys.modules['psutil'] = None + sys.modules['tkinter'] = None diff --git a/opencompass/datasets/xmgsm.py b/opencompass/datasets/xmgsm.py new file mode 100644 index 00000000..5580fe27 --- /dev/null +++ b/opencompass/datasets/xmgsm.py @@ -0,0 +1,96 @@ +import re + +from datasets import Dataset + +from opencompass.openicl.icl_evaluator import BaseEvaluator +from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path +import json +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class xMGSMSDataset(BaseDataset): + + @staticmethod + def load(path: str): + def read_jsonl(path: str): + data = [] + with open(path, 'r', encoding='utf-8') as f: + for line in f: + data.append(json.loads(line)) + return data + + path = get_data_path(path, local_mode=True) + # src_lines = open(path, 'r', encoding='utf-8').readlines() + data = {'question': [], 'answer': []} + all_data = read_jsonl(path) + for lines in all_data: + #question, answer = lines.strip().split('\t') + data['question'].append(lines['question']) + data['answer'].append(lines['answer_number']) + + dataset = Dataset.from_dict({ + 'question': data['question'], + 'answer': data['answer'] + }) + return dataset + + +LANG_TO_ANSWER_PREFIX = { + 'en': 'Answer', + 'bn': 'উত্তর', + 'de': 'Antwort', + 'es': 'Respuesta', + 'fr': 'Réponse', + 'ja': '答え', + 'ru': 'Ответ', + 'sw': 'Jibu', + 'te': 'సమాధానం', + 'th': 'คำตอบ', + 'zh': '答案', + 'sr': 'Одговор', + 'ar': 'الإجاب',# based on above instructions + 'hu': 'Válasz', + 'cs': 'Odpověď', + 'vi': 'Câu trả lời', + 'ko': '답', +} + + +def mgsm_postprocess(text: str, lang: str) -> str: + answer_prefix = LANG_TO_ANSWER_PREFIX[lang] + if answer_prefix not in text: + return '' + answer_text = text.split(answer_prefix)[-1].strip() + numbers = re.findall(r'\d+\.?\d*', answer_text.replace(',', '')) + return numbers[-1].rstrip('.') if numbers else '' + + +class xMGSM_Evaluator(BaseEvaluator): + + def score(self, predictions, references): + assert len(predictions) == len(references) + print('references', references) + print('predictions', predictions) + + num_correct, total = 0, 0 + details = {} + for index, (references_answer, predictions_answer) in enumerate( + zip(references, predictions)): + if str(references_answer) == str(predictions_answer): + is_correct = True + else: + is_correct = False + + num_correct += is_correct + total += 1 + details[str(index)] = { + 'references': references_answer, + 'predictions': predictions_answer, + 'correct': is_correct, + } + + accuracy = num_correct / total * 100 + final_result = {'accuracy': accuracy, 'details': details} + return final_result diff --git a/run.sh b/run.sh new file mode 100644 index 00000000..8a13ab2b --- /dev/null +++ b/run.sh @@ -0,0 +1 @@ +CUDA_VISIBLE_DEVICES=0 python3 run.py --models hf_llama3_1_8b_instruct --datasets xhumaneval_plus_gen -a vllm --max-num-worker 1 \ No newline at end of file diff --git a/start.sh b/start.sh new file mode 100644 index 00000000..028f06f2 --- /dev/null +++ b/start.sh @@ -0,0 +1,6 @@ +cd ../ +git lfs install +git clone https://huggingface.co/datasets/LLaMAX/BenchMAX_Function_Completion +mkdir -p ~/.cache/opencompass/data/xhumaneval_plus +mv BenchMAX_Function_Completion/* ~/.cache/opencompass/data/xhumaneval_plus/ +cd opencompass \ No newline at end of file