diff --git a/configs/datasets/needlebench/atc/atc.py b/configs/datasets/needlebench/atc/atc.py index b2eeebea..af49b8c9 100644 --- a/configs/datasets/needlebench/atc/atc.py +++ b/configs/datasets/needlebench/atc/atc.py @@ -31,7 +31,9 @@ needlebench_eval_cfg = dict( needle_num_list = list(range(2, 100, 3)) document_depth_percent_intervals = 20 repeats = 30 -names_path = './data/needlebench/names.json' + +path = 'opencompass/needlebench' +file_name = 'names.json' needlebench_atc_datasets_zh = [] needlebench_atc_datasets_en = [] @@ -44,7 +46,8 @@ for num_needles in needle_num_list: 'abbr': f'needlebench_atc_challenge' f'needle_{num_needles}_en_ordered', 'type': NeedleBenchATCOrderedDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': 'English', 'repeats': repeats, @@ -61,7 +64,8 @@ for num_needles in needle_num_list: 'abbr': f'needlebench_atc_challenge' f'needle_{num_needles}_zh_ordered', 'type': NeedleBenchATCOrderedDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': 'Chinese', 'repeats': repeats, @@ -77,7 +81,8 @@ for num_needles in needle_num_list: 'abbr': f'needlebench_atc_challenge' f'needle_{num_needles}_en', 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': 'English', 'repeats': repeats, @@ -93,7 +98,8 @@ for num_needles in needle_num_list: 'abbr': f'needlebench_atc_challenge' f'needle_{num_needles}_zh', 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': 'Chinese', 'repeats': repeats, diff --git a/configs/datasets/needlebench/atc/atc_choice.py b/configs/datasets/needlebench/atc/atc_choice.py index 10b0b5d7..42cb510b 100644 --- a/configs/datasets/needlebench/atc/atc_choice.py +++ b/configs/datasets/needlebench/atc/atc_choice.py @@ -61,7 +61,8 @@ few_shot_prompts = { # ----------------------- Prompt Settings ----------------------- # needle_num_list = list(range(2, 20, 1)) -names_path = './data/needlebench/names.json' +path = 'opencompass/needlebench' +file_name = 'names.json' repeats = 10 @@ -122,7 +123,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/configs/datasets/needlebench/atc/atc_choice_20.py b/configs/datasets/needlebench/atc/atc_choice_20.py index 95e88f60..3c003292 100644 --- a/configs/datasets/needlebench/atc/atc_choice_20.py +++ b/configs/datasets/needlebench/atc/atc_choice_20.py @@ -61,8 +61,8 @@ few_shot_prompts = { # ----------------------- Prompt Settings ----------------------- # needle_num_list = list(range(2, 20, 1)) -names_path = './data/needlebench/names.json' - +path = 'opencompass/needlebench' +file_name = 'names.json' repeats = 10 # Use Zero-Shot or not @@ -120,7 +120,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/configs/datasets/needlebench/atc/atc_choice_50.py b/configs/datasets/needlebench/atc/atc_choice_50.py index 03703a99..8ad6f9f6 100644 --- a/configs/datasets/needlebench/atc/atc_choice_50.py +++ b/configs/datasets/needlebench/atc/atc_choice_50.py @@ -30,7 +30,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py b/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py index c515a84a..828a51da 100644 --- a/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py +++ b/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py @@ -10,14 +10,38 @@ from opencompass.utils.text_postprocessors import first_option_postprocess few_shot_prompts = { 'single_choice_prompts': { 'single_choice_en_reasoning': [ - dict(role='HUMAN', prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n"), - dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C"), - dict(role='HUMAN', prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n"), - dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n-Jacob Oconnor's paternal grandmother is Robert Hill. \n- For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.\n\nTherefore, the eldest relative that 'Jacob Oconnor' can trace back to in the context is Mikayla Scott. The answer is: D"), - dict(role='HUMAN', prompt="Question: Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life.Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.Michael Martinez is not only Misty Moore's father but also Misty Moore's role model. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Barbara Fuentes' can trace back to in the context?\nA. Michael Martinez\nB. Jennifer Garcia\nC. Misty Moore\nD. Barbara Fuentes\n"), - dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n- Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life. \n- Michael Martinez is not only Misty Moore's father but also Misty Moore's role model.\n- Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.\n\nTherefore, the eldest relative that 'Barbara Fuentes' can trace back to in the context is Jennifer Garcia. The answer is: B"), - dict(role='HUMAN', prompt="Question: Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.Victor Dean considers Mary Gay as their grandfather.Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.David Hernandez is Jennifer Williams's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jennifer Williams' can trace back to in the context?\nA. Marcus Miller\nB. Carlos Smith\nC. Mary Gay\nD. Victor Dean\n"), - dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- David Hernandez is Jennifer Williams's mom.\n- Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.\n- Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.\n- Victor Dean considers Mary Gay as their grandfather. \n- Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.\n- Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.\n\nTherefore, the eldest relative that 'Jennifer Williams' can trace back to in the context is Marcus Miller. The answer is: A"), + dict( + role='HUMAN', + prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n", + ), + dict( + role='BOT', + prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C", + ), + dict( + role='HUMAN', + prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n", + ), + dict( + role='BOT', + prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n-Jacob Oconnor's paternal grandmother is Robert Hill. \n- For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.\n\nTherefore, the eldest relative that 'Jacob Oconnor' can trace back to in the context is Mikayla Scott. The answer is: D", + ), + dict( + role='HUMAN', + prompt="Question: Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life.Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.Michael Martinez is not only Misty Moore's father but also Misty Moore's role model. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Barbara Fuentes' can trace back to in the context?\nA. Michael Martinez\nB. Jennifer Garcia\nC. Misty Moore\nD. Barbara Fuentes\n", + ), + dict( + role='BOT', + prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n- Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life. \n- Michael Martinez is not only Misty Moore's father but also Misty Moore's role model.\n- Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.\n\nTherefore, the eldest relative that 'Barbara Fuentes' can trace back to in the context is Jennifer Garcia. The answer is: B", + ), + dict( + role='HUMAN', + prompt="Question: Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.Victor Dean considers Mary Gay as their grandfather.Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.David Hernandez is Jennifer Williams's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jennifer Williams' can trace back to in the context?\nA. Marcus Miller\nB. Carlos Smith\nC. Mary Gay\nD. Victor Dean\n", + ), + dict( + role='BOT', + prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- David Hernandez is Jennifer Williams's mom.\n- Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.\n- Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.\n- Victor Dean considers Mary Gay as their grandfather. \n- Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.\n- Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.\n\nTherefore, the eldest relative that 'Jennifer Williams' can trace back to in the context is Marcus Miller. The answer is: A", + ), dict(role='HUMAN', prompt='Question: {question}'), ], }, @@ -25,8 +49,8 @@ few_shot_prompts = { # ----------------------- Prompt Settings ----------------------- # needle_num_list = list(range(2, 50, 1)) -names_path = './data/needlebench/names.json' - +path = 'opencompass/needlebench' +file_name = 'names.json' repeats = 10 # Use Zero-Shot or not @@ -48,49 +72,54 @@ single_choice_prompts = needlebench_prompts['single_choice_prompts'] for _name in list(single_choice_prompts.keys()): if with_few_shot: assert few_shot_samples > 0 and few_shot_samples <= 4 - single_choice_prompts[_name] = \ - single_choice_prompts[_name][- few_shot_samples * 2 - 1:] + single_choice_prompts[_name] = single_choice_prompts[_name][ + -few_shot_samples * 2 - 1 : + ] # ----------------------- Dataset Settings ----------------------- # needlebench_datasets = [] -needlebench_atc_reader_cfg = dict(input_columns=['question'], - output_column='answer') +needlebench_atc_reader_cfg = dict(input_columns=['question'], output_column='answer') for _name in list(single_choice_prompts.keys()): needlebench_atc_infer_cfg = dict( prompt_template=dict( type=PromptTemplate, - template=dict( - round=(single_choice_prompts[_name])), + template=dict(round=(single_choice_prompts[_name])), ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer,), + inferencer=dict( + type=GenInferencer, + ), ) needlebench_atc_eval_cfg = dict( evaluator=dict(type=CircularEvaluator), - pred_postprocessor=dict(type=first_option_postprocess, options='ABCD')) + pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'), + ) for num_needles in needle_num_list: - abbr = (f'NeedleBenchATCDataset-' - f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}') + abbr = ( + f'NeedleBenchATCDataset-' + f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}' + ) language = 'English' if 'en' in _name else 'Chinese' if 'reasoning' in _name: abbr += '-Reasoning' dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name':file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, 'with_circular': with_circular_eval, 'reader_cfg': needlebench_atc_reader_cfg, 'infer_cfg': needlebench_atc_infer_cfg, - 'eval_cfg': needlebench_atc_eval_cfg + 'eval_cfg': needlebench_atc_eval_cfg, } needlebench_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/atc/atc_choice_80.py b/configs/datasets/needlebench/atc/atc_choice_80.py index b4a935cb..c3ade6e0 100644 --- a/configs/datasets/needlebench/atc/atc_choice_80.py +++ b/configs/datasets/needlebench/atc/atc_choice_80.py @@ -30,7 +30,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py b/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py index f05a4bdd..a4906b1f 100644 --- a/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py +++ b/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py @@ -25,8 +25,8 @@ few_shot_prompts = { # ----------------------- Prompt Settings ----------------------- # needle_num_list = list(range(2, 80, 1)) -names_path = './data/needlebench/names.json' - +path = 'opencompass/needlebench' +file_name = 'names.json' repeats = 10 # Use Zero-Shot or not @@ -84,7 +84,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py b/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py index e31aa7ab..9b4c782b 100644 --- a/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py +++ b/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = [20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -69,7 +71,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -85,7 +87,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -96,7 +98,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -112,7 +114,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -123,7 +125,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -139,7 +141,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -150,7 +152,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -166,12 +168,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -184,7 +186,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -200,7 +202,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -211,7 +213,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -227,7 +229,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -238,7 +240,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -254,7 +256,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -265,7 +267,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -281,6 +283,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py b/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py index 775e825f..63320d8f 100644 --- a/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py +++ b/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000]) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -64,8 +66,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_1000k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_1000k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -79,7 +80,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -88,8 +89,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_1000k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_1000k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -103,6 +103,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py b/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py index 49a1a40b..29d435bb 100644 --- a/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py +++ b/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py @@ -41,21 +41,23 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = [20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -64,7 +66,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_1000k', + f'Depth{int(depth_percent)}_origin_en_1000k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -78,7 +80,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -90,7 +92,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_1000k', + f'Depth{int(depth_percent)}_origin_zh_1000k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -104,6 +106,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py b/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py index 44c47731..ef817f24 100644 --- a/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py +++ b/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py @@ -41,16 +41,18 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000]) depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] @@ -58,7 +60,7 @@ document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -71,7 +73,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -87,7 +89,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -98,7 +100,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -114,7 +116,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -125,7 +127,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -141,7 +143,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -152,7 +154,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -168,12 +170,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -186,7 +188,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -202,7 +204,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -213,7 +215,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -229,7 +231,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -240,7 +242,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -256,7 +258,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -267,7 +269,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -283,6 +285,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py b/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py index b8d2f25f..c682441d 100644 --- a/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py +++ b/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000]) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -64,8 +66,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_128k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_128k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -79,7 +80,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -88,8 +89,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_128k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_128k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -103,6 +103,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py b/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py index 3a6a7836..15b4746b 100644 --- a/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py +++ b/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000]) depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -66,7 +68,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_128k', + f'Depth{int(depth_percent)}_origin_en_128k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -80,7 +82,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -92,7 +94,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_128k', + f'Depth{int(depth_percent)}_origin_zh_128k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -106,6 +108,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py b/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py index ba5741fe..a5504481 100644 --- a/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py +++ b/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [16000, 48000, 80000, 112000, 128000, 144000, 176000, 200000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -70,7 +72,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -86,7 +88,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -97,7 +99,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -113,7 +115,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -124,7 +126,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -140,7 +142,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -151,7 +153,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -167,12 +169,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -185,7 +187,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -201,7 +203,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -212,7 +214,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -228,7 +230,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -239,7 +241,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -255,7 +257,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -266,7 +268,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -282,6 +284,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py b/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py index 68e87ae9..9c7dadae 100644 --- a/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py +++ b/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = list([16000, 48000, 80000, 112000, 128000, 144000, 176000, 200000]) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -65,8 +67,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_200k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_200k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -80,7 +81,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -89,8 +90,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_200k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_200k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -104,6 +104,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py b/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py index d3badb6b..ca99ea5a 100644 --- a/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py +++ b/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [16000, 48000, 80000, 112000, 128000, 144000, 176000, 200000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -65,7 +67,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_200k', + f'Depth{int(depth_percent)}_origin_en_200k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -79,7 +81,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,7 +93,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_200k', + f'Depth{int(depth_percent)}_origin_zh_200k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -105,6 +107,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py b/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py index 0a7a0dd6..47208d82 100644 --- a/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py +++ b/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [32000, 128000, 256000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -70,7 +72,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -86,7 +88,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -97,7 +99,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -113,7 +115,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -124,7 +126,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -140,7 +142,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -151,7 +153,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -167,12 +169,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -185,7 +187,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -201,7 +203,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -212,7 +214,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -228,7 +230,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -239,7 +241,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -255,7 +257,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -266,7 +268,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -282,6 +284,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py b/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py index 8078d596..d8b4153e 100644 --- a/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py +++ b/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [32000, 128000, 256000] document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -65,8 +67,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_256k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_256k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -80,7 +81,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -89,8 +90,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_256k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_256k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -104,6 +104,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py b/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py index 88652fbf..4c8ec0e0 100644 --- a/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py +++ b/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [32000, 128000, 256000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -65,7 +67,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_256k', + f'Depth{int(depth_percent)}_origin_en_256k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -79,7 +81,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,7 +93,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_256k', + f'Depth{int(depth_percent)}_origin_zh_256k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -105,6 +107,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py b/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py index 143b1cc0..ea8428fb 100644 --- a/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py +++ b/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py @@ -41,16 +41,18 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000]) depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] @@ -58,7 +60,7 @@ document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -71,7 +73,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -87,7 +89,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -98,7 +100,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -114,7 +116,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -125,7 +127,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -141,7 +143,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -152,7 +154,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -168,12 +170,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -186,7 +188,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -202,7 +204,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -213,7 +215,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -229,7 +231,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -240,7 +242,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -256,7 +258,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -267,7 +269,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -283,6 +285,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py b/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py index ea5122f5..19d90b6e 100644 --- a/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py +++ b/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000]) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -64,8 +66,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_32k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_32k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -79,7 +80,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -88,8 +89,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_32k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_32k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -103,6 +103,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py b/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py index 1a6be8c0..d5440e7a 100644 --- a/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py +++ b/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000]) depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -66,7 +68,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_32k', + f'Depth{int(depth_percent)}_origin_en_32k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -80,7 +82,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -92,7 +94,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_32k', + f'Depth{int(depth_percent)}_origin_zh_32k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -106,6 +108,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py b/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py index bb813f15..0912d1c0 100644 --- a/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py +++ b/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(1000, 5000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -68,11 +70,11 @@ language = 'English' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -88,7 +90,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -97,11 +99,11 @@ needlebench_3needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -117,7 +119,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -126,11 +128,11 @@ needlebench_4needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -146,7 +148,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -155,11 +157,11 @@ needlebench_5needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -175,12 +177,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -191,11 +193,11 @@ language = 'Chinese' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -211,7 +213,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -220,11 +222,11 @@ needlebench_3needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -240,7 +242,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -249,11 +251,11 @@ needlebench_4needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -269,7 +271,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -278,11 +280,11 @@ needlebench_5needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -298,6 +300,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py b/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py index d29aa40e..b3c85b8b 100644 --- a/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py +++ b/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py @@ -41,34 +41,35 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(1000, 5000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' depths_float = generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type) + document_depth_percent_intervals, document_depth_percent_interval_type +) depths = [int(depth) for depth in depths_float] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_4k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_4k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -82,7 +83,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,8 +92,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_4k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_4k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -106,6 +106,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py b/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py index f8e03b52..095e1b33 100644 --- a/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py +++ b/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py @@ -41,33 +41,35 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(1000, 5000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_4k', + f'Depth{int(depth_percent)}_origin_en_4k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -81,7 +83,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,11 +93,11 @@ needle_file_name = 'needles.jsonl' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_4k', + f'Depth{int(depth_percent)}_origin_zh_4k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -109,6 +111,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py b/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py index f43bbb06..3b58b86f 100644 --- a/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py +++ b/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(5000, 9000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -68,11 +70,11 @@ language = 'English' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -88,7 +90,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -97,11 +99,11 @@ needlebench_3needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -117,7 +119,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -126,11 +128,11 @@ needlebench_4needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -146,7 +148,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -155,11 +157,11 @@ needlebench_5needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -175,12 +177,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -191,11 +193,11 @@ language = 'Chinese' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -211,7 +213,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -220,11 +222,11 @@ needlebench_3needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -240,7 +242,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -249,11 +251,11 @@ needlebench_4needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -269,7 +271,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -278,11 +280,11 @@ needlebench_5needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -298,6 +300,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py b/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py index 5cbea895..a7bf0e28 100644 --- a/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py +++ b/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py @@ -41,34 +41,35 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(5000, 9000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' depths_float = generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type) + document_depth_percent_intervals, document_depth_percent_interval_type +) depths = [int(depth) for depth in depths_float] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_8k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_8k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -82,7 +83,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,8 +92,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_8k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_8k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -106,6 +106,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py b/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py index 1bc4442f..67c2cf8c 100644 --- a/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py +++ b/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py @@ -41,36 +41,38 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(5000, 9000, 1000)) document_depth_percent_intervals_list = [1, 5, 10, 15, 20] document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' for document_depth_percent_intervals in document_depth_percent_intervals_list: depths_float = generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type) + document_depth_percent_intervals, document_depth_percent_interval_type + ) depths = [int(depth) for depth in depths_float] for original_context_length in context_lengths: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'_parallel_en_8k_batch{document_depth_percent_intervals}', + f'_parallel_en_8k_batch{document_depth_percent_intervals}', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -84,7 +86,7 @@ for document_depth_percent_intervals in document_depth_percent_intervals_list: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -94,14 +96,14 @@ needle_file_name = 'needles.jsonl' for document_depth_percent_intervals in document_depth_percent_intervals_list: depths_float = generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type) + document_depth_percent_intervals, document_depth_percent_interval_type + ) depths = [int(depth) for depth in depths_float] for original_context_length in context_lengths: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'_parallel_zh_8k_batch{document_depth_percent_intervals}', + f'_parallel_zh_8k_batch{document_depth_percent_intervals}', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -115,6 +117,6 @@ for document_depth_percent_intervals in document_depth_percent_intervals_list: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py b/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py index 368a0f54..9564a3cc 100644 --- a/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py +++ b/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py @@ -41,33 +41,35 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(5000, 9000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_8k', + f'Depth{int(depth_percent)}_origin_en_8k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -81,7 +83,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,11 +93,11 @@ needle_file_name = 'needles.jsonl' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_8k', + f'Depth{int(depth_percent)}_origin_zh_8k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -109,6 +111,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/configs/summarizers/needlebench.py b/configs/summarizers/needlebench.py index 69671db0..d1aeb984 100644 --- a/configs/summarizers/needlebench.py +++ b/configs/summarizers/needlebench.py @@ -23,9 +23,9 @@ def create_m_rs_names_list(context_lengths, depths, needle_counts, multi_needle_en_list.extend(names_list) elif language == 'zh': multi_needle_zh_list.extend(names_list) - names_dict['Multi-Needle-Reasoning(M-RS)'] = multi_needle_list - names_dict['Multi-Needle-Reasoning-EN'] = multi_needle_en_list - names_dict['Multi-Needle-Reasoning-ZH'] = multi_needle_zh_list + names_dict[f'Multi-Needle-Reasoning(M-RS)-{dataset_size.upper()}'] = multi_needle_list + names_dict[f'Multi-Needle-Reasoning-EN-{dataset_size.upper()}'] = multi_needle_en_list + names_dict[f'Multi-Needle-Reasoning-ZH-{dataset_size.upper()}'] = multi_needle_zh_list return names_dict @@ -56,9 +56,9 @@ def create_summarizer(context_lengths, depths, dataset_size, single_needle_en_list.extend(names_list) elif language == 'zh': single_needle_zh_list.extend(names_list) - names_dict['Single-Needle-Retrieval(S-RT)'] = single_needle_list - names_dict['Single-Needle-Retrieval-EN'] = single_needle_en_list - names_dict['Single-Needle-Retrieval-ZH'] = single_needle_zh_list + names_dict[f'Single-Needle-Retrieval(S-RT)-{dataset_size.upper()}'] = single_needle_list + names_dict[f'Single-Needle-Retrieval-EN-{dataset_size.upper()}'] = single_needle_en_list + names_dict[f'Single-Needle-Retrieval-ZH-{dataset_size.upper()}'] = single_needle_zh_list parallel_list = [] parallel_en_list = [] @@ -74,39 +74,39 @@ def create_summarizer(context_lengths, depths, dataset_size, parallel_en_list.extend(names_list) elif language == 'zh': parallel_zh_list.extend(names_list) - names_dict['Multi-Needle-Retrieval(M-RT)'] = parallel_list - names_dict['Multi-Needle-Retrieval-EN'] = parallel_en_list - names_dict['Multi-Needle-Retrieval-ZH'] = parallel_zh_list + names_dict[f'Multi-Needle-Retrieval(M-RT)-{dataset_size.upper()}'] = parallel_list + names_dict[f'Multi-Needle-Retrieval-EN-{dataset_size.upper()}'] = parallel_en_list + names_dict[f'Multi-Needle-Retrieval-ZH-{dataset_size.upper()}'] = parallel_zh_list summary_groups = [ {'name': key, 'subsets': value} for key, value in names_dict.items() ] summary_groups.append({ - 'name': 'NeedleBench-Overall-Score', - 'subsets': [['Single-Needle-Retrieval(S-RT)', 'naive_average'], - ['Multi-Needle-Reasoning(M-RS)', 'naive_average'], - ['Multi-Needle-Retrieval(M-RT)', 'average_score']], - 'weights': {'Single-Needle-Retrieval(S-RT)': 0.4, - 'Multi-Needle-Reasoning(M-RS)': 0.3, - 'Multi-Needle-Retrieval(M-RT)': 0.3}}) + 'name': f'NeedleBench-Overall-Score-{dataset_size.upper()}', + 'subsets': [[f'Single-Needle-Retrieval(S-RT)-{dataset_size.upper()}', 'naive_average'], + [f'Multi-Needle-Reasoning(M-RS)-{dataset_size.upper()}', 'naive_average'], + [f'Multi-Needle-Retrieval(M-RT)-{dataset_size.upper()}', 'average_score']], + 'weights': {f'Single-Needle-Retrieval(S-RT)-{dataset_size.upper()}': 0.4, + f'Multi-Needle-Reasoning(M-RS)-{dataset_size.upper()}': 0.3, + f'Multi-Needle-Retrieval(M-RT)-{dataset_size.upper()}': 0.3}}) summarizer_config = { 'type': NeedleBenchSummarizer, 'summary_groups': summary_groups, 'dataset_abbrs': [ - 'NeedleBench-Overall-Score', + f'NeedleBench-Overall-Score-{dataset_size.upper()}', f'--------- NeedleBench-{dataset_size.upper()}-Single-Needle-Retrieval ---------', - 'Single-Needle-Retrieval(S-RT)', - 'Single-Needle-Retrieval-EN', - 'Single-Needle-Retrieval-ZH', + f'Single-Needle-Retrieval(S-RT)-{dataset_size.upper()}', + f'Single-Needle-Retrieval-EN-{dataset_size.upper()}', + f'Single-Needle-Retrieval-ZH-{dataset_size.upper()}', f'--------- NeedleBench-{dataset_size.upper()}-Multi-Needle-Retrieval ---------', - 'Multi-Needle-Retrieval(M-RT)', - 'Multi-Needle-Retrieval-EN', - 'Multi-Needle-Retrieval-ZH', + f'Multi-Needle-Retrieval(M-RT)-{dataset_size.upper()}', + f'Multi-Needle-Retrieval-EN-{dataset_size.upper()}', + f'Multi-Needle-Retrieval-ZH-{dataset_size.upper()}', f'--------- NeedleBench-{dataset_size.upper()}-Multi-Needle-Reasoning ---------', - 'Multi-Needle-Reasoning(M-RS)', - 'Multi-Needle-Reasoning-EN', - 'Multi-Needle-Reasoning-ZH', + f'Multi-Needle-Reasoning(M-RS)-{dataset_size.upper()}', + f'Multi-Needle-Reasoning-EN-{dataset_size.upper()}', + f'Multi-Needle-Reasoning-ZH-{dataset_size.upper()}', f'2-Needle-EN-{dataset_size.upper()}', f'2-Needle-ZH-{dataset_size.upper()}', f'3-Needle-EN-{dataset_size.upper()}', diff --git a/opencompass/configs/datasets/needlebench/atc/atc.py b/opencompass/configs/datasets/needlebench/atc/atc.py index b2eeebea..af49b8c9 100644 --- a/opencompass/configs/datasets/needlebench/atc/atc.py +++ b/opencompass/configs/datasets/needlebench/atc/atc.py @@ -31,7 +31,9 @@ needlebench_eval_cfg = dict( needle_num_list = list(range(2, 100, 3)) document_depth_percent_intervals = 20 repeats = 30 -names_path = './data/needlebench/names.json' + +path = 'opencompass/needlebench' +file_name = 'names.json' needlebench_atc_datasets_zh = [] needlebench_atc_datasets_en = [] @@ -44,7 +46,8 @@ for num_needles in needle_num_list: 'abbr': f'needlebench_atc_challenge' f'needle_{num_needles}_en_ordered', 'type': NeedleBenchATCOrderedDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': 'English', 'repeats': repeats, @@ -61,7 +64,8 @@ for num_needles in needle_num_list: 'abbr': f'needlebench_atc_challenge' f'needle_{num_needles}_zh_ordered', 'type': NeedleBenchATCOrderedDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': 'Chinese', 'repeats': repeats, @@ -77,7 +81,8 @@ for num_needles in needle_num_list: 'abbr': f'needlebench_atc_challenge' f'needle_{num_needles}_en', 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': 'English', 'repeats': repeats, @@ -93,7 +98,8 @@ for num_needles in needle_num_list: 'abbr': f'needlebench_atc_challenge' f'needle_{num_needles}_zh', 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': 'Chinese', 'repeats': repeats, diff --git a/opencompass/configs/datasets/needlebench/atc/atc_choice.py b/opencompass/configs/datasets/needlebench/atc/atc_choice.py index 10b0b5d7..42cb510b 100644 --- a/opencompass/configs/datasets/needlebench/atc/atc_choice.py +++ b/opencompass/configs/datasets/needlebench/atc/atc_choice.py @@ -61,7 +61,8 @@ few_shot_prompts = { # ----------------------- Prompt Settings ----------------------- # needle_num_list = list(range(2, 20, 1)) -names_path = './data/needlebench/names.json' +path = 'opencompass/needlebench' +file_name = 'names.json' repeats = 10 @@ -122,7 +123,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/opencompass/configs/datasets/needlebench/atc/atc_choice_20.py b/opencompass/configs/datasets/needlebench/atc/atc_choice_20.py index 95e88f60..3c003292 100644 --- a/opencompass/configs/datasets/needlebench/atc/atc_choice_20.py +++ b/opencompass/configs/datasets/needlebench/atc/atc_choice_20.py @@ -61,8 +61,8 @@ few_shot_prompts = { # ----------------------- Prompt Settings ----------------------- # needle_num_list = list(range(2, 20, 1)) -names_path = './data/needlebench/names.json' - +path = 'opencompass/needlebench' +file_name = 'names.json' repeats = 10 # Use Zero-Shot or not @@ -120,7 +120,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/opencompass/configs/datasets/needlebench/atc/atc_choice_50.py b/opencompass/configs/datasets/needlebench/atc/atc_choice_50.py index 03703a99..8ad6f9f6 100644 --- a/opencompass/configs/datasets/needlebench/atc/atc_choice_50.py +++ b/opencompass/configs/datasets/needlebench/atc/atc_choice_50.py @@ -30,7 +30,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/opencompass/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py b/opencompass/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py index c515a84a..828a51da 100644 --- a/opencompass/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py +++ b/opencompass/configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py @@ -10,14 +10,38 @@ from opencompass.utils.text_postprocessors import first_option_postprocess few_shot_prompts = { 'single_choice_prompts': { 'single_choice_en_reasoning': [ - dict(role='HUMAN', prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n"), - dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C"), - dict(role='HUMAN', prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n"), - dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n-Jacob Oconnor's paternal grandmother is Robert Hill. \n- For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.\n\nTherefore, the eldest relative that 'Jacob Oconnor' can trace back to in the context is Mikayla Scott. The answer is: D"), - dict(role='HUMAN', prompt="Question: Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life.Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.Michael Martinez is not only Misty Moore's father but also Misty Moore's role model. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Barbara Fuentes' can trace back to in the context?\nA. Michael Martinez\nB. Jennifer Garcia\nC. Misty Moore\nD. Barbara Fuentes\n"), - dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n- Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life. \n- Michael Martinez is not only Misty Moore's father but also Misty Moore's role model.\n- Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.\n\nTherefore, the eldest relative that 'Barbara Fuentes' can trace back to in the context is Jennifer Garcia. The answer is: B"), - dict(role='HUMAN', prompt="Question: Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.Victor Dean considers Mary Gay as their grandfather.Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.David Hernandez is Jennifer Williams's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jennifer Williams' can trace back to in the context?\nA. Marcus Miller\nB. Carlos Smith\nC. Mary Gay\nD. Victor Dean\n"), - dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- David Hernandez is Jennifer Williams's mom.\n- Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.\n- Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.\n- Victor Dean considers Mary Gay as their grandfather. \n- Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.\n- Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.\n\nTherefore, the eldest relative that 'Jennifer Williams' can trace back to in the context is Marcus Miller. The answer is: A"), + dict( + role='HUMAN', + prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n", + ), + dict( + role='BOT', + prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C", + ), + dict( + role='HUMAN', + prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n", + ), + dict( + role='BOT', + prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n-Jacob Oconnor's paternal grandmother is Robert Hill. \n- For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.\n\nTherefore, the eldest relative that 'Jacob Oconnor' can trace back to in the context is Mikayla Scott. The answer is: D", + ), + dict( + role='HUMAN', + prompt="Question: Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life.Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.Michael Martinez is not only Misty Moore's father but also Misty Moore's role model. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Barbara Fuentes' can trace back to in the context?\nA. Michael Martinez\nB. Jennifer Garcia\nC. Misty Moore\nD. Barbara Fuentes\n", + ), + dict( + role='BOT', + prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n- Misty Moore plays the role of Barbara Fuentes's maternal grandfather in Barbara Fuentes's life. \n- Michael Martinez is not only Misty Moore's father but also Misty Moore's role model.\n- Jennifer Garcia, as Michael Martinez's grandmother, has a significant impact on Michael Martinez's upbringing.\n\nTherefore, the eldest relative that 'Barbara Fuentes' can trace back to in the context is Jennifer Garcia. The answer is: B", + ), + dict( + role='HUMAN', + prompt="Question: Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.Victor Dean considers Mary Gay as their grandfather.Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.David Hernandez is Jennifer Williams's mom. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jennifer Williams' can trace back to in the context?\nA. Marcus Miller\nB. Carlos Smith\nC. Mary Gay\nD. Victor Dean\n", + ), + dict( + role='BOT', + prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- David Hernandez is Jennifer Williams's mom.\n- Danielle Yates is not only David Hernandez's paternal grandmother but also David Hernandez's role model.\n- Victor Dean is not only Danielle Yates's maternal grandmother but also Danielle Yates's role model.\n- Victor Dean considers Mary Gay as their grandfather. \n- Carlos Smith, as Mary Gay's grandfather, has a significant impact on Mary Gay's upbringing.\n- Marcus Miller, as Carlos Smith's paternal grandfather, has a significant impact on Carlos Smith's upbringing.\n\nTherefore, the eldest relative that 'Jennifer Williams' can trace back to in the context is Marcus Miller. The answer is: A", + ), dict(role='HUMAN', prompt='Question: {question}'), ], }, @@ -25,8 +49,8 @@ few_shot_prompts = { # ----------------------- Prompt Settings ----------------------- # needle_num_list = list(range(2, 50, 1)) -names_path = './data/needlebench/names.json' - +path = 'opencompass/needlebench' +file_name = 'names.json' repeats = 10 # Use Zero-Shot or not @@ -48,49 +72,54 @@ single_choice_prompts = needlebench_prompts['single_choice_prompts'] for _name in list(single_choice_prompts.keys()): if with_few_shot: assert few_shot_samples > 0 and few_shot_samples <= 4 - single_choice_prompts[_name] = \ - single_choice_prompts[_name][- few_shot_samples * 2 - 1:] + single_choice_prompts[_name] = single_choice_prompts[_name][ + -few_shot_samples * 2 - 1 : + ] # ----------------------- Dataset Settings ----------------------- # needlebench_datasets = [] -needlebench_atc_reader_cfg = dict(input_columns=['question'], - output_column='answer') +needlebench_atc_reader_cfg = dict(input_columns=['question'], output_column='answer') for _name in list(single_choice_prompts.keys()): needlebench_atc_infer_cfg = dict( prompt_template=dict( type=PromptTemplate, - template=dict( - round=(single_choice_prompts[_name])), + template=dict(round=(single_choice_prompts[_name])), ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer,), + inferencer=dict( + type=GenInferencer, + ), ) needlebench_atc_eval_cfg = dict( evaluator=dict(type=CircularEvaluator), - pred_postprocessor=dict(type=first_option_postprocess, options='ABCD')) + pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'), + ) for num_needles in needle_num_list: - abbr = (f'NeedleBenchATCDataset-' - f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}') + abbr = ( + f'NeedleBenchATCDataset-' + f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}' + ) language = 'English' if 'en' in _name else 'Chinese' if 'reasoning' in _name: abbr += '-Reasoning' dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name':file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, 'with_circular': with_circular_eval, 'reader_cfg': needlebench_atc_reader_cfg, 'infer_cfg': needlebench_atc_infer_cfg, - 'eval_cfg': needlebench_atc_eval_cfg + 'eval_cfg': needlebench_atc_eval_cfg, } needlebench_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/atc/atc_choice_80.py b/opencompass/configs/datasets/needlebench/atc/atc_choice_80.py index b4a935cb..c3ade6e0 100644 --- a/opencompass/configs/datasets/needlebench/atc/atc_choice_80.py +++ b/opencompass/configs/datasets/needlebench/atc/atc_choice_80.py @@ -30,7 +30,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/opencompass/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py b/opencompass/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py index f05a4bdd..a4906b1f 100644 --- a/opencompass/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py +++ b/opencompass/configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py @@ -25,8 +25,8 @@ few_shot_prompts = { # ----------------------- Prompt Settings ----------------------- # needle_num_list = list(range(2, 80, 1)) -names_path = './data/needlebench/names.json' - +path = 'opencompass/needlebench' +file_name = 'names.json' repeats = 10 # Use Zero-Shot or not @@ -84,7 +84,8 @@ for _name in list(single_choice_prompts.keys()): dataset_dict = { 'abbr': abbr, 'type': NeedleBenchATCDataset, - 'path': names_path, + 'path': path, + 'file_name': file_name, 'num_needles': num_needles, 'language': language, 'repeats': repeats, diff --git a/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py b/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py index e31aa7ab..9b4c782b 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = [20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -69,7 +71,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -85,7 +87,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -96,7 +98,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -112,7 +114,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -123,7 +125,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -139,7 +141,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -150,7 +152,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -166,12 +168,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -184,7 +186,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -200,7 +202,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -211,7 +213,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -227,7 +229,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -238,7 +240,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -254,7 +256,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -265,7 +267,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_1000k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -281,6 +283,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py b/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py index 775e825f..63320d8f 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000]) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -64,8 +66,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_1000k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_1000k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -79,7 +80,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -88,8 +89,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_1000k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_1000k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -103,6 +103,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py b/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py index 49a1a40b..29d435bb 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py @@ -41,21 +41,23 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = [20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -64,7 +66,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_1000k', + f'Depth{int(depth_percent)}_origin_en_1000k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -78,7 +80,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -90,7 +92,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_1000k', + f'Depth{int(depth_percent)}_origin_zh_1000k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -104,6 +106,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py b/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py index 44c47731..ef817f24 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py @@ -41,16 +41,18 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000]) depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] @@ -58,7 +60,7 @@ document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -71,7 +73,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -87,7 +89,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -98,7 +100,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -114,7 +116,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -125,7 +127,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -141,7 +143,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -152,7 +154,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -168,12 +170,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -186,7 +188,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -202,7 +204,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -213,7 +215,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -229,7 +231,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -240,7 +242,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -256,7 +258,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -267,7 +269,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_128k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -283,6 +285,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py b/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py index b8d2f25f..c682441d 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000]) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -64,8 +66,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_128k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_128k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -79,7 +80,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -88,8 +89,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_128k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_128k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -103,6 +103,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py b/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py index 3a6a7836..15b4746b 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000]) depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -66,7 +68,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_128k', + f'Depth{int(depth_percent)}_origin_en_128k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -80,7 +82,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -92,7 +94,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_128k', + f'Depth{int(depth_percent)}_origin_zh_128k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -106,6 +108,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py b/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py index ba5741fe..a5504481 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [16000, 48000, 80000, 112000, 128000, 144000, 176000, 200000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -70,7 +72,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -86,7 +88,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -97,7 +99,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -113,7 +115,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -124,7 +126,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -140,7 +142,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -151,7 +153,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -167,12 +169,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -185,7 +187,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -201,7 +203,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -212,7 +214,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -228,7 +230,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -239,7 +241,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -255,7 +257,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -266,7 +268,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_200k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -282,6 +284,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py b/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py index 68e87ae9..9c7dadae 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = list([16000, 48000, 80000, 112000, 128000, 144000, 176000, 200000]) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -65,8 +67,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_200k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_200k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -80,7 +81,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -89,8 +90,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_200k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_200k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -104,6 +104,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py b/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py index d3badb6b..ca99ea5a 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [16000, 48000, 80000, 112000, 128000, 144000, 176000, 200000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -65,7 +67,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_200k', + f'Depth{int(depth_percent)}_origin_en_200k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -79,7 +81,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,7 +93,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_200k', + f'Depth{int(depth_percent)}_origin_zh_200k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -105,6 +107,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py b/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py index 0a7a0dd6..47208d82 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [32000, 128000, 256000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -70,7 +72,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -86,7 +88,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -97,7 +99,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -113,7 +115,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -124,7 +126,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -140,7 +142,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -151,7 +153,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -167,12 +169,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -185,7 +187,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -201,7 +203,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -212,7 +214,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -228,7 +230,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -239,7 +241,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -255,7 +257,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -266,7 +268,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_256k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -282,6 +284,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py b/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py index 8078d596..d8b4153e 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [32000, 128000, 256000] document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -65,8 +67,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_256k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_256k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -80,7 +81,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -89,8 +90,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_256k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_256k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -104,6 +104,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py b/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py index 88652fbf..4c8ec0e0 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000]) context_lengths = [32000, 128000, 256000] depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -65,7 +67,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_256k', + f'Depth{int(depth_percent)}_origin_en_256k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -79,7 +81,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,7 +93,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_256k', + f'Depth{int(depth_percent)}_origin_zh_256k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -105,6 +107,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py b/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py index 143b1cc0..ea8428fb 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py @@ -41,16 +41,18 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000]) depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] @@ -58,7 +60,7 @@ document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -71,7 +73,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -87,7 +89,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -98,7 +100,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -114,7 +116,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -125,7 +127,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -141,7 +143,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -152,7 +154,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -168,12 +170,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -186,7 +188,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -202,7 +204,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -213,7 +215,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -229,7 +231,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -240,7 +242,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -256,7 +258,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -267,7 +269,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_32k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -283,6 +285,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py b/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py index ea5122f5..19d90b6e 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py @@ -41,22 +41,24 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000]) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -64,8 +66,7 @@ depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_32k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_32k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -79,7 +80,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -88,8 +89,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_32k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_32k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -103,6 +103,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py b/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py index 1a6be8c0..d5440e7a 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000]) depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' @@ -66,7 +68,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_32k', + f'Depth{int(depth_percent)}_origin_en_32k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -80,7 +82,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -92,7 +94,7 @@ for original_context_length in context_lengths: for depth_percent in depths_list: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_32k', + f'Depth{int(depth_percent)}_origin_zh_32k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -106,6 +108,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py b/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py index bb813f15..0912d1c0 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_reasoning_4k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(1000, 5000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -68,11 +70,11 @@ language = 'English' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -88,7 +90,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -97,11 +99,11 @@ needlebench_3needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -117,7 +119,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -126,11 +128,11 @@ needlebench_4needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -146,7 +148,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -155,11 +157,11 @@ needlebench_5needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -175,12 +177,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -191,11 +193,11 @@ language = 'Chinese' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -211,7 +213,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -220,11 +222,11 @@ needlebench_3needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -240,7 +242,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -249,11 +251,11 @@ needlebench_4needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -269,7 +271,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -278,11 +280,11 @@ needlebench_5needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_4k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -298,6 +300,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py b/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py index d29aa40e..b3c85b8b 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_multi_retrieval_4k.py @@ -41,34 +41,35 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(1000, 5000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' depths_float = generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type) + document_depth_percent_intervals, document_depth_percent_interval_type +) depths = [int(depth) for depth in depths_float] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_4k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_4k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -82,7 +83,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,8 +92,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_4k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_4k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -106,6 +106,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py b/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py index f8e03b52..095e1b33 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_4k/needlebench_single_4k.py @@ -41,33 +41,35 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(1000, 5000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_4k', + f'Depth{int(depth_percent)}_origin_en_4k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -81,7 +83,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,11 +93,11 @@ needle_file_name = 'needles.jsonl' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_4k', + f'Depth{int(depth_percent)}_origin_zh_4k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -109,6 +111,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py b/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py index f43bbb06..3b58b86f 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_reasoning_8k.py @@ -41,23 +41,25 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchMultiEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(5000, 9000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' # ----------English Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needle_file_name = 'multi_needle_reasoning_en.json' @@ -68,11 +70,11 @@ language = 'English' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -88,7 +90,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_en_datasets.append(dataset_dict) @@ -97,11 +99,11 @@ needlebench_3needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -117,7 +119,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_en_datasets.append(dataset_dict) @@ -126,11 +128,11 @@ needlebench_4needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -146,7 +148,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_en_datasets.append(dataset_dict) @@ -155,11 +157,11 @@ needlebench_5needle_en_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_en_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -175,12 +177,12 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_en_datasets.append(dataset_dict) # ----------Chinese Version---------- -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['zh_finance.jsonl'] needle_file_name = 'multi_needle_reasoning_zh.json' @@ -191,11 +193,11 @@ language = 'Chinese' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -211,7 +213,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_2needle_zh_datasets.append(dataset_dict) @@ -220,11 +222,11 @@ needlebench_3needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -240,7 +242,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_3needle_zh_datasets.append(dataset_dict) @@ -249,11 +251,11 @@ needlebench_4needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -269,7 +271,7 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_4needle_zh_datasets.append(dataset_dict) @@ -278,11 +280,11 @@ needlebench_5needle_zh_datasets = [] for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', + f'Depth{int(depth_percent)}_{num_needles}needle_zh_8k', 'type': NeedleBenchMultiDataset, 'path': base_path, 'length': original_context_length, @@ -298,6 +300,6 @@ for original_context_length in context_lengths: 'diff': diff, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_5needle_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py b/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py index 5cbea895..a7bf0e28 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_8k.py @@ -41,34 +41,35 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(5000, 9000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' depths_float = generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type) + document_depth_percent_intervals, document_depth_percent_interval_type +) depths = [int(depth) for depth in depths_float] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_en_8k', + 'abbr': f'Length{original_context_length}' f'_parallel_en_8k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -82,7 +83,7 @@ for original_context_length in context_lengths: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,8 +92,7 @@ needlebench_zh_datasets = [] for original_context_length in context_lengths: dataset_dict = { - 'abbr': f'Length{original_context_length}' - f'_parallel_zh_8k', + 'abbr': f'Length{original_context_length}' f'_parallel_zh_8k', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -106,6 +106,6 @@ for original_context_length in context_lengths: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py b/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py index 1bc4442f..67c2cf8c 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_multi_retrieval_compare_batch_8k.py @@ -41,36 +41,38 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchParallelEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(5000, 9000, 1000)) document_depth_percent_intervals_list = [1, 5, 10, 15, 20] document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' for document_depth_percent_intervals in document_depth_percent_intervals_list: depths_float = generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type) + document_depth_percent_intervals, document_depth_percent_interval_type + ) depths = [int(depth) for depth in depths_float] for original_context_length in context_lengths: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'_parallel_en_8k_batch{document_depth_percent_intervals}', + f'_parallel_en_8k_batch{document_depth_percent_intervals}', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -84,7 +86,7 @@ for document_depth_percent_intervals in document_depth_percent_intervals_list: 'language': 'English', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -94,14 +96,14 @@ needle_file_name = 'needles.jsonl' for document_depth_percent_intervals in document_depth_percent_intervals_list: depths_float = generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type) + document_depth_percent_intervals, document_depth_percent_interval_type + ) depths = [int(depth) for depth in depths_float] for original_context_length in context_lengths: dataset_dict = { 'abbr': f'Length{original_context_length}' - f'_parallel_zh_8k_batch{document_depth_percent_intervals}', + f'_parallel_zh_8k_batch{document_depth_percent_intervals}', 'type': NeedleBenchParallelDataset, 'path': base_path, 'needle_file_name': needle_file_name, @@ -115,6 +117,6 @@ for document_depth_percent_intervals in document_depth_percent_intervals_list: 'language': 'Chinese', 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py b/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py index 368a0f54..9564a3cc 100644 --- a/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py +++ b/opencompass/configs/datasets/needlebench/needlebench_8k/needlebench_single_8k.py @@ -41,33 +41,35 @@ needlebench_infer_cfg = dict( dict(role='HUMAN', prompt='{prompt}'), dict(role='BOT', prompt='{answer}\n'), ] - ) ), + ), retriever=dict(type=ZeroRetriever), - inferencer=dict(type=GenInferencer)) + inferencer=dict(type=GenInferencer), +) needlebench_eval_cfg = dict( evaluator=dict(type=NeedleBenchOriginEvaluator), pred_postprocessor=dict(type=needlebench_postprocess), dataset_postprocessor=dict(type=needlebench_dataset_postprocess), - pred_role='BOT') + pred_role='BOT', +) context_lengths = list(range(5000, 9000, 1000)) document_depth_percent_intervals = 20 document_depth_percent_interval_type = 'linear' -base_path = './data/needlebench' +base_path = 'opencompass/needlebench' file_list = ['PaulGrahamEssays.jsonl'] needlebench_en_datasets = [] needle_file_name = 'needles.jsonl' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_en_8k', + f'Depth{int(depth_percent)}_origin_en_8k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -81,7 +83,7 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_en_datasets.append(dataset_dict) @@ -91,11 +93,11 @@ needle_file_name = 'needles.jsonl' for original_context_length in context_lengths: for depth_percent in generate_depth_percents( - document_depth_percent_intervals, - document_depth_percent_interval_type): + document_depth_percent_intervals, document_depth_percent_interval_type + ): dataset_dict = { 'abbr': f'Length{original_context_length}' - f'Depth{int(depth_percent)}_origin_zh_8k', + f'Depth{int(depth_percent)}_origin_zh_8k', 'type': NeedleBenchOriginDataset, 'path': base_path, 'length': original_context_length, @@ -109,6 +111,6 @@ for original_context_length in context_lengths: 'needle_file_name': needle_file_name, 'reader_cfg': needlebench_reader_cfg, 'infer_cfg': needlebench_infer_cfg, - 'eval_cfg': needlebench_eval_cfg + 'eval_cfg': needlebench_eval_cfg, } needlebench_zh_datasets.append(dataset_dict) diff --git a/opencompass/configs/summarizers/needlebench.py b/opencompass/configs/summarizers/needlebench.py index 69671db0..d1aeb984 100644 --- a/opencompass/configs/summarizers/needlebench.py +++ b/opencompass/configs/summarizers/needlebench.py @@ -23,9 +23,9 @@ def create_m_rs_names_list(context_lengths, depths, needle_counts, multi_needle_en_list.extend(names_list) elif language == 'zh': multi_needle_zh_list.extend(names_list) - names_dict['Multi-Needle-Reasoning(M-RS)'] = multi_needle_list - names_dict['Multi-Needle-Reasoning-EN'] = multi_needle_en_list - names_dict['Multi-Needle-Reasoning-ZH'] = multi_needle_zh_list + names_dict[f'Multi-Needle-Reasoning(M-RS)-{dataset_size.upper()}'] = multi_needle_list + names_dict[f'Multi-Needle-Reasoning-EN-{dataset_size.upper()}'] = multi_needle_en_list + names_dict[f'Multi-Needle-Reasoning-ZH-{dataset_size.upper()}'] = multi_needle_zh_list return names_dict @@ -56,9 +56,9 @@ def create_summarizer(context_lengths, depths, dataset_size, single_needle_en_list.extend(names_list) elif language == 'zh': single_needle_zh_list.extend(names_list) - names_dict['Single-Needle-Retrieval(S-RT)'] = single_needle_list - names_dict['Single-Needle-Retrieval-EN'] = single_needle_en_list - names_dict['Single-Needle-Retrieval-ZH'] = single_needle_zh_list + names_dict[f'Single-Needle-Retrieval(S-RT)-{dataset_size.upper()}'] = single_needle_list + names_dict[f'Single-Needle-Retrieval-EN-{dataset_size.upper()}'] = single_needle_en_list + names_dict[f'Single-Needle-Retrieval-ZH-{dataset_size.upper()}'] = single_needle_zh_list parallel_list = [] parallel_en_list = [] @@ -74,39 +74,39 @@ def create_summarizer(context_lengths, depths, dataset_size, parallel_en_list.extend(names_list) elif language == 'zh': parallel_zh_list.extend(names_list) - names_dict['Multi-Needle-Retrieval(M-RT)'] = parallel_list - names_dict['Multi-Needle-Retrieval-EN'] = parallel_en_list - names_dict['Multi-Needle-Retrieval-ZH'] = parallel_zh_list + names_dict[f'Multi-Needle-Retrieval(M-RT)-{dataset_size.upper()}'] = parallel_list + names_dict[f'Multi-Needle-Retrieval-EN-{dataset_size.upper()}'] = parallel_en_list + names_dict[f'Multi-Needle-Retrieval-ZH-{dataset_size.upper()}'] = parallel_zh_list summary_groups = [ {'name': key, 'subsets': value} for key, value in names_dict.items() ] summary_groups.append({ - 'name': 'NeedleBench-Overall-Score', - 'subsets': [['Single-Needle-Retrieval(S-RT)', 'naive_average'], - ['Multi-Needle-Reasoning(M-RS)', 'naive_average'], - ['Multi-Needle-Retrieval(M-RT)', 'average_score']], - 'weights': {'Single-Needle-Retrieval(S-RT)': 0.4, - 'Multi-Needle-Reasoning(M-RS)': 0.3, - 'Multi-Needle-Retrieval(M-RT)': 0.3}}) + 'name': f'NeedleBench-Overall-Score-{dataset_size.upper()}', + 'subsets': [[f'Single-Needle-Retrieval(S-RT)-{dataset_size.upper()}', 'naive_average'], + [f'Multi-Needle-Reasoning(M-RS)-{dataset_size.upper()}', 'naive_average'], + [f'Multi-Needle-Retrieval(M-RT)-{dataset_size.upper()}', 'average_score']], + 'weights': {f'Single-Needle-Retrieval(S-RT)-{dataset_size.upper()}': 0.4, + f'Multi-Needle-Reasoning(M-RS)-{dataset_size.upper()}': 0.3, + f'Multi-Needle-Retrieval(M-RT)-{dataset_size.upper()}': 0.3}}) summarizer_config = { 'type': NeedleBenchSummarizer, 'summary_groups': summary_groups, 'dataset_abbrs': [ - 'NeedleBench-Overall-Score', + f'NeedleBench-Overall-Score-{dataset_size.upper()}', f'--------- NeedleBench-{dataset_size.upper()}-Single-Needle-Retrieval ---------', - 'Single-Needle-Retrieval(S-RT)', - 'Single-Needle-Retrieval-EN', - 'Single-Needle-Retrieval-ZH', + f'Single-Needle-Retrieval(S-RT)-{dataset_size.upper()}', + f'Single-Needle-Retrieval-EN-{dataset_size.upper()}', + f'Single-Needle-Retrieval-ZH-{dataset_size.upper()}', f'--------- NeedleBench-{dataset_size.upper()}-Multi-Needle-Retrieval ---------', - 'Multi-Needle-Retrieval(M-RT)', - 'Multi-Needle-Retrieval-EN', - 'Multi-Needle-Retrieval-ZH', + f'Multi-Needle-Retrieval(M-RT)-{dataset_size.upper()}', + f'Multi-Needle-Retrieval-EN-{dataset_size.upper()}', + f'Multi-Needle-Retrieval-ZH-{dataset_size.upper()}', f'--------- NeedleBench-{dataset_size.upper()}-Multi-Needle-Reasoning ---------', - 'Multi-Needle-Reasoning(M-RS)', - 'Multi-Needle-Reasoning-EN', - 'Multi-Needle-Reasoning-ZH', + f'Multi-Needle-Reasoning(M-RS)-{dataset_size.upper()}', + f'Multi-Needle-Reasoning-EN-{dataset_size.upper()}', + f'Multi-Needle-Reasoning-ZH-{dataset_size.upper()}', f'2-Needle-EN-{dataset_size.upper()}', f'2-Needle-ZH-{dataset_size.upper()}', f'3-Needle-EN-{dataset_size.upper()}', diff --git a/opencompass/datasets/needlebench/atc.py b/opencompass/datasets/needlebench/atc.py index e7a95747..c715b19b 100644 --- a/opencompass/datasets/needlebench/atc.py +++ b/opencompass/datasets/needlebench/atc.py @@ -1,11 +1,13 @@ # flake8: noqa import json +import os import random from datasets import Dataset from opencompass.datasets.base import BaseDataset from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path @LOAD_DATASET.register_module() @@ -14,13 +16,20 @@ class NeedleBenchATCDataset(BaseDataset): @staticmethod def load( path, + file_name: str, num_needles: int, language: str, repeats: int, ): data = {'prompt': [], 'answer': []} + path = get_data_path(path) + if os.environ.get('DATASET_SOURCE') == 'HF': + from huggingface_hub import snapshot_download - with open(path, 'r', encoding='utf-8') as file: + path = snapshot_download(repo_id=path, repo_type='dataset') + file_path = os.path.join(path, file_name) + + with open(file_path, 'r', encoding='utf-8') as file: names_data = json.load(file) all_names = names_data[language].split(',') @@ -30,7 +39,16 @@ class NeedleBenchATCDataset(BaseDataset): if language == 'Chinese': relationship_terms = [ - '父亲', '母亲', '爸爸', '妈妈', '爷爷', '奶奶', '姥姥', '姥爷', '外公', '外婆' + '父亲', + '母亲', + '爸爸', + '妈妈', + '爷爷', + '奶奶', + '姥姥', + '姥爷', + '外公', + '外婆', ] relationship_templates = [ @@ -46,10 +64,16 @@ class NeedleBenchATCDataset(BaseDataset): elif language == 'English': relationship_terms = [ - 'father', 'mother', 'dad', 'mom', 'grandfather', - 'grandmother', 'maternal grandmother', - 'maternal grandfather', 'paternal grandfather', - 'paternal grandmother' + 'father', + 'mother', + 'dad', + 'mom', + 'grandfather', + 'grandmother', + 'maternal grandmother', + 'maternal grandfather', + 'paternal grandfather', + 'paternal grandmother', ] relationship_templates = [ @@ -96,21 +120,20 @@ class NeedleBenchATCDataset(BaseDataset): # Generating the prompt based on the language if language == 'Chinese': - prompt = (f""" + prompt = f""" 在上面提供的打乱的家族关系文本中，'{last_person}'的能够向上追溯到的最年长的亲人是谁？ 例如： 例子1.如果张强的父亲是马克，除此以外提供的文本中没有更多关于亲属关系的信息，那么在提供的文本中张强能够向上追溯到的最年长的亲人就是马克。 例子2.如果李明的姥姥是张红，而张红的父亲是张强，除此以外提供的文本中没有更多关于亲属关系的信息，那么在提供的文本中李明能够向上追溯到的最年长的亲人就是张强。 例子3.如果小明是张红的曾孙女，张红的祖母是王华，王华的父亲是王刚，除此以外提供的文本中没有更多关于亲属关系的信息，那么小明能够向上追溯到的最年长的亲人就是王刚。 -""") +""" elif language == 'English': - prompt = (f""" + prompt = f""" Given the scrambled family relationships described above, who is the eldest relative that '{last_person}' can trace back to in the context? For example: Example 1: If Zhang Qiang's father is Mark, and no further information about familial relationships is provided in the text, then the oldest relative Zhang Qiang can trace back to in the provided text is Mark. Example 2: If Li Ming's grandmother is Zhang Hong, and Zhang Hong's father is Zhang Qiang, and no further information about familial relationships is provided in the text, then the oldest relative Li Ming can trace back to in the provided text is Zhang Qiang. Example 3: If Xiao Ming is Zhang Hong's great-granddaughter, Zhang Hong's grandmother is Wang Hua, and Wang Hua's father is Wang Gang, and no further information about familial relationships is provided in the text, then the oldest relative Xiao Ming can trace back to in the provided text is Wang Gang.""" - ) else: prompt = 'Language not supported.' raise Exception('Unsupported language specified. ' @@ -135,13 +158,20 @@ class NeedleBenchATCOrderedDataset(BaseDataset): @staticmethod def load( path, + file_name, num_needles: int, language: str, repeats: int, ): data = {'prompt': [], 'answer': []} + path = get_data_path(path) + if os.environ.get('DATASET_SOURCE') == 'HF': + from huggingface_hub import snapshot_download - with open(path, 'r', encoding='utf-8') as file: + path = snapshot_download(repo_id=path, repo_type='dataset') + file_path = os.path.join(path, file_name) + + with open(file_path, 'r', encoding='utf-8') as file: names_data = json.load(file) all_names = names_data[language].split(',') @@ -151,7 +181,16 @@ class NeedleBenchATCOrderedDataset(BaseDataset): if language == 'Chinese': relationship_terms = [ - '父亲', '母亲', '爸爸', '妈妈', '爷爷', '奶奶', '姥姥', '姥爷', '外公', '外婆' + '父亲', + '母亲', + '爸爸', + '妈妈', + '爷爷', + '奶奶', + '姥姥', + '姥爷', + '外公', + '外婆', ] relationship_templates = [ @@ -167,10 +206,16 @@ class NeedleBenchATCOrderedDataset(BaseDataset): elif language == 'English': relationship_terms = [ - 'father', 'mother', 'dad', 'mom', 'grandfather', - 'grandmother', 'maternal grandmother', - 'maternal grandfather', 'paternal grandfather', - 'paternal grandmother' + 'father', + 'mother', + 'dad', + 'mom', + 'grandfather', + 'grandmother', + 'maternal grandmother', + 'maternal grandfather', + 'paternal grandfather', + 'paternal grandmother', ] relationship_templates = [ @@ -214,21 +259,20 @@ class NeedleBenchATCOrderedDataset(BaseDataset): # Generating the prompt based on the language if language == 'Chinese': - prompt = (f""" + prompt = f""" 在上面提供的打乱的家族关系文本中，'{last_person}'的能够向上追溯到的最年长的亲人是谁？ 例如： 例子1.如果张强的父亲是马克，除此以外提供的文本中没有更多关于亲属关系的信息，那么在提供的文本中张强能够向上追溯到的最年长的亲人就是马克。 例子2.如果李明的姥姥是张红，而张红的父亲是张强，除此以外提供的文本中没有更多关于亲属关系的信息，那么在提供的文本中李明能够向上追溯到的最年长的亲人就是张强。 例子3.如果小明是张红的曾孙女，张红的祖母是王华，王华的父亲是王刚，除此以外提供的文本中没有更多关于亲属关系的信息，那么小明能够向上追溯到的最年长的亲人就是王刚。 -""") +""" elif language == 'English': - prompt = (f""" + prompt = f""" Given the scrambled family relationships described above, who is the eldest relative that '{last_person}' can trace back to in the context? For example: Example 1: If Zhang Qiang's father is Mark, and no further information about familial relationships is provided in the text, then the oldest relative Zhang Qiang can trace back to in the provided text is Mark. Example 2: If Li Ming's grandmother is Zhang Hong, and Zhang Hong's father is Zhang Qiang, and no further information about familial relationships is provided in the text, then the oldest relative Li Ming can trace back to in the provided text is Zhang Qiang. Example 3: If Xiao Ming is Zhang Hong's great-granddaughter, Zhang Hong's grandmother is Wang Hua, and Wang Hua's father is Wang Gang, and no further information about familial relationships is provided in the text, then the oldest relative Xiao Ming can trace back to in the provided text is Wang Gang.""" - ) else: prompt = 'Language not supported.' raise Exception('Unsupported language specified. ' diff --git a/opencompass/datasets/needlebench/atc_choice.py b/opencompass/datasets/needlebench/atc_choice.py index af1d668e..1f586035 100644 --- a/opencompass/datasets/needlebench/atc_choice.py +++ b/opencompass/datasets/needlebench/atc_choice.py @@ -1,11 +1,13 @@ # flake8: noqa import copy import json +import os import random from datasets import Dataset from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path from ..base import BaseDataset @@ -46,11 +48,14 @@ def get_circular_example(entry, id): class NeedleBenchATCDataset(BaseDataset): @staticmethod - def load(path: str, - num_needles: int, - language: str, - repeats: int, - with_circular: bool = True): + def load( + path: str, + file_name: str, + num_needles: int, + language: str, + repeats: int, + with_circular: bool = True, + ): """NeedleBenthATC Dataset. Args: @@ -61,8 +66,14 @@ class NeedleBenchATCDataset(BaseDataset): """ data = [] entry = {} + path = get_data_path(path) + if os.environ.get('DATASET_SOURCE') == 'HF': + from huggingface_hub import snapshot_download - with open(path, 'r', encoding='utf-8') as file: + path = snapshot_download(repo_id=path, repo_type='dataset') + file_path = os.path.join(path, file_name) + + with open(file_path, 'r', encoding='utf-8') as file: names_data = json.load(file) all_names = names_data[language].split(',') @@ -73,7 +84,16 @@ class NeedleBenchATCDataset(BaseDataset): if language == 'Chinese': relationship_terms = [ - '父亲', '母亲', '爸爸', '妈妈', '爷爷', '奶奶', '姥姥', '姥爷', '外公', '外婆' + '父亲', + '母亲', + '爸爸', + '妈妈', + '爷爷', + '奶奶', + '姥姥', + '姥爷', + '外公', + '外婆', ] relationship_templates = [ @@ -89,10 +109,16 @@ class NeedleBenchATCDataset(BaseDataset): elif language == 'English': relationship_terms = [ - 'father', 'mother', 'dad', 'mom', 'grandfather', - 'grandmother', 'maternal grandmother', - 'maternal grandfather', 'paternal grandfather', - 'paternal grandmother' + 'father', + 'mother', + 'dad', + 'mom', + 'grandfather', + 'grandmother', + 'maternal grandmother', + 'maternal grandfather', + 'paternal grandfather', + 'paternal grandmother', ] relationship_templates = [ @@ -139,12 +165,11 @@ class NeedleBenchATCDataset(BaseDataset): # Generating the prompt based on the language if language == 'Chinese': - prompt = (f""" -在上面提供的打乱的家族关系文本中，'{last_person}'的能够向上追溯到的最年长的亲人是谁？""") + prompt = f""" +在上面提供的打乱的家族关系文本中，'{last_person}'的能够向上追溯到的最年长的亲人是谁？""" elif language == 'English': - prompt = (f""" + prompt = f""" Given the scrambled family relationships described above, who is the eldest relative that '{last_person}' can trace back to in the context?""" - ) else: prompt = 'Language not supported.' raise Exception('Unsupported language specified. ' @@ -158,7 +183,8 @@ Given the scrambled family relationships described above, who is the eldest rela additional_names_needed = max(4 - len(names), 0) additional_names = random.sample( [name for name in all_names if name not in names], - additional_names_needed) + additional_names_needed, + ) names.extend(additional_names) entry['options'] = names[0:4] diff --git a/opencompass/datasets/needlebench/multi.py b/opencompass/datasets/needlebench/multi.py index cbb96b1c..223fc216 100644 --- a/opencompass/datasets/needlebench/multi.py +++ b/opencompass/datasets/needlebench/multi.py @@ -4,11 +4,11 @@ import random import tiktoken from datasets import Dataset -from huggingface_hub import hf_hub_download from opencompass.datasets.base import BaseDataset from opencompass.openicl import BaseEvaluator from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path def get_random_needles(counter, file_path, needle_count): @@ -37,7 +37,7 @@ class NeedleBenchMultiDataset(BaseDataset): @staticmethod def load( - path: str, # depreciated + path: str, length: int, depth: int, tokenizer_model: str, @@ -152,25 +152,21 @@ class NeedleBenchMultiDataset(BaseDataset): return prompt - repo_id = 'opencompass/NeedleBench' file_names = [ 'PaulGrahamEssays.jsonl', 'multi_needle_reasoning_en.json', 'multi_needle_reasoning_zh.json', 'zh_finance.jsonl', 'zh_game.jsonl', 'zh_general.jsonl', 'zh_government.jsonl', 'zh_movie.jsonl', 'zh_tech.jsonl' ] - downloaded_files = [] - base_file_path = '' - for file_name in file_names: - file_path = hf_hub_download(repo_id=repo_id, - filename=file_name, - repo_type='dataset') - downloaded_files.append(file_path) - base_file_path = '/'.join(file_path.split('/')[:-1]) + path = get_data_path(path) + if os.environ.get('DATASET_SOURCE') == 'HF': + from huggingface_hub import snapshot_download + path = snapshot_download(repo_id=path, repo_type='dataset') + needle_file_path = os.path.join(path, needle_file_name) - needle_file_path = os.path.join(base_file_path, needle_file_name) - for file_path in downloaded_files: - if file_path.split('/')[-1] not in file_list: + for file_name in file_names: + file_path = os.path.join(path, file_name) + if file_name not in file_list: continue with open(file_path, 'r', encoding='utf-8') as f: diff --git a/opencompass/datasets/needlebench/origin.py b/opencompass/datasets/needlebench/origin.py index da3e6cc8..3359eae7 100644 --- a/opencompass/datasets/needlebench/origin.py +++ b/opencompass/datasets/needlebench/origin.py @@ -5,11 +5,11 @@ import re import tiktoken from datasets import Dataset -from huggingface_hub import hf_hub_download from opencompass.datasets.base import BaseDataset from opencompass.openicl import BaseEvaluator from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS +from opencompass.utils import get_data_path def get_random_line_by_language(counter, file_path, language): @@ -36,7 +36,7 @@ class NeedleBenchOriginDataset(BaseDataset): @staticmethod def load( - path: str, # depreciated + path: str, length: int, depth: int, tokenizer_model: str, @@ -128,33 +128,29 @@ class NeedleBenchOriginDataset(BaseDataset): return prompt - repo_id = 'opencompass/NeedleBench' file_names = [ - 'PaulGrahamEssays.jsonl', 'needles.jsonl', 'zh_finance.jsonl', + 'PaulGrahamEssays.jsonl', 'multi_needle_reasoning_en.json', + 'multi_needle_reasoning_zh.json', 'zh_finance.jsonl', 'zh_game.jsonl', 'zh_general.jsonl', 'zh_government.jsonl', 'zh_movie.jsonl', 'zh_tech.jsonl' ] + path = get_data_path(path) + if os.environ.get('DATASET_SOURCE') == 'HF': + from huggingface_hub import snapshot_download + path = snapshot_download(repo_id=path, repo_type='dataset') + needle_file_path = os.path.join(path, needle_file_name) - downloaded_files = [] - base_file_path = '' for file_name in file_names: - file_path = hf_hub_download(repo_id=repo_id, - filename=file_name, - repo_type='dataset') - downloaded_files.append(file_path) - base_file_path = '/'.join(file_path.split('/')[:-1]) - - for file_path in downloaded_files: - if file_path.split('/')[-1] not in file_list: + file_path = os.path.join(path, file_name) + if file_name not in file_list: continue + with open(file_path, 'r', encoding='utf-8') as f: lines_bak = [json.loads(line.strip()) for line in f] lines = lines_bak.copy() for counter in range(num_repeats_per_file): random.seed(counter) random.shuffle(lines) - needle_file_path = os.path.join(base_file_path, - needle_file_name) random_needle = get_random_line_by_language( counter, needle_file_path, language) needle = '\n' + random_needle['needle'] + '\n' diff --git a/opencompass/datasets/needlebench/parallel.py b/opencompass/datasets/needlebench/parallel.py index 59c971f1..8158afbb 100644 --- a/opencompass/datasets/needlebench/parallel.py +++ b/opencompass/datasets/needlebench/parallel.py @@ -1,21 +1,24 @@ import json +import os import random import tiktoken from datasets import Dataset -from huggingface_hub import hf_hub_download from opencompass.datasets.base import BaseDataset from opencompass.openicl import BaseEvaluator from opencompass.registry import LOAD_DATASET +from opencompass.utils import get_data_path -def get_unique_entries(file_path, - n, - language, - unique_arg1=False, - unique_arg2=False, - unique_combination=False): +def get_unique_entries( + file_path, + n, + language, + unique_arg1=False, + unique_arg2=False, + unique_combination=False, +): seen_arg1 = set() seen_arg2 = set() seen_combinations = set() @@ -38,9 +41,11 @@ def get_unique_entries(file_path, key2 = entry.get('arg2', '') if unique_arg2 else '' combination = (key1, key2) if unique_combination else '' - if (key1 not in seen_arg1 or not unique_arg1) and \ - (key2 not in seen_arg2 or not unique_arg2) and \ - (combination not in seen_combinations or not unique_combination): + if ((key1 not in seen_arg1 or not unique_arg1) # noqa: E501 + and (key2 not in seen_arg2 or not unique_arg2) + and # noqa: E501 + (combination not in seen_combinations + or not unique_combination)): # noqa: E501 seen_arg1.add(key1) seen_arg2.add(key2) seen_combinations.add(combination) @@ -57,7 +62,7 @@ class NeedleBenchParallelDataset(BaseDataset): @staticmethod def load( - path: str, # depreciated + path: str, needle_file_name: str, length: int, depths: list[int], @@ -72,30 +77,32 @@ class NeedleBenchParallelDataset(BaseDataset): data = {'prompt': [], 'answer': []} tokenizer = tiktoken.encoding_for_model(tokenizer_model) - repo_id = 'opencompass/NeedleBench' file_names = [ - 'PaulGrahamEssays.jsonl', 'needles.jsonl', 'zh_finance.jsonl', - 'zh_game.jsonl', 'zh_general.jsonl', 'zh_government.jsonl', - 'zh_movie.jsonl', 'zh_tech.jsonl' + 'PaulGrahamEssays.jsonl', + 'multi_needle_reasoning_en.json', + 'multi_needle_reasoning_zh.json', + 'zh_finance.jsonl', + 'zh_game.jsonl', + 'zh_general.jsonl', + 'zh_government.jsonl', + 'zh_movie.jsonl', + 'zh_tech.jsonl', ] + path = get_data_path(path) + if os.environ.get('DATASET_SOURCE') == 'HF': + from huggingface_hub import snapshot_download - downloaded_files = [] - for file_name in file_names: - file_path = hf_hub_download(repo_id=repo_id, - filename=file_name, - repo_type='dataset') - downloaded_files.append(file_path) + path = snapshot_download(repo_id=path, repo_type='dataset') + needle_file_path = os.path.join(path, needle_file_name) - for file in downloaded_files: - if file.split('/')[-1] == needle_file_name: - needle_file_path = file - - predefined_needles_bak = get_unique_entries(needle_file_path, - len(depths), - language, - unique_arg1=True, - unique_arg2=True, - unique_combination=True) + predefined_needles_bak = get_unique_entries( + needle_file_path, + len(depths), + language, + unique_arg1=True, + unique_arg2=True, + unique_combination=True, + ) def _generate_context(tokens_context, depths, needles): insertion_points = [ @@ -108,10 +115,12 @@ class NeedleBenchParallelDataset(BaseDataset): needle_tokens = _get_tokens_from_context(needle) current_insertion_point = min( insertion_points[i] + cumulative_inserted_length, - len(tokens_context)) + len(tokens_context), + ) - tokens_context = tokens_context[:current_insertion_point] + \ - needle_tokens + tokens_context[current_insertion_point:] + tokens_context = (tokens_context[:current_insertion_point] + + needle_tokens + + tokens_context[current_insertion_point:]) cumulative_inserted_length += len(needle_tokens) new_context = _decode_tokens(tokens_context) @@ -191,8 +200,9 @@ class NeedleBenchParallelDataset(BaseDataset): return prompt - for file_path in downloaded_files: - if file_path.split('/')[-1] not in file_list: + for file_name in file_names: + file_path = os.path.join(path, file_name) + if file_name not in file_list: continue with open(file_path, 'r', encoding='utf-8') as f: @@ -219,8 +229,8 @@ class NeedleBenchParallelDataset(BaseDataset): item['retrieval_question'].split("'")[1].split('。')[0] for item in predefined_needles ]) - retrieval_question = questions + "请按照'" + \ - answers_format + "'的格式回答。" + retrieval_question = (questions + "请按照'" + answers_format + + "'的格式回答。") elif language == 'English': questions = '、'.join([ item['retrieval_question'].split('?')[0] + '?' @@ -231,14 +241,14 @@ class NeedleBenchParallelDataset(BaseDataset): item['retrieval_question'].split("'")[1].split('.')[0] for item in predefined_needles ]) - retrieval_question = questions + \ - "Please answer in the format of '" + \ - answers_format + "'" + retrieval_question = (questions + + "Please answer in the format of '" + + answers_format + "'") context_length = length - length_buffer - target_length_per_record = context_length - \ - sum(len(tokens) for tokens - in _get_tokens_from_context(needles)) + target_length_per_record = context_length - sum( + len(tokens) + for tokens in _get_tokens_from_context(needles)) target_length_per_record = max(target_length_per_record, 0) accumulated_tokens = [] for line in lines: @@ -317,7 +327,8 @@ class NeedleBenchParallelEvaluator(BaseEvaluator): } result = { - **flattened_scores, 'details': details, - 'average_score': average_score + **flattened_scores, + 'details': details, + 'average_score': average_score, } return result diff --git a/opencompass/utils/datasets_info.py b/opencompass/utils/datasets_info.py index 617c796c..aa75ea4d 100644 --- a/opencompass/utils/datasets_info.py +++ b/opencompass/utils/datasets_info.py @@ -265,6 +265,12 @@ DATASETS_MAPPING = { "hf_id": "opencompass/xsum", "local": "./data/Xsum/dev.jsonl", }, + # Needlebench + "opencompass/needlebench": { + "ms_id": "", + "hf_id": "opencompass/needlebench", + "local": "./data/needlebench", + } } DATASETS_URL = { @@ -396,4 +402,8 @@ DATASETS_URL = { "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/mmlu_pro.zip", "md5": "e3200c7380f4cea5f13c768f2815fabb", }, + "/needlebench": { + "url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/needlebench.zip", + "md5": "b546da0397746eaff4d3ff0f20d6ede2", + } }