mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
[Feature] Longbench dataset update
This commit is contained in:
Linchen Xiao
GitHub
parent
928d0cfc3a
commit
87ffa71d68
@ -7,7 +7,7 @@ LongBench_2wikimqa_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_2wikimqa_infer_cfg = dict(
|
LongBench_2wikimqa_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_2wikimqa_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_2wikimqa_eval_cfg = dict(
|
LongBench_2wikimqa_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_2wikimqa_datasets = [
|
LongBench_2wikimqa_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBench2wikimqaDataset,
|
type=LongBench2wikimqaDataset,
|
||||||
abbr='LongBench_2wikimqa',
|
abbr='LongBench_2wikimqa',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='2wikimqa',
|
name='2wikimqa',
|
||||||
reader_cfg=LongBench_2wikimqa_reader_cfg,
|
reader_cfg=LongBench_2wikimqa_reader_cfg,
|
||||||
infer_cfg=LongBench_2wikimqa_infer_cfg,
|
infer_cfg=LongBench_2wikimqa_infer_cfg,
|
||||||
eval_cfg=LongBench_2wikimqa_eval_cfg)
|
eval_cfg=LongBench_2wikimqa_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_dureader_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_dureader_infer_cfg = dict(
|
LongBench_dureader_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_dureader_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=128)
|
inferencer=dict(type=GenInferencer, max_out_len=128),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_dureader_eval_cfg = dict(
|
LongBench_dureader_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
|
evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_dureader_datasets = [
|
LongBench_dureader_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchdureaderDataset,
|
type=LongBenchdureaderDataset,
|
||||||
abbr='LongBench_dureader',
|
abbr='LongBench_dureader',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='dureader',
|
name='dureader',
|
||||||
reader_cfg=LongBench_dureader_reader_cfg,
|
reader_cfg=LongBench_dureader_reader_cfg,
|
||||||
infer_cfg=LongBench_dureader_infer_cfg,
|
infer_cfg=LongBench_dureader_infer_cfg,
|
||||||
eval_cfg=LongBench_dureader_eval_cfg)
|
eval_cfg=LongBench_dureader_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_gov_report_reader_cfg = dict(
|
|||||||
input_columns=['context'],
|
input_columns=['context'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_gov_report_infer_cfg = dict(
|
LongBench_gov_report_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_gov_report_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=512)
|
inferencer=dict(type=GenInferencer, max_out_len=512),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_gov_report_eval_cfg = dict(
|
LongBench_gov_report_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator),
|
evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_gov_report_datasets = [
|
LongBench_gov_report_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchgov_reportDataset,
|
type=LongBenchgov_reportDataset,
|
||||||
abbr='LongBench_gov_report',
|
abbr='LongBench_gov_report',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='gov_report',
|
name='gov_report',
|
||||||
reader_cfg=LongBench_gov_report_reader_cfg,
|
reader_cfg=LongBench_gov_report_reader_cfg,
|
||||||
infer_cfg=LongBench_gov_report_infer_cfg,
|
infer_cfg=LongBench_gov_report_infer_cfg,
|
||||||
eval_cfg=LongBench_gov_report_eval_cfg)
|
eval_cfg=LongBench_gov_report_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_hotpotqa_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_hotpotqa_infer_cfg = dict(
|
LongBench_hotpotqa_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_hotpotqa_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_hotpotqa_eval_cfg = dict(
|
LongBench_hotpotqa_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_hotpotqa_datasets = [
|
LongBench_hotpotqa_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchhotpotqaDataset,
|
type=LongBenchhotpotqaDataset,
|
||||||
abbr='LongBench_hotpotqa',
|
abbr='LongBench_hotpotqa',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='hotpotqa',
|
name='hotpotqa',
|
||||||
reader_cfg=LongBench_hotpotqa_reader_cfg,
|
reader_cfg=LongBench_hotpotqa_reader_cfg,
|
||||||
infer_cfg=LongBench_hotpotqa_infer_cfg,
|
infer_cfg=LongBench_hotpotqa_infer_cfg,
|
||||||
eval_cfg=LongBench_hotpotqa_eval_cfg)
|
eval_cfg=LongBench_hotpotqa_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_lcc_reader_cfg = dict(
|
|||||||
input_columns=['context'],
|
input_columns=['context'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lcc_infer_cfg = dict(
|
LongBench_lcc_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_lcc_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Please complete the code given below. \n{context}Next line of code:\n'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Please complete the code given below. \n{context}Next line of code:\n',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lcc_eval_cfg = dict(
|
LongBench_lcc_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchCodeSimEvaluator),
|
evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lcc_datasets = [
|
LongBench_lcc_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchlccDataset,
|
type=LongBenchlccDataset,
|
||||||
abbr='LongBench_lcc',
|
abbr='LongBench_lcc',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='lcc',
|
name='lcc',
|
||||||
reader_cfg=LongBench_lcc_reader_cfg,
|
reader_cfg=LongBench_lcc_reader_cfg,
|
||||||
infer_cfg=LongBench_lcc_infer_cfg,
|
infer_cfg=LongBench_lcc_infer_cfg,
|
||||||
eval_cfg=LongBench_lcc_eval_cfg)
|
eval_cfg=LongBench_lcc_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,17 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchlshtDataset, lsht_postprocess
|
from opencompass.datasets import (
|
||||||
|
LongBenchClassificationEvaluator,
|
||||||
|
LongBenchlshtDataset,
|
||||||
|
lsht_postprocess,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_lsht_reader_cfg = dict(
|
LongBench_lsht_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='all_labels',
|
output_column='all_labels',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lsht_infer_cfg = dict(
|
LongBench_lsht_infer_cfg = dict(
|
||||||
@ -15,10 +19,15 @@ LongBench_lsht_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lsht_eval_cfg = dict(
|
LongBench_lsht_eval_cfg = dict(
|
||||||
@ -31,9 +40,10 @@ LongBench_lsht_datasets = [
|
|||||||
dict(
|
dict(
|
||||||
type=LongBenchlshtDataset,
|
type=LongBenchlshtDataset,
|
||||||
abbr='LongBench_lsht',
|
abbr='LongBench_lsht',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='lsht',
|
name='lsht',
|
||||||
reader_cfg=LongBench_lsht_reader_cfg,
|
reader_cfg=LongBench_lsht_reader_cfg,
|
||||||
infer_cfg=LongBench_lsht_infer_cfg,
|
infer_cfg=LongBench_lsht_infer_cfg,
|
||||||
eval_cfg=LongBench_lsht_eval_cfg)
|
eval_cfg=LongBench_lsht_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_multi_news_reader_cfg = dict(
|
|||||||
input_columns=['context'],
|
input_columns=['context'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multi_news_infer_cfg = dict(
|
LongBench_multi_news_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_multi_news_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=512)
|
inferencer=dict(type=GenInferencer, max_out_len=512),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multi_news_eval_cfg = dict(
|
LongBench_multi_news_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator),
|
evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multi_news_datasets = [
|
LongBench_multi_news_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchmulti_newsDataset,
|
type=LongBenchmulti_newsDataset,
|
||||||
abbr='LongBench_multi_news',
|
abbr='LongBench_multi_news',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='multi_news',
|
name='multi_news',
|
||||||
reader_cfg=LongBench_multi_news_reader_cfg,
|
reader_cfg=LongBench_multi_news_reader_cfg,
|
||||||
infer_cfg=LongBench_multi_news_infer_cfg,
|
infer_cfg=LongBench_multi_news_infer_cfg,
|
||||||
eval_cfg=LongBench_multi_news_eval_cfg)
|
eval_cfg=LongBench_multi_news_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_multifieldqa_en_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_en_infer_cfg = dict(
|
LongBench_multifieldqa_en_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_multifieldqa_en_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_en_eval_cfg = dict(
|
LongBench_multifieldqa_en_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_en_datasets = [
|
LongBench_multifieldqa_en_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchmultifieldqa_enDataset,
|
type=LongBenchmultifieldqa_enDataset,
|
||||||
abbr='LongBench_multifieldqa_en',
|
abbr='LongBench_multifieldqa_en',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='multifieldqa_en',
|
name='multifieldqa_en',
|
||||||
reader_cfg=LongBench_multifieldqa_en_reader_cfg,
|
reader_cfg=LongBench_multifieldqa_en_reader_cfg,
|
||||||
infer_cfg=LongBench_multifieldqa_en_infer_cfg,
|
infer_cfg=LongBench_multifieldqa_en_infer_cfg,
|
||||||
eval_cfg=LongBench_multifieldqa_en_eval_cfg)
|
eval_cfg=LongBench_multifieldqa_en_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_multifieldqa_zh_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_zh_infer_cfg = dict(
|
LongBench_multifieldqa_zh_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_multifieldqa_zh_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_zh_eval_cfg = dict(
|
LongBench_multifieldqa_zh_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator, language='zh'),
|
evaluator=dict(type=LongBenchF1Evaluator, language='zh'), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_zh_datasets = [
|
LongBench_multifieldqa_zh_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchmultifieldqa_zhDataset,
|
type=LongBenchmultifieldqa_zhDataset,
|
||||||
abbr='LongBench_multifieldqa_zh',
|
abbr='LongBench_multifieldqa_zh',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='multifieldqa_zh',
|
name='multifieldqa_zh',
|
||||||
reader_cfg=LongBench_multifieldqa_zh_reader_cfg,
|
reader_cfg=LongBench_multifieldqa_zh_reader_cfg,
|
||||||
infer_cfg=LongBench_multifieldqa_zh_infer_cfg,
|
infer_cfg=LongBench_multifieldqa_zh_infer_cfg,
|
||||||
eval_cfg=LongBench_multifieldqa_zh_eval_cfg)
|
eval_cfg=LongBench_multifieldqa_zh_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_musique_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_musique_infer_cfg = dict(
|
LongBench_musique_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_musique_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_musique_eval_cfg = dict(
|
LongBench_musique_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_musique_datasets = [
|
LongBench_musique_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchmusiqueDataset,
|
type=LongBenchmusiqueDataset,
|
||||||
abbr='LongBench_musique',
|
abbr='LongBench_musique',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='musique',
|
name='musique',
|
||||||
reader_cfg=LongBench_musique_reader_cfg,
|
reader_cfg=LongBench_musique_reader_cfg,
|
||||||
infer_cfg=LongBench_musique_infer_cfg,
|
infer_cfg=LongBench_musique_infer_cfg,
|
||||||
eval_cfg=LongBench_musique_eval_cfg)
|
eval_cfg=LongBench_musique_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_narrativeqa_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_narrativeqa_infer_cfg = dict(
|
LongBench_narrativeqa_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_narrativeqa_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=128)
|
inferencer=dict(type=GenInferencer, max_out_len=128),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_narrativeqa_eval_cfg = dict(
|
LongBench_narrativeqa_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_narrativeqa_datasets = [
|
LongBench_narrativeqa_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchnarrativeqaDataset,
|
type=LongBenchnarrativeqaDataset,
|
||||||
abbr='LongBench_narrativeqa',
|
abbr='LongBench_narrativeqa',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='narrativeqa',
|
name='narrativeqa',
|
||||||
reader_cfg=LongBench_narrativeqa_reader_cfg,
|
reader_cfg=LongBench_narrativeqa_reader_cfg,
|
||||||
infer_cfg=LongBench_narrativeqa_infer_cfg,
|
infer_cfg=LongBench_narrativeqa_infer_cfg,
|
||||||
eval_cfg=LongBench_narrativeqa_eval_cfg)
|
eval_cfg=LongBench_narrativeqa_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_passage_count_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_count_infer_cfg = dict(
|
LongBench_passage_count_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_passage_count_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_count_eval_cfg = dict(
|
LongBench_passage_count_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchCountEvaluator),
|
evaluator=dict(type=LongBenchCountEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_count_datasets = [
|
LongBench_passage_count_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchpassage_countDataset,
|
type=LongBenchpassage_countDataset,
|
||||||
abbr='LongBench_passage_count',
|
abbr='LongBench_passage_count',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='passage_count',
|
name='passage_count',
|
||||||
reader_cfg=LongBench_passage_count_reader_cfg,
|
reader_cfg=LongBench_passage_count_reader_cfg,
|
||||||
infer_cfg=LongBench_passage_count_infer_cfg,
|
infer_cfg=LongBench_passage_count_infer_cfg,
|
||||||
eval_cfg=LongBench_passage_count_eval_cfg)
|
eval_cfg=LongBench_passage_count_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,16 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_enDataset
|
from opencompass.datasets import (
|
||||||
|
LongBenchRetrievalEvaluator,
|
||||||
|
LongBenchpassage_retrieval_enDataset,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_en_reader_cfg = dict(
|
LongBench_passage_retrieval_en_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_en_infer_cfg = dict(
|
LongBench_passage_retrieval_en_infer_cfg = dict(
|
||||||
@ -15,24 +18,29 @@ LongBench_passage_retrieval_en_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like \"Paragraph 1\", \"Paragraph 2\", etc.\n\nThe answer is: '),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_en_eval_cfg = dict(
|
LongBench_passage_retrieval_en_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRetrievalEvaluator),
|
evaluator=dict(type=LongBenchRetrievalEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_en_datasets = [
|
LongBench_passage_retrieval_en_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchpassage_retrieval_enDataset,
|
type=LongBenchpassage_retrieval_enDataset,
|
||||||
abbr='LongBench_passage_retrieval_en',
|
abbr='LongBench_passage_retrieval_en',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='passage_retrieval_en',
|
name='passage_retrieval_en',
|
||||||
reader_cfg=LongBench_passage_retrieval_en_reader_cfg,
|
reader_cfg=LongBench_passage_retrieval_en_reader_cfg,
|
||||||
infer_cfg=LongBench_passage_retrieval_en_infer_cfg,
|
infer_cfg=LongBench_passage_retrieval_en_infer_cfg,
|
||||||
eval_cfg=LongBench_passage_retrieval_en_eval_cfg)
|
eval_cfg=LongBench_passage_retrieval_en_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,16 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_zhDataset
|
from opencompass.datasets import (
|
||||||
|
LongBenchRetrievalEvaluator,
|
||||||
|
LongBenchpassage_retrieval_zhDataset,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_zh_reader_cfg = dict(
|
LongBench_passage_retrieval_zh_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_zh_infer_cfg = dict(
|
LongBench_passage_retrieval_zh_infer_cfg = dict(
|
||||||
@ -15,24 +18,29 @@ LongBench_passage_retrieval_zh_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是\"段落1\"，\"段落2\"等格式\n\n答案是：'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_zh_eval_cfg = dict(
|
LongBench_passage_retrieval_zh_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'),
|
evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_zh_datasets = [
|
LongBench_passage_retrieval_zh_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchpassage_retrieval_zhDataset,
|
type=LongBenchpassage_retrieval_zhDataset,
|
||||||
abbr='LongBench_passage_retrieval_zh',
|
abbr='LongBench_passage_retrieval_zh',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='passage_retrieval_zh',
|
name='passage_retrieval_zh',
|
||||||
reader_cfg=LongBench_passage_retrieval_zh_reader_cfg,
|
reader_cfg=LongBench_passage_retrieval_zh_reader_cfg,
|
||||||
infer_cfg=LongBench_passage_retrieval_zh_infer_cfg,
|
infer_cfg=LongBench_passage_retrieval_zh_infer_cfg,
|
||||||
eval_cfg=LongBench_passage_retrieval_zh_eval_cfg)
|
eval_cfg=LongBench_passage_retrieval_zh_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_qasper_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qasper_infer_cfg = dict(
|
LongBench_qasper_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_qasper_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qasper_eval_cfg = dict(
|
LongBench_qasper_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qasper_datasets = [
|
LongBench_qasper_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchqasperDataset,
|
type=LongBenchqasperDataset,
|
||||||
abbr='LongBench_qasper',
|
abbr='LongBench_qasper',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='qasper',
|
name='qasper',
|
||||||
reader_cfg=LongBench_qasper_reader_cfg,
|
reader_cfg=LongBench_qasper_reader_cfg,
|
||||||
infer_cfg=LongBench_qasper_infer_cfg,
|
infer_cfg=LongBench_qasper_infer_cfg,
|
||||||
eval_cfg=LongBench_qasper_eval_cfg)
|
eval_cfg=LongBench_qasper_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_qmsum_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qmsum_infer_cfg = dict(
|
LongBench_qmsum_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_qmsum_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=512)
|
inferencer=dict(type=GenInferencer, max_out_len=512),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qmsum_eval_cfg = dict(
|
LongBench_qmsum_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator),
|
evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qmsum_datasets = [
|
LongBench_qmsum_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchqmsumDataset,
|
type=LongBenchqmsumDataset,
|
||||||
abbr='LongBench_qmsum',
|
abbr='LongBench_qmsum',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='qmsum',
|
name='qmsum',
|
||||||
reader_cfg=LongBench_qmsum_reader_cfg,
|
reader_cfg=LongBench_qmsum_reader_cfg,
|
||||||
infer_cfg=LongBench_qmsum_infer_cfg,
|
infer_cfg=LongBench_qmsum_infer_cfg,
|
||||||
eval_cfg=LongBench_qmsum_eval_cfg)
|
eval_cfg=LongBench_qmsum_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_repobench_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_repobench_infer_cfg = dict(
|
LongBench_repobench_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_repobench_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Please complete the code given below. \n{context}{input}Next line of code:\n'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Please complete the code given below. \n{context}{input}Next line of code:\n',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_repobench_eval_cfg = dict(
|
LongBench_repobench_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchCodeSimEvaluator),
|
evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_repobench_datasets = [
|
LongBench_repobench_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchrepobenchDataset,
|
type=LongBenchrepobenchDataset,
|
||||||
abbr='LongBench_repobench-p',
|
abbr='LongBench_repobench-p',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='repobench-p',
|
name='repobench-p',
|
||||||
reader_cfg=LongBench_repobench_reader_cfg,
|
reader_cfg=LongBench_repobench_reader_cfg,
|
||||||
infer_cfg=LongBench_repobench_infer_cfg,
|
infer_cfg=LongBench_repobench_infer_cfg,
|
||||||
eval_cfg=LongBench_repobench_eval_cfg)
|
eval_cfg=LongBench_repobench_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,17 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchRougeEvaluator, LongBenchsamsumDataset, samsum_postprocess
|
from opencompass.datasets import (
|
||||||
|
LongBenchRougeEvaluator,
|
||||||
|
LongBenchsamsumDataset,
|
||||||
|
samsum_postprocess,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_samsum_reader_cfg = dict(
|
LongBench_samsum_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_samsum_infer_cfg = dict(
|
LongBench_samsum_infer_cfg = dict(
|
||||||
@ -15,10 +19,15 @@ LongBench_samsum_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=128)
|
inferencer=dict(type=GenInferencer, max_out_len=128),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_samsum_eval_cfg = dict(
|
LongBench_samsum_eval_cfg = dict(
|
||||||
@ -31,9 +40,10 @@ LongBench_samsum_datasets = [
|
|||||||
dict(
|
dict(
|
||||||
type=LongBenchsamsumDataset,
|
type=LongBenchsamsumDataset,
|
||||||
abbr='LongBench_samsum',
|
abbr='LongBench_samsum',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='samsum',
|
name='samsum',
|
||||||
reader_cfg=LongBench_samsum_reader_cfg,
|
reader_cfg=LongBench_samsum_reader_cfg,
|
||||||
infer_cfg=LongBench_samsum_infer_cfg,
|
infer_cfg=LongBench_samsum_infer_cfg,
|
||||||
eval_cfg=LongBench_samsum_eval_cfg)
|
eval_cfg=LongBench_samsum_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,17 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchtrecDataset, trec_postprocess
|
from opencompass.datasets import (
|
||||||
|
LongBenchClassificationEvaluator,
|
||||||
|
LongBenchtrecDataset,
|
||||||
|
trec_postprocess,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_trec_reader_cfg = dict(
|
LongBench_trec_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='all_labels',
|
output_column='all_labels',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_trec_infer_cfg = dict(
|
LongBench_trec_infer_cfg = dict(
|
||||||
@ -15,10 +19,15 @@ LongBench_trec_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_trec_eval_cfg = dict(
|
LongBench_trec_eval_cfg = dict(
|
||||||
@ -31,9 +40,10 @@ LongBench_trec_datasets = [
|
|||||||
dict(
|
dict(
|
||||||
type=LongBenchtrecDataset,
|
type=LongBenchtrecDataset,
|
||||||
abbr='LongBench_trec',
|
abbr='LongBench_trec',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='trec',
|
name='trec',
|
||||||
reader_cfg=LongBench_trec_reader_cfg,
|
reader_cfg=LongBench_trec_reader_cfg,
|
||||||
infer_cfg=LongBench_trec_infer_cfg,
|
infer_cfg=LongBench_trec_infer_cfg,
|
||||||
eval_cfg=LongBench_trec_eval_cfg)
|
eval_cfg=LongBench_trec_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,17 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchF1Evaluator, LongBenchtriviaqaDataset, triviaqa_postprocess
|
from opencompass.datasets import (
|
||||||
|
LongBenchF1Evaluator,
|
||||||
|
LongBenchtriviaqaDataset,
|
||||||
|
triviaqa_postprocess,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_triviaqa_reader_cfg = dict(
|
LongBench_triviaqa_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_triviaqa_infer_cfg = dict(
|
LongBench_triviaqa_infer_cfg = dict(
|
||||||
@ -15,10 +19,15 @@ LongBench_triviaqa_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_triviaqa_eval_cfg = dict(
|
LongBench_triviaqa_eval_cfg = dict(
|
||||||
@ -31,9 +40,10 @@ LongBench_triviaqa_datasets = [
|
|||||||
dict(
|
dict(
|
||||||
type=LongBenchtriviaqaDataset,
|
type=LongBenchtriviaqaDataset,
|
||||||
abbr='LongBench_triviaqa',
|
abbr='LongBench_triviaqa',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='triviaqa',
|
name='triviaqa',
|
||||||
reader_cfg=LongBench_triviaqa_reader_cfg,
|
reader_cfg=LongBench_triviaqa_reader_cfg,
|
||||||
infer_cfg=LongBench_triviaqa_infer_cfg,
|
infer_cfg=LongBench_triviaqa_infer_cfg,
|
||||||
eval_cfg=LongBench_triviaqa_eval_cfg)
|
eval_cfg=LongBench_triviaqa_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_vcsum_reader_cfg = dict(
|
|||||||
input_columns=['context'],
|
input_columns=['context'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_vcsum_infer_cfg = dict(
|
LongBench_vcsum_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_vcsum_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=512)
|
inferencer=dict(type=GenInferencer, max_out_len=512),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_vcsum_eval_cfg = dict(
|
LongBench_vcsum_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
|
evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_vcsum_datasets = [
|
LongBench_vcsum_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchvcsumDataset,
|
type=LongBenchvcsumDataset,
|
||||||
abbr='LongBench_vcsum',
|
abbr='LongBench_vcsum',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='vcsum',
|
name='vcsum',
|
||||||
reader_cfg=LongBench_vcsum_reader_cfg,
|
reader_cfg=LongBench_vcsum_reader_cfg,
|
||||||
infer_cfg=LongBench_vcsum_infer_cfg,
|
infer_cfg=LongBench_vcsum_infer_cfg,
|
||||||
eval_cfg=LongBench_vcsum_eval_cfg)
|
eval_cfg=LongBench_vcsum_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_2wikimqa_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_2wikimqa_infer_cfg = dict(
|
LongBench_2wikimqa_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_2wikimqa_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_2wikimqa_eval_cfg = dict(
|
LongBench_2wikimqa_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_2wikimqa_datasets = [
|
LongBench_2wikimqa_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBench2wikimqaDataset,
|
type=LongBench2wikimqaDataset,
|
||||||
abbr='LongBench_2wikimqa',
|
abbr='LongBench_2wikimqa',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='2wikimqa',
|
name='2wikimqa',
|
||||||
reader_cfg=LongBench_2wikimqa_reader_cfg,
|
reader_cfg=LongBench_2wikimqa_reader_cfg,
|
||||||
infer_cfg=LongBench_2wikimqa_infer_cfg,
|
infer_cfg=LongBench_2wikimqa_infer_cfg,
|
||||||
eval_cfg=LongBench_2wikimqa_eval_cfg)
|
eval_cfg=LongBench_2wikimqa_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_dureader_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_dureader_infer_cfg = dict(
|
LongBench_dureader_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_dureader_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=128)
|
inferencer=dict(type=GenInferencer, max_out_len=128),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_dureader_eval_cfg = dict(
|
LongBench_dureader_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
|
evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_dureader_datasets = [
|
LongBench_dureader_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchdureaderDataset,
|
type=LongBenchdureaderDataset,
|
||||||
abbr='LongBench_dureader',
|
abbr='LongBench_dureader',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='dureader',
|
name='dureader',
|
||||||
reader_cfg=LongBench_dureader_reader_cfg,
|
reader_cfg=LongBench_dureader_reader_cfg,
|
||||||
infer_cfg=LongBench_dureader_infer_cfg,
|
infer_cfg=LongBench_dureader_infer_cfg,
|
||||||
eval_cfg=LongBench_dureader_eval_cfg)
|
eval_cfg=LongBench_dureader_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_gov_report_reader_cfg = dict(
|
|||||||
input_columns=['context'],
|
input_columns=['context'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_gov_report_infer_cfg = dict(
|
LongBench_gov_report_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_gov_report_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=512)
|
inferencer=dict(type=GenInferencer, max_out_len=512),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_gov_report_eval_cfg = dict(
|
LongBench_gov_report_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator),
|
evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_gov_report_datasets = [
|
LongBench_gov_report_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchgov_reportDataset,
|
type=LongBenchgov_reportDataset,
|
||||||
abbr='LongBench_gov_report',
|
abbr='LongBench_gov_report',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='gov_report',
|
name='gov_report',
|
||||||
reader_cfg=LongBench_gov_report_reader_cfg,
|
reader_cfg=LongBench_gov_report_reader_cfg,
|
||||||
infer_cfg=LongBench_gov_report_infer_cfg,
|
infer_cfg=LongBench_gov_report_infer_cfg,
|
||||||
eval_cfg=LongBench_gov_report_eval_cfg)
|
eval_cfg=LongBench_gov_report_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_hotpotqa_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_hotpotqa_infer_cfg = dict(
|
LongBench_hotpotqa_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_hotpotqa_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_hotpotqa_eval_cfg = dict(
|
LongBench_hotpotqa_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_hotpotqa_datasets = [
|
LongBench_hotpotqa_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchhotpotqaDataset,
|
type=LongBenchhotpotqaDataset,
|
||||||
abbr='LongBench_hotpotqa',
|
abbr='LongBench_hotpotqa',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='hotpotqa',
|
name='hotpotqa',
|
||||||
reader_cfg=LongBench_hotpotqa_reader_cfg,
|
reader_cfg=LongBench_hotpotqa_reader_cfg,
|
||||||
infer_cfg=LongBench_hotpotqa_infer_cfg,
|
infer_cfg=LongBench_hotpotqa_infer_cfg,
|
||||||
eval_cfg=LongBench_hotpotqa_eval_cfg)
|
eval_cfg=LongBench_hotpotqa_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_lcc_reader_cfg = dict(
|
|||||||
input_columns=['context'],
|
input_columns=['context'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lcc_infer_cfg = dict(
|
LongBench_lcc_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_lcc_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Please complete the code given below. \n{context}Next line of code:\n'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Please complete the code given below. \n{context}Next line of code:\n',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lcc_eval_cfg = dict(
|
LongBench_lcc_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchCodeSimEvaluator),
|
evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lcc_datasets = [
|
LongBench_lcc_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchlccDataset,
|
type=LongBenchlccDataset,
|
||||||
abbr='LongBench_lcc',
|
abbr='LongBench_lcc',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='lcc',
|
name='lcc',
|
||||||
reader_cfg=LongBench_lcc_reader_cfg,
|
reader_cfg=LongBench_lcc_reader_cfg,
|
||||||
infer_cfg=LongBench_lcc_infer_cfg,
|
infer_cfg=LongBench_lcc_infer_cfg,
|
||||||
eval_cfg=LongBench_lcc_eval_cfg)
|
eval_cfg=LongBench_lcc_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,17 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchlshtDataset, lsht_postprocess
|
from opencompass.datasets import (
|
||||||
|
LongBenchClassificationEvaluator,
|
||||||
|
LongBenchlshtDataset,
|
||||||
|
lsht_postprocess,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_lsht_reader_cfg = dict(
|
LongBench_lsht_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='all_labels',
|
output_column='all_labels',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lsht_infer_cfg = dict(
|
LongBench_lsht_infer_cfg = dict(
|
||||||
@ -15,10 +19,15 @@ LongBench_lsht_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_lsht_eval_cfg = dict(
|
LongBench_lsht_eval_cfg = dict(
|
||||||
@ -31,9 +40,10 @@ LongBench_lsht_datasets = [
|
|||||||
dict(
|
dict(
|
||||||
type=LongBenchlshtDataset,
|
type=LongBenchlshtDataset,
|
||||||
abbr='LongBench_lsht',
|
abbr='LongBench_lsht',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='lsht',
|
name='lsht',
|
||||||
reader_cfg=LongBench_lsht_reader_cfg,
|
reader_cfg=LongBench_lsht_reader_cfg,
|
||||||
infer_cfg=LongBench_lsht_infer_cfg,
|
infer_cfg=LongBench_lsht_infer_cfg,
|
||||||
eval_cfg=LongBench_lsht_eval_cfg)
|
eval_cfg=LongBench_lsht_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_multi_news_reader_cfg = dict(
|
|||||||
input_columns=['context'],
|
input_columns=['context'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multi_news_infer_cfg = dict(
|
LongBench_multi_news_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_multi_news_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=512)
|
inferencer=dict(type=GenInferencer, max_out_len=512),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multi_news_eval_cfg = dict(
|
LongBench_multi_news_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator),
|
evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multi_news_datasets = [
|
LongBench_multi_news_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchmulti_newsDataset,
|
type=LongBenchmulti_newsDataset,
|
||||||
abbr='LongBench_multi_news',
|
abbr='LongBench_multi_news',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='multi_news',
|
name='multi_news',
|
||||||
reader_cfg=LongBench_multi_news_reader_cfg,
|
reader_cfg=LongBench_multi_news_reader_cfg,
|
||||||
infer_cfg=LongBench_multi_news_infer_cfg,
|
infer_cfg=LongBench_multi_news_infer_cfg,
|
||||||
eval_cfg=LongBench_multi_news_eval_cfg)
|
eval_cfg=LongBench_multi_news_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_multifieldqa_en_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_en_infer_cfg = dict(
|
LongBench_multifieldqa_en_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_multifieldqa_en_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_en_eval_cfg = dict(
|
LongBench_multifieldqa_en_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_en_datasets = [
|
LongBench_multifieldqa_en_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchmultifieldqa_enDataset,
|
type=LongBenchmultifieldqa_enDataset,
|
||||||
abbr='LongBench_multifieldqa_en',
|
abbr='LongBench_multifieldqa_en',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='multifieldqa_en',
|
name='multifieldqa_en',
|
||||||
reader_cfg=LongBench_multifieldqa_en_reader_cfg,
|
reader_cfg=LongBench_multifieldqa_en_reader_cfg,
|
||||||
infer_cfg=LongBench_multifieldqa_en_infer_cfg,
|
infer_cfg=LongBench_multifieldqa_en_infer_cfg,
|
||||||
eval_cfg=LongBench_multifieldqa_en_eval_cfg)
|
eval_cfg=LongBench_multifieldqa_en_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_multifieldqa_zh_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_zh_infer_cfg = dict(
|
LongBench_multifieldqa_zh_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_multifieldqa_zh_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_zh_eval_cfg = dict(
|
LongBench_multifieldqa_zh_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator, language='zh'),
|
evaluator=dict(type=LongBenchF1Evaluator, language='zh'), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_multifieldqa_zh_datasets = [
|
LongBench_multifieldqa_zh_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchmultifieldqa_zhDataset,
|
type=LongBenchmultifieldqa_zhDataset,
|
||||||
abbr='LongBench_multifieldqa_zh',
|
abbr='LongBench_multifieldqa_zh',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='multifieldqa_zh',
|
name='multifieldqa_zh',
|
||||||
reader_cfg=LongBench_multifieldqa_zh_reader_cfg,
|
reader_cfg=LongBench_multifieldqa_zh_reader_cfg,
|
||||||
infer_cfg=LongBench_multifieldqa_zh_infer_cfg,
|
infer_cfg=LongBench_multifieldqa_zh_infer_cfg,
|
||||||
eval_cfg=LongBench_multifieldqa_zh_eval_cfg)
|
eval_cfg=LongBench_multifieldqa_zh_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_musique_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_musique_infer_cfg = dict(
|
LongBench_musique_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_musique_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_musique_eval_cfg = dict(
|
LongBench_musique_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_musique_datasets = [
|
LongBench_musique_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchmusiqueDataset,
|
type=LongBenchmusiqueDataset,
|
||||||
abbr='LongBench_musique',
|
abbr='LongBench_musique',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='musique',
|
name='musique',
|
||||||
reader_cfg=LongBench_musique_reader_cfg,
|
reader_cfg=LongBench_musique_reader_cfg,
|
||||||
infer_cfg=LongBench_musique_infer_cfg,
|
infer_cfg=LongBench_musique_infer_cfg,
|
||||||
eval_cfg=LongBench_musique_eval_cfg)
|
eval_cfg=LongBench_musique_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_narrativeqa_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_narrativeqa_infer_cfg = dict(
|
LongBench_narrativeqa_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_narrativeqa_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=128)
|
inferencer=dict(type=GenInferencer, max_out_len=128),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_narrativeqa_eval_cfg = dict(
|
LongBench_narrativeqa_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_narrativeqa_datasets = [
|
LongBench_narrativeqa_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchnarrativeqaDataset,
|
type=LongBenchnarrativeqaDataset,
|
||||||
abbr='LongBench_narrativeqa',
|
abbr='LongBench_narrativeqa',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='narrativeqa',
|
name='narrativeqa',
|
||||||
reader_cfg=LongBench_narrativeqa_reader_cfg,
|
reader_cfg=LongBench_narrativeqa_reader_cfg,
|
||||||
infer_cfg=LongBench_narrativeqa_infer_cfg,
|
infer_cfg=LongBench_narrativeqa_infer_cfg,
|
||||||
eval_cfg=LongBench_narrativeqa_eval_cfg)
|
eval_cfg=LongBench_narrativeqa_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_passage_count_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_count_infer_cfg = dict(
|
LongBench_passage_count_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_passage_count_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: '),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_count_eval_cfg = dict(
|
LongBench_passage_count_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchCountEvaluator),
|
evaluator=dict(type=LongBenchCountEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_count_datasets = [
|
LongBench_passage_count_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchpassage_countDataset,
|
type=LongBenchpassage_countDataset,
|
||||||
abbr='LongBench_passage_count',
|
abbr='LongBench_passage_count',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='passage_count',
|
name='passage_count',
|
||||||
reader_cfg=LongBench_passage_count_reader_cfg,
|
reader_cfg=LongBench_passage_count_reader_cfg,
|
||||||
infer_cfg=LongBench_passage_count_infer_cfg,
|
infer_cfg=LongBench_passage_count_infer_cfg,
|
||||||
eval_cfg=LongBench_passage_count_eval_cfg)
|
eval_cfg=LongBench_passage_count_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,16 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_enDataset
|
from opencompass.datasets import (
|
||||||
|
LongBenchRetrievalEvaluator,
|
||||||
|
LongBenchpassage_retrieval_enDataset,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_en_reader_cfg = dict(
|
LongBench_passage_retrieval_en_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_en_infer_cfg = dict(
|
LongBench_passage_retrieval_en_infer_cfg = dict(
|
||||||
@ -15,24 +18,29 @@ LongBench_passage_retrieval_en_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like \"Paragraph 1\", \"Paragraph 2\", etc.\n\nThe answer is: '),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_en_eval_cfg = dict(
|
LongBench_passage_retrieval_en_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRetrievalEvaluator),
|
evaluator=dict(type=LongBenchRetrievalEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_en_datasets = [
|
LongBench_passage_retrieval_en_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchpassage_retrieval_enDataset,
|
type=LongBenchpassage_retrieval_enDataset,
|
||||||
abbr='LongBench_passage_retrieval_en',
|
abbr='LongBench_passage_retrieval_en',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='passage_retrieval_en',
|
name='passage_retrieval_en',
|
||||||
reader_cfg=LongBench_passage_retrieval_en_reader_cfg,
|
reader_cfg=LongBench_passage_retrieval_en_reader_cfg,
|
||||||
infer_cfg=LongBench_passage_retrieval_en_infer_cfg,
|
infer_cfg=LongBench_passage_retrieval_en_infer_cfg,
|
||||||
eval_cfg=LongBench_passage_retrieval_en_eval_cfg)
|
eval_cfg=LongBench_passage_retrieval_en_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,16 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchRetrievalEvaluator, LongBenchpassage_retrieval_zhDataset
|
from opencompass.datasets import (
|
||||||
|
LongBenchRetrievalEvaluator,
|
||||||
|
LongBenchpassage_retrieval_zhDataset,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_zh_reader_cfg = dict(
|
LongBench_passage_retrieval_zh_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_zh_infer_cfg = dict(
|
LongBench_passage_retrieval_zh_infer_cfg = dict(
|
||||||
@ -15,24 +18,29 @@ LongBench_passage_retrieval_zh_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是\"段落1\"，\"段落2\"等格式\n\n答案是：'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_zh_eval_cfg = dict(
|
LongBench_passage_retrieval_zh_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'),
|
evaluator=dict(type=LongBenchRetrievalEvaluator, language='zh'), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_passage_retrieval_zh_datasets = [
|
LongBench_passage_retrieval_zh_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchpassage_retrieval_zhDataset,
|
type=LongBenchpassage_retrieval_zhDataset,
|
||||||
abbr='LongBench_passage_retrieval_zh',
|
abbr='LongBench_passage_retrieval_zh',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='passage_retrieval_zh',
|
name='passage_retrieval_zh',
|
||||||
reader_cfg=LongBench_passage_retrieval_zh_reader_cfg,
|
reader_cfg=LongBench_passage_retrieval_zh_reader_cfg,
|
||||||
infer_cfg=LongBench_passage_retrieval_zh_infer_cfg,
|
infer_cfg=LongBench_passage_retrieval_zh_infer_cfg,
|
||||||
eval_cfg=LongBench_passage_retrieval_zh_eval_cfg)
|
eval_cfg=LongBench_passage_retrieval_zh_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_qasper_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qasper_infer_cfg = dict(
|
LongBench_qasper_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_qasper_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qasper_eval_cfg = dict(
|
LongBench_qasper_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchF1Evaluator),
|
evaluator=dict(type=LongBenchF1Evaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qasper_datasets = [
|
LongBench_qasper_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchqasperDataset,
|
type=LongBenchqasperDataset,
|
||||||
abbr='LongBench_qasper',
|
abbr='LongBench_qasper',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='qasper',
|
name='qasper',
|
||||||
reader_cfg=LongBench_qasper_reader_cfg,
|
reader_cfg=LongBench_qasper_reader_cfg,
|
||||||
infer_cfg=LongBench_qasper_infer_cfg,
|
infer_cfg=LongBench_qasper_infer_cfg,
|
||||||
eval_cfg=LongBench_qasper_eval_cfg)
|
eval_cfg=LongBench_qasper_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_qmsum_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qmsum_infer_cfg = dict(
|
LongBench_qmsum_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_qmsum_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=512)
|
inferencer=dict(type=GenInferencer, max_out_len=512),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qmsum_eval_cfg = dict(
|
LongBench_qmsum_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator),
|
evaluator=dict(type=LongBenchRougeEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_qmsum_datasets = [
|
LongBench_qmsum_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchqmsumDataset,
|
type=LongBenchqmsumDataset,
|
||||||
abbr='LongBench_qmsum',
|
abbr='LongBench_qmsum',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='qmsum',
|
name='qmsum',
|
||||||
reader_cfg=LongBench_qmsum_reader_cfg,
|
reader_cfg=LongBench_qmsum_reader_cfg,
|
||||||
infer_cfg=LongBench_qmsum_infer_cfg,
|
infer_cfg=LongBench_qmsum_infer_cfg,
|
||||||
eval_cfg=LongBench_qmsum_eval_cfg)
|
eval_cfg=LongBench_qmsum_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_repobench_reader_cfg = dict(
|
|||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_repobench_infer_cfg = dict(
|
LongBench_repobench_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_repobench_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Please complete the code given below. \n{context}{input}Next line of code:\n'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Please complete the code given below. \n{context}{input}Next line of code:\n',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_repobench_eval_cfg = dict(
|
LongBench_repobench_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchCodeSimEvaluator),
|
evaluator=dict(type=LongBenchCodeSimEvaluator), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_repobench_datasets = [
|
LongBench_repobench_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchrepobenchDataset,
|
type=LongBenchrepobenchDataset,
|
||||||
abbr='LongBench_repobench-p',
|
abbr='LongBench_repobench-p',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='repobench-p',
|
name='repobench-p',
|
||||||
reader_cfg=LongBench_repobench_reader_cfg,
|
reader_cfg=LongBench_repobench_reader_cfg,
|
||||||
infer_cfg=LongBench_repobench_infer_cfg,
|
infer_cfg=LongBench_repobench_infer_cfg,
|
||||||
eval_cfg=LongBench_repobench_eval_cfg)
|
eval_cfg=LongBench_repobench_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,17 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchRougeEvaluator, LongBenchsamsumDataset, samsum_postprocess
|
from opencompass.datasets import (
|
||||||
|
LongBenchRougeEvaluator,
|
||||||
|
LongBenchsamsumDataset,
|
||||||
|
samsum_postprocess,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_samsum_reader_cfg = dict(
|
LongBench_samsum_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_samsum_infer_cfg = dict(
|
LongBench_samsum_infer_cfg = dict(
|
||||||
@ -15,10 +19,15 @@ LongBench_samsum_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=128)
|
inferencer=dict(type=GenInferencer, max_out_len=128),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_samsum_eval_cfg = dict(
|
LongBench_samsum_eval_cfg = dict(
|
||||||
@ -31,9 +40,10 @@ LongBench_samsum_datasets = [
|
|||||||
dict(
|
dict(
|
||||||
type=LongBenchsamsumDataset,
|
type=LongBenchsamsumDataset,
|
||||||
abbr='LongBench_samsum',
|
abbr='LongBench_samsum',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='samsum',
|
name='samsum',
|
||||||
reader_cfg=LongBench_samsum_reader_cfg,
|
reader_cfg=LongBench_samsum_reader_cfg,
|
||||||
infer_cfg=LongBench_samsum_infer_cfg,
|
infer_cfg=LongBench_samsum_infer_cfg,
|
||||||
eval_cfg=LongBench_samsum_eval_cfg)
|
eval_cfg=LongBench_samsum_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,17 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchClassificationEvaluator, LongBenchtrecDataset, trec_postprocess
|
from opencompass.datasets import (
|
||||||
|
LongBenchClassificationEvaluator,
|
||||||
|
LongBenchtrecDataset,
|
||||||
|
trec_postprocess,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_trec_reader_cfg = dict(
|
LongBench_trec_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='all_labels',
|
output_column='all_labels',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_trec_infer_cfg = dict(
|
LongBench_trec_infer_cfg = dict(
|
||||||
@ -15,10 +19,15 @@ LongBench_trec_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=64)
|
inferencer=dict(type=GenInferencer, max_out_len=64),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_trec_eval_cfg = dict(
|
LongBench_trec_eval_cfg = dict(
|
||||||
@ -31,9 +40,10 @@ LongBench_trec_datasets = [
|
|||||||
dict(
|
dict(
|
||||||
type=LongBenchtrecDataset,
|
type=LongBenchtrecDataset,
|
||||||
abbr='LongBench_trec',
|
abbr='LongBench_trec',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='trec',
|
name='trec',
|
||||||
reader_cfg=LongBench_trec_reader_cfg,
|
reader_cfg=LongBench_trec_reader_cfg,
|
||||||
infer_cfg=LongBench_trec_infer_cfg,
|
infer_cfg=LongBench_trec_infer_cfg,
|
||||||
eval_cfg=LongBench_trec_eval_cfg)
|
eval_cfg=LongBench_trec_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -1,13 +1,17 @@
|
|||||||
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
||||||
from opencompass.openicl.icl_retriever import ZeroRetriever
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
||||||
from opencompass.openicl.icl_inferencer import GenInferencer
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
||||||
from opencompass.datasets import LongBenchF1Evaluator, LongBenchtriviaqaDataset, triviaqa_postprocess
|
from opencompass.datasets import (
|
||||||
|
LongBenchF1Evaluator,
|
||||||
|
LongBenchtriviaqaDataset,
|
||||||
|
triviaqa_postprocess,
|
||||||
|
)
|
||||||
|
|
||||||
LongBench_triviaqa_reader_cfg = dict(
|
LongBench_triviaqa_reader_cfg = dict(
|
||||||
input_columns=['context', 'input'],
|
input_columns=['context', 'input'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_triviaqa_infer_cfg = dict(
|
LongBench_triviaqa_infer_cfg = dict(
|
||||||
@ -15,10 +19,15 @@ LongBench_triviaqa_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=32)
|
inferencer=dict(type=GenInferencer, max_out_len=32),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_triviaqa_eval_cfg = dict(
|
LongBench_triviaqa_eval_cfg = dict(
|
||||||
@ -31,9 +40,10 @@ LongBench_triviaqa_datasets = [
|
|||||||
dict(
|
dict(
|
||||||
type=LongBenchtriviaqaDataset,
|
type=LongBenchtriviaqaDataset,
|
||||||
abbr='LongBench_triviaqa',
|
abbr='LongBench_triviaqa',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='triviaqa',
|
name='triviaqa',
|
||||||
reader_cfg=LongBench_triviaqa_reader_cfg,
|
reader_cfg=LongBench_triviaqa_reader_cfg,
|
||||||
infer_cfg=LongBench_triviaqa_infer_cfg,
|
infer_cfg=LongBench_triviaqa_infer_cfg,
|
||||||
eval_cfg=LongBench_triviaqa_eval_cfg)
|
eval_cfg=LongBench_triviaqa_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -7,7 +7,7 @@ LongBench_vcsum_reader_cfg = dict(
|
|||||||
input_columns=['context'],
|
input_columns=['context'],
|
||||||
output_column='answers',
|
output_column='answers',
|
||||||
train_split='test',
|
train_split='test',
|
||||||
test_split='test'
|
test_split='test',
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_vcsum_infer_cfg = dict(
|
LongBench_vcsum_infer_cfg = dict(
|
||||||
@ -15,24 +15,29 @@ LongBench_vcsum_infer_cfg = dict(
|
|||||||
type=PromptTemplate,
|
type=PromptTemplate,
|
||||||
template=dict(
|
template=dict(
|
||||||
round=[
|
round=[
|
||||||
dict(role='HUMAN', prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：'),
|
dict(
|
||||||
], )),
|
role='HUMAN',
|
||||||
|
prompt='下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
retriever=dict(type=ZeroRetriever),
|
retriever=dict(type=ZeroRetriever),
|
||||||
inferencer=dict(type=GenInferencer, max_out_len=512)
|
inferencer=dict(type=GenInferencer, max_out_len=512),
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_vcsum_eval_cfg = dict(
|
LongBench_vcsum_eval_cfg = dict(
|
||||||
evaluator=dict(type=LongBenchRougeEvaluator, language='zh'),
|
evaluator=dict(type=LongBenchRougeEvaluator, language='zh'), pred_role='BOT'
|
||||||
pred_role='BOT'
|
|
||||||
)
|
)
|
||||||
|
|
||||||
LongBench_vcsum_datasets = [
|
LongBench_vcsum_datasets = [
|
||||||
dict(
|
dict(
|
||||||
type=LongBenchvcsumDataset,
|
type=LongBenchvcsumDataset,
|
||||||
abbr='LongBench_vcsum',
|
abbr='LongBench_vcsum',
|
||||||
path='THUDM/LongBench',
|
path='opencompass/Longbench',
|
||||||
name='vcsum',
|
name='vcsum',
|
||||||
reader_cfg=LongBench_vcsum_reader_cfg,
|
reader_cfg=LongBench_vcsum_reader_cfg,
|
||||||
infer_cfg=LongBench_vcsum_infer_cfg,
|
infer_cfg=LongBench_vcsum_infer_cfg,
|
||||||
eval_cfg=LongBench_vcsum_eval_cfg)
|
eval_cfg=LongBench_vcsum_eval_cfg,
|
||||||
|
)
|
||||||
]
|
]
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBench2wikimqaDataset(BaseDataset):
|
class LongBench2wikimqaDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchdureaderDataset(BaseDataset):
|
class LongBenchdureaderDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchgov_reportDataset(BaseDataset):
|
class LongBenchgov_reportDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchhotpotqaDataset(BaseDataset):
|
class LongBenchhotpotqaDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchlccDataset(BaseDataset):
|
class LongBenchlccDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchlshtDataset(BaseDataset):
|
class LongBenchlshtDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchmulti_newsDataset(BaseDataset):
|
class LongBenchmulti_newsDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchmultifieldqa_enDataset(BaseDataset):
|
class LongBenchmultifieldqa_enDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchmultifieldqa_zhDataset(BaseDataset):
|
class LongBenchmultifieldqa_zhDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchmusiqueDataset(BaseDataset):
|
class LongBenchmusiqueDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchnarrativeqaDataset(BaseDataset):
|
class LongBenchnarrativeqaDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchpassage_countDataset(BaseDataset):
|
class LongBenchpassage_countDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchpassage_retrieval_enDataset(BaseDataset):
|
class LongBenchpassage_retrieval_enDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchpassage_retrieval_zhDataset(BaseDataset):
|
class LongBenchpassage_retrieval_zhDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchqasperDataset(BaseDataset):
|
class LongBenchqasperDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchqmsumDataset(BaseDataset):
|
class LongBenchqmsumDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchrepobenchDataset(BaseDataset):
|
class LongBenchrepobenchDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchsamsumDataset(BaseDataset):
|
class LongBenchsamsumDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchtrecDataset(BaseDataset):
|
class LongBenchtrecDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchtriviaqaDataset(BaseDataset):
|
class LongBenchtriviaqaDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -10,11 +10,12 @@ from ..base import BaseDataset
|
|||||||
class LongBenchvcsumDataset(BaseDataset):
|
class LongBenchvcsumDataset(BaseDataset):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def load(**kwargs):
|
def load(path: str, name: str):
|
||||||
if 'data_files' in kwargs:
|
path = get_data_path(path)
|
||||||
kwargs['data_files'] = get_data_path(kwargs['data_files'],
|
dataset = load_dataset(path=path,
|
||||||
local_mode=True)
|
name=name,
|
||||||
dataset = load_dataset(**kwargs)
|
data_dir=path,
|
||||||
|
trust_remote_code=True)
|
||||||
split = 'test'
|
split = 'test'
|
||||||
raw_data = []
|
raw_data = []
|
||||||
for i in range(len(dataset[split])):
|
for i in range(len(dataset[split])):
|
||||||
|
|||||||
@ -265,6 +265,12 @@ DATASETS_MAPPING = {
|
|||||||
"hf_id": "opencompass/xsum",
|
"hf_id": "opencompass/xsum",
|
||||||
"local": "./data/Xsum/dev.jsonl",
|
"local": "./data/Xsum/dev.jsonl",
|
||||||
},
|
},
|
||||||
|
# Longbench
|
||||||
|
"opencompass/Longbench": {
|
||||||
|
"ms_id": "",
|
||||||
|
"hf_id": "THUDM/LongBench",
|
||||||
|
"local": "./data/Longbench",
|
||||||
|
},
|
||||||
# Needlebench
|
# Needlebench
|
||||||
"opencompass/needlebench": {
|
"opencompass/needlebench": {
|
||||||
"ms_id": "",
|
"ms_id": "",
|
||||||
@ -402,6 +408,10 @@ DATASETS_URL = {
|
|||||||
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/mmlu_pro.zip",
|
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/mmlu_pro.zip",
|
||||||
"md5": "e3200c7380f4cea5f13c768f2815fabb",
|
"md5": "e3200c7380f4cea5f13c768f2815fabb",
|
||||||
},
|
},
|
||||||
|
"/Longbench": {
|
||||||
|
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/Longbench.zip",
|
||||||
|
"md5": "ab0cb9e520ae5cfb899bf38b564249bb",
|
||||||
|
},
|
||||||
"/needlebench": {
|
"/needlebench": {
|
||||||
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/needlebench.zip",
|
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/needlebench.zip",
|
||||||
"md5": "b546da0397746eaff4d3ff0f20d6ede2",
|
"md5": "b546da0397746eaff4d3ff0f20d6ede2",
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user