mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
a4b54048ae
* [Feature] Add Ruler datasets * pre-commit fixed * Add model specific tokenizer to dataset * pre-commit modified * remove unused import * fix linting * add trust_remote to tokenizer load * lint fix * comments resolved * fix lint * Add readme * Fix lint * ruler refactorize * fix lint * lint fix * updated * lint fix * fix wonderwords import issue * prompt modified * update * readme updated * update * ruler dataset added * Update --------- Co-authored-by: tonysy <sy.zhangbuaa@gmail.com>
35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
from opencompass.datasets.ruler.ruler_cwe import RulerCweDataset
|
|
from opencompass.datasets.ruler.ruler_cwe import RulerCweEvaluator
|
|
from opencompass.openicl.icl_prompt_template import PromptTemplate
|
|
from opencompass.openicl.icl_retriever import ZeroRetriever
|
|
from opencompass.openicl.icl_inferencer import GenInferencer
|
|
|
|
# CWE Dataset
|
|
cwe_datasets = [
|
|
{
|
|
'abbr': 'ruler_cwe',
|
|
'type': RulerCweDataset,
|
|
'freq_cw': 30,
|
|
'freq_ucw': 3,
|
|
'num_cw': 10,
|
|
'tokens_to_generate': 120,
|
|
'reader_cfg': dict(input_columns=['prompt'], output_column='answer'),
|
|
'infer_cfg': dict(
|
|
prompt_template=dict(
|
|
type=PromptTemplate,
|
|
template=dict(
|
|
round=[
|
|
dict(role='HUMAN', prompt='{prompt}'),
|
|
dict(role='BOT', prompt='{answer}\n'),
|
|
]
|
|
),
|
|
),
|
|
retriever=dict(type=ZeroRetriever),
|
|
inferencer=dict(type=GenInferencer),
|
|
),
|
|
'eval_cfg': dict(
|
|
evaluator=dict(type=RulerCweEvaluator),
|
|
),
|
|
}
|
|
]
|