mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
a4b54048ae
* [Feature] Add Ruler datasets * pre-commit fixed * Add model specific tokenizer to dataset * pre-commit modified * remove unused import * fix linting * add trust_remote to tokenizer load * lint fix * comments resolved * fix lint * Add readme * Fix lint * ruler refactorize * fix lint * lint fix * updated * lint fix * fix wonderwords import issue * prompt modified * update * readme updated * update * ruler dataset added * Update --------- Co-authored-by: tonysy <sy.zhangbuaa@gmail.com>
28 lines
720 B
Python
28 lines
720 B
Python
default_ruler_tasks = [
|
|
'ruler_niah_single_1',
|
|
'ruler_niah_single_2',
|
|
'ruler_niah_single_3',
|
|
'ruler_niah_multikey_1',
|
|
'ruler_niah_multikey_2',
|
|
'ruler_niah_multikey_3',
|
|
'ruler_niah_multivalue',
|
|
'ruler_niah_multiquery',
|
|
'ruler_vt',
|
|
'ruler_fwe',
|
|
'ruler_cwe',
|
|
'ruler_qa_squad',
|
|
'ruler_qa_hotpotqa',
|
|
]
|
|
context_window_sizes = ['4k', '8k', '16k', '32k', '128k', '1m']
|
|
|
|
ruler_summary_groups = []
|
|
for context_window_size in context_window_sizes:
|
|
ruler_summary_groups.append(
|
|
{
|
|
'name': f'ruler_{context_window_size}',
|
|
'subsets': [
|
|
f'{task}_{context_window_size}' for task in default_ruler_tasks
|
|
],
|
|
}
|
|
)
|