jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Fix] Fix ruler_16k_gen (#1643)

Browse Source
This commit is contained in:
Chang Lan 2024-10-29 02:58:43 -07:00 committed by GitHub
parent 8172af49bb
commit 46affab882
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 18 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configs/datasets/ruler/ruler_16k_gen.py
View File

@ -21,7 +21,7 @@ ruler_datasets = []
# Different seq length
for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
for dataset in import_datasets:
for dataset in import_ds:
tmp_dataset = dataset.deepcopy()
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES

31
configs/eval_ruler.py
View File

@ -1,29 +1,32 @@
from mmengine.config import read_base
from opencompass.partitioners import (
NaivePartitioner,
NumWorkerPartitioner,
)
from mmengine.config import read_base
from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
with read_base():
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
models as qwen2_7b_instruct_model,
from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets # CWE
from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets # FWE
from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets # Niah
from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets # QA
from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets # VT
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
models as internlm2_5_7b_chat_1m,
)
from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import (
models as llama3_8b_instruct_model,
)
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
models as internlm2_5_7b_chat_1m,
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
models as qwen2_7b_instruct_model,
)
from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets # Niah
from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets # VT
from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets # FWE
from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets # CWE
from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets # QA
from opencompass.configs.summarizers.groups.ruler import ruler_summary_groups
import_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
import_datasets = sum(
[niah_datasets, vt_datasets, fwe_datasets, cwe_datasets, qa_datasets], []
)
# Evaluation config
NUM_SAMPLES = 500
@ -84,9 +87,7 @@ eval = dict(
summarizer = dict(
dataset_abbrs=abbr_suffixs,
summary_groups=sum(
[v for k, v in locals().items() if k.endswith('_summary_groups')], []
),
summary_groups=sum([ruler_summary_groups], []),
)

2
opencompass/configs/datasets/ruler/ruler_16k_gen.py
View File

@ -21,7 +21,7 @@ ruler_datasets = []
# Different seq length
for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
for dataset in import_datasets:
for dataset in import_ds:
tmp_dataset = dataset.deepcopy()
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES