mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
[Fix] Fix ruler_16k_gen (#1643)
This commit is contained in:
parent
8172af49bb
commit
46affab882
@ -21,7 +21,7 @@ ruler_datasets = []
|
|||||||
|
|
||||||
# Different seq length
|
# Different seq length
|
||||||
for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
|
for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
|
||||||
for dataset in import_datasets:
|
for dataset in import_ds:
|
||||||
tmp_dataset = dataset.deepcopy()
|
tmp_dataset = dataset.deepcopy()
|
||||||
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
|
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
|
||||||
tmp_dataset['num_samples'] = NUM_SAMPLES
|
tmp_dataset['num_samples'] = NUM_SAMPLES
|
||||||
|
@ -1,29 +1,32 @@
|
|||||||
|
from mmengine.config import read_base
|
||||||
|
|
||||||
from opencompass.partitioners import (
|
from opencompass.partitioners import (
|
||||||
NaivePartitioner,
|
NaivePartitioner,
|
||||||
NumWorkerPartitioner,
|
NumWorkerPartitioner,
|
||||||
)
|
)
|
||||||
from mmengine.config import read_base
|
|
||||||
from opencompass.runners import LocalRunner
|
from opencompass.runners import LocalRunner
|
||||||
from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
|
from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
|
||||||
|
|
||||||
with read_base():
|
with read_base():
|
||||||
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
|
from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets # CWE
|
||||||
models as qwen2_7b_instruct_model,
|
from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets # FWE
|
||||||
|
from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets # Niah
|
||||||
|
from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets # QA
|
||||||
|
from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets # VT
|
||||||
|
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
|
||||||
|
models as internlm2_5_7b_chat_1m,
|
||||||
)
|
)
|
||||||
from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import (
|
from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import (
|
||||||
models as llama3_8b_instruct_model,
|
models as llama3_8b_instruct_model,
|
||||||
)
|
)
|
||||||
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
|
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
|
||||||
models as internlm2_5_7b_chat_1m,
|
models as qwen2_7b_instruct_model,
|
||||||
)
|
)
|
||||||
from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets # Niah
|
|
||||||
from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets # VT
|
|
||||||
from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets # FWE
|
|
||||||
from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets # CWE
|
|
||||||
from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets # QA
|
|
||||||
from opencompass.configs.summarizers.groups.ruler import ruler_summary_groups
|
from opencompass.configs.summarizers.groups.ruler import ruler_summary_groups
|
||||||
|
|
||||||
import_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
|
import_datasets = sum(
|
||||||
|
[niah_datasets, vt_datasets, fwe_datasets, cwe_datasets, qa_datasets], []
|
||||||
|
)
|
||||||
|
|
||||||
# Evaluation config
|
# Evaluation config
|
||||||
NUM_SAMPLES = 500
|
NUM_SAMPLES = 500
|
||||||
@ -84,9 +87,7 @@ eval = dict(
|
|||||||
|
|
||||||
summarizer = dict(
|
summarizer = dict(
|
||||||
dataset_abbrs=abbr_suffixs,
|
dataset_abbrs=abbr_suffixs,
|
||||||
summary_groups=sum(
|
summary_groups=sum([ruler_summary_groups], []),
|
||||||
[v for k, v in locals().items() if k.endswith('_summary_groups')], []
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -21,7 +21,7 @@ ruler_datasets = []
|
|||||||
|
|
||||||
# Different seq length
|
# Different seq length
|
||||||
for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
|
for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
|
||||||
for dataset in import_datasets:
|
for dataset in import_ds:
|
||||||
tmp_dataset = dataset.deepcopy()
|
tmp_dataset = dataset.deepcopy()
|
||||||
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
|
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
|
||||||
tmp_dataset['num_samples'] = NUM_SAMPLES
|
tmp_dataset['num_samples'] = NUM_SAMPLES
|
||||||
|
Loading…
Reference in New Issue
Block a user