[Fix] Fix ruler_16k_gen (#1643)

This commit is contained in:
Chang Lan 2024-10-29 02:58:43 -07:00 committed by GitHub
parent 8172af49bb
commit 46affab882
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 18 additions and 17 deletions

View File

@ -21,7 +21,7 @@ ruler_datasets = []
# Different seq length # Different seq length
for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
for dataset in import_datasets: for dataset in import_ds:
tmp_dataset = dataset.deepcopy() tmp_dataset = dataset.deepcopy()
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES tmp_dataset['num_samples'] = NUM_SAMPLES

View File

@ -1,29 +1,32 @@
from mmengine.config import read_base
from opencompass.partitioners import ( from opencompass.partitioners import (
NaivePartitioner, NaivePartitioner,
NumWorkerPartitioner, NumWorkerPartitioner,
) )
from mmengine.config import read_base
from opencompass.runners import LocalRunner from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
with read_base(): with read_base():
from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import ( from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets # CWE
models as qwen2_7b_instruct_model, from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets # FWE
from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets # Niah
from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets # QA
from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets # VT
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import (
models as internlm2_5_7b_chat_1m,
) )
from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import ( from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import (
models as llama3_8b_instruct_model, models as llama3_8b_instruct_model,
) )
from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat_1m import ( from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import (
models as internlm2_5_7b_chat_1m, models as qwen2_7b_instruct_model,
) )
from opencompass.configs.datasets.ruler.ruler_niah_gen import niah_datasets # Niah
from opencompass.configs.datasets.ruler.ruler_vt_gen import vt_datasets # VT
from opencompass.configs.datasets.ruler.ruler_fwe_gen import fwe_datasets # FWE
from opencompass.configs.datasets.ruler.ruler_cwe_gen import cwe_datasets # CWE
from opencompass.configs.datasets.ruler.ruler_qa_gen import qa_datasets # QA
from opencompass.configs.summarizers.groups.ruler import ruler_summary_groups from opencompass.configs.summarizers.groups.ruler import ruler_summary_groups
import_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), []) import_datasets = sum(
[niah_datasets, vt_datasets, fwe_datasets, cwe_datasets, qa_datasets], []
)
# Evaluation config # Evaluation config
NUM_SAMPLES = 500 NUM_SAMPLES = 500
@ -84,9 +87,7 @@ eval = dict(
summarizer = dict( summarizer = dict(
dataset_abbrs=abbr_suffixs, dataset_abbrs=abbr_suffixs,
summary_groups=sum( summary_groups=sum([ruler_summary_groups], []),
[v for k, v in locals().items() if k.endswith('_summary_groups')], []
),
) )

View File

@ -21,7 +21,7 @@ ruler_datasets = []
# Different seq length # Different seq length
for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs): for max_seq_len, abbr_suffix in zip(max_seq_lens, abbr_suffixs):
for dataset in import_datasets: for dataset in import_ds:
tmp_dataset = dataset.deepcopy() tmp_dataset = dataset.deepcopy()
tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix tmp_dataset['abbr'] = tmp_dataset['abbr'] + '_' + abbr_suffix
tmp_dataset['num_samples'] = NUM_SAMPLES tmp_dataset['num_samples'] = NUM_SAMPLES