mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
e039f3efa0
* [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * Update mmlu-cf * Update mmlu-cf * Update mmlu-cf * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * [Feature] Support MMLU-CF Benchmark * Remove outside configs --------- Co-authored-by: liushz <qq1791167085@163.com>
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
from mmengine.config import read_base
|
|
|
|
with read_base():
|
|
from opencompass.configs.datasets.mmlu_cf.mmlu_cf_gen_040615 import mmlu_cf_datasets
|
|
|
|
from opencompass.configs.models.qwen2_5.hf_qwen2_5_7b_instruct import models as hf_qwen2_5_7b_instruct_model
|
|
from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import models as lmdeploy_llama3_8b_instruct_model
|
|
|
|
from opencompass.configs.summarizers.mmlu_cf import summarizer
|
|
|
|
|
|
datasets = sum([v for k, v in locals().items() if k.endswith('_datasets') or k == 'datasets'], [])
|
|
models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
|
|
|
|
|
|
from opencompass.runners import LocalRunner
|
|
from opencompass.partitioners import NaivePartitioner, NumWorkerPartitioner
|
|
from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
|
|
|
|
infer = dict(
|
|
partitioner=dict(type=NumWorkerPartitioner, num_worker=8),
|
|
runner=dict(
|
|
type=LocalRunner,
|
|
max_num_workers=8,
|
|
task=dict(type=OpenICLInferTask)
|
|
),
|
|
)
|
|
|
|
eval = dict(
|
|
partitioner=dict(type=NaivePartitioner, n=10),
|
|
runner=dict(
|
|
type=LocalRunner,
|
|
max_num_workers=256,
|
|
task=dict(type=OpenICLEvalTask)
|
|
),
|
|
)
|
|
|
|
work_dir = 'outputs/debug/mmlu_cf'
|