mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
[Doc] quick start swap tabs (#1263)
* [doc] quick start swap tabs * update docs * update * update * update * update * update * update * update
This commit is contained in:
Fengzhe Zhou
GitHub
parent
68ca48496b
commit
1d3a26c732
8
configs/datasets/demo/demo_cmmlu_base_ppl.py
Normal file
8
configs/datasets/demo/demo_cmmlu_base_ppl.py
Normal file
@ -0,0 +1,8 @@
|
||||
from mmengine import read_base
|
||||
|
||||
with read_base():
|
||||
from ..cmmlu.cmmlu_ppl_041cbf import cmmlu_datasets
|
||||
|
||||
for d in cmmlu_datasets:
|
||||
d['abbr'] = 'demo_' + d['abbr']
|
||||
d['reader_cfg']['test_range'] = '[0:4]'
|
||||
8
configs/datasets/demo/demo_cmmlu_chat_gen.py
Normal file
8
configs/datasets/demo/demo_cmmlu_chat_gen.py
Normal file
@ -0,0 +1,8 @@
|
||||
from mmengine import read_base
|
||||
|
||||
with read_base():
|
||||
from ..cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
|
||||
|
||||
for d in cmmlu_datasets:
|
||||
d['abbr'] = 'demo_' + d['abbr']
|
||||
d['reader_cfg']['test_range'] = '[0:4]'
|
||||
7
configs/datasets/demo/demo_gsm8k_base_gen.py
Normal file
7
configs/datasets/demo/demo_gsm8k_base_gen.py
Normal file
@ -0,0 +1,7 @@
|
||||
from mmengine import read_base
|
||||
|
||||
with read_base():
|
||||
from ..gsm8k.gsm8k_gen_17d0dc import gsm8k_datasets
|
||||
|
||||
gsm8k_datasets[0]['abbr'] = 'demo_' + gsm8k_datasets[0]['abbr']
|
||||
gsm8k_datasets[0]['reader_cfg']['test_range'] = '[0:64]'
|
||||
7
configs/datasets/demo/demo_gsm8k_chat_gen.py
Normal file
7
configs/datasets/demo/demo_gsm8k_chat_gen.py
Normal file
@ -0,0 +1,7 @@
|
||||
from mmengine import read_base
|
||||
|
||||
with read_base():
|
||||
from ..gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
|
||||
|
||||
gsm8k_datasets[0]['abbr'] = 'demo_' + gsm8k_datasets[0]['abbr']
|
||||
gsm8k_datasets[0]['reader_cfg']['test_range'] = '[0:64]'
|
||||
7
configs/datasets/demo/demo_math_base_gen.py
Normal file
7
configs/datasets/demo/demo_math_base_gen.py
Normal file
@ -0,0 +1,7 @@
|
||||
from mmengine import read_base
|
||||
|
||||
with read_base():
|
||||
from ..math.math_4shot_base_gen_db136b import math_datasets
|
||||
|
||||
math_datasets[0]['abbr'] = 'demo_' + math_datasets[0]['abbr']
|
||||
math_datasets[0]['reader_cfg']['test_range'] = '[0:64]'
|
||||
7
configs/datasets/demo/demo_math_chat_gen.py
Normal file
7
configs/datasets/demo/demo_math_chat_gen.py
Normal file
@ -0,0 +1,7 @@
|
||||
from mmengine import read_base
|
||||
|
||||
with read_base():
|
||||
from ..math.math_0shot_gen_393424 import math_datasets
|
||||
|
||||
math_datasets[0]['abbr'] = 'demo_' + math_datasets[0]['abbr']
|
||||
math_datasets[0]['reader_cfg']['test_range'] = '[0:64]'
|
||||
10
configs/eval_base_demo.py
Normal file
10
configs/eval_base_demo.py
Normal file
@ -0,0 +1,10 @@
|
||||
from mmengine.config import read_base
|
||||
|
||||
with read_base():
|
||||
from .datasets.demo.demo_gsm8k_base_gen import gsm8k_datasets
|
||||
from .datasets.demo.demo_math_base_gen import math_datasets
|
||||
from .models.qwen.hf_qwen2_1_5b import models as hf_qwen2_1_5b_models
|
||||
from .models.hf_internlm.hf_internlm2_1_8b import models as hf_internlm2_1_8b_models
|
||||
|
||||
datasets = gsm8k_datasets + math_datasets
|
||||
models = hf_qwen2_1_5b_models + hf_internlm2_1_8b_models
|
||||
10
configs/eval_chat_demo.py
Normal file
10
configs/eval_chat_demo.py
Normal file
@ -0,0 +1,10 @@
|
||||
from mmengine.config import read_base
|
||||
|
||||
with read_base():
|
||||
from .datasets.demo.demo_gsm8k_chat_gen import gsm8k_datasets
|
||||
from .datasets.demo.demo_math_chat_gen import math_datasets
|
||||
from .models.qwen.hf_qwen2_1_5b_instruct import models as hf_qwen2_1_5b_instruct_models
|
||||
from .models.hf_internlm.hf_internlm2_chat_1_8b import models as hf_internlm2_chat_1_8b_models
|
||||
|
||||
datasets = gsm8k_datasets + math_datasets
|
||||
models = hf_qwen2_1_5b_instruct_models + hf_internlm2_chat_1_8b_models
|
||||
@ -1,10 +0,0 @@
|
||||
from mmengine.config import read_base
|
||||
|
||||
with read_base():
|
||||
from .datasets.siqa.siqa_gen import siqa_datasets
|
||||
from .datasets.winograd.winograd_ppl import winograd_datasets
|
||||
from .models.opt.hf_opt_125m import opt125m
|
||||
from .models.opt.hf_opt_350m import opt350m
|
||||
|
||||
datasets = [*siqa_datasets, *winograd_datasets]
|
||||
models = [opt125m, opt350m]
|
||||
@ -8,6 +8,5 @@ models = [
|
||||
max_out_len=1024,
|
||||
batch_size=8,
|
||||
run_cfg=dict(num_gpus=1),
|
||||
stop_words=['</s>', '<|im_end|>'],
|
||||
)
|
||||
]
|
||||
|
||||
@ -8,6 +8,5 @@ models = [
|
||||
max_out_len=1024,
|
||||
batch_size=8,
|
||||
run_cfg=dict(num_gpus=1),
|
||||
stop_words=['</s>', '<|im_end|>'],
|
||||
)
|
||||
]
|
||||
|
||||
@ -8,6 +8,5 @@ models = [
|
||||
max_out_len=1024,
|
||||
batch_size=8,
|
||||
run_cfg=dict(num_gpus=1),
|
||||
stop_words=['</s>', '<|im_end|>'],
|
||||
)
|
||||
]
|
||||
|
||||
@ -8,6 +8,5 @@ models = [
|
||||
max_out_len=1024,
|
||||
batch_size=8,
|
||||
run_cfg=dict(num_gpus=2),
|
||||
stop_words=['</s>', '<|im_end|>'],
|
||||
)
|
||||
]
|
||||
|
||||
@ -8,6 +8,5 @@ models = [
|
||||
max_out_len=1024,
|
||||
batch_size=8,
|
||||
run_cfg=dict(num_gpus=2),
|
||||
stop_words=['</s>', '<|im_end|>'],
|
||||
)
|
||||
]
|
||||
|
||||
@ -8,6 +8,5 @@ models = [
|
||||
max_out_len=1024,
|
||||
batch_size=8,
|
||||
run_cfg=dict(num_gpus=1),
|
||||
stop_words=['</s>', '<|im_end|>'],
|
||||
)
|
||||
]
|
||||
|
||||
@ -8,6 +8,5 @@ models = [
|
||||
max_out_len=1024,
|
||||
batch_size=8,
|
||||
run_cfg=dict(num_gpus=1),
|
||||
stop_words=['</s>', '<|im_end|>'],
|
||||
)
|
||||
]
|
||||
|
||||
@ -3,9 +3,9 @@ from opencompass.models import HuggingFaceCausalLM
|
||||
|
||||
_meta_template = dict(
|
||||
round=[
|
||||
dict(role='HUMAN', begin='[UNUSED_TOKEN_146]user\n', end='[UNUSED_TOKEN_145]\n'),
|
||||
dict(role='SYSTEM', begin='[UNUSED_TOKEN_146]system\n', end='[UNUSED_TOKEN_145]\n'),
|
||||
dict(role='BOT', begin='[UNUSED_TOKEN_146]assistant\n', end='[UNUSED_TOKEN_145]\n', generate=True),
|
||||
dict(role='HUMAN', begin='<|im_start|>user\n', end='<|im_end|>\n'),
|
||||
dict(role='SYSTEM', begin='<|im_start|>system\n', end='<|im_end|>\n'),
|
||||
dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
|
||||
],
|
||||
)
|
||||
|
||||
@ -30,6 +30,6 @@ models = [
|
||||
batch_size=8,
|
||||
meta_template=_meta_template,
|
||||
run_cfg=dict(num_gpus=2, num_procs=1),
|
||||
end_str='[UNUSED_TOKEN_145]',
|
||||
end_str='<|im_end|>',
|
||||
)
|
||||
]
|
||||
|
||||
@ -3,9 +3,9 @@ from opencompass.models import HuggingFaceCausalLM
|
||||
|
||||
_meta_template = dict(
|
||||
round=[
|
||||
dict(role='HUMAN', begin='[UNUSED_TOKEN_146]user\n', end='[UNUSED_TOKEN_145]\n'),
|
||||
dict(role='SYSTEM', begin='[UNUSED_TOKEN_146]system\n', end='[UNUSED_TOKEN_145]\n'),
|
||||
dict(role='BOT', begin='[UNUSED_TOKEN_146]assistant\n', end='[UNUSED_TOKEN_145]\n', generate=True),
|
||||
dict(role='HUMAN', begin='<|im_start|>user\n', end='<|im_end|>\n'),
|
||||
dict(role='SYSTEM', begin='<|im_start|>system\n', end='<|im_end|>\n'),
|
||||
dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
|
||||
],
|
||||
)
|
||||
|
||||
@ -30,6 +30,6 @@ models = [
|
||||
batch_size=8,
|
||||
meta_template=_meta_template,
|
||||
run_cfg=dict(num_gpus=1, num_procs=1),
|
||||
end_str='[UNUSED_TOKEN_145]',
|
||||
end_str='<|im_end|>',
|
||||
)
|
||||
]
|
||||
|
||||
@ -22,7 +22,43 @@ For larger parameterized models like Llama-7B, refer to other examples provided
|
||||
In OpenCompass, each evaluation task consists of the model to be evaluated and the dataset. The entry point for evaluation is `run.py`. Users can select the model and dataset to be tested either via command line or configuration files.
|
||||
|
||||
`````{tabs}
|
||||
````{tab} Command Line (Custom HF Model)
|
||||
|
||||
For HuggingFace models, users can set model parameters directly through the command line without additional configuration files. For instance, for the `facebook/opt-125m` model, you can evaluate it with the following command:
|
||||
|
||||
```bash
|
||||
python run.py --datasets siqa_gen winograd_ppl \
|
||||
--hf-type base \
|
||||
--hf-path facebook/opt-125m
|
||||
```
|
||||
|
||||
Note that in this way, OpenCompass only evaluates one model at a time, while other ways can evaluate multiple models at once.
|
||||
|
||||
```{caution}
|
||||
`--hf-num-gpus` does not stand for the actual number of GPUs to use in evaluation, but the minimum required number of GPUs for this model. [More](faq.md#how-does-opencompass-allocate-gpus)
|
||||
```
|
||||
|
||||
:::{dropdown} More detailed example
|
||||
:animate: fade-in-slide-down
|
||||
```bash
|
||||
python run.py --datasets siqa_gen winograd_ppl \
|
||||
--hf-type base \ # HuggingFace model type, base or chat
|
||||
--hf-path facebook/opt-125m \ # HuggingFace model path
|
||||
--tokenizer-path facebook/opt-125m \ # HuggingFace tokenizer path (if the same as the model path, can be omitted)
|
||||
--tokenizer-kwargs padding_side='left' truncation='left' trust_remote_code=True \ # Arguments to construct the tokenizer
|
||||
--model-kwargs device_map='auto' \ # Arguments to construct the model
|
||||
--max-seq-len 2048 \ # Maximum sequence length the model can accept
|
||||
--max-out-len 100 \ # Maximum number of tokens to generate
|
||||
--min-out-len 100 \ # Minimum number of tokens to generate
|
||||
--batch-size 64 \ # Batch size
|
||||
--hf-num-gpus 1 # Number of GPUs required to run the model
|
||||
```
|
||||
```{seealso}
|
||||
For all HuggingFace related parameters supported by `run.py`, please read [Launching Evaluation Task](../user_guides/experimentation.md#launching-an-evaluation-task).
|
||||
```
|
||||
:::
|
||||
|
||||
````
|
||||
````{tab} Command Line
|
||||
|
||||
Users can combine the models and datasets they want to test using `--models` and `--datasets`.
|
||||
@ -74,44 +110,6 @@ If you want to evaluate other models, please check out the "Command Line (Custom
|
||||
|
||||
````
|
||||
|
||||
````{tab} Command Line (Custom HF Model)
|
||||
|
||||
For HuggingFace models, users can set model parameters directly through the command line without additional configuration files. For instance, for the `facebook/opt-125m` model, you can evaluate it with the following command:
|
||||
|
||||
```bash
|
||||
python run.py --datasets siqa_gen winograd_ppl \
|
||||
--hf-type base \
|
||||
--hf-path facebook/opt-125m
|
||||
```
|
||||
|
||||
Note that in this way, OpenCompass only evaluates one model at a time, while other ways can evaluate multiple models at once.
|
||||
|
||||
```{caution}
|
||||
`--hf-num-gpus` does not stand for the actual number of GPUs to use in evaluation, but the minimum required number of GPUs for this model. [More](faq.md#how-does-opencompass-allocate-gpus)
|
||||
```
|
||||
|
||||
:::{dropdown} More detailed example
|
||||
:animate: fade-in-slide-down
|
||||
```bash
|
||||
python run.py --datasets siqa_gen winograd_ppl \
|
||||
--hf-type base \ # HuggingFace model type, base or chat
|
||||
--hf-path facebook/opt-125m \ # HuggingFace model path
|
||||
--tokenizer-path facebook/opt-125m \ # HuggingFace tokenizer path (if the same as the model path, can be omitted)
|
||||
--tokenizer-kwargs padding_side='left' truncation='left' trust_remote_code=True \ # Arguments to construct the tokenizer
|
||||
--model-kwargs device_map='auto' \ # Arguments to construct the model
|
||||
--max-seq-len 2048 \ # Maximum sequence length the model can accept
|
||||
--max-out-len 100 \ # Maximum number of tokens to generate
|
||||
--min-out-len 100 \ # Minimum number of tokens to generate
|
||||
--batch-size 64 \ # Batch size
|
||||
--hf-num-gpus 1 # Number of GPUs required to run the model
|
||||
```
|
||||
```{seealso}
|
||||
For all HuggingFace related parameters supported by `run.py`, please read [Launching Evaluation Task](../user_guides/experimentation.md#launching-an-evaluation-task).
|
||||
```
|
||||
:::
|
||||
|
||||
|
||||
````
|
||||
````{tab} Configuration File
|
||||
|
||||
In addition to configuring the experiment through the command line, OpenCompass also allows users to write the full configuration of the experiment in a configuration file and run it directly through `run.py`. The configuration file is organized in Python format and must include the `datasets` and `models` fields.
|
||||
|
||||
174
docs/zh_cn/get_started/extra-installation.md
Normal file
174
docs/zh_cn/get_started/extra-installation.md
Normal file
@ -0,0 +1,174 @@
|
||||
# 其他安装说明
|
||||
|
||||
欢迎使用浏览器自带的搜索功能。
|
||||
|
||||
## 推理后端
|
||||
|
||||
- LMDeploy
|
||||
|
||||
```bash
|
||||
pip install lmdeploy
|
||||
```
|
||||
|
||||
- VLLM
|
||||
|
||||
```bash
|
||||
pip install vllm
|
||||
```
|
||||
|
||||
OpenCompass 开发者所使用的 CUDA 版本为 11.8，一个能够应对 2024.07 之前绝大部分模型的依赖版本如下：
|
||||
|
||||
```bash
|
||||
export VLLM_VERSION=0.4.3
|
||||
export LMDEPLOY_VERSION=0.4.1
|
||||
export FLASH_ATTN_VERSION=2.5.7
|
||||
export XFORMERS_VERSION=0.0.25.post1
|
||||
export TORCH_VERSION=2.2.2
|
||||
export TORCHVISION_VERSION=0.17.2
|
||||
export TORCHAUDIO_VERSION=2.2.2
|
||||
export TRITON_VERSION=2.1.0
|
||||
export PYTHON_VERSION=310
|
||||
|
||||
|
||||
pip3 install "https://github.com/InternLM/lmdeploy/releases/download/v${LMDEPLOY_VERSION}/lmdeploy-${LMDEPLOY_VERSION}+cu118-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}-manylinux2014_x86_64.whl" --extra-index-url https://download.pytorch.org/whl/cu118
|
||||
pip3 install "https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cu118-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}-manylinux1_x86_64.whl" --extra-index-url https://download.pytorch.org/whl/cu118
|
||||
pip3 install "https://github.com/Dao-AILab/flash-attention/releases/download/v${FLASH_ATTN_VERSION}/flash_attn-${FLASH_ATTN_VERSION}+cu118torch2.2cxx11abiFALSE-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}-linux_x86_64.whl" --extra-index-url https://download.pytorch.org/whl/cu118
|
||||
pip3 install xformers==${XFORMERS_VERSION} --extra-index-url https://download.pytorch.org/whl/cu118
|
||||
pip3 install torch==${TORCH_VERSION} torchvision==${TORCHVISION_VERSION} torchaudio==${TORCHAUDIO_VERSION} --index-url https://download.pytorch.org/whl/cu118
|
||||
pip3 install triton==${TRITON_VERSION} --extra-index-url https://download.pytorch.org/whl/cu118
|
||||
```
|
||||
|
||||
请注意，在安装过程中，后一条 `pip install` 命令可能会覆盖前一条命令中部分依赖的版本。并且在最终安装完成后，可能有的软件依赖会不满足，但是对 lmdeploy / vllm / xformers 等有需求的模型都是可以跑起来的。很神秘。
|
||||
|
||||
## 模型
|
||||
|
||||
- LLAMA (参数，原生, 非 HF 格式)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/facebookresearch/llama.git
|
||||
cd llama
|
||||
pip install -r requirements.txt
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
- Vicuna (参数)
|
||||
|
||||
```bash
|
||||
pip install "fschat[model_worker,webui]
|
||||
```
|
||||
|
||||
- Baichuan / Baichuan2 (参数)
|
||||
|
||||
```bash
|
||||
pip install "transformers<=4.33.3"
|
||||
```
|
||||
|
||||
- ChatGLM-3 / GLM-4 (参数)
|
||||
|
||||
```bash
|
||||
pip install "transformers<=4.41.2"
|
||||
```
|
||||
|
||||
- GPT-3.5-Turbo / GPT-4-Turbo / GPT-4 / GPT-4o (API)
|
||||
|
||||
```bash
|
||||
pip install openai
|
||||
```
|
||||
|
||||
- Claude (API)
|
||||
|
||||
```bash
|
||||
pip install anthropic
|
||||
```
|
||||
|
||||
- 字节豆包 (API)
|
||||
|
||||
```bash
|
||||
pip install volcengine-python-sdk
|
||||
```
|
||||
|
||||
- 腾讯混元 (API)
|
||||
|
||||
```bash
|
||||
pip install tencentcloud-sdk-python
|
||||
```
|
||||
|
||||
- 讯飞星火 (API)
|
||||
|
||||
```bash
|
||||
pip install spark_ai_python "sseclient-py==1.7.2" websocket-client
|
||||
```
|
||||
|
||||
- 智谱 (API)
|
||||
|
||||
```bash
|
||||
pip install zhipuai
|
||||
```
|
||||
|
||||
- 通义千问 (API)
|
||||
|
||||
```bash
|
||||
pip install dashscope
|
||||
```
|
||||
|
||||
## 数据集
|
||||
|
||||
- HumanEval
|
||||
|
||||
```bash
|
||||
git clone git@github.com:open-compass/human-eval.git
|
||||
cd human-eval && pip install -e .
|
||||
```
|
||||
|
||||
该代码库 fork 自 https://github.com/openai/human-eval.git，并且已经注释了 `human_eval/execution.py` **第48-57行** 的提示。该提示告知了直接运行 LLM 生成的代码会有风险。
|
||||
|
||||
- HumanEvalX / HumanEval+ / MBPP+
|
||||
|
||||
```bash
|
||||
git clone --recurse-submodules git@github.com:open-compass/human-eval.git
|
||||
cd human-eval
|
||||
pip install -e .
|
||||
pip install -e evalplus
|
||||
```
|
||||
|
||||
- AlpacaEval
|
||||
|
||||
```bash
|
||||
pip install alpaca-eval==0.6 scikit-learn==1.5
|
||||
```
|
||||
|
||||
- CIBench
|
||||
|
||||
```bash
|
||||
pip install -r requirements/agent.txt
|
||||
```
|
||||
|
||||
- T-Eval
|
||||
|
||||
```bash
|
||||
pip install lagent==0.1.2
|
||||
```
|
||||
|
||||
- APPS / TACO
|
||||
|
||||
```bash
|
||||
pip install pyext
|
||||
```
|
||||
|
||||
- IFEval
|
||||
|
||||
```bash
|
||||
pip install langdetect
|
||||
```
|
||||
|
||||
- NPHardEval
|
||||
|
||||
```bash
|
||||
pip install networkx
|
||||
```
|
||||
|
||||
- LawBench
|
||||
|
||||
```bash
|
||||
pip install cn2an
|
||||
```
|
||||
@ -35,58 +35,9 @@
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
3. 安装 humaneval（可选）：
|
||||
3. 如果需要使用推理后端，或者进行 API 模型测试，或者进行 代码、智能体、主观 等数据集的评测，请参考 [其他安装说明](./extra-installation.md)
|
||||
|
||||
如果你需要**在 humaneval 数据集上评估模型代码能力**，请执行此步骤，否则忽略这一步。
|
||||
|
||||
<details>
|
||||
<summary><b>点击查看详细</b></summary>
|
||||
|
||||
```bash
|
||||
git clone https://github.com/openai/human-eval.git
|
||||
cd human-eval
|
||||
pip install -r requirements.txt
|
||||
pip install -e .
|
||||
cd ..
|
||||
```
|
||||
|
||||
请仔细阅读 `human_eval/execution.py` **第48-57行**的注释，了解执行模型生成的代码可能存在的风险，如果接受这些风险，请取消**第58行**的注释，启用代码执行评测。
|
||||
|
||||
</details>
|
||||
|
||||
4. 安装 Llama（可选）：
|
||||
|
||||
如果你需要**使用官方实现评测 Llama / Llama-2 / Llama-2-chat 模型**，请执行此步骤，否则忽略这一步。
|
||||
|
||||
<details>
|
||||
<summary><b>点击查看详细</b></summary>
|
||||
|
||||
```bash
|
||||
git clone https://github.com/facebookresearch/llama.git
|
||||
cd llama
|
||||
pip install -r requirements.txt
|
||||
pip install -e .
|
||||
cd ..
|
||||
```
|
||||
|
||||
你可以在 `configs/models` 下找到所有 Llama / Llama-2 / Llama-2-chat 模型的配置文件示例。([示例](https://github.com/open-compass/opencompass/blob/eb4822a94d624a4e16db03adeb7a59bbd10c2012/configs/models/llama2_7b_chat.py))
|
||||
|
||||
</details>
|
||||
|
||||
5. 安装 alpaca-eval（可选）：
|
||||
|
||||
如果你需要**使用官方alpaca-eval实现评测 alpaca-eval 数据集**，请执行此步骤，否则忽略这一步。
|
||||
|
||||
<details>
|
||||
<summary><b>点击查看详细</b></summary>
|
||||
|
||||
```bash
|
||||
pip install alpaca-eval
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
# 数据集准备
|
||||
## 数据集准备
|
||||
|
||||
OpenCompass 支持的数据集主要包括两个部分：
|
||||
|
||||
|
||||
@ -12,22 +12,82 @@
|
||||
|
||||
**可视化**：评估完成后，OpenCompass 将结果整理成易读的表格，并将其保存为 CSV 和 TXT 文件。你也可以激活飞书状态上报功能，此后可以在飞书客户端中及时获得评测状态报告。
|
||||
|
||||
接下来，我们将展示 OpenCompass 的基础用法，展示预训练模型 [OPT-125M](https://huggingface.co/facebook/opt-125m) 和 [OPT-350M](https://huggingface.co/facebook/opt-350m) 在 [SIQA](https://huggingface.co/datasets/social_i_qa) 和 [Winograd](https://huggingface.co/datasets/winograd_wsc) 基准任务上的评估。它们的配置文件可以在 [configs/eval_demo.py](https://github.com/open-compass/opencompass/blob/main/configs/eval_demo.py) 中找到。
|
||||
接下来，我们将展示 OpenCompass 的基础用法，展示基座模型模型 [InternLM2-1.8B](https://huggingface.co/internlm/internlm2-1_8b) 和对话模型 [InternLM2-Chat-1.8B](https://huggingface.co/internlm/internlm2-chat-1_8b)、[Qwen2-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2-1.5B-Instruct) 在 [GSM8K](https://github.com/openai/grade-school-math) 和 [MATH](https://github.com/hendrycks/math) 下采样数据集上的评估。它们的配置文件可以在 [configs/eval_chat_demo.py](https://github.com/open-compass/opencompass/blob/main/configs/eval_chat_demo.py) 和 [configs/eval_base_demo.py](https://github.com/open-compass/opencompass/blob/main/configs/eval_base_demo.py) 中找到。
|
||||
|
||||
在运行此实验之前，请确保您已在本地安装了 OpenCompass。这个例子可以在一台 _GTX-1660-6G_ GPU 下成功运行。
|
||||
对于参数更大的模型，如 Llama-7B，请参考 [configs 目录](https://github.com/open-compass/opencompass/tree/main/configs) 中提供的其他示例。
|
||||
在运行此实验之前，请确保您已在本地安装了 OpenCompass。这个例子 (应该) 可以在一台 _GTX-1660-6G_ GPU 下成功运行。
|
||||
|
||||
对于参数更大的模型，如 Llama3-8B，请参考 [configs 目录](https://github.com/open-compass/opencompass/tree/main/configs) 中提供的其他示例。
|
||||
|
||||
## 配置评估任务
|
||||
|
||||
在 OpenCompass 中，每个评估任务由待评估的模型和数据集组成。评估的入口点是 `run.py`。用户可以通过命令行或配置文件选择要测试的模型和数据集。
|
||||
|
||||
对于对话模型
|
||||
|
||||
`````{tabs}
|
||||
````{tab} 命令行（自定义 HF 模型）
|
||||
|
||||
对于 HuggingFace 模型，用户可以通过命令行直接设置模型参数，无需额外的配置文件。例如，对于 `internlm/internlm2-chat-1_8b` 模型，您可以使用以下命令进行评估：
|
||||
|
||||
```bash
|
||||
python run.py \
|
||||
--datasets demo_gsm8k_chat_gen demo_math_chat_gen \
|
||||
--hf-type chat \
|
||||
--hf-path internlm/internlm2-chat-1_8b \
|
||||
--debug
|
||||
```
|
||||
|
||||
请注意，通过这种方式，OpenCompass 一次只评估一个模型，而其他方式可以一次评估多个模型。
|
||||
|
||||
:::{dropdown} HF 模型完整参数列表
|
||||
:animate: fade-in-slide-down
|
||||
| 命令行参数 | 描述 | 样例数值 |
|
||||
| --- | --- | --- |
|
||||
| `--hf-type` | HuggingFace 模型类型，可选值为 `chat` 或 `base` | chat |
|
||||
| `--hf-path` | HuggingFace 模型路径 | internlm/internlm2-chat-1_8b |
|
||||
| `--model-kwargs` | 构建模型的参数 | device_map='auto' |
|
||||
| `--tokenizer-path` | HuggingFace tokenizer 路径（如果与模型路径相同，可以省略） | internlm/internlm2-chat-1_8b |
|
||||
| `--tokenizer-kwargs` | 构建 tokenizer 的参数 | padding_side='left' truncation='left' trust_remote_code=True |
|
||||
| `--generation-kwargs` | 生成的参数 | do_sample=True top_k=50 top_p=0.95 |
|
||||
| `--max-seq-len` | 模型可以接受的最大序列长度 | 2048 |
|
||||
| `--max-out-len` | 生成的最大 token 数 | 100 |
|
||||
| `--min-out-len` | 生成的最小 token 数 | 1 |
|
||||
| `--batch-size` | 批量大小 | 64 |
|
||||
| `--hf-num-gpus` | 运行一个模型实例所需的 GPU 数量 | 1 |
|
||||
| `--stop-words` | 停用词列表 | '<\|im_end\|>' '<\|im_start\|>' |
|
||||
| `--pad-token-id` | 填充 token 的 ID | 0 |
|
||||
| `--peft-path` | (例如) LoRA 模型的路径 | internlm/internlm2-chat-1_8b |
|
||||
| `--peft-kwargs` | (例如) 构建 LoRA 模型的参数 | trust_remote_code=True |
|
||||
:::
|
||||
|
||||
:::{dropdown} 更复杂的命令样例
|
||||
:animate: fade-in-slide-down
|
||||
|
||||
例如一个占用 2 卡进行测试的 Qwen1.5-14B-Chat, 开启数据采样，模型的命令如下：
|
||||
|
||||
```bash
|
||||
python run.py --datasets demo_gsm8k_chat_gen demo_math_chat_gen \
|
||||
--hf-type chat \
|
||||
--hf-path Qwen/Qwen1.5-14B-Chat \
|
||||
--max-out-len 1024 \
|
||||
--min-out-len 1 \
|
||||
--hf-num-gpus 2 \
|
||||
--generation-kwargs do_sample=True temperature=0.6 \
|
||||
--stop-words '<|im_end|>' '<|im_start|>' \
|
||||
--debug
|
||||
```
|
||||
:::
|
||||
|
||||
````
|
||||
````{tab} 命令行
|
||||
|
||||
用户可以使用 `--models` 和 `--datasets` 结合想测试的模型和数据集。
|
||||
|
||||
```bash
|
||||
python run.py --models hf_opt_125m hf_opt_350m --datasets siqa_gen winograd_ppl
|
||||
python run.py \
|
||||
--models hf_internlm2_chat_1_8b hf_qwen2_1_5b_instruct \
|
||||
--datasets demo_gsm8k_chat_gen demo_math_chat_gen \
|
||||
--debug
|
||||
```
|
||||
|
||||
模型和数据集的配置文件预存于 `configs/models` 和 `configs/datasets` 中。用户可以使用 `tools/list_configs.py` 查看或过滤当前可用的模型和数据集配置。
|
||||
@ -71,93 +131,49 @@ python tools/list_configs.py llama mmlu
|
||||
|
||||
:::
|
||||
|
||||
````
|
||||
|
||||
````{tab} 命令行（自定义 HF 模型）
|
||||
|
||||
对于 HuggingFace 模型，用户可以通过命令行直接设置模型参数，无需额外的配置文件。例如，对于 `facebook/opt-125m` 模型，您可以使用以下命令进行评估：
|
||||
|
||||
```bash
|
||||
python run.py --datasets siqa_gen winograd_ppl \
|
||||
--hf-type base \
|
||||
--hf-path facebook/opt-125m
|
||||
```
|
||||
|
||||
请注意，通过这种方式，OpenCompass 一次只评估一个模型，而其他方式可以一次评估多个模型。
|
||||
|
||||
```{caution}
|
||||
`--hf-num-gpus` 不代表实际用于评估的 GPU 数量，而是该模型所需的最少 GPU 数量。[更多](faq.md#opencompass-如何分配-gpu)
|
||||
```
|
||||
|
||||
|
||||
|
||||
:::{dropdown} 更详细的示例
|
||||
:animate: fade-in-slide-down
|
||||
```bash
|
||||
python run.py --datasets siqa_gen winograd_ppl \
|
||||
--hf-type base \ # HuggingFace 模型类型, base 或 chat
|
||||
--hf-path facebook/opt-125m \ # HuggingFace 模型路径
|
||||
--tokenizer-path facebook/opt-125m \ # HuggingFace tokenizer 路径（如果与模型路径相同，可以省略）
|
||||
--tokenizer-kwargs padding_side='left' truncation='left' trust_remote_code=True \ # 构建 tokenizer 的参数
|
||||
--model-kwargs device_map='auto' \ # 构建模型的参数
|
||||
--max-seq-len 2048 \ # 模型可以接受的最大序列长度
|
||||
--max-out-len 100 \ # 生成的最大 token 数
|
||||
--min-out-len 100 \ # 生成的最小 token 数
|
||||
--batch-size 64 \ # 批量大小
|
||||
--hf-num-gpus 1 # 运行模型所需的 GPU 数量
|
||||
```
|
||||
```{seealso}
|
||||
有关 `run.py` 支持的所有与 HuggingFace 相关的参数，请阅读 [评测任务发起](../user_guides/experimentation.md#评测任务发起)
|
||||
```
|
||||
:::
|
||||
|
||||
|
||||
````
|
||||
````{tab} 配置文件
|
||||
|
||||
除了通过命令行配置实验外，OpenCompass 还允许用户在配置文件中编写实验的完整配置，并通过 `run.py` 直接运行它。配置文件是以 Python 格式组织的，并且必须包括 `datasets` 和 `models` 字段。
|
||||
|
||||
本次测试配置在 [configs/eval_demo.py](https://github.com/open-compass/opencompass/blob/main/configs/eval_demo.py) 中。此配置通过 [继承机制](../user_guides/config.md#继承机制) 引入所需的数据集和模型配置，并以所需格式组合 `datasets` 和 `models` 字段。
|
||||
本次测试配置在 [configs/eval_chat_demo.py](https://github.com/open-compass/opencompass/blob/main/configs/eval_chat_demo.py) 中。此配置通过 [继承机制](../user_guides/config.md#继承机制) 引入所需的数据集和模型配置，并以所需格式组合 `datasets` 和 `models` 字段。
|
||||
|
||||
```python
|
||||
from mmengine.config import read_base
|
||||
|
||||
with read_base():
|
||||
from .datasets.siqa.siqa_gen import siqa_datasets
|
||||
from .datasets.winograd.winograd_ppl import winograd_datasets
|
||||
from .models.opt.hf_opt_125m import opt125m
|
||||
from .models.opt.hf_opt_350m import opt350m
|
||||
from .datasets.demo.demo_gsm8k_chat_gen import gsm8k_datasets
|
||||
from .datasets.demo.demo_math_chat_gen import math_datasets
|
||||
from .models.qwen.hf_qwen2_1_5b_instruct import models as hf_qwen2_1_5b_instruct_models
|
||||
from .models.hf_internlm.hf_internlm2_chat_1_8b import models as hf_internlm2_chat_1_8b_models
|
||||
|
||||
datasets = [*siqa_datasets, *winograd_datasets]
|
||||
models = [opt125m, opt350m]
|
||||
datasets = gsm8k_datasets + math_datasets
|
||||
models = hf_qwen2_1_5b_instruct_models + hf_internlm2_chat_1_8b_models
|
||||
```
|
||||
|
||||
运行任务时，我们只需将配置文件的路径传递给 `run.py`：
|
||||
|
||||
```bash
|
||||
python run.py configs/eval_demo.py
|
||||
python run.py configs/eval_chat_demo.py --debug
|
||||
```
|
||||
|
||||
:::{dropdown} 关于 `models`
|
||||
:animate: fade-in-slide-down
|
||||
|
||||
OpenCompass 提供了一系列预定义的模型配置，位于 `configs/models` 下。以下是与 [opt-350m](https://github.com/open-compass/opencompass/blob/main/configs/models/opt/hf_opt_350m.py)（`configs/models/opt/hf_opt_350m.py`）相关的配置片段：
|
||||
OpenCompass 提供了一系列预定义的模型配置，位于 `configs/models` 下。以下是与 [InternLM2-Chat-1.8B](https://github.com/open-compass/opencompass/blob/main/configs/models/hf_internlm/hf_internlm2_chat_1_8b.py)（`configs/models/hf_internlm/hf_internlm2_chat_1_8b.py`）相关的配置片段：
|
||||
|
||||
```python
|
||||
# 使用 `HuggingFaceBaseModel` 评估由 HuggingFace 的 `AutoModelForCausalLM` 支持的模型
|
||||
from opencompass.models import HuggingFaceBaseModel
|
||||
# 使用 `HuggingFacewithChatTemplate` 评估由 HuggingFace 的 `AutoModelForCausalLM` 支持的对话模型
|
||||
from opencompass.models import HuggingFacewithChatTemplate
|
||||
|
||||
models = [
|
||||
# OPT-350M
|
||||
dict(
|
||||
type=HuggingFaceBaseModel,
|
||||
# `HuggingFaceBaseModel` 的初始化参数
|
||||
path='facebook/opt-350m',
|
||||
# 下面是所有模型的共同参数，不特定于 HuggingFaceBaseModel
|
||||
abbr='opt-350m-hf', # 模型的缩写
|
||||
max_out_len=1024, # 生成的最大 token 数
|
||||
batch_size=32, # 批量大小
|
||||
run_cfg=dict(num_gpus=1), # 该模型所需的 GPU 数量
|
||||
type=HuggingFacewithChatTemplate,
|
||||
abbr='internlm2-chat-1.8b-hf', # 模型的缩写
|
||||
path='internlm/internlm2-chat-1_8b', # 模型的 HuggingFace 路径
|
||||
max_out_len=1024, # 生成的最大 token 数
|
||||
batch_size=8, # 批量大小
|
||||
run_cfg=dict(num_gpus=1), # 该模型所需的 GPU 数量
|
||||
)
|
||||
]
|
||||
```
|
||||
@ -174,25 +190,25 @@ models = [
|
||||
|
||||
与模型类似，数据集的配置文件也提供在 `configs/datasets` 下。用户可以在命令行中使用 `--datasets`，或通过继承在配置文件中导入相关配置
|
||||
|
||||
下面是来自 `configs/eval_demo.py` 的与数据集相关的配置片段：
|
||||
下面是来自 `configs/eval_chat_demo.py` 的与数据集相关的配置片段：
|
||||
|
||||
```python
|
||||
from mmengine.config import read_base # 使用 mmengine.read_base() 读取基本配置
|
||||
|
||||
with read_base():
|
||||
# 直接从预设的数据集配置中读取所需的数据集配置
|
||||
from .datasets.winograd.winograd_ppl import winograd_datasets # 读取 Winograd 配置，基于 PPL（困惑度）进行评估
|
||||
from .datasets.siqa.siqa_gen import siqa_datasets # 读取 SIQA 配置，基于生成进行评估
|
||||
from .datasets.demo.demo_gsm8k_chat_gen import gsm8k_datasets # 读取 GSM8K 配置，使用 4-shot，基于生成式进行评估
|
||||
from .datasets.demo.demo_math_chat_gen import math_datasets # 读取 MATH 配置，使用 0-shot，基于生成式进行评估
|
||||
|
||||
datasets = [*siqa_datasets, *winograd_datasets] # 最终的配置需要包含所需的评估数据集列表 'datasets'
|
||||
datasets = gsm8k_datasets + math_datasets # 最终的配置需要包含所需的评估数据集列表 'datasets'
|
||||
```
|
||||
|
||||
数据集配置通常有两种类型：'ppl' 和 'gen'，分别指示使用的评估方法。其中 `ppl` 表示辨别性评估，`gen` 表示生成性评估。
|
||||
数据集配置通常有两种类型：`ppl` 和 `gen`，分别指示使用的评估方法。其中 `ppl` 表示辨别性评估，`gen` 表示生成性评估。对话模型仅使用 `gen` 生成式评估。
|
||||
|
||||
此外，[configs/datasets/collections](https://github.com/open-compass/opencompass/blob/main/configs/datasets/collections) 收录了各种数据集集合，方便进行综合评估。OpenCompass 通常使用 [`base_medium.py`](https://github.com/open-compass/opencompass/blob/main/configs/datasets/collections/base_medium.py) 进行全面的模型测试。要复制结果，只需导入该文件，例如：
|
||||
此外，[configs/datasets/collections](https://github.com/open-compass/opencompass/blob/main/configs/datasets/collections) 收录了各种数据集集合，方便进行综合评估。OpenCompass 通常使用 [`chat_OC15.py`](https://github.com/open-compass/opencompass/blob/main/configs/dataset_collections/chat_OC15.py) 进行全面的模型测试。要复制结果，只需导入该文件，例如：
|
||||
|
||||
```bash
|
||||
python run.py --models hf_llama_7b --datasets base_medium
|
||||
python run.py --models hf_internlm2_chat_1_8b --datasets chat_OC15 --debug
|
||||
```
|
||||
|
||||
```{seealso}
|
||||
@ -200,6 +216,133 @@ python run.py --models hf_llama_7b --datasets base_medium
|
||||
```
|
||||
:::
|
||||
|
||||
````
|
||||
`````
|
||||
|
||||
对于基座模型
|
||||
|
||||
`````{tabs}
|
||||
````{tab} 命令行（自定义 HF 模型）
|
||||
|
||||
对于 HuggingFace 模型，用户可以通过命令行直接设置模型参数，无需额外的配置文件。例如，对于 `internlm/internlm2-1_8b` 模型，您可以使用以下命令进行评估：
|
||||
|
||||
```bash
|
||||
python run.py \
|
||||
--datasets demo_gsm8k_base_gen demo_math_base_gen \
|
||||
--hf-type base \
|
||||
--hf-path internlm/internlm2-1_8b \
|
||||
--debug
|
||||
```
|
||||
|
||||
请注意，通过这种方式，OpenCompass 一次只评估一个模型，而其他方式可以一次评估多个模型。
|
||||
|
||||
:::{dropdown} 更复杂的命令样例
|
||||
:animate: fade-in-slide-down
|
||||
|
||||
例如一个占用 2 卡进行测试的 Qwen1.5-14B, 开启数据采样，模型的命令如下：
|
||||
|
||||
```bash
|
||||
python run.py --datasets demo_gsm8k_base_gen demo_math_base_gen \
|
||||
--hf-type chat \
|
||||
--hf-path Qwen/Qwen1.5-14B \
|
||||
--max-out-len 1024 \
|
||||
--min-out-len 1 \
|
||||
--hf-num-gpus 2 \
|
||||
--generation-kwargs do_sample=True temperature=0.6 \
|
||||
--debug
|
||||
```
|
||||
:::
|
||||
|
||||
````
|
||||
````{tab} 命令行
|
||||
|
||||
用户可以使用 `--models` 和 `--datasets` 结合想测试的模型和数据集。
|
||||
|
||||
```bash
|
||||
python run.py \
|
||||
--models hf_internlm2_1_8b hf_qwen2_1_5b \
|
||||
--datasets demo_gsm8k_base_gen demo_math_base_gen \
|
||||
--debug
|
||||
```
|
||||
|
||||
````
|
||||
````{tab} 配置文件
|
||||
|
||||
除了通过命令行配置实验外，OpenCompass 还允许用户在配置文件中编写实验的完整配置，并通过 `run.py` 直接运行它。配置文件是以 Python 格式组织的，并且必须包括 `datasets` 和 `models` 字段。
|
||||
|
||||
本次测试配置在 [configs/eval_base_demo.py](https://github.com/open-compass/opencompass/blob/main/configs/eval_base_demo.py) 中。此配置通过 [继承机制](../user_guides/config.md#继承机制) 引入所需的数据集和模型配置，并以所需格式组合 `datasets` 和 `models` 字段。
|
||||
|
||||
```python
|
||||
from mmengine.config import read_base
|
||||
|
||||
with read_base():
|
||||
from .datasets.demo.demo_gsm8k_base_gen import gsm8k_datasets
|
||||
from .datasets.demo.demo_math_base_gen import math_datasets
|
||||
from .models.qwen.hf_qwen2_1_5b import models as hf_qwen2_1_5b_models
|
||||
from .models.hf_internlm.hf_internlm2_1_8b import models as hf_internlm2_1_8b_models
|
||||
|
||||
datasets = gsm8k_datasets + math_datasets
|
||||
models = hf_qwen2_1_5b_models + hf_internlm2_1_8b_models
|
||||
```
|
||||
|
||||
运行任务时，我们只需将配置文件的路径传递给 `run.py`：
|
||||
|
||||
```bash
|
||||
python run.py configs/eval_base_demo.py --debug
|
||||
```
|
||||
|
||||
:::{dropdown} 关于 `models`
|
||||
:animate: fade-in-slide-down
|
||||
|
||||
OpenCompass 提供了一系列预定义的模型配置，位于 `configs/models` 下。以下是与 [InternLM2-1.8B](https://github.com/open-compass/opencompass/blob/main/configs/models/hf_internlm/hf_internlm2_1_8b.py)（`configs/models/hf_internlm/hf_internlm2_1_8b.py`）相关的配置片段：
|
||||
|
||||
```python
|
||||
# 使用 `HuggingFaceBaseModel` 评估由 HuggingFace 的 `AutoModelForCausalLM` 支持的基座模型
|
||||
from opencompass.models import HuggingFaceBaseModel
|
||||
|
||||
models = [
|
||||
dict(
|
||||
type=HuggingFaceBaseModel,
|
||||
abbr='internlm2-1.8b-hf', # 模型的缩写
|
||||
path='internlm/internlm2-1_8b', # 模型的 HuggingFace 路径
|
||||
max_out_len=1024, # 生成的最大 token 数
|
||||
batch_size=8, # 批量大小
|
||||
run_cfg=dict(num_gpus=1), # 该模型所需的 GPU 数量
|
||||
)
|
||||
]
|
||||
```
|
||||
|
||||
使用配置时，我们可以通过命令行参数 `--models` 指定相关文件，或使用继承机制将模型配置导入到配置文件中的 `models` 列表中。
|
||||
|
||||
```{seealso}
|
||||
有关模型配置的更多信息，请参见 [准备模型](../user_guides/models.md)。
|
||||
```
|
||||
:::
|
||||
|
||||
:::{dropdown} 关于 `datasets`
|
||||
:animate: fade-in-slide-down
|
||||
|
||||
与模型类似，数据集的配置文件也提供在 `configs/datasets` 下。用户可以在命令行中使用 `--datasets`，或通过继承在配置文件中导入相关配置
|
||||
|
||||
下面是来自 `configs/eval_base_demo.py` 的与数据集相关的配置片段：
|
||||
|
||||
```python
|
||||
from mmengine.config import read_base # 使用 mmengine.read_base() 读取基本配置
|
||||
|
||||
with read_base():
|
||||
# 直接从预设的数据集配置中读取所需的数据集配置
|
||||
from .datasets.demo.demo_gsm8k_base_gen import gsm8k_datasets # 读取 GSM8K 配置，使用 4-shot，基于生成式进行评估
|
||||
from .datasets.demo.demo_math_base_gen import math_datasets # 读取 MATH 配置，使用 0-shot，基于生成式进行评估
|
||||
|
||||
datasets = gsm8k_datasets + math_datasets # 最终的配置需要包含所需的评估数据集列表 'datasets'
|
||||
```
|
||||
|
||||
数据集配置通常有两种类型：`ppl` 和 `gen`，分别指示使用的评估方法。其中 `ppl` 表示判别性评估，`gen` 表示生成性评估。基座模型对于 "选择题" 类型的数据集会使用 `ppl` 判别性评估，其他则会使用 `gen` 生成式评估。
|
||||
|
||||
```{seealso}
|
||||
您可以从 [配置数据集](../user_guides/datasets.md) 中找到更多信息。
|
||||
```
|
||||
:::
|
||||
|
||||
````
|
||||
`````
|
||||
@ -208,27 +351,27 @@ python run.py --models hf_llama_7b --datasets base_medium
|
||||
OpenCompass 通常假定运行环境网络是可用的。如果您遇到网络问题或希望在离线环境中运行 OpenCompass，请参阅 [FAQ - 网络 - Q1](./faq.md#网络) 寻求解决方案。
|
||||
```
|
||||
|
||||
接下来的部分将使用基于配置的方法作为示例来解释其他特征。
|
||||
接下来的部分将使用基于配置的方法，评测对话模型，作为示例来解释其他特征。
|
||||
|
||||
## 启动评估
|
||||
|
||||
由于 OpenCompass 默认并行启动评估过程，我们可以在第一次运行时以 `--debug` 模式启动评估，并检查是否存在问题。在 `--debug` 模式下，任务将按顺序执行，并实时打印输出。
|
||||
由于 OpenCompass 默认并行启动评估过程，我们可以在第一次运行时以 `--debug` 模式启动评估，并检查是否存在问题。包括在前述的所有文档中，我们都使用了 `--debug` 开关。在 `--debug` 模式下，任务将按顺序执行，并实时打印输出。
|
||||
|
||||
```bash
|
||||
python run.py configs/eval_demo.py -w outputs/demo --debug
|
||||
python run.py configs/eval_chat_demo.py -w outputs/demo --debug
|
||||
```
|
||||
|
||||
预训练模型 'facebook/opt-350m' 和 'facebook/opt-125m' 将在首次运行期间从 HuggingFace 自动下载。
|
||||
如果一切正常，您应该看到屏幕上显示 “Starting inference process”：
|
||||
对话默写 'internlm/internlm2-chat-1_8b' 和 'Qwen/Qwen2-1.5B-Instruct' 将在首次运行期间从 HuggingFace 自动下载。
|
||||
如果一切正常，您应该看到屏幕上显示 “Starting inference process”，且进度条开始前进：
|
||||
|
||||
```bash
|
||||
[2023-07-12 18:23:55,076] [opencompass.openicl.icl_inferencer.icl_gen_inferencer] [INFO] Starting inference process...
|
||||
```
|
||||
|
||||
然后，您可以按 `ctrl+c` 中断程序，并以正常模式运行以下命令：
|
||||
然后，您可以按 `Ctrl+C` 中断程序，并以正常模式运行以下命令：
|
||||
|
||||
```bash
|
||||
python run.py configs/eval_demo.py -w outputs/demo
|
||||
python run.py configs/eval_chat_demo.py -w outputs/demo
|
||||
```
|
||||
|
||||
在正常模式下，评估任务将在后台并行执行，其输出将被重定向到输出目录 `outputs/demo/{TIMESTAMP}`。前端的进度条只指示已完成任务的数量，而不考虑其成功或失败。**任何后端任务失败都只会在终端触发警告消息。**
|
||||
@ -238,14 +381,13 @@ python run.py configs/eval_demo.py -w outputs/demo
|
||||
以下是与评估相关的一些参数，可以帮助您根据环境配置更有效的推理任务：
|
||||
|
||||
- `-w outputs/demo`：保存评估日志和结果的工作目录。在这种情况下，实验结果将保存到 `outputs/demo/{TIMESTAMP}`。
|
||||
- `-r`：重用现有的推理结果，并跳过已完成的任务。如果后面跟随时间戳，将重用工作空间路径下该时间戳的结果；否则，将重用指定工作空间路径下的最新结果。
|
||||
- `-r {TIMESTAMP/latest}`：重用现有的推理结果，并跳过已完成的任务。如果后面跟随时间戳，将重用工作空间路径下该时间戳的结果；若给定 latest 或干脆不指定，将重用指定工作空间路径下的最新结果。
|
||||
- `--mode all`：指定任务的特定阶段。
|
||||
- all：（默认）执行完整评估，包括推理和评估。
|
||||
- infer：在每个数据集上执行推理。
|
||||
- eval：根据推理结果进行评估。
|
||||
- viz：仅显示评估结果。
|
||||
- `--max-partition-size 40000`：数据集分片大小。一些数据集可能很大，使用此参数可以将它们分成多个子任务以有效利用资源。但是，如果分片过细，由于模型加载时间较长，整体速度可能会变慢。
|
||||
- `--max-num-workers 32`：并行任务的最大数量。在如 Slurm 之类的分布式环境中，此参数指定提交任务的最大数量。在本地环境中，它指定同时执行的任务的最大数量。请注意，实际的并行任务数量取决于可用的 GPU 资源，可能不等于这个数字。
|
||||
- `--max-num-workers 8`：并行任务的最大数量。在如 Slurm 之类的分布式环境中，此参数指定提交任务的最大数量。在本地环境中，它指定同时执行的任务的最大数量。请注意，实际的并行任务数量取决于可用的 GPU 资源，可能不等于这个数字。
|
||||
|
||||
如果您不是在本地机器上执行评估，而是使用 Slurm 集群，您可以指定以下参数：
|
||||
|
||||
@ -264,10 +406,10 @@ python run.py configs/eval_demo.py -w outputs/demo
|
||||
评估完成后，评估结果表格将打印如下：
|
||||
|
||||
```text
|
||||
dataset version metric mode opt350m opt125m
|
||||
--------- --------- -------- ------ --------- ---------
|
||||
siqa e78df3 accuracy gen 21.55 12.44
|
||||
winograd b6c7ed accuracy ppl 51.23 49.82
|
||||
dataset version metric mode qwen2-1.5b-instruct-hf internlm2-chat-1.8b-hf
|
||||
---------- --------- -------- ------ ------------------------ ------------------------
|
||||
demo_gsm8k 1d7fe4 accuracy gen 56.25 32.81
|
||||
demo_math 393424 accuracy gen 18.75 14.06
|
||||
```
|
||||
|
||||
所有运行输出将定向到 `outputs/demo/` 目录，结构如下：
|
||||
@ -280,9 +422,9 @@ outputs/default/
|
||||
│ ├── logs # 推理和评估阶段的日志文件
|
||||
│ │ ├── eval
|
||||
│ │ └── infer
|
||||
│ ├── predictions # 每个任务的推理结果
|
||||
│ ├── results # 每个任务的评估结果
|
||||
│ └── summary # 单个实验的汇总评估结果
|
||||
│ ├── predictions # 每个任务的推理结果
|
||||
│ ├── results # 每个任务的评估结果
|
||||
│ └── summary # 单个实验的汇总评估结果
|
||||
├── ...
|
||||
```
|
||||
|
||||
@ -297,6 +439,6 @@ outputs/default/
|
||||
- [配置数据集](../user_guides/datasets.md)
|
||||
- [准备模型](../user_guides/models.md)
|
||||
- [任务运行和监控](../user_guides/experimentation.md)
|
||||
- [如何调Prompt](../prompt/overview.md)
|
||||
- [如何调 Prompt](../prompt/overview.md)
|
||||
- [结果展示](../user_guides/summarizer.md)
|
||||
- [学习配置文件](../user_guides/config.md)
|
||||
|
||||
@ -18,17 +18,25 @@ from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
|
||||
from opencompass.utils import get_logger, match_files
|
||||
|
||||
|
||||
def match_cfg_file(workdir: str,
|
||||
def match_cfg_file(workdir: Union[str, List[str]],
|
||||
pattern: Union[str, List[str]]) -> List[Tuple[str, str]]:
|
||||
"""Match the config file in workdir recursively given the pattern.
|
||||
|
||||
Additionally, if the pattern itself points to an existing file, it will be
|
||||
directly returned.
|
||||
"""
|
||||
def _mf_with_multi_workdirs(workdir, pattern, fuzzy=False):
|
||||
if isinstance(workdir, str):
|
||||
workdir = [workdir]
|
||||
files = []
|
||||
for wd in workdir:
|
||||
files += match_files(wd, pattern, fuzzy=fuzzy)
|
||||
return files
|
||||
|
||||
if isinstance(pattern, str):
|
||||
pattern = [pattern]
|
||||
pattern = [p + '.py' if not p.endswith('.py') else p for p in pattern]
|
||||
files = match_files(workdir, pattern, fuzzy=False)
|
||||
files = _mf_with_multi_workdirs(workdir, pattern, fuzzy=False)
|
||||
if len(files) != len(pattern):
|
||||
nomatched = []
|
||||
ambiguous = []
|
||||
@ -37,7 +45,7 @@ def match_cfg_file(workdir: str,
|
||||
'You may use tools/list_configs.py to list or '
|
||||
'locate the configurations.\n')
|
||||
for p in pattern:
|
||||
files = match_files(workdir, p, fuzzy=False)
|
||||
files = _mf_with_multi_workdirs(workdir, p, fuzzy=False)
|
||||
if len(files) == 0:
|
||||
nomatched.append([p[:-3]])
|
||||
elif len(files) > 1:
|
||||
@ -101,7 +109,10 @@ def get_config_from_arg(args) -> Config:
|
||||
raise ValueError('You must specify "--datasets" or "--custom-dataset-path" if you do not specify a config file path.')
|
||||
datasets = []
|
||||
if args.datasets:
|
||||
datasets_dir = os.path.join(args.config_dir, 'datasets')
|
||||
datasets_dir = [
|
||||
os.path.join(args.config_dir, 'datasets'),
|
||||
os.path.join(args.config_dir, 'dataset_collections')
|
||||
]
|
||||
for dataset_arg in args.datasets:
|
||||
if '/' in dataset_arg:
|
||||
dataset_name, dataset_suffix = dataset_arg.split('/', 1)
|
||||
@ -150,6 +161,7 @@ def get_config_from_arg(args) -> Config:
|
||||
model_kwargs=args.model_kwargs,
|
||||
tokenizer_path=args.tokenizer_path,
|
||||
tokenizer_kwargs=args.tokenizer_kwargs,
|
||||
generation_kwargs=args.generation_kwargs,
|
||||
peft_path=args.peft_path,
|
||||
peft_kwargs=args.peft_kwargs,
|
||||
max_seq_len=args.max_seq_len,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user