mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
fb69ba5eb8
* update * update * update * update * update * update * update * update * update * update * update * update * update * update * update * update * update * update * update * update --------- Co-authored-by: zhulin1 <zhulin1@pjlab.org.cn>
106 lines
6.2 KiB
YAML
106 lines
6.2 KiB
YAML
name: daily_run_test
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: '56 16 * * *'
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
CONDA_ENV: opencompass_regression
|
|
PIP_CACHE_PATH: /cpfs01/user/qa-llm-cicd/.cache/pip
|
|
USERSPACE_PREFIX: /cpfs01/user/qa-llm-cicd
|
|
HF_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub
|
|
DATEASET_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/llm-evaluation-datasets
|
|
HF_DATASETS_OFFLINE: 1
|
|
TRANSFORMERS_OFFLINE: 1
|
|
HF_HUB_OFFLINE: 1
|
|
|
|
jobs:
|
|
daily_run_test:
|
|
runs-on: self-hosted
|
|
environment: 'prod'
|
|
timeout-minutes: 240 #4hours
|
|
steps:
|
|
- name: Clone repository
|
|
uses: actions/checkout@v2
|
|
- name: Prepare - create conda env and install torch
|
|
run: |
|
|
eval "$(conda shell.bash hook)"
|
|
conda create -y --name ${{env.CONDA_ENV}} python=3.10
|
|
conda activate ${{env.CONDA_ENV}}
|
|
pip install /cpfs01/user/qa-llm-cicd/packages/lmdeploy-0.5.0+cu118-cp310-cp310-manylinux2014_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118
|
|
pip install /cpfs01/user/qa-llm-cicd/packages/vllm-0.5.2+cu118-cp310-cp310-manylinux1_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}}
|
|
FLASH_ATTENTION_FORCE_BUILD=TRUE pip install /cpfs01/user/qa-llm-cicd/packages/flash_attn-2.5.8+cu118torch2.2cxx11abiFALSE-cp310-cp310-linux_x86_64.whl --index-url https://download.pytorch.org/whl/cu118
|
|
pip install torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2 --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118
|
|
pip install xformers==0.0.25.post1 --cache-dir ${{env.PIP_CACHE_PATH}} --extra-index-url https://download.pytorch.org/whl/cu118
|
|
conda info --envs
|
|
- name: Prepare - Pip install code
|
|
run: |
|
|
eval "$(conda shell.bash hook)"
|
|
conda activate ${{env.CONDA_ENV}}
|
|
pip install -e . --cache-dir ${{env.PIP_CACHE_PATH}}
|
|
pip install human_eval transformers protobuf pytest --cache-dir ${{env.PIP_CACHE_PATH}}
|
|
conda info --envs
|
|
- name: Prepare - prepare data and hf model
|
|
run: |
|
|
ln -s ${{env.DATEASET_CACHE_PATH}} data
|
|
rm -rf ~/.cache/huggingface/hub -f && mkdir ~/.cache -p && mkdir ~/.cache/huggingface -p
|
|
ln -s ${{env.HF_CACHE_PATH}} ~/.cache/huggingface/hub
|
|
- name: Run chat model test
|
|
run: |
|
|
eval "$(conda shell.bash hook)"
|
|
conda activate ${{env.CONDA_ENV}}
|
|
conda info --envs
|
|
python3 run.py .github/scripts/eval_regression_chat.py --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/chat --reuse
|
|
rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/chat/*/summary regression_result_daily
|
|
python -m pytest -m chat -s -v --color=yes .github/scripts/oc_score_assert.py
|
|
- name: Run base model test
|
|
run: |
|
|
eval "$(conda shell.bash hook)"
|
|
conda activate ${{env.CONDA_ENV}}
|
|
conda info --envs
|
|
python3 run.py .github/scripts/eval_regression_base.py --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/base --reuse
|
|
rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/base/*/summary regression_result_daily
|
|
python -m pytest -m base -s -v --color=yes .github/scripts/oc_score_assert.py
|
|
- name: Run command testcase
|
|
run: |
|
|
eval "$(conda shell.bash hook)"
|
|
conda activate ${{env.CONDA_ENV}}
|
|
conda info --envs
|
|
export from_tf=TRUE
|
|
python tools/list_configs.py internlm2_5 mmlu
|
|
python run.py --models hf_internlm2_5_7b --datasets race_ppl --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd1 --reuse
|
|
rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd1/*/summary regression_result_daily
|
|
python -m pytest -m case1 -s -v --color=yes .github/scripts/oc_score_assert.py
|
|
python run.py --models hf_internlm2_5_7b_chat --datasets race_gen -a lmdeploy --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd2 --reuse
|
|
rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd2/*/summary regression_result_daily
|
|
python -m pytest -m case2 -s -v --color=yes .github/scripts/oc_score_assert.py
|
|
python run.py --datasets race_ppl --hf-type base --hf-path internlm/internlm2_5-7b --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd3 --reuse
|
|
rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd3/*/summary regression_result_daily
|
|
python -m pytest -m case3 -s -v --color=yes .github/scripts/oc_score_assert.py
|
|
python run.py --datasets race_gen --hf-type chat --hf-path internlm/internlm2_5-7b-chat --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd4 --reuse
|
|
rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd4/*/summary regression_result_daily
|
|
python -m pytest -m case4 -s -v --color=yes .github/scripts/oc_score_assert.py
|
|
- name: Remove Conda Env
|
|
if: always()
|
|
run: |
|
|
rm -rf regression_result_daily
|
|
eval "$(conda shell.bash hook)"
|
|
conda env remove -y --name ${{env.CONDA_ENV}}
|
|
conda info --envs
|
|
|
|
notify_to_feishu:
|
|
if: ${{ always() && !cancelled() && contains(needs.*.result, 'failure') && (github.ref_name == 'develop' || github.ref_name == 'main') }}
|
|
needs: [daily_run_test]
|
|
environment: 'prod'
|
|
timeout-minutes: 5
|
|
runs-on: self-hosted
|
|
steps:
|
|
- name: notify
|
|
run: |
|
|
curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Opencompass- Daily test failed","content":[[{"tag":"text","text":"branch: ${{github.ref_name}}, run action: ${{github.workflow}} failed. "},{"tag":"a","text":"Please click here for details ","href":"https://github.com/'${{ github.repository }}'/actions/runs/'${GITHUB_RUN_ID}'"},{"tag":"at","user_id":"'${{ secrets.USER_ID }}'"}]]}}}}' ${{ secrets.WEBHOOK_URL }}
|