name: daily_run_test

on:
  workflow_dispatch:
  schedule:
    - cron:  '56 16 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  CONDA_ENV: opencompass_regression
  PIP_CACHE_PATH: /cpfs01/user/qa-llm-cicd/.cache/pip
  USERSPACE_PREFIX: /cpfs01/user/qa-llm-cicd
  HF_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub
  HF_DATASETS_OFFLINE: 1
  TRANSFORMERS_OFFLINE: 1
  HF_HUB_OFFLINE: 1

jobs:
  daily_run_test:
    runs-on: self-hosted
    environment: 'prod'
    timeout-minutes: 240 #4hours
    steps:
      - name: Clone repository
        uses: actions/checkout@v2
      - name: Prepare - create conda env and install torch
        run: |
          eval "$(conda shell.bash hook)"
          conda create -y --name ${{env.CONDA_ENV}} python=3.10
          conda activate ${{env.CONDA_ENV}}
          pip install /cpfs01/user/qa-llm-cicd/packages/lmdeploy-0.5.0+cu118-cp310-cp310-manylinux2014_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118
          pip install /cpfs01/user/qa-llm-cicd/packages/vllm-0.5.0.post1+cu118-cp310-cp310-manylinux1_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118
          FLASH_ATTENTION_FORCE_BUILD=TRUE pip install /cpfs01/user/qa-llm-cicd/packages/flash_attn-2.5.8+cu118torch2.2cxx11abiFALSE-cp310-cp310-linux_x86_64.whl --index-url https://download.pytorch.org/whl/cu118
          pip install torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2 --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118
          pip install xformers==0.0.25.post1 --cache-dir ${{env.PIP_CACHE_PATH}} --extra-index-url https://download.pytorch.org/whl/cu118
          conda info --envs
      - name: Prepare - Pip install code
        run: |
          eval "$(conda shell.bash hook)"
          conda activate ${{env.CONDA_ENV}}
          pip install -e . --cache-dir ${{env.PIP_CACHE_PATH}}
          pip install human_eval transformers protobuf pytest --cache-dir ${{env.PIP_CACHE_PATH}}
          conda info --envs
      - name: Prepare - prepare data and hf model
        run: |
          cp -r ${{env.USERSPACE_PREFIX}}/data .
          rm -rf ~/.cache/huggingface/hub -f && mkdir ~/.cache -p && mkdir ~/.cache/huggingface -p
          ln -s ${{env.HF_CACHE_PATH}} ~/.cache/huggingface/hub
      - name:  Run chat model test
        run: |
          eval "$(conda shell.bash hook)"
          conda activate ${{env.CONDA_ENV}}
          conda info --envs
          rm -rf regression_result_daily
          export from_tf=TRUE
          rm -rf /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/*/summary
          python3 run.py .github/scripts/eval_regression_chat.py --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }} --reuse
          cp -r /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/*/summary regression_result_daily
          python -m pytest -m chat -s -v --color=yes .github/scripts/oc_score_assert.py
      - name:  Run base model test
        run: |
          eval "$(conda shell.bash hook)"
          conda activate ${{env.CONDA_ENV}}
          conda info --envs
          rm -rf regression_result_daily
          export from_tf=TRUE
          rm -rf /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/*/summary
          python3 run.py .github/scripts/eval_regression_base.py --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }} --reuse
          cp -r /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/*/summary regression_result_daily
          python -m pytest -m base -s -v --color=yes .github/scripts/oc_score_assert.py
      - name:  Remove Conda Env
        if: always()
        run: |
          cp -r regression_result_daily/* /cpfs01/user/qa-llm-cicd/report
          eval "$(conda shell.bash hook)"
          conda env remove -y --name ${{env.CONDA_ENV}}
          conda info --envs

  notify_to_feishu:
    if: ${{ always() && !cancelled() && contains(needs.*.result, 'failure') && (github.ref_name == 'develop' || github.ref_name == 'main') }}
    needs: [daily_run_test]
    environment: 'prod'
    timeout-minutes: 5
    runs-on: self-hosted
    steps:
      - name: notify
        run: |
          curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Opencompass- Daily test failed","content":[[{"tag":"text","text":"branch: ${{github.ref_name}}, run action: ${{github.workflow}} failed. "},{"tag":"a","text":"Please click here for details ","href":"https://github.com/'${{ github.repository }}'/actions/runs/'${GITHUB_RUN_ID}'"},{"tag":"at","user_id":"'${{ secrets.USER_ID }}'"}]]}}}}'  ${{ secrets.WEBHOOK_URL }}