上传文件至 /
This commit is contained in:
commit
a232c01c8f
22
config.yaml
Normal file
22
config.yaml
Normal file
@ -0,0 +1,22 @@
|
||||
model: "HaiRuo-7B-General"
|
||||
model_version: "V1.0.0.0DEV"
|
||||
conda_env: "agent-common"
|
||||
tensor_parallel_size: "1"
|
||||
visible_gpu_index: "1"
|
||||
model_port: "20004"
|
||||
gpu_memory_utilization: "0.9"
|
||||
dtype: "float16"
|
||||
model_server_ip: "100.200.128.151"
|
||||
yellow_block_server_ip: "100.200.128.83"
|
||||
yellow_block_server_port: "6233"
|
||||
yellow_block_conda_env: "agent-common"
|
||||
|
||||
# Constant definitions
|
||||
media_download_path: "/data/media"
|
||||
model_path: "/data/models"
|
||||
model_name: "{{model}}-{{model_version}}"
|
||||
model_name_path: "{{model_path}}/{{model_name}}"
|
||||
model_tar: "{{model_name}}.tar.gz"
|
||||
miniconda_path: "/data/miniconda3/envs/"
|
||||
yellow_block_path: "/data/app/dev/ihp-model-ops/test/model_service_v2"
|
||||
automated_deployment_path: "/data/jenkins_script/automated_deployment/script"
|
||||
419
deploy_model_start_1.sh
Normal file
419
deploy_model_start_1.sh
Normal file
@ -0,0 +1,419 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Color definitions
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
CYAN='\033[36m'
|
||||
RESET='\033[0m'
|
||||
|
||||
:<<'COMMENT'
|
||||
检测介质下载路径中是否包含以下介质：模型权重文件和conda环境tar包，如果不存在则退出。
|
||||
检测服务器节点目标conda环境是否存在，如果不存在则创建。
|
||||
检测模型服务是否已启动，如果模型服务已启动，则终止模型服务。
|
||||
启动模型服务
|
||||
检查模型服务是否启动成功
|
||||
执行curl命令验证模型服务
|
||||
将黄块注册脚本和黄块curl命令脚本scp到黄块服务ip
|
||||
ssh登录黄块服务ip进行黄块注册，执行黄块curl命令
|
||||
MEDIA_DOWNLOAD_PATH="/data/media"
|
||||
MODEL_PATH="/data/models"
|
||||
#MODEL=$1 # "HaiRuo-7B-General"
|
||||
#MODEL_VERSION=$2 # "V1.0.0.0"
|
||||
MODEL_NAME="$MODEL-$MODEL_VERSION" # "HaiRuo-7B-General-V1.0.0.0" # 模型名称 外部传参
|
||||
MODEL_NAME_PATH="$MODEL_PATH/$MODEL_NAME"
|
||||
MODEL_TAR="$MODEL_NAME.tar.gz"
|
||||
MINICONDA_PATH="/data/miniconda3/envs/"
|
||||
#CONDA_ENV=$3 # "conda-HaiRuo-7B-General-V1.0.0.0" # conda环境名称 外部传参
|
||||
#VISIBLE_GPU_INDEX=$4
|
||||
#MODEL_PORT=$5
|
||||
#MODEL_SERVER_IP=$6
|
||||
#YELLOW_BLOCK_SERVER_IP=$7
|
||||
#YELLOW_BLOCK_SERVER_PORT=$8
|
||||
YELLOW_BLOCK_PATH="/data/app/dev/ihp-model-ops/test/model_service_v2"
|
||||
#YELLOW_BLOCK_CONDA_ENV=$9
|
||||
COMMENT
|
||||
|
||||
# 定义帮助信息
|
||||
function print_help() {
|
||||
echo "Usage: $0 [options]"
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -h, --help 显示此帮助信息"
|
||||
echo " --model <model> 模型名称，例如 'HaiRuo-7B-General'"
|
||||
echo " --model-version <version> 模型版本，例如 'V1.0.0.0'"
|
||||
echo " --conda-env <conda_env> Conda环境名称，例如 'conda-HaiRuo-7B-General-V1.0.0.0'"
|
||||
echo " --tensor-parallel-size <size> 张量并行的GPU卡数量，例如 '2'"
|
||||
echo " --gpu-index <gpu_index> 可用的GPU卡索引，例如 '0,1'"
|
||||
echo " --model-port <model_port> 模型服务端口，例如 '20004'"
|
||||
echo " --model-server-ip <server_ip> 模型服务节点IP地址，例如 '127.0.0.1'"
|
||||
echo " --yellow-block-server-ip <ip> 黄块服务节点IP地址，例如 '127.0.0.1'"
|
||||
echo " --yellow-block-server-port <port> 黄块服务节点SSH端口，例如 '22'"
|
||||
echo " --yellow-block-conda-env <env> 黄块Conda环境名称，例如 'agent-common'"
|
||||
echo
|
||||
echo "Example:"
|
||||
echo " bash $0 --model 'HaiRuo-7B-General' --model-version 'V1.0.0.0' --conda-env 'conda-HaiRuo-7B-General-V1.0.0.0' --tensor-parallel-size 2 --gpu-index 0,1 --model-port 20004 --model-server-ip 127.0.0.1 --yellow-block-server-ip 127.0.0.1 --yellow-block-server-port 22 --yellow-block-conda-env 'conda-yellow-block'"
|
||||
}
|
||||
|
||||
# 初始化变量
|
||||
MODEL=""
|
||||
MODEL_VERSION=""
|
||||
CONDA_ENV=""
|
||||
TENSOR_PARALLEL_SIZE=""
|
||||
VISIBLE_GPU_INDEX=""
|
||||
MODEL_PORT=""
|
||||
MODEL_SERVER_IP=""
|
||||
YELLOW_BLOCK_SERVER_IP=""
|
||||
YELLOW_BLOCK_SERVER_PORT=""
|
||||
YELLOW_BLOCK_CONDA_ENV=""
|
||||
|
||||
# Constant definitions
|
||||
MEDIA_DOWNLOAD_PATH="/data/media"
|
||||
MODEL_PATH="/data/models"
|
||||
MODEL_NAME="$MODEL-$MODEL_VERSION"
|
||||
MODEL_NAME_PATH="$MODEL_PATH/$MODEL_NAME"
|
||||
MODEL_TAR="$MODEL_NAME.tar.gz"
|
||||
MINICONDA_PATH="/data/miniconda3/envs/"
|
||||
YELLOW_BLOCK_PATH="/data/app/dev/ihp-model-ops/test/model_service_v2"
|
||||
|
||||
# 检查是否有参数
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "No arguments provided. Use -h or --help for help."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 处理参数
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
print_help
|
||||
exit 0
|
||||
;;
|
||||
--model)
|
||||
shift
|
||||
MODEL=$1
|
||||
MODEL_NAME="$MODEL-$MODEL_VERSION"
|
||||
MODEL_NAME_PATH="$MODEL_PATH/$MODEL_NAME"
|
||||
MODEL_TAR="$MODEL_NAME.tar.gz"
|
||||
shift
|
||||
;;
|
||||
--model-version)
|
||||
shift
|
||||
MODEL_VERSION=$1
|
||||
MODEL_NAME="$MODEL-$MODEL_VERSION"
|
||||
MODEL_NAME_PATH="$MODEL_PATH/$MODEL_NAME"
|
||||
MODEL_TAR="$MODEL_NAME.tar.gz"
|
||||
shift
|
||||
;;
|
||||
--conda-env)
|
||||
shift
|
||||
CONDA_ENV=$1
|
||||
shift
|
||||
;;
|
||||
--tensor-parallel-size)
|
||||
shift
|
||||
TENSOR_PARALLEL_SIZE=$1
|
||||
shift
|
||||
;;
|
||||
--gpu-index)
|
||||
shift
|
||||
VISIBLE_GPU_INDEX=$1
|
||||
shift
|
||||
;;
|
||||
--model-port)
|
||||
shift
|
||||
MODEL_PORT=$1
|
||||
shift
|
||||
;;
|
||||
--model-server-ip)
|
||||
shift
|
||||
MODEL_SERVER_IP=$1
|
||||
shift
|
||||
;;
|
||||
--yellow-block-server-ip)
|
||||
shift
|
||||
YELLOW_BLOCK_SERVER_IP=$1
|
||||
shift
|
||||
;;
|
||||
--yellow-block-server-port)
|
||||
shift
|
||||
YELLOW_BLOCK_SERVER_PORT=$1
|
||||
shift
|
||||
;;
|
||||
--yellow-block-conda-env)
|
||||
shift
|
||||
YELLOW_BLOCK_CONDA_ENV=$1
|
||||
shift
|
||||
;;
|
||||
*) # 未知选项
|
||||
echo "未知选项: $1"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# 打印参数值（调试用）
|
||||
echo "MODEL: $MODEL"
|
||||
echo "MODEL_VERSION: $MODEL_VERSION"
|
||||
echo "MODEL_NAME: $MODEL_NAME"
|
||||
echo "MODEL_NAME_PATH: $MODEL_NAME_PATH"
|
||||
echo "MODEL_TAR: $MODEL_TAR"
|
||||
echo "CONDA_ENV: $CONDA_ENV"
|
||||
echo "TENSOR_PARALLEL_SIZE: $TENSOR_PARALLEL_SIZE"
|
||||
echo "VISIBLE_GPU_INDEX: $VISIBLE_GPU_INDEX"
|
||||
echo "MODEL_PORT: $MODEL_PORT"
|
||||
echo "MODEL_SERVER_IP: $MODEL_SERVER_IP"
|
||||
echo "YELLOW_BLOCK_SERVER_IP: $YELLOW_BLOCK_SERVER_IP"
|
||||
echo "YELLOW_BLOCK_SERVER_PORT: $YELLOW_BLOCK_SERVER_PORT"
|
||||
echo "YELLOW_BLOCK_CONDA_ENV: $YELLOW_BLOCK_CONDA_ENV"
|
||||
echo "MEDIA_DOWNLOAD_PATH: $MEDIA_DOWNLOAD_PATH"
|
||||
echo "MODEL_PATH: $MODEL_PATH"
|
||||
echo "MINICONDA_PATH: $MINICONDA_PATH"
|
||||
echo "YELLOW_BLOCK_PATH: $YELLOW_BLOCK_PATH"
|
||||
|
||||
# 检查介质下载路径中是否包含以下介质：模型权重文件和conda环境tar包。
|
||||
check_media_list() {
|
||||
echo -e "${CYAN}Checking media files...${RESET}"
|
||||
cd "$MEDIA_DOWNLOAD_PATH"
|
||||
if [ -f "$MODEL_TAR" ]; then
|
||||
echo -e "${GREEN}The media file $MEDIA_DOWNLOAD_PATH/$MODEL_TAR exists.${RESET}"
|
||||
else
|
||||
echo -e "${RED}The media file $MEDIA_DOWNLOAD_PATH/$MODEL_TAR does not exist.${RESET}"
|
||||
fi
|
||||
if [ -f "$CONDA_ENV.tar.gz" ]; then
|
||||
echo -e "${GREEN}The media file $MEDIA_DOWNLOAD_PATH/$CONDA_ENV.tar.gz exists.${RESET}"
|
||||
else
|
||||
echo -e "${RED}The media file $MEDIA_DOWNLOAD_PATH/$CONDA_ENV.tar.gz does not exist.${RESET}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 检查服务器节点中模型权重文件是否存在
|
||||
check_model_path() {
|
||||
cd "$MODEL_PATH"
|
||||
if [ ! -d "$MODEL_NAME" ]; then
|
||||
echo -e "${RED}The model $MODEL_NAME does not exist.${RESET}"
|
||||
return 1
|
||||
else
|
||||
echo -e "${GREEN}The model $MODEL_NAME exists.${RESET}"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# 部署模型
|
||||
deploy_model() {
|
||||
echo -e "${CYAN}Start deploying model.${RESET}"
|
||||
cd "$MODEL_PATH"
|
||||
cp "$MEDIA_DOWNLOAD_PATH/$MODEL_TAR" ./
|
||||
tar -zxf "$MODEL_TAR"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Model deployed successfully.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Model deployment failed.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
ll "$MODEL_NAME"
|
||||
chown -R inspur:inspur "$MODEL_NAME"
|
||||
rm -rf "$MODEL_TAR"
|
||||
}
|
||||
|
||||
# 检查服务器节点中conda环境是否存在
|
||||
check_conda_env() {
|
||||
cd "$MINICONDA_PATH"
|
||||
if [ ! -d "$CONDA_ENV" ]; then
|
||||
echo -e "${RED}The conda environment $CONDA_ENV does not exist.${RESET}"
|
||||
return 1
|
||||
else
|
||||
echo -e "${GREEN}The conda environment $CONDA_ENV exists.${RESET}"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# 将介质下载路径中的conda环境压缩包解压至/data/miniconda3/envs/$CONDA_ENV目录下，修改用户和用户组。
|
||||
deploy_conda_env() {
|
||||
cd "$MINICONDA_PATH"
|
||||
mkdir "$CONDA_ENV"
|
||||
cd "$MEDIA_DOWNLOAD_PATH"
|
||||
tar -zxf "$CONDA_ENV.tar.gz" -C "$MINICONDA_PATH/$CONDA_ENV"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}The environment $CONDA_ENV has been created.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Failed to create environment $CONDA_ENV.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
chown -R inspur:inspur "$MINICONDA_PATH/$CONDA_ENV"
|
||||
}
|
||||
|
||||
# 调用conda环境中的check.sh脚本，检查模型服务进程
|
||||
check_model_service_process() {
|
||||
cd "$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
check_result=$(bash check.sh)
|
||||
# 检查输出内容
|
||||
if [[ "$check_result" == *"SUCC"* ]]; then
|
||||
return 0
|
||||
elif [[ "$check_result" == *"FAIL"* ]]; then
|
||||
return 1
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 调用业务代码中的stop.sh脚本，停止模型服务
|
||||
stop_model_service() {
|
||||
cd "$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
bash stop.sh > /dev/null 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}$MODEL_NAME service stopped successfully.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Failed to stop $MODEL_NAME service.${RESET}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 激活conda环境，调用业务代码中的start.sh脚本，启动模型服务
|
||||
start_model_service(){
|
||||
cd "$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
bash "start.sh" ${VISIBLE_GPU_INDEX} ${CONDA_ENV} ${MODEL_NAME_PATH} ${TENSOR_PARALLEL_SIZE} ${MODEL_SERVER_IP} ${MODEL_PORT}
|
||||
}
|
||||
|
||||
# 轮询检测模型服务是否已启动成功
|
||||
check_model_started() {
|
||||
local start_time=$(date +%s)
|
||||
local timeout=300
|
||||
local interval=5
|
||||
while true; do
|
||||
sleep $interval
|
||||
if check_model_service_process; then
|
||||
echo -e "${GREEN}$MODEL_NAME service started successfully.${RESET}"
|
||||
break
|
||||
fi
|
||||
local current_time=$(date +%s)
|
||||
local elapsed_time=$(($current_time - $start_time))
|
||||
if [ $elapsed_time -ge $timeout ]; then
|
||||
echo -e "${RED}Failed to start $MODEL_NAME service.${RESET}"
|
||||
echo -e "${RED}Please check the log under $MINICONDA_PATH/$CONDA_ENV/script/$MODEL${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
update_model_curl_sh() {
|
||||
local script_file="model_curl.sh"
|
||||
local model_server_ip=$1
|
||||
local model_port=$2
|
||||
|
||||
cd "$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
# 使用 sed 命令替换变量值
|
||||
sed -i "s|^export MODEL_SERVER_IP=.*|export MODEL_SERVER_IP=$model_server_ip|" "$script_file"
|
||||
sed -i "s|^export MODEL_PORT=.*|export MODEL_PORT=$model_port|" "$script_file"
|
||||
echo -e "${GREEN}Updated $script_file with the provided values.${RESET}"
|
||||
}
|
||||
|
||||
# 验证模型服务
|
||||
model_curl_verification(){
|
||||
echo -e "${GREEN}Start to execute the curl command to verify the model${RESET}"
|
||||
update_model_curl_sh $MODEL_SERVER_IP $MODEL_PORT
|
||||
cd "$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
cat model_curl.sh
|
||||
echo -e "${GREEN}The curl command returns the following results:${RESET}"
|
||||
bash model_curl.sh
|
||||
}
|
||||
|
||||
yellow_block_registration_curl(){
|
||||
echo -e "${CYAN}Start to execute yellow block registration and curl command${RESET}"
|
||||
if [ ! -d "$YELLOW_BLOCK_PATH" ]; then
|
||||
echo -e "${RED}Directory $YELLOW_BLOCK_PATH does not exist. Please deploy the start yellow block first.${RESET}"
|
||||
else
|
||||
echo -e "${GREEN}Directory $YELLOW_BLOCK_PATH exists.${RESET}"
|
||||
cd "$YELLOW_BLOCK_PATH"
|
||||
echo -e "${GREEN}Start to execute the yellow block registration${RESET}"
|
||||
bash yellow_block_register.sh
|
||||
echo -e "${GREEN}Start to execute the curl command to verify the yellow block${RESET}"
|
||||
cat yellow_block_curl.sh
|
||||
echo -e "${GREEN}The curl command returns the following results:${RESET}"
|
||||
bash yellow_block_curl.sh
|
||||
fi
|
||||
}
|
||||
|
||||
update_yellow_block_register_sh() {
|
||||
local script_file="yellow_block_register.sh"
|
||||
local yellow_block_conda_env=$1
|
||||
local model_server_ip=$2
|
||||
local model_port=$3
|
||||
|
||||
cd "$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
# 使用 sed 命令替换变量值
|
||||
sed -i "s|^export YELLOW_BLOCK_CONDA_ENV=.*|export YELLOW_BLOCK_CONDA_ENV=$yellow_block_conda_env|" "$script_file"
|
||||
sed -i "s|^export MODEL_SERVER_IP=.*|export MODEL_SERVER_IP=$model_server_ip|" "$script_file"
|
||||
sed -i "s|^export MODEL_PORT=.*|export MODEL_PORT=$model_port|" "$script_file"
|
||||
echo -e "${GREEN}Updated $script_file with the provided values.${RESET}"
|
||||
}
|
||||
|
||||
cp_registration_curl(){
|
||||
# Check if the Model IP and Yellow Block IP are the same
|
||||
if [ "$MODEL_SERVER_IP" == "$YELLOW_BLOCK_SERVER_IP" ]; then
|
||||
echo -e "\n${GREEN}Model IP and Yellow Block IP are the same. Using cp to copy files.${RESET}"
|
||||
update_yellow_block_register_sh $YELLOW_BLOCK_CONDA_ENV $MODEL_SERVER_IP $MODEL_PORT
|
||||
# Use cp to copy files, forcing overwrite
|
||||
cd "$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
cp -f yellow_block_* "$YELLOW_BLOCK_PATH"
|
||||
yellow_block_registration_curl
|
||||
else
|
||||
echo -e "\n${GREEN}Model IP and Yellow Block IP are different. Using scp to copy files.${RESET}"
|
||||
update_yellow_block_register_sh $YELLOW_BLOCK_CONDA_ENV $MODEL_SERVER_IP $MODEL_PORT
|
||||
# Use scp to copy files to the Yellow Block server
|
||||
cd "$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
scp -P "$YELLOW_BLOCK_SERVER_PORT" yellow_block_* "root@$YELLOW_BLOCK_SERVER_IP":"$YELLOW_BLOCK_PATH"
|
||||
ssh -p "$YELLOW_BLOCK_SERVER_PORT" "root@$YELLOW_BLOCK_SERVER_IP" << 'EOF'
|
||||
# Color definitions
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
CYAN='\033[36m'
|
||||
RESET='\033[0m'
|
||||
|
||||
YELLOW_BLOCK_PATH="/data/app/dev/ihp-model-ops/test/model_service_v2"
|
||||
|
||||
echo -e "${CYAN}Start to execute yellow block registration and curl command${RESET}"
|
||||
if [ ! -d "$YELLOW_BLOCK_PATH" ]; then
|
||||
echo -e "${RED}Directory $YELLOW_BLOCK_PATH does not exist. Please deploy the start yellow block first.${RESET}"
|
||||
else
|
||||
echo -e "${GREEN}Directory $YELLOW_BLOCK_PATH exists.${RESET}"
|
||||
cd "$YELLOW_BLOCK_PATH"
|
||||
echo -e "${GREEN}Start to execute the yellow block registration${RESET}"
|
||||
bash yellow_block_register.sh
|
||||
echo -e "${GREEN}Start to execute the curl command to verify the yellow block${RESET}"
|
||||
cat yellow_block_curl.sh
|
||||
echo -e "${GREEN}The curl command returns the following results:${RESET}"
|
||||
bash yellow_block_curl.sh
|
||||
fi
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# Main function
|
||||
main(){
|
||||
echo -e "${CYAN}======================$MODEL_NAME======================${RESET}"
|
||||
check_media_list
|
||||
if ! check_model_path; then
|
||||
deploy_model
|
||||
fi
|
||||
if ! check_conda_env; then
|
||||
deploy_conda_env
|
||||
fi
|
||||
if check_model_service_process; then
|
||||
echo -e "${GREEN}The model service process already exists, starting to stop the process.${RESET}"
|
||||
stop_model_service
|
||||
else
|
||||
echo -e "${GREEN}The model service process does not exist. Start to start the model service.${RESET}"
|
||||
fi
|
||||
sleep 5
|
||||
start_model_service
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${CYAN}The model service is starting...${RESET}"
|
||||
fi
|
||||
check_model_started
|
||||
model_curl_verification
|
||||
cp_registration_curl
|
||||
echo -e "\n${CYAN}======================$MODEL_NAME======================${RESET}"
|
||||
}
|
||||
|
||||
# Execute the main function
|
||||
main
|
||||
364
deploy_model_start_2.sh
Normal file
364
deploy_model_start_2.sh
Normal file
@ -0,0 +1,364 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Color definitions
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
CYAN='\033[36m'
|
||||
RESET='\033[0m'
|
||||
|
||||
:<<'COMMENT'
|
||||
检测介质下载路径中是否包含以下介质：模型权重文件和conda环境tar包，如果不存在则退出。
|
||||
检测服务器节点目标conda环境是否存在，如果不存在则创建。
|
||||
检测模型服务是否已启动，如果模型服务已启动，则终止模型服务。
|
||||
启动模型服务
|
||||
检查模型服务是否启动成功
|
||||
执行curl命令验证模型服务
|
||||
将黄块注册脚本和黄块curl命令脚本scp到黄块服务ip
|
||||
ssh登录黄块服务ip进行黄块注册，执行黄块curl命令
|
||||
COMMENT
|
||||
|
||||
# 定义帮助信息
|
||||
function print_help() {
|
||||
echo "Usage: $0"
|
||||
echo
|
||||
echo "This script reads configuration from config.yaml and performs the necessary setup."
|
||||
echo
|
||||
echo "Example:"
|
||||
echo " bash $0"
|
||||
}
|
||||
|
||||
# 加载 YAML 配置
|
||||
function load_config() {
|
||||
local config_file="$1"
|
||||
MODEL=$(yq eval '.model' "$config_file")
|
||||
MODEL_VERSION=$(yq eval '.model_version' "$config_file")
|
||||
CONDA_ENV=$(yq eval '.conda_env' "$config_file")
|
||||
TENSOR_PARALLEL_SIZE=$(yq eval '.tensor_parallel_size' "$config_file")
|
||||
VISIBLE_GPU_INDEX=$(yq eval '.visible_gpu_index' "$config_file")
|
||||
MODEL_PORT=$(yq eval '.model_port' "$config_file")
|
||||
GPU_MEMORY_UTILIZATION=$(yq eval '.gpu_memory_utilization' "$config_file")
|
||||
DTYPE=$(yq eval '.dtype' "$config_file")
|
||||
MODEL_SERVER_IP=$(yq eval '.model_server_ip' "$config_file")
|
||||
YELLOW_BLOCK_SERVER_IP=$(yq eval '.yellow_block_server_ip' "$config_file")
|
||||
YELLOW_BLOCK_SERVER_PORT=$(yq eval '.yellow_block_server_port' "$config_file")
|
||||
YELLOW_BLOCK_CONDA_ENV=$(yq eval '.yellow_block_conda_env' "$config_file")
|
||||
|
||||
MEDIA_DOWNLOAD_PATH=$(yq eval '.media_download_path' "$config_file")
|
||||
MODEL_PATH=$(yq eval '.model_path' "$config_file")
|
||||
MODEL_NAME=$(yq eval '.model_name' "$config_file" | sed "s|{{model}}|$MODEL|g" | sed "s|{{model_version}}|$MODEL_VERSION|g")
|
||||
MODEL_NAME_PATH=$(yq eval '.model_name_path' "$config_file" | sed "s|{{model_path}}|$MODEL_PATH|g" | sed "s|{{model_name}}|$MODEL_NAME|g")
|
||||
MODEL_TAR=$(yq eval '.model_tar' "$config_file" | sed "s|{{model_name}}|$MODEL_NAME|g")
|
||||
MINICONDA_PATH=$(yq eval '.miniconda_path' "$config_file")
|
||||
YELLOW_BLOCK_PATH=$(yq eval '.yellow_block_path' "$config_file")
|
||||
AUTOMATED_DEPLOYMENT_PATH=$(yq eval '.automated_deployment_path' "$config_file")
|
||||
}
|
||||
|
||||
# 检查配置文件是否存在
|
||||
cd /data/jenkins_script/automated_deployment
|
||||
if [ ! -f "config.yaml" ]; then
|
||||
echo "Error: config.yaml not found in the current directory."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 加载配置
|
||||
load_config "config.yaml"
|
||||
|
||||
# 打印参数值（调试用）
|
||||
echo "MODEL: $MODEL"
|
||||
echo "MODEL_VERSION: $MODEL_VERSION"
|
||||
echo "MODEL_NAME: $MODEL_NAME"
|
||||
echo "MODEL_NAME_PATH: $MODEL_NAME_PATH"
|
||||
echo "MODEL_TAR: $MODEL_TAR"
|
||||
echo "CONDA_ENV: $CONDA_ENV"
|
||||
echo "TENSOR_PARALLEL_SIZE: $TENSOR_PARALLEL_SIZE"
|
||||
echo "VISIBLE_GPU_INDEX: $VISIBLE_GPU_INDEX"
|
||||
echo "MODEL_PORT: $MODEL_PORT"
|
||||
echo "GPU_MEMORY_UTILIZATION: $GPU_MEMORY_UTILIZATION"
|
||||
echo "DTYPE: $DTYPE"
|
||||
echo "MODEL_SERVER_IP: $MODEL_SERVER_IP"
|
||||
echo "YELLOW_BLOCK_SERVER_IP: $YELLOW_BLOCK_SERVER_IP"
|
||||
echo "YELLOW_BLOCK_SERVER_PORT: $YELLOW_BLOCK_SERVER_PORT"
|
||||
echo "YELLOW_BLOCK_CONDA_ENV: $YELLOW_BLOCK_CONDA_ENV"
|
||||
echo "MEDIA_DOWNLOAD_PATH: $MEDIA_DOWNLOAD_PATH"
|
||||
echo "MODEL_PATH: $MODEL_PATH"
|
||||
echo "MINICONDA_PATH: $MINICONDA_PATH"
|
||||
echo "YELLOW_BLOCK_PATH: $YELLOW_BLOCK_PATH"
|
||||
echo "AUTOMATED_DEPLOYMENT_PATH: $AUTOMATED_DEPLOYMENT_PATH"
|
||||
|
||||
check_and_enter_script() {
|
||||
local script_name="$1"
|
||||
local path1="$MINICONDA_PATH/$CONDA_ENV/script/$MODEL"
|
||||
local path2="$AUTOMATED_DEPLOYMENT_PATH"
|
||||
|
||||
# 检查第一个路径是否存在脚本
|
||||
if [ -f "$path1/$script_name" ]; then
|
||||
cd "$path1" || { echo "无法进入目录 $path1"; return 1; }
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 检查第二个路径是否存在脚本
|
||||
if [ -f "$path2/$script_name" ]; then
|
||||
cd "$path2" || { echo "无法进入目录 $path2"; return 1; }
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 如果两个路径都不存在脚本
|
||||
return 1
|
||||
}
|
||||
|
||||
# 检查介质下载路径中是否包含以下介质：模型权重文件和conda环境tar包。
|
||||
check_media_list() {
|
||||
echo -e "${CYAN}Checking media files...${RESET}"
|
||||
cd "$MEDIA_DOWNLOAD_PATH"
|
||||
if [ -f "$MODEL_TAR" ]; then
|
||||
echo -e "${GREEN}The media file $MEDIA_DOWNLOAD_PATH/$MODEL_TAR exists.${RESET}"
|
||||
else
|
||||
echo -e "${RED}The media file $MEDIA_DOWNLOAD_PATH/$MODEL_TAR does not exist.${RESET}"
|
||||
fi
|
||||
if [ -f "$CONDA_ENV.tar.gz" ]; then
|
||||
echo -e "${GREEN}The media file $MEDIA_DOWNLOAD_PATH/$CONDA_ENV.tar.gz exists.${RESET}"
|
||||
else
|
||||
echo -e "${RED}The media file $MEDIA_DOWNLOAD_PATH/$CONDA_ENV.tar.gz does not exist.${RESET}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 检查服务器节点中模型权重文件是否存在
|
||||
check_model_path() {
|
||||
cd "$MODEL_PATH"
|
||||
if [ ! -d "$MODEL_NAME" ]; then
|
||||
echo -e "${RED}The model $MODEL_NAME does not exist.${RESET}"
|
||||
return 1
|
||||
else
|
||||
echo -e "${GREEN}The model $MODEL_NAME exists.${RESET}"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# 部署模型
|
||||
deploy_model() {
|
||||
echo -e "${CYAN}Start deploying model.${RESET}"
|
||||
cd "$MODEL_PATH"
|
||||
cp "$MEDIA_DOWNLOAD_PATH/$MODEL_TAR" ./
|
||||
tar -zxf "$MODEL_TAR"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Model deployed successfully.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Model deployment failed.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
ll "$MODEL_NAME"
|
||||
chown -R inspur:inspur "$MODEL_NAME"
|
||||
rm -rf "$MODEL_TAR"
|
||||
}
|
||||
|
||||
# 检查服务器节点中conda环境是否存在
|
||||
check_conda_env() {
|
||||
cd "$MINICONDA_PATH"
|
||||
if [ ! -d "$CONDA_ENV" ]; then
|
||||
echo -e "${RED}The conda environment $CONDA_ENV does not exist.${RESET}"
|
||||
return 1
|
||||
else
|
||||
echo -e "${GREEN}The conda environment $CONDA_ENV exists.${RESET}"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# 将介质下载路径中的conda环境压缩包解压至/data/miniconda3/envs/$CONDA_ENV目录下，修改用户和用户组。
|
||||
deploy_conda_env() {
|
||||
cd "$MINICONDA_PATH"
|
||||
mkdir "$CONDA_ENV"
|
||||
cd "$MEDIA_DOWNLOAD_PATH"
|
||||
tar -zxf "$CONDA_ENV.tar.gz" -C "$MINICONDA_PATH/$CONDA_ENV"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}The environment $CONDA_ENV has been created.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Failed to create environment $CONDA_ENV.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
chown -R inspur:inspur "$MINICONDA_PATH/$CONDA_ENV"
|
||||
}
|
||||
|
||||
# 调用conda环境中的check.sh脚本，检查模型服务进程
|
||||
check_model_service_process() {
|
||||
check_and_enter_script "check.sh"
|
||||
check_result=$(bash check.sh ${MODEL})
|
||||
# 检查输出内容
|
||||
if [[ "$check_result" == *"SUCC"* ]]; then
|
||||
return 0
|
||||
elif [[ "$check_result" == *"FAIL"* ]]; then
|
||||
return 1
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 调用业务代码中的stop.sh脚本，停止模型服务
|
||||
stop_model_service() {
|
||||
check_and_enter_script "stop.sh"
|
||||
bash stop.sh ${MODEL} > /dev/null 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}$MODEL_NAME service stopped successfully.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Failed to stop $MODEL_NAME service.${RESET}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 激活conda环境，调用业务代码中的start.sh脚本，启动模型服务
|
||||
start_model_service(){
|
||||
check_and_enter_script "start.sh"
|
||||
bash "start.sh" ${VISIBLE_GPU_INDEX} ${CONDA_ENV} ${MODEL} ${MODEL_NAME_PATH} ${TENSOR_PARALLEL_SIZE} ${MODEL_SERVER_IP} ${MODEL_PORT} ${GPU_MEMORY_UTILIZATION} ${DTYPE}
|
||||
}
|
||||
|
||||
# 轮询检测模型服务是否已启动成功
|
||||
check_model_started() {
|
||||
local start_time=$(date +%s)
|
||||
local timeout=300
|
||||
local interval=5
|
||||
while true; do
|
||||
sleep $interval
|
||||
if check_model_service_process; then
|
||||
echo -e "${GREEN}$MODEL_NAME service started successfully.${RESET}"
|
||||
break
|
||||
fi
|
||||
local current_time=$(date +%s)
|
||||
local elapsed_time=$(($current_time - $start_time))
|
||||
if [ $elapsed_time -ge $timeout ]; then
|
||||
echo -e "${RED}Failed to start $MODEL_NAME service.${RESET}"
|
||||
echo -e "${RED}Please check the log under $MINICONDA_PATH/$CONDA_ENV/script/$MODEL${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
update_model_curl_sh() {
|
||||
local script_file="model_curl.sh"
|
||||
local model_server_ip=$1
|
||||
local model_port=$2
|
||||
local model=$3
|
||||
|
||||
check_and_enter_script $script_file
|
||||
# 使用 sed 命令替换变量值
|
||||
sed -i "s|^export MODEL_SERVER_IP=.*|export MODEL_SERVER_IP=$model_server_ip|" "$script_file"
|
||||
sed -i "s|^export MODEL_PORT=.*|export MODEL_PORT=$model_port|" "$script_file"
|
||||
sed -i "s|^export MODEL=.*|export MODEL=$model|" "$script_file"
|
||||
echo -e "${GREEN}Updated $script_file with the provided values.${RESET}"
|
||||
}
|
||||
|
||||
# 验证模型服务
|
||||
model_curl_verification(){
|
||||
echo -e "${GREEN}Start to execute the curl command to verify the model${RESET}"
|
||||
update_model_curl_sh $MODEL_SERVER_IP $MODEL_PORT $MODEL
|
||||
# check_and_enter_script "model_curl.sh"
|
||||
cat model_curl.sh
|
||||
echo -e "${GREEN}The curl command returns the following results:${RESET}"
|
||||
bash model_curl.sh
|
||||
}
|
||||
|
||||
yellow_block_registration_curl(){
|
||||
echo -e "${CYAN}Start to execute yellow block registration and curl command${RESET}"
|
||||
if [ ! -d "$YELLOW_BLOCK_PATH" ]; then
|
||||
echo -e "${RED}Directory $YELLOW_BLOCK_PATH does not exist. Please deploy the start yellow block first.${RESET}"
|
||||
else
|
||||
echo -e "${GREEN}Directory $YELLOW_BLOCK_PATH exists.${RESET}"
|
||||
cd "$YELLOW_BLOCK_PATH"
|
||||
echo -e "${GREEN}Start to execute the yellow block registration${RESET}"
|
||||
bash yellow_block_register.sh
|
||||
echo -e "${GREEN}Start to execute the curl command to verify the yellow block${RESET}"
|
||||
cat yellow_block_curl.sh
|
||||
echo -e "${GREEN}The curl command returns the following results:${RESET}"
|
||||
bash yellow_block_curl.sh
|
||||
fi
|
||||
}
|
||||
|
||||
update_yellow_block_register_sh() {
|
||||
local script_file="yellow_block_register.sh"
|
||||
local yellow_block_conda_env=$1
|
||||
local model_server_ip=$2
|
||||
local model_port=$3
|
||||
local model=$4
|
||||
|
||||
check_and_enter_script $script_file
|
||||
# 使用 sed 命令替换变量值
|
||||
sed -i "s|^export YELLOW_BLOCK_CONDA_ENV=.*|export YELLOW_BLOCK_CONDA_ENV=$yellow_block_conda_env|" "$script_file"
|
||||
sed -i "s|^export MODEL_SERVER_IP=.*|export MODEL_SERVER_IP=$model_server_ip|" "$script_file"
|
||||
sed -i "s|^export MODEL_PORT=.*|export MODEL_PORT=$model_port|" "$script_file"
|
||||
sed -i "s|^export MODEL=.*|export MODEL=$model|" "$script_file"
|
||||
# echo -e "${GREEN}Updated $script_file with the provided values.${RESET}"
|
||||
}
|
||||
|
||||
update_yellow_block_curl_sh() {
|
||||
local script_file="yellow_block_curl.sh"
|
||||
local model=$1
|
||||
|
||||
check_and_enter_script $script_file
|
||||
# 使用 sed 命令替换变量值
|
||||
sed -i "s|^export MODEL=.*|export MODEL=$model|" "$script_file"
|
||||
# echo -e "${GREEN}Updated $script_file with the provided values.${RESET}"
|
||||
}
|
||||
|
||||
cp_registration_curl(){
|
||||
update_yellow_block_register_sh $YELLOW_BLOCK_CONDA_ENV $MODEL_SERVER_IP $MODEL_PORT $MODEL
|
||||
update_yellow_block_curl_sh $MODEL
|
||||
# Check if the Model IP and Yellow Block IP are the same
|
||||
if [ "$MODEL_SERVER_IP" == "$YELLOW_BLOCK_SERVER_IP" ]; then
|
||||
echo -e "\n${GREEN}Model IP and Yellow Block IP are the same. Using cp to copy files.${RESET}"
|
||||
cp -f yellow_block_* "$YELLOW_BLOCK_PATH"
|
||||
yellow_block_registration_curl
|
||||
else
|
||||
echo -e "\n${GREEN}Model IP and Yellow Block IP are different. Using scp to copy files.${RESET}"
|
||||
scp -P "$YELLOW_BLOCK_SERVER_PORT" yellow_block_* "root@$YELLOW_BLOCK_SERVER_IP":"$YELLOW_BLOCK_PATH"
|
||||
ssh -p "$YELLOW_BLOCK_SERVER_PORT" "root@$YELLOW_BLOCK_SERVER_IP" << 'EOF'
|
||||
# Color definitions
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
CYAN='\033[36m'
|
||||
RESET='\033[0m'
|
||||
|
||||
YELLOW_BLOCK_PATH="/data/app/dev/ihp-model-ops/test/model_service_v2"
|
||||
|
||||
echo -e "${CYAN}Start to execute yellow block registration and curl command${RESET}"
|
||||
if [ ! -d "$YELLOW_BLOCK_PATH" ]; then
|
||||
echo -e "${RED}Directory $YELLOW_BLOCK_PATH does not exist. Please deploy the start yellow block first.${RESET}"
|
||||
else
|
||||
echo -e "${GREEN}Directory $YELLOW_BLOCK_PATH exists.${RESET}"
|
||||
cd "$YELLOW_BLOCK_PATH"
|
||||
echo -e "${GREEN}Start to execute the yellow block registration${RESET}"
|
||||
bash yellow_block_register.sh
|
||||
echo -e "${GREEN}Start to execute the curl command to verify the yellow block${RESET}"
|
||||
cat yellow_block_curl.sh
|
||||
echo -e "${GREEN}The curl command returns the following results:${RESET}"
|
||||
bash yellow_block_curl.sh
|
||||
fi
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# Main function
|
||||
main(){
|
||||
echo -e "${CYAN}======================$MODEL_NAME======================${RESET}"
|
||||
check_media_list
|
||||
if ! check_model_path; then
|
||||
deploy_model
|
||||
fi
|
||||
if ! check_conda_env; then
|
||||
deploy_conda_env
|
||||
fi
|
||||
if check_model_service_process; then
|
||||
echo -e "${GREEN}The model service process already exists, starting to stop the process.${RESET}"
|
||||
stop_model_service
|
||||
else
|
||||
echo -e "${GREEN}The model service process does not exist. Start to start the model service.${RESET}"
|
||||
fi
|
||||
sleep 5
|
||||
start_model_service
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${CYAN}The model service is starting...${RESET}"
|
||||
fi
|
||||
check_model_started
|
||||
model_curl_verification
|
||||
cp_registration_curl
|
||||
echo -e "\n${CYAN}======================$MODEL_NAME======================${RESET}"
|
||||
}
|
||||
|
||||
# Execute the main function
|
||||
main
|
||||
309
deploy_red_agent_service.sh
Normal file
309
deploy_red_agent_service.sh
Normal file
@ -0,0 +1,309 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Color definitions
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
CYAN='\033[36m'
|
||||
RESET='\033[0m'
|
||||
|
||||
:<<'COMMENT'
|
||||
通用红块场景自动化部署脚本
|
||||
检测介质下载路径中是否包含以下介质：代码包和conda环境tar包，如果不存在则退出。
|
||||
检测服务器节点目标conda环境是否存在，如果不存在则创建。
|
||||
检测红块场景代码是否存在。
|
||||
检测红块场景服务是否已启动，如果红块场景服务已启动，则终止红块场景服务。
|
||||
备份红块场景代码
|
||||
部署场景代码
|
||||
启动场景代码
|
||||
检查场景代码是否启动成功
|
||||
执行curl命令验证场景代码
|
||||
# Define the directory path for the red block scene
|
||||
MEDIA_DOWNLOAD_PATH="/data/media"
|
||||
RED_PATH="/data/redserver/red-agent-service"
|
||||
RED_SCENE=$1 # 场景代码名称 外部传参
|
||||
RED_SCENE_PATH="$RED_PATH/$RED_SCENE"
|
||||
RED_SCENE_VERSION=$2 # 场景代码版本 外部传参
|
||||
RED_SCENE_TAR="$RED_SCENE-$RED_SCENE_VERSION.tar.gz"
|
||||
MINICONDA_PATH="/data/miniconda3/envs/"
|
||||
CONDA_ENV=$3 # conda环境名称 外部传参
|
||||
COMMENT
|
||||
|
||||
# 定义帮助信息
|
||||
function print_help() {
|
||||
echo "Usage: $0 [options]"
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -h, --help 显示此帮助信息"
|
||||
echo " --red-scene <scene> 场景代码名称，例如 'scene1'"
|
||||
echo " --red-scene-version <version> 场景代码版本，例如 'V1.0.0.0'"
|
||||
echo " --conda-env <conda_env> Conda环境名称，例如 'conda-scene1-V1.0.0.0'"
|
||||
echo
|
||||
echo "Example:"
|
||||
echo " bash $0 --red-scene 'scene1' --red-scene-version 'V1.0.0.0' --conda-env 'conda-scene1-V1.0.0.0'"
|
||||
}
|
||||
|
||||
# 初始化变量
|
||||
MEDIA_DOWNLOAD_PATH="/data/media"
|
||||
RED_PATH="/data/redserver/red-agent-service"
|
||||
RED_SCENE=""
|
||||
RED_SCENE_PATH=""
|
||||
RED_SCENE_VERSION=""
|
||||
RED_SCENE_TAR=""
|
||||
MINICONDA_PATH="/data/miniconda3/envs/"
|
||||
CONDA_ENV=""
|
||||
|
||||
# 检查是否有参数
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "No arguments provided. Use -h or --help for help."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 处理参数
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
print_help
|
||||
exit 0
|
||||
;;
|
||||
--red-scene)
|
||||
shift
|
||||
RED_SCENE=$1
|
||||
RED_SCENE_PATH="$RED_PATH/$RED_SCENE"
|
||||
shift
|
||||
;;
|
||||
--red-scene-version)
|
||||
shift
|
||||
RED_SCENE_VERSION=$1
|
||||
RED_SCENE_TAR="$RED_SCENE-$RED_SCENE_VERSION.tar.gz"
|
||||
shift
|
||||
;;
|
||||
--conda-env)
|
||||
shift
|
||||
CONDA_ENV=$1
|
||||
shift
|
||||
;;
|
||||
*) # 未知选项
|
||||
echo "未知选项: $1"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# 打印参数值（调试用）
|
||||
echo "MEDIA_DOWNLOAD_PATH: $MEDIA_DOWNLOAD_PATH"
|
||||
echo "RED_PATH: $RED_PATH"
|
||||
echo "RED_SCENE: $RED_SCENE"
|
||||
echo "RED_SCENE_PATH: $RED_SCENE_PATH"
|
||||
echo "RED_SCENE_VERSION: $RED_SCENE_VERSION"
|
||||
echo "RED_SCENE_TAR: $RED_SCENE_TAR"
|
||||
echo "MINICONDA_PATH: $MINICONDA_PATH"
|
||||
echo "CONDA_ENV: $CONDA_ENV"
|
||||
|
||||
# 检查介质下载路径中是否包含以下介质：代码包和conda环境tar包，如果不存在则退出。
|
||||
check_media_list() {
|
||||
echo -e "${CYAN}Checking media files...${RESET}"
|
||||
cd "$MEDIA_DOWNLOAD_PATH"
|
||||
if [ -f "$RED_SCENE_TAR" ]; then
|
||||
echo -e "${GREEN}The media file $RED_SCENE_TAR exists.${RESET}"
|
||||
else
|
||||
echo -e "${RED}The media file $RED_SCENE_TAR does not exist.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
if [ -f "$CONDA_ENV.tar.gz" ]; then
|
||||
echo -e "${GREEN}The media file $CONDA_ENV.tar.gz exists.${RESET}"
|
||||
else
|
||||
echo -e "${RED}The media file $CONDA_ENV.tar.gz does not exist.${RESET}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 如果当前服务器节点中已存在红块场景代码，则备份红块场景代码。
|
||||
backup_red_block() {
|
||||
cd "$RED_PATH"
|
||||
local current_time=$(date +"%Y%m%d%H%M%S")
|
||||
if [ ! -d "$RED_SCENE-bak" ]; then
|
||||
mkdir "$RED_SCENE-bak"
|
||||
fi
|
||||
mv "$RED_SCENE" "$RED_SCENE-bak/$RED_SCENE-$current_time"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Red block code backup succeeded.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Red block code backup failed.${RESET}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 将介质下载路径中的红块场景代码压缩包拷贝至/data/redserver/red-agent-service目录下，并解压，重命名，修改用户和用户组。
|
||||
deploy_red_block() {
|
||||
echo -e "${CYAN}Start deploying code.${RESET}"
|
||||
cd "$RED_PATH"
|
||||
cp "$MEDIA_DOWNLOAD_PATH/$RED_SCENE_TAR" ./
|
||||
tar -zxf "$RED_SCENE_TAR"
|
||||
mv "$RED_SCENE-$RED_SCENE_VERSION" "$RED_SCENE"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Red block code deployed successfully.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Red block code deployment failed.${RESET}"
|
||||
fi
|
||||
chown -R inspur:inspur "$RED_SCENE_PATH"
|
||||
rm -rf "$RED_SCENE_TAR"
|
||||
}
|
||||
|
||||
# 检查服务器节点中conda环境是否存在
|
||||
check_conda_env() {
|
||||
cd "$MINICONDA_PATH"
|
||||
if [ ! -d "$CONDA_ENV" ]; then
|
||||
echo -e "${RED}The environment $CONDA_ENV does not exist.${RESET}"
|
||||
return 1
|
||||
else
|
||||
echo -e "${GREEN}The environment $CONDA_ENV exists.${RESET}"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# 将介质下载路径中的conda环境压缩包解压至/data/miniconda3/envs/$CONDA_ENV目录下，修改用户和用户组。
|
||||
deploy_conda_env() {
|
||||
cd "$MINICONDA_PATH"
|
||||
mkdir "$CONDA_ENV"
|
||||
cd "$MEDIA_DOWNLOAD_PATH"
|
||||
tar -zxf "$CONDA_ENV.tar.gz" -C "$MINICONDA_PATH/$CONDA_ENV"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}The environment $CONDA_ENV has been created.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Failed to create environment $CONDA_ENV.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
chown -R inspur:inspur "$MINICONDA_PATH/$CONDA_ENV"
|
||||
}
|
||||
|
||||
# 调用业务代码中的check.sh脚本，检查红块服务是否已启动成功。
|
||||
check_red_block_service_process() {
|
||||
cd "$RED_SCENE_PATH"
|
||||
check_result=$(bash check.sh)
|
||||
# 检查输出内容
|
||||
if [[ "$check_result" == *"SUCC"* ]]; then
|
||||
return 0
|
||||
elif [[ "$check_result" == *"FAIL"* ]]; then
|
||||
return 1
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 调用业务代码中的stop.sh脚本，终止红块服务。
|
||||
stop_red_block() {
|
||||
cd "$RED_SCENE_PATH"
|
||||
bash stop.sh > /dev/null 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Red block service stopped successfully.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Failed to stop red block service.${RESET}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 激活conda环境，调用业务代码中的start.sh脚本，启动红块服务。
|
||||
start_red_block() {
|
||||
source /etc/profile
|
||||
echo -e "${CYAN}HAIRUO_ENV=$HAIRUO_ENV${RESET}"
|
||||
su - inspur << EOF
|
||||
source /data/miniconda3/etc/profile.d/conda.sh
|
||||
conda activate $CONDA_ENV
|
||||
cd "$RED_SCENE_PATH"
|
||||
bash start.sh
|
||||
EOF
|
||||
}
|
||||
|
||||
# 轮询检测红块服务是否已启动成功
|
||||
check_red_block_started() {
|
||||
local start_time=$(date +%s)
|
||||
local timeout=120
|
||||
local interval=5
|
||||
while true; do
|
||||
sleep $interval
|
||||
if check_red_block_service_process; then
|
||||
echo -e "${GREEN}Red block service started successfully.${RESET}"
|
||||
break
|
||||
fi
|
||||
local current_time=$(date +%s)
|
||||
local elapsed_time=$(($current_time - $start_time))
|
||||
if [ $elapsed_time -ge $timeout ]; then
|
||||
echo -e "${RED}Failed to start red block service.${RESET}"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# 回滚红块场景代码，暂时未用到
|
||||
rollback_red_block(){
|
||||
cd "$RED_PATH"
|
||||
if [ ! -d "$RED_SCENE-bak" ]; then
|
||||
echo -e "${RED}The backup directory does not exist and cannot be rolled back${RESET}"
|
||||
else
|
||||
echo -e "${GREEN}The backup directory exists, start rollback${RESET}"
|
||||
rm -rf "$RED_SCENE"
|
||||
local latest_backup=$(ls -td $RED_SCENE-bak/$RED_SCENE-* | head -1)
|
||||
mv "$latest_backup" "$RED_SCENE"
|
||||
fi
|
||||
}
|
||||
|
||||
# 调用业务代码中的curl.sh脚本，验证红块场景代码
|
||||
red_block_curl_verification(){
|
||||
echo -e "${GREEN}Start to execute the curl command to verify the red block${RESET}"
|
||||
cd "$RED_SCENE_PATH"
|
||||
cat curl.sh
|
||||
echo -e "${GREEN}The curl command returns the following results:${RESET}"
|
||||
bash curl.sh
|
||||
}
|
||||
|
||||
# Main function
|
||||
main() {
|
||||
echo -e "${CYAN}======================$RED_SCENE======================${RESET}"
|
||||
check_media_list
|
||||
if ! check_conda_env; then
|
||||
deploy_conda_env
|
||||
fi
|
||||
if [ -d "$RED_PATH" ]; then
|
||||
if [ -d "$RED_SCENE_PATH" ]; then
|
||||
echo -e "${GREEN}Directory $RED_SCENE_PATH exists.${RESET}"
|
||||
if check_red_block_service_process; then
|
||||
echo -e "${GREEN}Red block service process already exists, starting to stop the process.${RESET}"
|
||||
stop_red_block
|
||||
else
|
||||
echo -e "${RED}Red block service process does not exist. Start to start the model service.${RESET}"
|
||||
fi
|
||||
backup_red_block
|
||||
else
|
||||
echo -e "${RED}Directory $RED_SCENE_PATH does not exist.${RESET}"
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}Directory $RED_PATH does not exist.${RESET}"
|
||||
mkdir -p "$RED_PATH"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Directory $RED_PATH created successfully.${RESET}"
|
||||
else
|
||||
echo -e "${RED}Failed to create directory $RED_PATH.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
deploy_red_block
|
||||
start_red_block
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Red block service is starting.${RESET}"
|
||||
fi
|
||||
check_red_block_started
|
||||
:<<'COMMENT'
|
||||
if [ $? -eq 1 ]; then
|
||||
echo -e "${RED}Start executing rollback function.${RESET}"
|
||||
rollback_red_block
|
||||
start_red_block
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Red block service is starting.${RESET}"
|
||||
fi
|
||||
check_red_block_started
|
||||
fi
|
||||
COMMENT
|
||||
red_block_curl_verification
|
||||
echo -e "\n${CYAN}======================$RED_SCENE======================${RESET}"
|
||||
}
|
||||
|
||||
# Execute the main function
|
||||
main
|
||||
63
deploy_s3cmd_download.sh
Normal file
63
deploy_s3cmd_download.sh
Normal file
@ -0,0 +1,63 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 颜色定义
|
||||
CYAN="\033[1;36m"
|
||||
RED="\033[0;31m"
|
||||
GREEN="\033[1;32m"
|
||||
RESET="\033[0m"
|
||||
|
||||
:<<'COMMENT'
|
||||
输入OSS内部桶介质存储路径列表
|
||||
自动识别目录或文件
|
||||
将其同步到本地
|
||||
COMMENT
|
||||
|
||||
# 定义本地存储路径
|
||||
LOCAL_PATH="/data/media"
|
||||
mkdir -p "$LOCAL_PATH" # 确保本地路径存在
|
||||
|
||||
# OSS内部桶和外部桶的配置文件
|
||||
INTERNAL_S3CFG=~/.s3cfg-internal
|
||||
|
||||
# 检查配置文件是否存在
|
||||
if [ ! -f "$INTERNAL_S3CFG" ]; then
|
||||
echo -e "${RED}Internal S3 configuration file not found.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查是否传入了OSS路径参数
|
||||
if [ $# -eq 0 ]; then
|
||||
echo -e "${RED}No OSS paths provided. Please provide OSS paths as arguments.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 遍历OSS路径列表
|
||||
for oss_path in "$@"; do
|
||||
echo -e "${CYAN}Processing OSS path: $oss_path${RESET}"
|
||||
|
||||
# 检查路径是否有效
|
||||
if [[ ! $oss_path =~ ^s3:// ]]; then
|
||||
echo -e "${RED}Skipping invalid OSS path: $oss_path${RESET}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# 提取目标路径的本地路径
|
||||
local_target="$LOCAL_PATH/$(basename "$oss_path")"
|
||||
if [[ $oss_path =~ /$ ]]; then
|
||||
# 如果是目录，确保尾部有斜杠
|
||||
local_target="$LOCAL_PATH/$(basename "$oss_path")/"
|
||||
mkdir -p "$local_target" # 确保本地目录存在
|
||||
fi
|
||||
|
||||
# 使用 s3cmd sync 同步文件或目录
|
||||
echo -e "${GREEN}Syncing $oss_path to local path $local_target${RESET}"
|
||||
s3cmd -c "$INTERNAL_S3CFG" sync "$oss_path" "$local_target" # | grep "Downloaded"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}Successfully synced $oss_path to $local_target${RESET}"
|
||||
else
|
||||
echo -e "${RED}Error: Failed to sync $oss_path.${RESET}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo -e "${GREEN}Script execution completed.${RESET}"
|
||||
Loading…
Reference in New Issue
Block a user