增加音频驱动服务

configs/cfg.yml

@@ -1,10 +1,10 @@
 # 数字人
 dighthuman:
      dev:
-         dh_webui: http://100.200.128.72:14040/agentstore/api/v1/multimodal_models/dh/dighthuman
+         dh_webui: http://100.200.128.72:14041/agentstore/api/v1/multimodal_models/dh/dighthuman
      test:
-         dh_webui: http://100.200.128.72:14040/agentstore/api/v1/multimodal_models/dh/dighthuman
+         dh_webui: http://100.200.128.72:14041/agentstore/api/v1/multimodal_models/dh/dighthuman
      prod:
-         dh_webui: http://100.200.128.72:14040/agentstore/api/v1/multimodal_models/dh/dighthuman
+         dh_webui: http://100.200.128.72:14041/agentstore/api/v1/multimodal_models/dh/dighthuman
      model_name: HaiRuo-AudioVisual-ST
      static_token: 7c3eafb5-2d6e-100d-ab0f-7b2c1cdafb3c

sadtalker-server/sadtalker.conf.py

@@ -0,0 +1,30 @@
+    # -*- coding: utf-8 -*-
+
+import os
+
+path_of_current_file = os.path.abspath(__file__)
+path_of_current_dir = os.path.split(path_of_current_file)[0]
+
+# worker_class为sync会报错
+# uvicorn.workers.UvicornWorker
+worker_class = 'uvicorn.workers.UvicornWorker'
+# workers = multiprocessing.cpu_count() * 2 + 1
+workers = 1     # 按需启动的进程数
+threads = 1     # 各进程包含的线程数
+
+chdir = path_of_current_dir
+
+worker_connections = 0
+timeout = 0
+max_requests = 0
+graceful_timeout = 0
+
+loglevel = 'info'
+access_log_format = '%(t)s %(p)s %(h)s "%(r)s" %(s)s %(L)s %(b)s %(f)s" "%(a)s"'
+reload = True
+debug = False
+bind = "%s:%s" % ("0.0.0.0", 14041)
+pidfile = '%s/sadtalker.pid' % (path_of_current_dir)
+errorlog = '%s/logs/sadtalker.log' % (path_of_current_dir)
+accesslog = '%s/logs/sadtalker_access.log' % (path_of_current_dir)
+proc_name = "sadtalker_api"

sadtalker-server/sadtalker_server.py

@@ -0,0 +1,84 @@
+# -*- encoding: utf-8 -*-
+'''
+@Email   :   liaoxiju@inspur.com
+'''
+import os
+import re
+import sys
+import uvicorn
+from fastapi import FastAPI, Request
+from modelscope.pipelines import pipeline
+import requests
+
+app = FastAPI()
+
+#--env MODEL_PATH="/data/models/HaiRuo-3B-SDXL-Base-TypeB-V1.0.0.0"
+
+model_path = os.getenv('MODEL_PATH','./wwd123/sadtalker')
+
+inference = pipeline('talking-head', model=model_path, model_revision='v1.0.0')
+
+def upload_to_s3(filepath):
+    os.system(f's3cmd put --expiry-days 10 {filepath} -r s3://ihp/tmp/liao/service/sadtalker/results/')
+
+def download_wav(wav_url):
+    try:
+        res = requests.get(wav_url)
+        content = res.content
+        tmp_dir = tempfile.TemporaryDirectory(dir="results").name
+        if not os.path.exists(tmp_dir):
+            os.system(f"mkdir -p {tmp_dir}")
+        wav_path = os.path.join(tmp_dir, "input_audio.wav")
+        fout = open(wav_path, "wb")
+        fout.write(content)
+        fout.close()
+        return wav_path, tmp_dir
+    except:
+        import traceback
+        print("download error", traceback.format_exc())
+        return None
+
+#@app.post("/hairuo/audiodrivenvido")
+@app.post("/agentstore/api/v1/multimodal_models/dh/dighthuman")
+async def digithuman(request: Request, req: dict):
+    '''
+    获取音频url-wav音频，然后使用sadtalker
+    '''
+    body_data = await request.body()
+    body_data = body_data.decode("utf-8")
+    try:   
+        save_path = "results"
+        if not os.path.exists(save_path):
+            os.system(f"mkdir -p {save_path}")
+
+        audio_url = req.get("audio_url")
+        wav_path, tmp_dir = download_wav(audio_url)
+        source_image = 'nv.jpg'
+        
+        video_path = inference(source_image, driven_audio=wav_path, **kwargs)
+
+        upload_to_s3(video_path)
+
+        ##清除临时文件
+        os.system(f"rm -rf {tmp_dir}")
+        os.system(f"rm {video_path}")
+
+        video_url = os.path.join("https://ihp.oss.cn-north-4.inspurcloudoss.com/tmp/liao/service/sadtalker/", video_path)
+        resp = {
+            "code": "0",
+            "message": "success",
+            "result": video_url
+        }
+        return resp
+        
+    except:
+        resp = {     
+                     "code": "1",
+                     "message": "audiodrivenvido model call error",
+                     "result": ""
+                    }
+        return resp
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=14041)

sadtalker-server/start_sadtalker.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+sconfig_common
+
+# 检查命令是否存在
+if npu-smi >/dev/null 2>&1; then
+    echo "npu-smi命令存在, 使用ASCEND_RT_VISIBLE_DEVICES执行相关操作..."
+    # 在这里执行需要的操作
+    ASCEND_RT_VISIBLE_DEVICES=1 gunicorn --env MODEL_PATH="./wwd123/sadtalker" sadtalker_server:app -n sadtalker_api -c sadtalker.conf.py --daemon
+    exit 0
+else
+    echo "npu-smi命令不存在, 跳过该部分脚本。"
+fi
+
+# 检查命令是否存在
+if cnmon >/dev/null 2>&1; then
+    echo "cnmon命令存在, 使用MLU_VISIBLE_DEVICES执行相关操作..."
+    # 在这里执行需要的操作
+    MLU_VISIBLE_DEVICES=1 gunicorn --env MODEL_PATH="./wwd123/sadtalker" sadtalker:app -n sadtalker_api -c sadtalker.conf.py --daemon
+    exit 0
+else
+    echo "cnmon命令不存在, 跳过该部分脚本。"
+fi
+
+CUDA_VISIBLE_DEVICES=1 gunicorn --env MODEL_PATH="./wwd123/sadtalker" sadtalker_server:app -n sadtalker_api -c sadtalker.conf.py --daemon

sadtalker-server/stop_sadtalker.sh

@@ -0,0 +1 @@
+ps -ef | grep sadtalker_api | grep -v grep | awk '{print $2}'| xargs kill -9