From 45efdc994da33947ccfb388842a94328fabb9031 Mon Sep 17 00:00:00 2001 From: Maxime SHE <72982050+chengyingshe@users.noreply.github.com> Date: Thu, 5 Sep 2024 17:51:16 +0800 Subject: [PATCH] [Feature] Add an attribute api_key into TurboMindAPIModel default None (#1475) Co-authored-by: Maxime Add an attribute api_key into TurboMindAPIModel default None then we can set the api_key while using lmdeploy to deploy the llm model --- configs/eval_internlm_chat_lmdeploy_apiserver.py | 2 ++ opencompass/models/turbomind_api.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/configs/eval_internlm_chat_lmdeploy_apiserver.py b/configs/eval_internlm_chat_lmdeploy_apiserver.py index cc815580..b69b9eab 100644 --- a/configs/eval_internlm_chat_lmdeploy_apiserver.py +++ b/configs/eval_internlm_chat_lmdeploy_apiserver.py @@ -28,6 +28,7 @@ internlm_chat_20b = dict( type=TurboMindAPIModel, abbr='internlm-chat-20b-turbomind', api_addr='http://0.0.0.0:23333', + api_key='internlm-chat-20b', # api_key max_out_len=100, max_seq_len=2048, batch_size=8, @@ -40,6 +41,7 @@ internlm_chat_7b = dict( type=TurboMindAPIModel, abbr='internlm-chat-7b-turbomind', api_addr='http://0.0.0.0:23333', + api_key='interlm-chat-7b', # api_key max_out_len=100, max_seq_len=2048, batch_size=16, diff --git a/opencompass/models/turbomind_api.py b/opencompass/models/turbomind_api.py index a4efd367..fb3f0ac2 100644 --- a/opencompass/models/turbomind_api.py +++ b/opencompass/models/turbomind_api.py @@ -40,6 +40,7 @@ class TurboMindAPIModel(BaseModel): def __init__(self, api_addr: str = 'http://0.0.0.0:23333', + api_key: str | None = None, max_seq_len: int = 2048, meta_template: Optional[Dict] = None, end_str: Optional[str] = None, @@ -48,7 +49,7 @@ class TurboMindAPIModel(BaseModel): max_seq_len=max_seq_len, meta_template=meta_template) from lmdeploy.serve.openai.api_client import APIClient - self.chatbot = APIClient(api_addr) + self.chatbot = APIClient(api_addr, api_key) self.model_name = self.chatbot.available_models[0] self.logger = get_logger() self.template_parser = LMTemplateParser(meta_template)