From 3ee430a2c0761bfc341bbab91965a0c07a37faae Mon Sep 17 00:00:00 2001 From: JensenDong Date: Wed, 8 Jan 2025 23:53:34 +0800 Subject: [PATCH] update support of yayi api --- configs/api_examples/eval_api_yayi.py | 163 ++++++++++++++++++++++++++ opencompass/models/yayi_api.py | 60 +++++----- 2 files changed, 190 insertions(+), 33 deletions(-) create mode 100644 configs/api_examples/eval_api_yayi.py diff --git a/configs/api_examples/eval_api_yayi.py b/configs/api_examples/eval_api_yayi.py new file mode 100644 index 00000000..45a55fe2 --- /dev/null +++ b/configs/api_examples/eval_api_yayi.py @@ -0,0 +1,163 @@ +from mmengine.config import read_base +import os.path as osp +from opencompass.partitioners import NaivePartitioner, NumWorkerPartitioner +from opencompass.partitioners import SizePartitioner +from opencompass.runners import LocalRunner, VOLCRunner +from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask + + +####################################################################### +# PART 0 Essential Configs # +####################################################################### +with read_base(): + # Datasets Part + # Knowledge + from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_cot_gen_08c1de import ( + mmlu_pro_datasets, + ) + # General Reasoning + from opencompass.configs.datasets.gpqa.gpqa_openai_simple_evals_gen_5aeece import ( + gpqa_datasets, + ) + from opencompass.configs.datasets.bbh.bbh_0shot_nocot_gen_925fc4 import ( + bbh_datasets, + ) + from opencompass.configs.datasets.humaneval.humaneval_openai_sample_evals_gen_dcae0e import ( + humaneval_datasets, + ) + + # Instruction Following + from opencompass.configs.datasets.IFEval.IFEval_gen_353ae7 import ( + ifeval_datasets, + ) + from opencompass.configs.datasets.livecodebench.livecodebench_gen_a4f90b import ( + LCBCodeGeneration_dataset, + ) + + # Math + from opencompass.configs.datasets.aime2024.aime2024_gen_6e39a4 import ( + aime2024_datasets, + ) + from opencompass.configs.datasets.math.math_prm800k_500_0shot_cot_gen import ( + math_datasets, + ) + + # Summary Groups + from opencompass.configs.summarizers.groups.bbh import bbh_summary_groups + from opencompass.configs.summarizers.groups.mmlu_pro import ( + mmlu_pro_summary_groups, + ) +from opencompass.models import Yayi + +####################################################################### +# PART 1 Datasets List # +####################################################################### +# Datasets list for evaluation +# Only take LCB generation for evaluation +datasets = sum( + (v for k, v in locals().items() if k.endswith('_datasets')), [] +) + [LCBCodeGeneration_dataset] + +####################################################################### +# PART 2 Datset Summarizer # +####################################################################### + +core_summary_groups = [ + { + 'name': 'core_average', + 'subsets': [ + ['IFEval', 'Prompt-level-strict-accuracy'], + ['bbh', 'naive_average'], + ['math_prm800k_500', 'accuracy'], + ['aime2024', 'accuracy'], + ['GPQA_diamond', 'accuracy'], + ['mmlu_pro', 'naive_average'], + ['openai_humaneval', 'humaneval_pass@1'], + ['lcb_code_generation', 'pass@1'], + ], + }, +] + + +summarizer = dict( + dataset_abbrs=[ + ['core_average', 'naive_average'], + '', + 'Instruction Following', + ['IFEval', 'Prompt-level-strict-accuracy'], + '', + 'General Reasoning', + ['bbh', 'naive_average'], + ['GPQA_diamond', 'accuracy'], + '', + 'Math Calculation', + ['math_prm800k_500', 'accuracy'], + ['aime2024', 'accuracy'], + '', + 'Knowledge', + ['mmlu_pro', 'naive_average'], + '', + 'Code', + ['openai_humaneval', 'humaneval_pass@1'], + ['lcb_code_generation', 'pass@1'], + ], + summary_groups=sum( + [v for k, v in locals().items() if k.endswith('_summary_groups')], [] + ), +) + +####################################################################### +# PART 3 Models List # +####################################################################### + +api_meta_template = dict( + round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True), + ], + reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')], +) + +models = [ + dict( + abbr="YAYI", + type=Yayi, + path="YAYI", + url="https://tilake.wenge.com/saas-gateway/******", + url_path="/******", + x_tilake_app_key="xxxxxx", # Please give you app_key + x_tilake_app_secret="xxxxxxxxx", # Please give you app_secret + x_tilake_ca_sginature_method="HmacSHA256", + meta_template=api_meta_template, + batch_size=1, + ) +] + +####################################################################### +# PART 4 Inference/Evaluation Configuaration # +####################################################################### + +# Local Runner +infer = dict( + partitioner=dict(type=SizePartitioner, max_task_size=100, gen_task_coef=1), + runner=dict( + type=LocalRunner, + max_num_workers=16, + retry=3, # Modify if needed + task=dict(type=OpenICLInferTask), + ), +) + +# Eval with local runner +eval = dict( + partitioner=dict(type=NaivePartitioner, n=10), + runner=dict( + type=LocalRunner, max_num_workers=16, task=dict(type=OpenICLEvalTask) + ), +) + + +####################################################################### +# PART 5 Utils Configuaration # +####################################################################### +work_dir = f"outputs/YAYI/" diff --git a/opencompass/models/yayi_api.py b/opencompass/models/yayi_api.py index 73ba3ea0..0ed83769 100644 --- a/opencompass/models/yayi_api.py +++ b/opencompass/models/yayi_api.py @@ -70,19 +70,23 @@ def get_current_time_gmt_format(): class Yayi(BaseAPIModel): - """Model wrapper around SenseTime. + """Model wrapper around Yayi. Args: - path (str): The name of SenseTime model. - e.g. `nova-ptc-xl-v1` - key (str): Authorization key. + path (str): The name of Yayi model. + url (str): The base URL for the API. + url_path (str): The specific path for the API endpoint. + x_tilake_app_key (str): The application key for authentication. + x_tilake_app_secret (str): The application secret for authentication. + x_tilake_ca_sginature_method (str): The signature method for authentication. query_per_second (int): The maximum queries allowed per second - between two consecutive calls of the API. Defaults to 1. - max_seq_len (int): Unused here. + between two consecutive calls of the API. Defaults to 2. + max_seq_len (int): The maximum sequence length. Defaults to 8192. meta_template (Dict, optional): The model's meta prompt template if needed, in case the requirement of injecting or wrapping of any meta instructions. - retry (int): Number of retires if the API call fails. Defaults to 2. + retry (int): Number of retries if the API call fails. Defaults to 2. + temperature (float): The temperature for the model's response. Defaults to 0.0. """ def __init__( @@ -94,10 +98,10 @@ class Yayi(BaseAPIModel): x_tilake_app_secret: str, x_tilake_ca_sginature_method: str, query_per_second: int = 2, - max_seq_len: int = 2048, + max_seq_len: int = 8192, meta_template: Optional[Dict] = None, retry: int = 2, - temperature: float = 0.4, + temperature: float = 0.0, ): super().__init__( path=path, @@ -116,14 +120,8 @@ class Yayi(BaseAPIModel): self.model = path def generate_signature(self, method, accept, content_type, date, url_path): - """生成签名. - - :param method: - :param accept: - :param content_type: - :param date: - :param url_path: - :return: + """ + 生成签名. """ string_to_sign = (method + '\n' + accept + '\n' + content_type + '\n' + date + '\n' + url_path) @@ -134,11 +132,8 @@ class Yayi(BaseAPIModel): return encode_base64_string(signature) def generate_header(self, content_type, accept, date, signature): - """生成请求头参数. - - :param content_type: - :param accept: - :return: + """ + 生成请求头参数. """ headers = { 'x-tilake-app-key': self.X_TILAKE_APP_KEY, @@ -155,7 +150,7 @@ class Yayi(BaseAPIModel): def generate( self, inputs: List[PromptType], - max_out_len: int = 512, + max_out_len: int = 8192, ) -> List[str]: """Generate results given a list of inputs. @@ -178,7 +173,7 @@ class Yayi(BaseAPIModel): def _generate( self, input: PromptType, - max_out_len: int = 512, + max_out_len: int = 8192, ) -> str: """Generate results given an input. @@ -199,7 +194,7 @@ class Yayi(BaseAPIModel): messages = [] msg_buffer, last_role = [], None for item in input: - item['role'] = 'yayi' if item['role'] == 'BOT' else 'user' + item['role'] = 'assistant' if item['role'] == 'BOT' else 'user' if item['role'] != last_role and last_role is not None: messages.append({ 'content': '\n'.join(msg_buffer), @@ -218,14 +213,12 @@ class Yayi(BaseAPIModel): accept = '*/*' method = 'POST' data = { - 'id': '001', # 请求id，无需修改。 - 'model': self.model, + 'id': "001", 'messages': messages, - 'max_new_tokens': max_out_len, # max_new_tokens及以下参数可根据实际任务进行调整。 - 'temperature': self.temperature, - 'presence_penalty': 0.85, - 'frequency_penalty': 0.16, - 'do_sample': True, + 'max_new_tokens': 8192, + 'temperature': 0.0, + 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'top_p': 1.0, 'top_k': -1, } @@ -242,6 +235,7 @@ class Yayi(BaseAPIModel): signature=signature_str) try: + print(data) response = requests.post(self.url, json=data, headers=headers) except Exception as e: print(e) @@ -253,7 +247,7 @@ class Yayi(BaseAPIModel): continue print(response) try: - return response['data']['choices'][0]['message']['content'] + return response['choices'][0]['message']['content'] except Exception as e: print(e) continue