update support of yayi api

configs/api_examples/eval_api_yayi.py

@@ -0,0 +1,163 @@
+from mmengine.config import read_base
+import os.path as osp
+from opencompass.partitioners import NaivePartitioner, NumWorkerPartitioner
+from opencompass.partitioners import SizePartitioner
+from opencompass.runners import LocalRunner, VOLCRunner
+from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
+
+
+#######################################################################
+#                          PART 0  Essential Configs                  #
+#######################################################################
+with read_base():
+    # Datasets Part
+    # Knowledge
+    from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_cot_gen_08c1de import (
+        mmlu_pro_datasets,
+    )
+    # General Reasoning
+    from opencompass.configs.datasets.gpqa.gpqa_openai_simple_evals_gen_5aeece import (
+        gpqa_datasets,
+    )
+    from opencompass.configs.datasets.bbh.bbh_0shot_nocot_gen_925fc4 import (
+        bbh_datasets,
+    )
+    from opencompass.configs.datasets.humaneval.humaneval_openai_sample_evals_gen_dcae0e import (
+        humaneval_datasets,
+    )
+
+    # Instruction Following
+    from opencompass.configs.datasets.IFEval.IFEval_gen_353ae7 import (
+        ifeval_datasets,
+    )
+    from opencompass.configs.datasets.livecodebench.livecodebench_gen_a4f90b import (
+        LCBCodeGeneration_dataset,
+    )
+
+    # Math
+    from opencompass.configs.datasets.aime2024.aime2024_gen_6e39a4 import (
+        aime2024_datasets,
+    )
+    from opencompass.configs.datasets.math.math_prm800k_500_0shot_cot_gen import (
+        math_datasets,
+    )
+
+    # Summary Groups
+    from opencompass.configs.summarizers.groups.bbh import bbh_summary_groups
+    from opencompass.configs.summarizers.groups.mmlu_pro import (
+        mmlu_pro_summary_groups,
+    )
+from opencompass.models import Yayi
+
+#######################################################################
+#                          PART 1  Datasets List                      #
+#######################################################################
+# Datasets list for evaluation
+# Only take LCB generation for evaluation
+datasets = sum(
+    (v for k, v in locals().items() if k.endswith('_datasets')), []
+) + [LCBCodeGeneration_dataset]
+
+#######################################################################
+#                       PART 2  Datset Summarizer                     #
+#######################################################################
+
+core_summary_groups = [
+    {
+        'name': 'core_average',
+        'subsets': [
+            ['IFEval', 'Prompt-level-strict-accuracy'],
+            ['bbh', 'naive_average'],
+            ['math_prm800k_500', 'accuracy'],
+            ['aime2024', 'accuracy'],
+            ['GPQA_diamond', 'accuracy'],
+            ['mmlu_pro', 'naive_average'],
+            ['openai_humaneval', 'humaneval_pass@1'],
+            ['lcb_code_generation', 'pass@1'],
+        ],
+    },
+]
+
+
+summarizer = dict(
+    dataset_abbrs=[
+        ['core_average', 'naive_average'],
+        '',
+        'Instruction Following',
+        ['IFEval', 'Prompt-level-strict-accuracy'],
+        '',
+        'General Reasoning',
+        ['bbh', 'naive_average'],
+        ['GPQA_diamond', 'accuracy'],
+        '',
+        'Math Calculation',
+        ['math_prm800k_500', 'accuracy'],
+        ['aime2024', 'accuracy'],
+        '',
+        'Knowledge',
+        ['mmlu_pro', 'naive_average'],
+        '',
+        'Code',
+        ['openai_humaneval', 'humaneval_pass@1'],
+        ['lcb_code_generation', 'pass@1'],
+    ],
+    summary_groups=sum(
+        [v for k, v in locals().items() if k.endswith('_summary_groups')], []
+    ),
+)
+
+#######################################################################
+#                        PART 3  Models  List                         #
+#######################################################################
+
+api_meta_template = dict(
+    round=[
+        dict(role='HUMAN', api_role='HUMAN'),
+        dict(role='BOT', api_role='BOT', generate=True),
+    ],
+    reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
+)
+
+models = [
+    dict(
+        abbr="YAYI",
+        type=Yayi,
+        path="YAYI",
+        url="https://tilake.wenge.com/saas-gateway/******",
+        url_path="/******",
+        x_tilake_app_key="xxxxxx", # Please give you app_key
+        x_tilake_app_secret="xxxxxxxxx", # Please give you app_secret
+        x_tilake_ca_sginature_method="HmacSHA256",
+        meta_template=api_meta_template,
+        batch_size=1,
+    )
+]
+
+#######################################################################
+#                 PART 4  Inference/Evaluation Configuaration         #
+#######################################################################
+
+# Local Runner
+infer = dict(
+    partitioner=dict(type=SizePartitioner, max_task_size=100, gen_task_coef=1),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=16,
+        retry=3,  # Modify if needed
+        task=dict(type=OpenICLInferTask),
+    ),
+)
+
+# Eval with local runner
+eval = dict(
+    partitioner=dict(type=NaivePartitioner, n=10),
+    runner=dict(
+        type=LocalRunner, max_num_workers=16, task=dict(type=OpenICLEvalTask)
+    ),
+)
+
+
+#######################################################################
+#                      PART 5  Utils Configuaration                   #
+#######################################################################
+work_dir = f"outputs/YAYI/"

opencompass/models/yayi_api.py

@@ -70,19 +70,23 @@ def get_current_time_gmt_format():
 
 
 class Yayi(BaseAPIModel):
-    """Model wrapper around SenseTime.
+    """Model wrapper around Yayi.
 
     Args:
-        path (str): The name of SenseTime model.
+        path (str): The name of Yayi model.
-            e.g. `nova-ptc-xl-v1`
+        url (str): The base URL for the API.
-        key (str): Authorization key.
+        url_path (str): The specific path for the API endpoint.
+        x_tilake_app_key (str): The application key for authentication.
+        x_tilake_app_secret (str): The application secret for authentication.
+        x_tilake_ca_sginature_method (str): The signature method for authentication.
         query_per_second (int): The maximum queries allowed per second
-            between two consecutive calls of the API. Defaults to 1.
+            between two consecutive calls of the API. Defaults to 2.
-        max_seq_len (int): Unused here.
+        max_seq_len (int): The maximum sequence length. Defaults to 8192.
         meta_template (Dict, optional): The model's meta prompt
             template if needed, in case the requirement of injecting or
             wrapping of any meta instructions.
-        retry (int): Number of retires if the API call fails. Defaults to 2.
+        retry (int): Number of retries if the API call fails. Defaults to 2.
+        temperature (float): The temperature for the model's response. Defaults to 0.0.
     """
 
     def __init__(
@@ -94,10 +98,10 @@ class Yayi(BaseAPIModel):
         x_tilake_app_secret: str,
         x_tilake_ca_sginature_method: str,
         query_per_second: int = 2,
-        max_seq_len: int = 2048,
+        max_seq_len: int = 8192,
         meta_template: Optional[Dict] = None,
         retry: int = 2,
-        temperature: float = 0.4,
+        temperature: float = 0.0,
     ):
         super().__init__(
             path=path,
@@ -116,14 +120,8 @@ class Yayi(BaseAPIModel):
         self.model = path
 
     def generate_signature(self, method, accept, content_type, date, url_path):
-        """生成签名.
+        """
-
+        生成签名.
-        :param method:
-        :param accept:
-        :param content_type:
-        :param date:
-        :param url_path:
-        :return:
         """
         string_to_sign = (method + '\n' + accept + '\n' + content_type + '\n' +
                           date + '\n' + url_path)
@@ -134,11 +132,8 @@ class Yayi(BaseAPIModel):
         return encode_base64_string(signature)
 
     def generate_header(self, content_type, accept, date, signature):
-        """生成请求头参数.
+        """
-
+        生成请求头参数.
-        :param content_type:
-        :param accept:
-        :return:
         """
         headers = {
             'x-tilake-app-key': self.X_TILAKE_APP_KEY,
@@ -155,7 +150,7 @@ class Yayi(BaseAPIModel):
     def generate(
         self,
         inputs: List[PromptType],
-        max_out_len: int = 512,
+        max_out_len: int = 8192,
     ) -> List[str]:
         """Generate results given a list of inputs.
 
@@ -178,7 +173,7 @@ class Yayi(BaseAPIModel):
     def _generate(
         self,
         input: PromptType,
-        max_out_len: int = 512,
+        max_out_len: int = 8192,
     ) -> str:
         """Generate results given an input.
 
@@ -199,7 +194,7 @@ class Yayi(BaseAPIModel):
             messages = []
             msg_buffer, last_role = [], None
             for item in input:
-                item['role'] = 'yayi' if item['role'] == 'BOT' else 'user'
+                item['role'] = 'assistant' if item['role'] == 'BOT' else 'user'
                 if item['role'] != last_role and last_role is not None:
                     messages.append({
                         'content': '\n'.join(msg_buffer),
@@ -218,14 +213,12 @@ class Yayi(BaseAPIModel):
         accept = '*/*'
         method = 'POST'
         data = {
-            'id': '001',  # 请求id，无需修改。
+            'id': "001",
-            'model': self.model,
             'messages': messages,
-            'max_new_tokens': max_out_len,  # max_new_tokens及以下参数可根据实际任务进行调整。
+            'max_new_tokens': 8192,
-            'temperature': self.temperature,
+            'temperature': 0.0,
-            'presence_penalty': 0.85,
+            'presence_penalty': 0.0,
-            'frequency_penalty': 0.16,
+            'frequency_penalty': 0.0,
-            'do_sample': True,
             'top_p': 1.0,
             'top_k': -1,
         }
@@ -242,6 +235,7 @@ class Yayi(BaseAPIModel):
                                            signature=signature_str)
 
             try:
+                print(data)
                 response = requests.post(self.url, json=data, headers=headers)
             except Exception as e:
                 print(e)
@@ -253,7 +247,7 @@ class Yayi(BaseAPIModel):
                 continue
             print(response)
             try:
-                return response['data']['choices'][0]['message']['content']
+                return response['choices'][0]['message']['content']
             except Exception as e:
                 print(e)
                 continue