Merge branch 'open-compass:main' into main

configs/api_examples/eval_api_nanbeige.py

@@ -0,0 +1,36 @@
+from mmengine.config import read_base
+from opencompass.models import Nanbeige
+from opencompass.partitioners import NaivePartitioner
+from opencompass.runners.local_api import LocalAPIRunner
+from opencompass.tasks import OpenICLInferTask
+
+
+with read_base():
+    from ..summarizers.medium import summarizer
+    from ..datasets.ceval.ceval_gen import ceval_datasets
+
+datasets = [
+    *ceval_datasets,
+]
+
+models = [
+    dict(
+        abbr='nanbeige-plus',
+        type=Nanbeige,
+        path='nanbeige-plus',
+        key="xxxxxx",
+        query_per_second=1,
+        max_out_len=2048,
+        batch_size=8),
+]
+
+infer = dict(
+    partitioner=dict(type=NaivePartitioner),
+    runner=dict(
+        type=LocalAPIRunner,
+        max_num_workers=2,
+        concurrent_users=2,
+        task=dict(type=OpenICLInferTask)),
+)
+
+work_dir ="./output/nanbeige-plus"

configs/datasets/gsm8k/gsm8k_agent_gen_3ac57d.py

@@ -1,55 +0,0 @@
-from opencompass.openicl.icl_prompt_template import PromptTemplate
-from opencompass.openicl.icl_retriever import ZeroRetriever
-from opencompass.openicl.icl_inferencer import AgentInferencer
-from opencompass.datasets import (
-    GSM8KDataset,
-    gsm8k_postprocess,
-    gsm8k_dataset_postprocess,
-    Gsm8kAgentEvaluator,
-)
-
-gsm8k_reader_cfg = dict(input_columns=["question"], output_column="answer")
-
-gsm8k_infer_cfg = dict(
-    prompt_template=dict(
-        type=PromptTemplate,
-        template=dict(
-            round=[
-                # # ################################### NEW SHOT ###################################
-                dict(role='HUMAN', prompt='Mark\'s basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws.  Their opponents score double the 2 pointers but half the 3 pointers and free throws.  What\'s the total number of points scored by both teams added together?'),
-                dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:def solution():\n    mark_pointers_2 = 25 * 2\n    mark_pointers_3 = 8 * 3\n    mark_free_throws = 10 * 1\n    mark_points_scored = mark_pointers_2 + mark_pointers_3 + mark_free_throws\n    opponents_pointers_2 = mark_pointers_2 * 2\n    opponents_pointers_3 = mark_pointers_3 / 2\n    opponents_free_throws = mark_free_throws / 2\n    opponents_points_scored = opponents_pointers_2 + opponents_pointers_3 + opponents_free_throws\n    total_points_scored = mark_points_scored + opponents_points_scored\n    result = total_points_scored\n    return result'),
-                dict(role='SYSTEM', prompt='Response:210'),
-                dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 210'),
-
-                dict(role='HUMAN', prompt='Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles?'),
-                dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:def solution():\n    marbles = 60\n    num_increased_marbles = marbles * 2 / 5\n    num_total_marbles = marbles + num_increased_marbles\n    frisbees = marbles / 2\n    num_increased_frisbees = frisbees * 2 / 5\n    num_total_frisbees = frisbees + num_increased_frisbees\n    deck_cards = frisbees - 20\n    num_increased_deck_cards = deck_cards * 2 / 5\n    num_total_deck_cards = deck_cards + num_increased_deck_cards\n    num_total = num_total_marbles + num_total_frisbees + num_total_deck_cards\n    result = num_total\n    return result'),
-                dict(role='SYSTEM', prompt='Response:140'),
-                dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 140'),
-
-                dict(role='HUMAN', prompt='A group of 4 fruit baskets contains 9 apples, 15 oranges, and 14 bananas in the first three baskets and 2 less of each fruit in the fourth basket. How many fruits are there?'),
-                dict(role='BOT', prompt="""Tool:PythonInterpreter\nTool Input:def solution():\n    num_fruits_per_first_three_basket = 9 + 15 + 14\n    num_fruits_first_three_basket = num_fruits_per_first_three_basket * 3\n    num_apple_fourth_basket = 9 - 2\n    num_orange_fourth_basket = 15 - 2\n    num_banana_fourth_basket = 14 - 2\n    num_fruits_fourth_basket = num_apple_fourth_basket + num_orange_fourth_basket + num_banana_fourth_basket\n    num_fruits_total = num_fruits_first_three_basket + num_fruits_fourth_basket\n    result = num_fruits_total\n    return result"""),
-                dict(role='SYSTEM', prompt='Response:146'),
-                dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 146'),
-
-                dict(role='HUMAN', prompt='{question}'),
-            ])),
-    retriever=dict(type=ZeroRetriever),
-    inferencer=dict(type=AgentInferencer),
-)
-
-gsm8k_eval_cfg = dict(
-    evaluator=dict(type=Gsm8kAgentEvaluator),
-    pred_postprocessor=dict(type=gsm8k_postprocess),
-    dataset_postprocessor=dict(type=gsm8k_dataset_postprocess),
-)
-
-gsm8k_datasets = [
-    dict(
-        abbr='gsm8k-agent',
-        type=GSM8KDataset,
-        path='./data/gsm8k',
-        reader_cfg=gsm8k_reader_cfg,
-        infer_cfg=gsm8k_infer_cfg,
-        eval_cfg=gsm8k_eval_cfg,
-    )
-]

configs/datasets/gsm8k/gsm8k_agent_gen_c3dff3.py

@@ -18,8 +18,8 @@ gsm8k_infer_cfg = dict(
                 # # ################################### NEW SHOT ###################################
                 dict(role='HUMAN', prompt='Mark\'s basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws.  Their opponents score double the 2 pointers but half the 3 pointers and free throws.  What\'s the total number of points scored by both teams added together?'),
                 dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:```python\ndef solution():\n    mark_pointers_2 = 25 * 2\n    mark_pointers_3 = 8 * 3\n    mark_free_throws = 10 * 1\n    mark_points_scored = mark_pointers_2 + mark_pointers_3 + mark_free_throws\n    opponents_pointers_2 = mark_pointers_2 * 2\n    opponents_pointers_3 = mark_pointers_3 / 2\n    opponents_free_throws = mark_free_throws / 2\n    opponents_points_scored = opponents_pointers_2 + opponents_pointers_3 + opponents_free_throws\n    total_points_scored = mark_points_scored + opponents_points_scored\n    result = total_points_scored\n    return result\n```'),
-                dict(role='SYSTEM', prompt='Response:210'),
+                dict(role='SYSTEM', prompt='Response:201'),
-                dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 210'),
+                dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 201'),
 
                 dict(role='HUMAN', prompt='Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles?'),
                 dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:```python\ndef solution():\n    marbles = 60\n    num_increased_marbles = marbles * 2 / 5\n    num_total_marbles = marbles + num_increased_marbles\n    frisbees = marbles / 2\n    num_increased_frisbees = frisbees * 2 / 5\n    num_total_frisbees = frisbees + num_increased_frisbees\n    deck_cards = frisbees - 20\n    num_increased_deck_cards = deck_cards * 2 / 5\n    num_total_deck_cards = deck_cards + num_increased_deck_cards\n    num_total = num_total_marbles + num_total_frisbees + num_total_deck_cards\n    result = num_total\n    return result\n```'),

opencompass/models/__init__.py

@@ -18,6 +18,7 @@ from .minimax_api import MiniMax  # noqa: F401
 from .mixtral import Mixtral  # noqa: F401
 from .modelscope import ModelScope, ModelScopeCausalLM  # noqa: F401, F403
 from .moonshot_api import MoonShot  # noqa: F401
+from .nanbeige_api import Nanbeige  # noqa: F401
 from .openai_api import OpenAI  # noqa: F401
 from .pangu_api import PanGu  # noqa: F401
 from .qwen_api import Qwen  # noqa: F401

opencompass/models/nanbeige_api.py

@@ -0,0 +1,148 @@
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Dict, List, Optional, Union
+
+import requests
+
+from opencompass.utils.prompt import PromptList
+
+from .base_api import BaseAPIModel
+
+PromptType = Union[PromptList, str]
+
+
+class Nanbeige(BaseAPIModel):
+    """Model wrapper around Nanbeige.
+
+    Documentations:
+
+    Args:
+        path (str): Model name, e.g. `nanbeige-plus`
+        key (str): Provide API Key
+        url (str): Provided URL
+        query_per_second (int): The maximum queries allowed per second
+            between two consecutive calls of the API. Defaults to 2.
+        max_seq_len (int): Unused here.
+        meta_template (Dict, optional): The model's meta prompt
+            template if needed, in case the requirement of injecting or
+            wrapping of any meta instructions.
+        retry (int): Number of retires if the API call fails. Defaults to 2.
+    """
+
+    def __init__(self,
+                 path: str,
+                 key: str,
+                 url: str = None,
+                 query_per_second: int = 2,
+                 max_seq_len: int = 2048,
+                 meta_template: Optional[Dict] = None,
+                 retry: int = 3):
+        super().__init__(path=path,
+                         max_seq_len=max_seq_len,
+                         query_per_second=query_per_second,
+                         meta_template=meta_template,
+                         retry=retry)
+        self.headers = {
+            'Authorization': 'Bearer ' + key,
+            'Content-Type': 'application/json',
+        }
+        self.model = path
+        self.url = url if url is not None \
+            else 'http://stardustlm.zhipin.com/api/gpt/open/chat/send/sync'
+
+    def generate(
+        self,
+        inputs: List[str or PromptList],
+        max_out_len: int = 512,
+    ) -> List[str]:
+        """Generate results given a list of inputs.
+
+        Args:
+            inputs (List[str or PromptList]): A list of strings or PromptDicts.
+                The PromptDict should be organized in OpenCompass'
+                API format.
+            max_out_len (int): The maximum length of the output.
+
+        Returns:
+            List[str]: A list of generated strings.
+        """
+        with ThreadPoolExecutor() as executor:
+            results = list(
+                executor.map(self._generate, inputs,
+                             [max_out_len] * len(inputs)))
+        self.flush()
+        return results
+
+    def _generate(
+        self,
+        input: str or PromptList,
+        max_out_len: int = 512,
+    ) -> str:
+        """Generate results given an input.
+
+        Args:
+            inputs (str or PromptList): A string or PromptDict.
+                The PromptDict should be organized in OpenCompass'
+                API format.
+            max_out_len (int): The maximum length of the output.
+
+        Returns:
+            str: The generated string.
+        """
+        assert isinstance(input, (str, PromptList))
+
+        if isinstance(input, str):
+            messages = [{'sender_type': 'USER', 'text': input}]
+        else:
+            messages = []
+            for item in input:
+                msg = {'text': item['prompt']}
+                if item['role'] == 'HUMAN':
+                    msg['sender_type'] = 'USER'
+                elif item['role'] == 'BOT':
+                    msg['sender_type'] = 'BOT'
+
+                messages.append(msg)
+
+        data = {
+            'model': self.model,
+            'messages': messages,
+        }
+
+        max_num_retries = 0
+        while max_num_retries < self.retry:
+            self.acquire()
+            raw_response = requests.request('POST',
+                                            url=self.url,
+                                            headers=self.headers,
+                                            json=data)
+            self.release()
+
+            if raw_response is None:
+                print('Connection error, reconnect.')
+                # if connect error, frequent requests will casuse
+                # continuous unstable network, therefore wait here
+                # to slow down the request
+                self.wait()
+                continue
+
+            if raw_response.status_code != 200:
+                print('请求失败：', raw_response)
+                print('失败信息：', raw_response.text)
+                max_num_retries += 1
+                continue
+
+            response = raw_response.json()
+            if response['stardustCode'] == 0:
+                return response['reply']
+
+            # exceed concurrency limit
+            if response['stardustCode'] == 20035:
+                print(response)
+                time.sleep(2)
+                continue
+
+            print(response)
+            max_num_retries += 1
+
+        raise RuntimeError(raw_response.text)

opencompass/models/openai_api.py

@@ -334,6 +334,7 @@ class OpenAIAllesAPIN(OpenAI):
                  path: str,
                  url: str,
                  key: str,
+                 temperature: float = 1.0,
                  query_per_second: int = 1,
                  rpm_verbose: bool = False,
                  max_seq_len: int = 2048,
@@ -346,6 +347,7 @@ class OpenAIAllesAPIN(OpenAI):
                          meta_template=meta_template,
                          retry=retry)
         self.url = url
+        self.temperature = temperature
         self.headers = {
             'alles-apin-token': key,
             'content-type': 'application/json',
@@ -387,11 +389,12 @@ class OpenAIAllesAPIN(OpenAI):
             # model can be response with user and system
             # when it comes with agent involved.
             assert msg['role'] in ['user', 'system']
+
         data = {
             'model': self.path,
             'messages': messages,
+            'temperature': temperature
         }
-
         for _ in range(self.retry):
             self.wait()
             raw_response = requests.post(self.url,

requirements/agent.txt

@@ -1,5 +1,4 @@
 antlr4-python3-runtime==4.11
-git+ssh://git@gitlab.pjlab.org.cn:1122/openmmlab/bigmodel/ilagent.git@czh/eval_gen
 ipykernel
 ipython
 json5