diff --git a/configs/api_examples/eval_api_nanbeige.py b/configs/api_examples/eval_api_nanbeige.py new file mode 100644 index 00000000..1d941067 --- /dev/null +++ b/configs/api_examples/eval_api_nanbeige.py @@ -0,0 +1,36 @@ +from mmengine.config import read_base +from opencompass.models import Nanbeige +from opencompass.partitioners import NaivePartitioner +from opencompass.runners.local_api import LocalAPIRunner +from opencompass.tasks import OpenICLInferTask + + +with read_base(): + from ..summarizers.medium import summarizer + from ..datasets.ceval.ceval_gen import ceval_datasets + +datasets = [ + *ceval_datasets, +] + +models = [ + dict( + abbr='nanbeige-plus', + type=Nanbeige, + path='nanbeige-plus', + key="xxxxxx", + query_per_second=1, + max_out_len=2048, + batch_size=8), +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalAPIRunner, + max_num_workers=2, + concurrent_users=2, + task=dict(type=OpenICLInferTask)), +) + +work_dir ="./output/nanbeige-plus" \ No newline at end of file diff --git a/configs/datasets/gsm8k/gsm8k_agent_gen_3ac57d.py b/configs/datasets/gsm8k/gsm8k_agent_gen_3ac57d.py deleted file mode 100644 index d8f7248d..00000000 --- a/configs/datasets/gsm8k/gsm8k_agent_gen_3ac57d.py +++ /dev/null @@ -1,55 +0,0 @@ -from opencompass.openicl.icl_prompt_template import PromptTemplate -from opencompass.openicl.icl_retriever import ZeroRetriever -from opencompass.openicl.icl_inferencer import AgentInferencer -from opencompass.datasets import ( - GSM8KDataset, - gsm8k_postprocess, - gsm8k_dataset_postprocess, - Gsm8kAgentEvaluator, -) - -gsm8k_reader_cfg = dict(input_columns=["question"], output_column="answer") - -gsm8k_infer_cfg = dict( - prompt_template=dict( - type=PromptTemplate, - template=dict( - round=[ - # # ################################### NEW SHOT ################################### - dict(role='HUMAN', prompt='Mark\'s basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws. Their opponents score double the 2 pointers but half the 3 pointers and free throws. What\'s the total number of points scored by both teams added together?'), - dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:def solution():\n mark_pointers_2 = 25 * 2\n mark_pointers_3 = 8 * 3\n mark_free_throws = 10 * 1\n mark_points_scored = mark_pointers_2 + mark_pointers_3 + mark_free_throws\n opponents_pointers_2 = mark_pointers_2 * 2\n opponents_pointers_3 = mark_pointers_3 / 2\n opponents_free_throws = mark_free_throws / 2\n opponents_points_scored = opponents_pointers_2 + opponents_pointers_3 + opponents_free_throws\n total_points_scored = mark_points_scored + opponents_points_scored\n result = total_points_scored\n return result'), - dict(role='SYSTEM', prompt='Response:210'), - dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 210'), - - dict(role='HUMAN', prompt='Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles?'), - dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:def solution():\n marbles = 60\n num_increased_marbles = marbles * 2 / 5\n num_total_marbles = marbles + num_increased_marbles\n frisbees = marbles / 2\n num_increased_frisbees = frisbees * 2 / 5\n num_total_frisbees = frisbees + num_increased_frisbees\n deck_cards = frisbees - 20\n num_increased_deck_cards = deck_cards * 2 / 5\n num_total_deck_cards = deck_cards + num_increased_deck_cards\n num_total = num_total_marbles + num_total_frisbees + num_total_deck_cards\n result = num_total\n return result'), - dict(role='SYSTEM', prompt='Response:140'), - dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 140'), - - dict(role='HUMAN', prompt='A group of 4 fruit baskets contains 9 apples, 15 oranges, and 14 bananas in the first three baskets and 2 less of each fruit in the fourth basket. How many fruits are there?'), - dict(role='BOT', prompt="""Tool:PythonInterpreter\nTool Input:def solution():\n num_fruits_per_first_three_basket = 9 + 15 + 14\n num_fruits_first_three_basket = num_fruits_per_first_three_basket * 3\n num_apple_fourth_basket = 9 - 2\n num_orange_fourth_basket = 15 - 2\n num_banana_fourth_basket = 14 - 2\n num_fruits_fourth_basket = num_apple_fourth_basket + num_orange_fourth_basket + num_banana_fourth_basket\n num_fruits_total = num_fruits_first_three_basket + num_fruits_fourth_basket\n result = num_fruits_total\n return result"""), - dict(role='SYSTEM', prompt='Response:146'), - dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 146'), - - dict(role='HUMAN', prompt='{question}'), - ])), - retriever=dict(type=ZeroRetriever), - inferencer=dict(type=AgentInferencer), -) - -gsm8k_eval_cfg = dict( - evaluator=dict(type=Gsm8kAgentEvaluator), - pred_postprocessor=dict(type=gsm8k_postprocess), - dataset_postprocessor=dict(type=gsm8k_dataset_postprocess), -) - -gsm8k_datasets = [ - dict( - abbr='gsm8k-agent', - type=GSM8KDataset, - path='./data/gsm8k', - reader_cfg=gsm8k_reader_cfg, - infer_cfg=gsm8k_infer_cfg, - eval_cfg=gsm8k_eval_cfg, - ) -] diff --git a/configs/datasets/gsm8k/gsm8k_agent_gen_1f182e.py b/configs/datasets/gsm8k/gsm8k_agent_gen_c3dff3.py similarity index 97% rename from configs/datasets/gsm8k/gsm8k_agent_gen_1f182e.py rename to configs/datasets/gsm8k/gsm8k_agent_gen_c3dff3.py index cc81ac12..fd412a21 100644 --- a/configs/datasets/gsm8k/gsm8k_agent_gen_1f182e.py +++ b/configs/datasets/gsm8k/gsm8k_agent_gen_c3dff3.py @@ -18,8 +18,8 @@ gsm8k_infer_cfg = dict( # # ################################### NEW SHOT ################################### dict(role='HUMAN', prompt='Mark\'s basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws. Their opponents score double the 2 pointers but half the 3 pointers and free throws. What\'s the total number of points scored by both teams added together?'), dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:```python\ndef solution():\n mark_pointers_2 = 25 * 2\n mark_pointers_3 = 8 * 3\n mark_free_throws = 10 * 1\n mark_points_scored = mark_pointers_2 + mark_pointers_3 + mark_free_throws\n opponents_pointers_2 = mark_pointers_2 * 2\n opponents_pointers_3 = mark_pointers_3 / 2\n opponents_free_throws = mark_free_throws / 2\n opponents_points_scored = opponents_pointers_2 + opponents_pointers_3 + opponents_free_throws\n total_points_scored = mark_points_scored + opponents_points_scored\n result = total_points_scored\n return result\n```'), - dict(role='SYSTEM', prompt='Response:210'), - dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 210'), + dict(role='SYSTEM', prompt='Response:201'), + dict(role='BOT', prompt='Thought: According to the response, I got the answer\nFinalAnswer: 201'), dict(role='HUMAN', prompt='Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles?'), dict(role='BOT', prompt='Tool:PythonInterpreter\nTool Input:```python\ndef solution():\n marbles = 60\n num_increased_marbles = marbles * 2 / 5\n num_total_marbles = marbles + num_increased_marbles\n frisbees = marbles / 2\n num_increased_frisbees = frisbees * 2 / 5\n num_total_frisbees = frisbees + num_increased_frisbees\n deck_cards = frisbees - 20\n num_increased_deck_cards = deck_cards * 2 / 5\n num_total_deck_cards = deck_cards + num_increased_deck_cards\n num_total = num_total_marbles + num_total_frisbees + num_total_deck_cards\n result = num_total\n return result\n```'), diff --git a/opencompass/models/__init__.py b/opencompass/models/__init__.py index 331c66eb..41c3e127 100644 --- a/opencompass/models/__init__.py +++ b/opencompass/models/__init__.py @@ -18,6 +18,7 @@ from .minimax_api import MiniMax # noqa: F401 from .mixtral import Mixtral # noqa: F401 from .modelscope import ModelScope, ModelScopeCausalLM # noqa: F401, F403 from .moonshot_api import MoonShot # noqa: F401 +from .nanbeige_api import Nanbeige # noqa: F401 from .openai_api import OpenAI # noqa: F401 from .pangu_api import PanGu # noqa: F401 from .qwen_api import Qwen # noqa: F401 diff --git a/opencompass/models/nanbeige_api.py b/opencompass/models/nanbeige_api.py new file mode 100644 index 00000000..d39c34bf --- /dev/null +++ b/opencompass/models/nanbeige_api.py @@ -0,0 +1,148 @@ +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Dict, List, Optional, Union + +import requests + +from opencompass.utils.prompt import PromptList + +from .base_api import BaseAPIModel + +PromptType = Union[PromptList, str] + + +class Nanbeige(BaseAPIModel): + """Model wrapper around Nanbeige. + + Documentations: + + Args: + path (str): Model name, e.g. `nanbeige-plus` + key (str): Provide API Key + url (str): Provided URL + query_per_second (int): The maximum queries allowed per second + between two consecutive calls of the API. Defaults to 2. + max_seq_len (int): Unused here. + meta_template (Dict, optional): The model's meta prompt + template if needed, in case the requirement of injecting or + wrapping of any meta instructions. + retry (int): Number of retires if the API call fails. Defaults to 2. + """ + + def __init__(self, + path: str, + key: str, + url: str = None, + query_per_second: int = 2, + max_seq_len: int = 2048, + meta_template: Optional[Dict] = None, + retry: int = 3): + super().__init__(path=path, + max_seq_len=max_seq_len, + query_per_second=query_per_second, + meta_template=meta_template, + retry=retry) + self.headers = { + 'Authorization': 'Bearer ' + key, + 'Content-Type': 'application/json', + } + self.model = path + self.url = url if url is not None \ + else 'http://stardustlm.zhipin.com/api/gpt/open/chat/send/sync' + + def generate( + self, + inputs: List[str or PromptList], + max_out_len: int = 512, + ) -> List[str]: + """Generate results given a list of inputs. + + Args: + inputs (List[str or PromptList]): A list of strings or PromptDicts. + The PromptDict should be organized in OpenCompass' + API format. + max_out_len (int): The maximum length of the output. + + Returns: + List[str]: A list of generated strings. + """ + with ThreadPoolExecutor() as executor: + results = list( + executor.map(self._generate, inputs, + [max_out_len] * len(inputs))) + self.flush() + return results + + def _generate( + self, + input: str or PromptList, + max_out_len: int = 512, + ) -> str: + """Generate results given an input. + + Args: + inputs (str or PromptList): A string or PromptDict. + The PromptDict should be organized in OpenCompass' + API format. + max_out_len (int): The maximum length of the output. + + Returns: + str: The generated string. + """ + assert isinstance(input, (str, PromptList)) + + if isinstance(input, str): + messages = [{'sender_type': 'USER', 'text': input}] + else: + messages = [] + for item in input: + msg = {'text': item['prompt']} + if item['role'] == 'HUMAN': + msg['sender_type'] = 'USER' + elif item['role'] == 'BOT': + msg['sender_type'] = 'BOT' + + messages.append(msg) + + data = { + 'model': self.model, + 'messages': messages, + } + + max_num_retries = 0 + while max_num_retries < self.retry: + self.acquire() + raw_response = requests.request('POST', + url=self.url, + headers=self.headers, + json=data) + self.release() + + if raw_response is None: + print('Connection error, reconnect.') + # if connect error, frequent requests will casuse + # continuous unstable network, therefore wait here + # to slow down the request + self.wait() + continue + + if raw_response.status_code != 200: + print('请求失败：', raw_response) + print('失败信息：', raw_response.text) + max_num_retries += 1 + continue + + response = raw_response.json() + if response['stardustCode'] == 0: + return response['reply'] + + # exceed concurrency limit + if response['stardustCode'] == 20035: + print(response) + time.sleep(2) + continue + + print(response) + max_num_retries += 1 + + raise RuntimeError(raw_response.text) diff --git a/opencompass/models/openai_api.py b/opencompass/models/openai_api.py index 3228ec2f..40a85ea8 100644 --- a/opencompass/models/openai_api.py +++ b/opencompass/models/openai_api.py @@ -334,6 +334,7 @@ class OpenAIAllesAPIN(OpenAI): path: str, url: str, key: str, + temperature: float = 1.0, query_per_second: int = 1, rpm_verbose: bool = False, max_seq_len: int = 2048, @@ -346,6 +347,7 @@ class OpenAIAllesAPIN(OpenAI): meta_template=meta_template, retry=retry) self.url = url + self.temperature = temperature self.headers = { 'alles-apin-token': key, 'content-type': 'application/json', @@ -387,11 +389,12 @@ class OpenAIAllesAPIN(OpenAI): # model can be response with user and system # when it comes with agent involved. assert msg['role'] in ['user', 'system'] + data = { 'model': self.path, 'messages': messages, + 'temperature': temperature } - for _ in range(self.retry): self.wait() raw_response = requests.post(self.url, diff --git a/requirements/agent.txt b/requirements/agent.txt index 10be1148..23c8b6a4 100644 --- a/requirements/agent.txt +++ b/requirements/agent.txt @@ -1,5 +1,4 @@ antlr4-python3-runtime==4.11 -git+ssh://git@gitlab.pjlab.org.cn:1122/openmmlab/bigmodel/ilagent.git@czh/eval_gen ipykernel ipython json5