mirror of
https://github.com/open-compass/opencompass.git
synced 2025-05-30 16:03:24 +08:00
Merge 3ee430a2c0 into 408f5caff4
This commit is contained in:
Yao Dong
GitHub
commit
deec91dc54
163
configs/api_examples/eval_api_yayi.py
Normal file
163
configs/api_examples/eval_api_yayi.py
Normal file
@ -0,0 +1,163 @@
|
||||
from mmengine.config import read_base
|
||||
import os.path as osp
|
||||
from opencompass.partitioners import NaivePartitioner, NumWorkerPartitioner
|
||||
from opencompass.partitioners import SizePartitioner
|
||||
from opencompass.runners import LocalRunner, VOLCRunner
|
||||
from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask
|
||||
|
||||
|
||||
#######################################################################
|
||||
# PART 0 Essential Configs #
|
||||
#######################################################################
|
||||
with read_base():
|
||||
# Datasets Part
|
||||
# Knowledge
|
||||
from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_cot_gen_08c1de import (
|
||||
mmlu_pro_datasets,
|
||||
)
|
||||
# General Reasoning
|
||||
from opencompass.configs.datasets.gpqa.gpqa_openai_simple_evals_gen_5aeece import (
|
||||
gpqa_datasets,
|
||||
)
|
||||
from opencompass.configs.datasets.bbh.bbh_0shot_nocot_gen_925fc4 import (
|
||||
bbh_datasets,
|
||||
)
|
||||
from opencompass.configs.datasets.humaneval.humaneval_openai_sample_evals_gen_dcae0e import (
|
||||
humaneval_datasets,
|
||||
)
|
||||
|
||||
# Instruction Following
|
||||
from opencompass.configs.datasets.IFEval.IFEval_gen_353ae7 import (
|
||||
ifeval_datasets,
|
||||
)
|
||||
from opencompass.configs.datasets.livecodebench.livecodebench_gen_a4f90b import (
|
||||
LCBCodeGeneration_dataset,
|
||||
)
|
||||
|
||||
# Math
|
||||
from opencompass.configs.datasets.aime2024.aime2024_gen_6e39a4 import (
|
||||
aime2024_datasets,
|
||||
)
|
||||
from opencompass.configs.datasets.math.math_prm800k_500_0shot_cot_gen import (
|
||||
math_datasets,
|
||||
)
|
||||
|
||||
# Summary Groups
|
||||
from opencompass.configs.summarizers.groups.bbh import bbh_summary_groups
|
||||
from opencompass.configs.summarizers.groups.mmlu_pro import (
|
||||
mmlu_pro_summary_groups,
|
||||
)
|
||||
from opencompass.models import Yayi
|
||||
|
||||
#######################################################################
|
||||
# PART 1 Datasets List #
|
||||
#######################################################################
|
||||
# Datasets list for evaluation
|
||||
# Only take LCB generation for evaluation
|
||||
datasets = sum(
|
||||
(v for k, v in locals().items() if k.endswith('_datasets')), []
|
||||
) + [LCBCodeGeneration_dataset]
|
||||
|
||||
#######################################################################
|
||||
# PART 2 Datset Summarizer #
|
||||
#######################################################################
|
||||
|
||||
core_summary_groups = [
|
||||
{
|
||||
'name': 'core_average',
|
||||
'subsets': [
|
||||
['IFEval', 'Prompt-level-strict-accuracy'],
|
||||
['bbh', 'naive_average'],
|
||||
['math_prm800k_500', 'accuracy'],
|
||||
['aime2024', 'accuracy'],
|
||||
['GPQA_diamond', 'accuracy'],
|
||||
['mmlu_pro', 'naive_average'],
|
||||
['openai_humaneval', 'humaneval_pass@1'],
|
||||
['lcb_code_generation', 'pass@1'],
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
summarizer = dict(
|
||||
dataset_abbrs=[
|
||||
['core_average', 'naive_average'],
|
||||
'',
|
||||
'Instruction Following',
|
||||
['IFEval', 'Prompt-level-strict-accuracy'],
|
||||
'',
|
||||
'General Reasoning',
|
||||
['bbh', 'naive_average'],
|
||||
['GPQA_diamond', 'accuracy'],
|
||||
'',
|
||||
'Math Calculation',
|
||||
['math_prm800k_500', 'accuracy'],
|
||||
['aime2024', 'accuracy'],
|
||||
'',
|
||||
'Knowledge',
|
||||
['mmlu_pro', 'naive_average'],
|
||||
'',
|
||||
'Code',
|
||||
['openai_humaneval', 'humaneval_pass@1'],
|
||||
['lcb_code_generation', 'pass@1'],
|
||||
],
|
||||
summary_groups=sum(
|
||||
[v for k, v in locals().items() if k.endswith('_summary_groups')], []
|
||||
),
|
||||
)
|
||||
|
||||
#######################################################################
|
||||
# PART 3 Models List #
|
||||
#######################################################################
|
||||
|
||||
api_meta_template = dict(
|
||||
round=[
|
||||
dict(role='HUMAN', api_role='HUMAN'),
|
||||
dict(role='BOT', api_role='BOT', generate=True),
|
||||
],
|
||||
reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
|
||||
)
|
||||
|
||||
models = [
|
||||
dict(
|
||||
abbr="YAYI",
|
||||
type=Yayi,
|
||||
path="YAYI",
|
||||
url="https://tilake.wenge.com/saas-gateway/******",
|
||||
url_path="/******",
|
||||
x_tilake_app_key="xxxxxx", # Please give you app_key
|
||||
x_tilake_app_secret="xxxxxxxxx", # Please give you app_secret
|
||||
x_tilake_ca_sginature_method="HmacSHA256",
|
||||
meta_template=api_meta_template,
|
||||
batch_size=1,
|
||||
)
|
||||
]
|
||||
|
||||
#######################################################################
|
||||
# PART 4 Inference/Evaluation Configuaration #
|
||||
#######################################################################
|
||||
|
||||
# Local Runner
|
||||
infer = dict(
|
||||
partitioner=dict(type=SizePartitioner, max_task_size=100, gen_task_coef=1),
|
||||
runner=dict(
|
||||
type=LocalRunner,
|
||||
max_num_workers=16,
|
||||
retry=3, # Modify if needed
|
||||
task=dict(type=OpenICLInferTask),
|
||||
),
|
||||
)
|
||||
|
||||
# Eval with local runner
|
||||
eval = dict(
|
||||
partitioner=dict(type=NaivePartitioner, n=10),
|
||||
runner=dict(
|
||||
type=LocalRunner, max_num_workers=16, task=dict(type=OpenICLEvalTask)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
#######################################################################
|
||||
# PART 5 Utils Configuaration #
|
||||
#######################################################################
|
||||
work_dir = f"outputs/YAYI/"
|
||||
@ -64,19 +64,23 @@ def get_current_time_gmt_format():
|
||||
|
||||
|
||||
class Yayi(BaseAPIModel):
|
||||
"""Model wrapper around SenseTime.
|
||||
"""Model wrapper around Yayi.
|
||||
|
||||
Args:
|
||||
path (str): The name of SenseTime model.
|
||||
e.g. `nova-ptc-xl-v1`
|
||||
key (str): Authorization key.
|
||||
path (str): The name of Yayi model.
|
||||
url (str): The base URL for the API.
|
||||
url_path (str): The specific path for the API endpoint.
|
||||
x_tilake_app_key (str): The application key for authentication.
|
||||
x_tilake_app_secret (str): The application secret for authentication.
|
||||
x_tilake_ca_sginature_method (str): The signature method for authentication.
|
||||
query_per_second (int): The maximum queries allowed per second
|
||||
between two consecutive calls of the API. Defaults to 1.
|
||||
max_seq_len (int): Unused here.
|
||||
between two consecutive calls of the API. Defaults to 2.
|
||||
max_seq_len (int): The maximum sequence length. Defaults to 8192.
|
||||
meta_template (Dict, optional): The model's meta prompt
|
||||
template if needed, in case the requirement of injecting or
|
||||
wrapping of any meta instructions.
|
||||
retry (int): Number of retires if the API call fails. Defaults to 2.
|
||||
retry (int): Number of retries if the API call fails. Defaults to 2.
|
||||
temperature (float): The temperature for the model's response. Defaults to 0.0.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@ -88,10 +92,10 @@ class Yayi(BaseAPIModel):
|
||||
x_tilake_app_secret: str,
|
||||
x_tilake_ca_sginature_method: str,
|
||||
query_per_second: int = 2,
|
||||
max_seq_len: int = 2048,
|
||||
max_seq_len: int = 8192,
|
||||
meta_template: Optional[Dict] = None,
|
||||
retry: int = 2,
|
||||
temperature: float = 0.4,
|
||||
temperature: float = 0.0,
|
||||
):
|
||||
super().__init__(
|
||||
path=path,
|
||||
@ -110,14 +114,8 @@ class Yayi(BaseAPIModel):
|
||||
self.model = path
|
||||
|
||||
def generate_signature(self, method, accept, content_type, date, url_path):
|
||||
"""生成签名.
|
||||
|
||||
:param method:
|
||||
:param accept:
|
||||
:param content_type:
|
||||
:param date:
|
||||
:param url_path:
|
||||
:return:
|
||||
"""
|
||||
生成签名.
|
||||
"""
|
||||
string_to_sign = (method + '\n' + accept + '\n' + content_type + '\n' +
|
||||
date + '\n' + url_path)
|
||||
@ -128,11 +126,8 @@ class Yayi(BaseAPIModel):
|
||||
return encode_base64_string(signature)
|
||||
|
||||
def generate_header(self, content_type, accept, date, signature):
|
||||
"""生成请求头参数.
|
||||
|
||||
:param content_type:
|
||||
:param accept:
|
||||
:return:
|
||||
"""
|
||||
生成请求头参数.
|
||||
"""
|
||||
headers = {
|
||||
'x-tilake-app-key': self.X_TILAKE_APP_KEY,
|
||||
@ -149,7 +144,7 @@ class Yayi(BaseAPIModel):
|
||||
def generate(
|
||||
self,
|
||||
inputs: List[PromptType],
|
||||
max_out_len: int = 512,
|
||||
max_out_len: int = 8192,
|
||||
) -> List[str]:
|
||||
"""Generate results given a list of inputs.
|
||||
|
||||
@ -172,7 +167,7 @@ class Yayi(BaseAPIModel):
|
||||
def _generate(
|
||||
self,
|
||||
input: PromptType,
|
||||
max_out_len: int = 512,
|
||||
max_out_len: int = 8192,
|
||||
) -> str:
|
||||
"""Generate results given an input.
|
||||
|
||||
@ -193,7 +188,7 @@ class Yayi(BaseAPIModel):
|
||||
messages = []
|
||||
msg_buffer, last_role = [], None
|
||||
for item in input:
|
||||
item['role'] = 'yayi' if item['role'] == 'BOT' else 'user'
|
||||
item['role'] = 'assistant' if item['role'] == 'BOT' else 'user'
|
||||
if item['role'] != last_role and last_role is not None:
|
||||
messages.append({
|
||||
'content': '\n'.join(msg_buffer),
|
||||
@ -212,14 +207,12 @@ class Yayi(BaseAPIModel):
|
||||
accept = '*/*'
|
||||
method = 'POST'
|
||||
data = {
|
||||
'id': '001', # 请求id，无需修改。
|
||||
'model': self.model,
|
||||
'id': "001",
|
||||
'messages': messages,
|
||||
'max_new_tokens': max_out_len, # max_new_tokens及以下参数可根据实际任务进行调整。
|
||||
'temperature': self.temperature,
|
||||
'presence_penalty': 0.85,
|
||||
'frequency_penalty': 0.16,
|
||||
'do_sample': True,
|
||||
'max_new_tokens': 8192,
|
||||
'temperature': 0.0,
|
||||
'presence_penalty': 0.0,
|
||||
'frequency_penalty': 0.0,
|
||||
'top_p': 1.0,
|
||||
'top_k': -1,
|
||||
}
|
||||
@ -236,6 +229,7 @@ class Yayi(BaseAPIModel):
|
||||
signature=signature_str)
|
||||
|
||||
try:
|
||||
print(data)
|
||||
response = requests.post(self.url, json=data, headers=headers)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
@ -247,7 +241,7 @@ class Yayi(BaseAPIModel):
|
||||
continue
|
||||
print(response)
|
||||
try:
|
||||
return response['data']['choices'][0]['message']['content']
|
||||
return response['choices'][0]['message']['content']
|
||||
except Exception as e:
|
||||
print(e)
|
||||
continue
|
||||
|
||||
Loading…
Reference in New Issue
Block a user