OpenCompass/opencompass/models/unigpt_api.py

import hashlib
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Union

import requests

from opencompass.utils.prompt import PromptList

from .base_api import BaseAPIModel

PromptType = Union[PromptList, str]


def get_sign(appkey, udid, timestamp, secret):
    original_str = f'{appkey}{udid}{timestamp}{secret}'
    sign = ''
    try:
        md = hashlib.sha256()
        md.update(original_str.encode('utf-8'))
        bytes_result = md.digest()
        for byte in bytes_result:
            hex_value = format(byte, '02X')
            sign += hex_value.upper()
    except Exception as e:
        print(e)
    return sign


class UniGPT(BaseAPIModel):

    def __init__(
        self,
        path: str,
        appkey: str,
        secret: str,
        url: str,
        query_per_second: int = 2,
        max_seq_len: int = 2048,
        meta_template: Optional[Dict] = None,
        retry: int = 2,
        temperature: float = 0.2,
    ):  # noqa E125
        super().__init__(
            path=path,
            max_seq_len=max_seq_len,
            query_per_second=query_per_second,
            meta_template=meta_template,
            retry=retry,
        )

        self.appkey = appkey
        self.secret = secret
        self.udid = str(uuid.uuid1())
        self.url = url
        self.model = path
        self.temperature = temperature

    def generate(self,
                 inputs: List[PromptType],
                 max_out_len: int = 512) -> List[str]:
        """Generate results given a list of inputs.

        Args:
            inputs (List[PromptType]): A list of strings or PromptDicts.
                The PromptDict should be organized in OpenCompass'
                API format.
            max_out_len (int): The maximum length of the output.

        Returns:
            List[str]: A list of generated strings.
        """
        with ThreadPoolExecutor() as executor:
            results = list(
                executor.map(self._generate, inputs,
                             [max_out_len] * len(inputs)))
        self.flush()
        return results

    def _generate(self, input: PromptType, max_out_len: int = 512) -> str:
        """Generate results given an input.

        Args:
            inputs (PromptType): A string or PromptDict.
                The PromptDict should be organized in OpenCompass'
                API format.
            max_out_len (int): The maximum length of the output.

        Returns:
            str: The generated string.
        """

        assert isinstance(input, (str, PromptList))

        if isinstance(input, str):
            messages = [{'role': 'user', 'content': input}]
        else:
            messages = []
            for item in input:
                msg = {'content': item['prompt']}
                if item['role'] == 'HUMAN':
                    msg['role'] = 'user'
                elif item['role'] == 'BOT':
                    msg['role'] = 'assistant'
                elif item['role'] == 'SYSTEM':
                    msg['role'] = 'system'
                messages.append(msg)

        data = {
            'model': self.path,
            'temperature': self.temperature,
            'messages': messages,
            'max_tokens': max_out_len,
        }

        timestamp = str(int(time.time()) * 1000)
        headers = {
            'appkey': self.appkey,
            'sign': get_sign(self.appkey, self.udid, timestamp, self.secret),
            'stream': 'false',
            'timestamp': timestamp,
            'udid': self.udid,
            'censor': 'none',
        }

        for _ in range(self.retry):
            try:
                response = requests.post(self.url, json=data, headers=headers)
            except Exception as e:
                print(e)
                continue
            if response is None or response.status_code != 200:
                code = response.status_code if response else -1
                print(f'request err, status_code: {code}')
                time.sleep(10)
                continue
            try:
                response = response.json()
            except Exception as e:
                print(e)
                continue
            print(response)
            if response.get('errorCode') == '8500502':
                return 'context_length_exceeded'
            return response['result']['choices'][0]['message']['content']
        raise RuntimeError(f'Failed to respond in {self.retry} retrys')