jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Update] Unset disallowed_special token for Openai model (#1960)

Browse Source
This commit is contained in:
Linchen Xiao 2025-03-18 20:24:07 +08:00 committed by GitHub
parent c98599271b
commit b9de8b0e2b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
opencompass/models/openai_api.py
View File

@ -399,7 +399,7 @@ class OpenAI(BaseAPIModel):
self.logger.info(
f'Successfully load default tiktoken tokenizer: '
f' {default_tokenizer}')
return len(enc.encode(prompt))
return len(enc.encode(prompt, disallowed_special=()))
def _bin_trim(self, prompt: str, num_token: int, mode: str) -> str:
"""Get a suffix of prompt which is no longer than num_token tokens.