From b9de8b0e2b47f9561395c6e5fd23bd4ca1e5e4f6 Mon Sep 17 00:00:00 2001 From: Linchen Xiao Date: Tue, 18 Mar 2025 20:24:07 +0800 Subject: [PATCH] [Update] Unset disallowed_special token for Openai model (#1960) --- opencompass/models/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencompass/models/openai_api.py b/opencompass/models/openai_api.py index d5ac02d8..f46de71c 100644 --- a/opencompass/models/openai_api.py +++ b/opencompass/models/openai_api.py @@ -399,7 +399,7 @@ class OpenAI(BaseAPIModel): self.logger.info( f'Successfully load default tiktoken tokenizer: ' f' {default_tokenizer}') - return len(enc.encode(prompt)) + return len(enc.encode(prompt, disallowed_special=())) def _bin_trim(self, prompt: str, num_token: int, mode: str) -> str: """Get a suffix of prompt which is no longer than num_token tokens.