diff --git a/opencompass/models/turbomind_with_tf_above_v4_33.py b/opencompass/models/turbomind_with_tf_above_v4_33.py index 7d4e3891..f2a367b2 100644 --- a/opencompass/models/turbomind_with_tf_above_v4_33.py +++ b/opencompass/models/turbomind_with_tf_above_v4_33.py @@ -73,7 +73,11 @@ class TurboMindModelwithChatTemplate(BaseModel): else: assert isinstance(generation_config.eos_token_id, list) for token_id in generation_config.eos_token_id: - potential_stop_words.append(self.tokenizer.decode(token_id)) + stop_word = self.tokenizer.decode(token_id) + if stop_word.startswith(' '): + self.logger.warning(f'stop_word "{stop_word}" contains blanks, which will be stripped') + stop_word = stop_word.strip() + potential_stop_words.append(stop_word) if self.tokenizer.eos_token is not None: potential_stop_words.append(self.tokenizer.eos_token) potential_stop_words = list(set(potential_stop_words))