jiangzhs/OpenCompass
1
0
Fork 0
mirror of https://github.com/open-compass/opencompass.git synced 2025-05-30 16:03:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Update] strip stop_words (#1635)

Browse Source
This commit is contained in:
Lyu Han 2024-10-24 20:39:20 +08:00 committed by GitHub
parent 662dddf41a
commit fb12c3f98a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
opencompass/models/turbomind_with_tf_above_v4_33.py
View File

@ -73,7 +73,11 @@ class TurboMindModelwithChatTemplate(BaseModel):
else: else:
assert isinstance(generation_config.eos_token_id, list) assert isinstance(generation_config.eos_token_id, list)
for token_id in generation_config.eos_token_id: for token_id in generation_config.eos_token_id:
potential_stop_words.append(self.tokenizer.decode(token_id)) stop_word = self.tokenizer.decode(token_id)
if stop_word.startswith(' '):
self.logger.warning(f'stop_word "{stop_word}" contains blanks, which will be stripped')
stop_word = stop_word.strip()
potential_stop_words.append(stop_word)
if self.tokenizer.eos_token is not None: if self.tokenizer.eos_token is not None:
potential_stop_words.append(self.tokenizer.eos_token) potential_stop_words.append(self.tokenizer.eos_token)
potential_stop_words = list(set(potential_stop_words)) potential_stop_words = list(set(potential_stop_words))