This commit is contained in:
lyuxiang.lx
2024-12-16 10:37:10 +08:00
parent ac70560364
commit 3581caec76
8 changed files with 24 additions and 22 deletions

View File

@@ -269,6 +269,7 @@ class QwenTokenizer():
text = self.tokenizer.batch_decode([tokens], skip_special_tokens=self.skip_special_tokens)[0]
return text
@lru_cache(maxsize=None)
def get_qwen_tokenizer(
token_path: str,