Update common.py

This commit is contained in:
0xCAFEBABE0
2024-12-16 13:57:38 +08:00
committed by GitHub
parent 1d8d94de82
commit bcc58cb4cb

View File

@@ -161,6 +161,7 @@ def is_only_punctuation(text):
punctuation_pattern = r'^[\p{P}\p{S}]*$' punctuation_pattern = r'^[\p{P}\p{S}]*$'
return bool(regex.fullmatch(punctuation_pattern, text)) return bool(regex.fullmatch(punctuation_pattern, text))
def mask_to_bias(mask: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: def mask_to_bias(mask: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
assert mask.dtype == torch.bool assert mask.dtype == torch.bool
assert dtype in [torch.float32, torch.bfloat16, torch.float16] assert dtype in [torch.float32, torch.bfloat16, torch.float16]