mirror of
https://github.com/Steffo99/unimore-bda-6.git
synced 2024-11-29 19:24:19 +00:00
11 lines
315 B
Python
11 lines
315 B
Python
import tensorflow
|
|
|
|
from .base import BaseTokenizer
|
|
|
|
|
|
class LowercaseTokenizer(BaseTokenizer):
|
|
def tokenize_builtins(self, text: str) -> list[str]:
|
|
return text.lower().split()
|
|
|
|
def tokenize_tensorflow(self, text: tensorflow.Tensor) -> tensorflow.Tensor:
|
|
return tensorflow.strings.lower(text)
|