mirror of
https://github.com/Steffo99/unimore-bda-6.git
synced 2024-11-25 17:24:20 +00:00
13 lines
319 B
Python
13 lines
319 B
Python
import typing as t
|
|
|
|
from .base import BaseTokenizer
|
|
|
|
|
|
class PlainTokenizer(BaseTokenizer):
|
|
"""
|
|
Tokenizer which just splits the text into tokens by separating them at whitespaces with `str.split`.
|
|
"""
|
|
|
|
def tokenize(self, text: str) -> t.Iterator[str]:
|
|
tokens = text.split()
|
|
return tokens
|