mirror of
https://github.com/Steffo99/unimore-bda-6.git
synced 2024-11-22 07:54:19 +00:00
26 lines
818 B
Python
26 lines
818 B
Python
import typing as t
|
|
import abc
|
|
from ..database.datatypes import TextReview, TokenizedReview
|
|
|
|
|
|
class BaseTokenizer(metaclass=abc.ABCMeta):
|
|
"""
|
|
The base for all tokenizers in this project.
|
|
"""
|
|
|
|
def __repr__(self):
|
|
return f"<{self.__class__.__qualname__}>"
|
|
|
|
@abc.abstractmethod
|
|
def tokenize(self, text: str) -> t.Iterator[str]:
|
|
"""
|
|
Convert a text `str` into another `str` containing a series of whitespace-separated tokens.
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
def tokenize_review(self, review: TextReview) -> TokenizedReview:
|
|
"""
|
|
Apply `.tokenize` to the text of a `TextReview`, converting it in a `TokenizedReview`.
|
|
"""
|
|
tokens = self.tokenize(review.text)
|
|
return TokenizedReview(rating=review.rating, tokens=tokens)
|