1
Fork 0
mirror of https://github.com/Steffo99/unimore-bda-6.git synced 2024-11-26 01:34:20 +00:00
bda-6-steffo/unimore_bda_6/tokenization/__init__.py

15 lines
213 B
Python

from . import nltk_based
from . import potts_based
all_tokenizers = [
nltk_based.nltk_tokenizer,
potts_based.potts_tokenizer,
]
__all__ = (
"nltk_based",
"potts_based",
"all_tokenizers",
)