2023-02-02 16:24:11 +00:00
|
|
|
from . import nltk_based
|
|
|
|
|
from . import potts_based
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
all_tokenizers = [
|
2023-02-03 01:10:00 +00:00
|
|
|
nltk_based.nltk_tokenizer,
|
|
|
|
|
potts_based.potts_tokenizer,
|
2023-02-02 16:24:11 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
__all__ = (
|
|
|
|
|
"nltk_based",
|
|
|
|
|
"potts_based",
|
|
|
|
|
"all_tokenizers",
|
|
|
|
|
)
|
