bda-6-steffo/unimore_bda_6/tokenization/nltk_based.py

import nltk
import nltk.sentiment.util


def nltk_tokenizer(text: str) -> list[str]:
    """
    Convert a text string into a list of tokens.
    """
    tokens = nltk.word_tokenize(text)
    nltk.sentiment.util.mark_negation(tokens, shallow=True)
    return tokens


__all__ = (
    "nltk_tokenizer",
)
Refactor things to work better 2023-02-02 16:24:11 +00:00			`import nltk`
			`import nltk.sentiment.util`


Fix VanillaSA to work with iterators 2023-02-03 01:10:00 +00:00			`def nltk_tokenizer(text: str) -> list[str]:`
Refactor things to work better 2023-02-02 16:24:11 +00:00			`"""`
			`Convert a text string into a list of tokens.`
			`"""`
			`tokens = nltk.word_tokenize(text)`
			`nltk.sentiment.util.mark_negation(tokens, shallow=True)`
			`return tokens`


			`__all__ = (`
Fix VanillaSA to work with iterators 2023-02-03 01:10:00 +00:00			`"nltk_tokenizer",`
Refactor things to work better 2023-02-02 16:24:11 +00:00			`)`