import unittest
import re
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()


class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "word_tokenize"), 
            "`word_tokenize` is not declared"
        )
        _require_success(
            self,
            user_code.word_tokenize == word_tokenize,
            "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
        )
        _require_success(
            self, 
            hasattr(user_code, "stopwords"), 
            "`stopwords` is not declared"
        )
        _require_success(
            self,
            user_code.stopwords == stopwords,
            "`stopwords` is declared, but it's not `nltk.corpus.stopwords`"
        )
        _require_success(
            self, 
            hasattr(user_code, "WordNetLemmatizer"), 
            "`WordNetLemmatizer` is not declared"
        )
        _require_success(
            self,
            user_code.WordNetLemmatizer == WordNetLemmatizer,
            "`WordNetLemmatizer` is declared, but it's not `nltk.stem.WordNetLemmatizer`"
        )
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        self._testMethodName = "imports are correct"

    # Test that required NLTK resources are downloaded
    def test_nltk_downloads(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
                "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'stopwords' *\)", user_code_text),
                "Code for downloading `'stopwords'` shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'wordnet' *\)", user_code_text),
                "Code for downloading `'wordnet'` is not provided"
            )
        self._testMethodName = "NLTK resources are downloaded"

    # Test original text
    def test_text(self):
        import user_code
        expected = "The leaves on the tree were turning a bright red, indicating that fall was leaving its mark."
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _check_success(
            self,
            user_code.text == expected,
            "`text` contains correct value",
            "`text` shouldn't be modified"
        )

    # Test lowercase conversion
    def test_text_lower(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text, str), 
            "`text` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _check_success(
            self,
            user_code.text_lower == user_code.text.lower(),
            "`text_lower` is computed correctly",
            "`text_lower` is not the lowercase of `text`"
        )

    # Test tokenization of text_lower
    def test_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text_lower, str), 
            "`text_lower` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tokens == word_tokenize(user_code.text_lower),
            "`tokens` is computed correctly",
            "`text_lower` is not correctly tokenized into words"
        )

    # Test stop_words set
    def test_stop_words(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "stop_words"), 
            "`stop_words` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.stop_words, set), 
            "`stop_words` is not a set"
        )
        _check_success(
            self,
            user_code.stop_words == set(stopwords.words('english')),
            "`stop_words` contains correct English stopwords set",
            "`stop_words` doesn't contain English stopwords"
        )

    # Test filtered_tokens filtering
    def test_filtered_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.tokens, list), 
            "`tokens` is not a list"
        )
        _require_success(
            self, 
            hasattr(user_code, "stop_words"), 
            "`stop_words` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.stop_words, set), 
            "`stop_words` is not a set"
        )
        _require_success(
            self, 
            hasattr(user_code, "filtered_tokens"), 
            "`filtered_tokens` is not declared"
        )
        expected = [tok for tok in user_code.tokens if tok not in user_code.stop_words]
        _check_success(
            self,
            user_code.filtered_tokens == expected,
            "`filtered_tokens` is computed correctly",
            "stop words in `tokens` are not correctly filtered out"
        )

    # Test lemmatizer instance
    def test_lemmatizer(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "lemmatizer"), 
            "`lemmatizer` is not declared"
        )
        _check_success(
            self,
            isinstance(user_code.lemmatizer, WordNetLemmatizer),
            "`lemmatizer` is defined correctly",
            "`lemmatizer` is not an instance of `WordNetLemmatizer`"
        )

    # Test lemmatizing results
    def test_lemmatized_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "lemmatizer"), 
            "`lemmatizer` is not declared"
        )
        _require_success(
            self,
            isinstance(user_code.lemmatizer, WordNetLemmatizer),
            "`lemmatizer` is not an instance of `WordNetLemmatizer`"
        )
        _require_success(
            self, 
            hasattr(user_code, "filtered_tokens"), 
            "`filtered_tokens` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.filtered_tokens, list), 
            "`filtered_tokens` is not a list"
        )
        _require_success(
            self, 
            hasattr(user_code, "lemmatized_tokens"), 
            "`lemmatized_tokens` is not declared"
        )
        expected = [user_code.lemmatizer.lemmatize(tok) for tok in user_code.filtered_tokens]
        _check_success(
            self,
            user_code.lemmatized_tokens == expected,
            "`lemmatized_tokens` is computed correctly",
            "`filtered_tokens` are not correctly lemmatized"
        )

    # Test print statement
    def test_print(self):
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *\( *\\"Lemmatized tokens:\\" *, *lemmatized_tokens *\)", user_code_text),
            "print statement is correct",
            "print statement shouldn't be modified"
        )


if __name__ == '__main__':
    nltk.download('punkt_tab')
    nltk.download('stopwords')
    nltk.download('wordnet')
    unittest.main()

test_main.py

Tutustu luonnollisen kielen käsittelyn (NLP) perusteisiin oppimalla keskeiset tekstin esikäsittelytekniikat ja menetelmät tekstidatan esittämiseen. Saat käytännön kokemusta työkaluista, joilla puhdistetaan, analysoidaan ja tulkitaan tekstuaalista tietoa. Kehitä taidot, joilla muutat raakakielen jäsennellyiksi oivalluksiksi, ja luot vahvan pohjan tekoälyn ja koneoppimisen edistyneille sovelluksille.

Perehdy tekstin esikäsittelyn perusteisiin raakatekstin valmistelussa analyysia varten. Opiskele tekstin tokenisointia, stop-sanojen suodatusta sekä tokenisoinnin mukauttamista säännöllisten lausekkeiden avulla.

Opi, miten sanoja voidaan palauttaa perusmuotoonsa stemmauksen ja lemmatisaation avulla. Hallitse sanaluokkien tunnistus tekstin kieliopillisen kontekstin rikastamiseksi ja sovella sanaluokkatietoista lemmatisaatiota.

Opi, miten teksti voidaan esittää numeroina vektoriavaruusmallien avulla. Saat käytännön kokemusta toteuttamalla ja mukauttamalla kahta suosittua vektoriavaruusmallia: sanapussimalli ja TF-IDF.

Hanki vankka ymmärrys sanaesityksistä ja siitä, miten ne tallentavat semanttista merkitystä. Tutustu Word2Vecin CBoW- ja Skip-gram-arkkitehtuureihin ja toteuta ne itse.

Haaste: Lemmatisaatio

Ratkaisu

Awesome!

Haaste: Lemmatisaatio

Ratkaisu