import unittest
import re
import nltk
from nltk import pos_tag
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet as wn


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()

# Function that SHOULD NOT be modified
def get_wordnet_pos(treebank_tag):
    if treebank_tag.startswith('J'):
        return wn.ADJ
    elif treebank_tag.startswith('V'):
        return wn.VERB
    elif treebank_tag.startswith('R'):
        return wn.ADV
    else:
        return wn.NOUN

class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "word_tokenize"), 
            "`word_tokenize` is not declared"
        )
        _require_success(
            self,
            user_code.word_tokenize == word_tokenize,
            "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
        )
        _require_success(
            self, 
            hasattr(user_code, "stopwords"), 
            "`stopwords` is not declared"
        )
        _require_success(
            self,
            user_code.stopwords == stopwords,
            "`stopwords` is declared, but it's not `nltk.corpus.stopwords`"
        )
        _require_success(
            self, 
            hasattr(user_code, "WordNetLemmatizer"), 
            "`WordNetLemmatizer` is not declared"
        )
        _require_success(
            self,
            user_code.WordNetLemmatizer == WordNetLemmatizer,
            "`WordNetLemmatizer` is declared, but it's not `nltk.stem.WordNetLemmatizer`"
        )
        _require_success(
            self, 
            hasattr(user_code, "pos_tag"), 
            "`pos_tag` is not declared"
        )
        _require_success(
            self,
            user_code.pos_tag == pos_tag,
            "`pos_tag` is declared, but it's not `nltk.pos_tag`"
        )
        _require_success(
            self, 
            hasattr(user_code, "wn"), 
            "`wn` is not declared"
        )
        _require_success(
            self,
            user_code.wn == wn,
            "`wn` is declared, but it's not `nltk.corpus.wordnet`"
        )
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        self._testMethodName = "imports are correct"

    # Test that required NLTK resources are downloaded
    def test_nltk_downloads(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
            _require_success(
                self,
                re.search(r"nltk *\. *download *$ *'punkt_tab' *$", user_code_text),
                "Code for downloading `'punkt_tab'` shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *$ *'stopwords' *$", user_code_text),
                "Code for downloading `'stopwords'` shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *$ *'averaged_perceptron_tagger_eng' *$", user_code_text),
                "Code for downloading `'averaged_perceptron_tagger_eng'` shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *$ *'wordnet' *$", user_code_text),
                "Code for downloading `'wordnet'` is not provided"
            )
        self._testMethodName = "NLTK resources are downloaded"

    # Test get_wordnet_pos
    def test_get_wordnet_pos(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "get_wordnet_pos"), 
            "`get_wordnet_pos` is not declared"
        )
        _require_success(
            self, 
            callable(user_code.get_wordnet_pos), 
            "`get_wordnet_pos` is not a function"
        )
        pos_tags = pos_tags = ['CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'JJ', 'JJR', 'JJS', 'LS', 'MD', 'NN', 'NNS', 'NNP', 'NNPS', 'PDT', 'POS', 'PRP', 'PRP$', 'RB', 'RBR', 'RBS', 'RP', 'SYM', 'TO', 'UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'WDT', 'WP', 'WP$', 'WRB']
        _check_success(
            self, 
            list(map(user_code.get_wordnet_pos, pos_tags)) == list(map(get_wordnet_pos, pos_tags)),
            "`get_wordnet_pos` is correctly defined",
            "`get_wordnet_pos` shouldn't be modified"
        )
    
    # Test original text
    def test_text(self):
        import user_code
        expected = """Late in the evening, as the stars began to twinkle in the clear night sky, 
an old man sat quietly on the porch, reminiscing about his youthful adventures."""
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _check_success(
            self,
            user_code.text == expected,
            "`text` contains correct value",
            "`text` shouldn't be modified"
        )

    # Test lowercase conversion
    def test_text_lower(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text, str), 
            "`text` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _check_success(
            self,
            user_code.text_lower == user_code.text.lower(),
            "`text_lower` is computed correctly",
            "`text_lower` is not the lowercase of `text`"
        )

    # Test tokenization of text_lower
    def test_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text_lower, str), 
            "`text_lower` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tokens == word_tokenize(user_code.text_lower),
            "`tokens` is computed correctly",
            "`text_lower` is not correctly tokenized into words"
        )

    # Test stop_words set
    def test_stop_words(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "stop_words"), 
            "`stop_words` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.stop_words, set), 
            "`stop_words` is not a set"
        )
        _check_success(
            self,
            user_code.stop_words == set(stopwords.words('english')),
            "`stop_words` contains correct English stopwords set",
            "`stop_words` doesn't contain English stopwords"
        )

    # Test filtered_tokens filtering
    def test_filtered_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.tokens, list), 
            "`tokens` is not a list"
        )
        _require_success(
            self, 
            hasattr(user_code, "stop_words"), 
            "`stop_words` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.stop_words, set), 
            "`stop_words` is not a set"
        )
        _require_success(
            self, 
            hasattr(user_code, "filtered_tokens"), 
            "`filtered_tokens` is not declared"
        )
        expected = [tok for tok in user_code.tokens if tok not in user_code.stop_words]
        _check_success(
            self,
            user_code.filtered_tokens == expected,
            "`filtered_tokens` is computed correctly",
            "stopword in `tokens` are not correctly filtered out"
        )

    # Test tagged_tokens tagging
    def test_tagged_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "filtered_tokens"), 
            "`filtered_tokens` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.filtered_tokens, list), 
            "`filtered_tokens` is not a list"
        )
        _require_success(
            self, 
            hasattr(user_code, "tagged_tokens"), 
            "`tagged_tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tagged_tokens == pos_tag(user_code.filtered_tokens),
            "`tagged_tokens` is computed correctly",
            "`filtered_tokens` are not correctly pos tagged"
        )

    # Test lemmatizer instance
    def test_lemmatizer(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "lemmatizer"), 
            "`lemmatizer` is not declared"
        )
        _check_success(
            self,
            isinstance(user_code.lemmatizer, WordNetLemmatizer),
            "`lemmatizer` is defined correctly",
            "`lemmatizer` is not an instance of `WordNetLemmatizer`"
        )

    # Test lemmatizing results
    def test_lemmatized_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "lemmatizer"), 
            "`lemmatizer` is not declared"
        )
        _require_success(
            self,
            isinstance(user_code.lemmatizer, WordNetLemmatizer),
            "`lemmatizer` is not an instance of `WordNetLemmatizer`"
        )
        _require_success(
            self, 
            hasattr(user_code, "filtered_tokens"), 
            "`filtered_tokens` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.filtered_tokens, list), 
            "`filtered_tokens` is not a list"
        )
        _require_success(
            self, 
            hasattr(user_code, "lemmatized_tokens"), 
            "`lemmatized_tokens` is not declared"
        )
        expected = [user_code.lemmatizer.lemmatize(tok, get_wordnet_pos(tag)) for tok, tag in user_code.tagged_tokens]
        _check_success(
            self,
            user_code.lemmatized_tokens == expected,
            "`lemmatized_tokens` is computed correctly",
            "`filtered_tokens` are not correctly lemmatized"
        )

    # Test print statement
    def test_print(self):
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *$ *' ' *\. *join *\( *lemmatized_tokens *$", user_code_text),
            "print statement is correct",
            "print statement shouldn't be modified"
        )


if __name__ == '__main__':
    nltk.download('punkt_tab')
    nltk.download('stopwords')
    nltk.download('averaged_perceptron_tagger_eng')
    nltk.download('wordnet')
    unittest.main()

test_main.py

Verken de basisprincipes van Natural Language Processing (NLP) door essentiële technieken voor tekstvoorbewerking en methoden voor het representeren van tekstgegevens te leren. Doe praktische ervaring op met de tools die worden gebruikt om tekstuele informatie te reinigen, analyseren en interpreteren. Ontwikkel de vaardigheden die nodig zijn om ruwe taal om te zetten in gestructureerde inzichten, waarmee een solide basis wordt gelegd voor geavanceerde toepassingen in kunstmatige intelligentie en machine learning.

Verdiep u in de grondbeginselen van tekstvoorbewerking om ruwe tekst voor analyse voor te bereiden. Leer hoe u tekst segmenteert, stopwoorden verwijdert en tokenisatie aanpast met reguliere expressies.

Ontdek hoe woorden kunnen worden teruggebracht tot hun basisvormen met behulp van stemming en lemmatizatie. Beheers het toekennen van woordsoorten om tekst te verrijken met grammaticale context en pas lemmatizatie toe met kennis van woordsoorten.

Leer hoe tekst kan worden weergegeven met getallen door middel van vectorruimte modellen. Doe praktische ervaring op met het implementeren en aanpassen van twee populaire vectorruimte modellen: bag-of-words en TF-IDF.

Verkrijg een grondig begrip van woordembeddings en hoe deze semantische betekenis vastleggen. Verken CBoW- en Skip-gram-architecturen die worden gebruikt in Word2Vec, en implementeer deze zelfstandig.

Uitdaging: Lemmatisatie met POS-Tagging

Oplossing

Awesome!

Uitdaging: Lemmatisatie met POS-Tagging

Oplossing