import unittest
import re
import nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag
import pandas as pd


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()


class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "word_tokenize"), 
            "`word_tokenize` is not declared"
        )
        _require_success(
            self,
            user_code.word_tokenize == word_tokenize,
            "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
        )
        _require_success(
            self, 
            hasattr(user_code, "pos_tag"), 
            "`pos_tag` is not declared"
        )
        _require_success(
            self,
            user_code.pos_tag == pos_tag,
            "`pos_tag` is declared, but it's not `nltk.pos_tag`"
        )
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        self._testMethodName = "imports are correct"

    # Test that required NLTK resources are downloaded
    def test_nltk_downloads(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
                "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'averaged_perceptron_tagger_eng' *\)", user_code_text),
                "Code for downloading `'averaged_perceptron_tagger_eng'` is not provided"
            )
        self._testMethodName = "NLTK resources are downloaded"

    # Test original text
    def test_text(self):
        import user_code
        expected = "The central bank raised interest rates by 0.5 percentage points yesterday."
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _check_success(
            self,
            user_code.text == expected,
            "`text` contains correct value",
            "`text` shouldn't be modified"
        )

    # Test lowercase conversion
    def test_text_lower(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text, str), 
            "`text` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _check_success(
            self,
            user_code.text_lower == user_code.text.lower(),
            "`text_lower` is computed correctly",
            "`text_lower` is not the lowercase of `text`"
        )

    # Test tokenization of text_lower
    def test_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text_lower, str), 
            "`text_lower` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tokens == word_tokenize(user_code.text_lower),
            "`tokens` is computed correctly",
            "`text_lower` is not correctly tokenized into words"
        )

    # Test pos tagging results
    def test_tagged_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.tokens, list), 
            "`tokens` is not a list"
        )
        _require_success(
            self, 
            hasattr(user_code, "tagged_tokens"), 
            "`tagged_tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tagged_tokens == pos_tag(user_code.tokens),
            "`tagged_tokens` is computed correctly",
            "`tokens` are not correctly pos tagged"
        )

    # Test print statement
    def test_dataframe_print(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "pd"),
            "`pd` is not declared"
        )
        _require_success(
            self,
            user_code.pd == pd,
            "`pd` is declared, but it's not `pandas` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *\( *pd *. *DataFrame *\( *tagged_tokens *, *columns *= *\[ *'Token', *'POS tag' *\] *\) *\)", user_code_text),
            "print statement is correct",
            "print statement is either incorrect, or original parts were modified"
        )


if __name__ == '__main__':
    nltk.download('punkt_tab')
    nltk.download('averaged_perceptron_tagger_eng')
    unittest.main()

test_main.py

Verken de basisprincipes van Natural Language Processing (NLP) door essentiële technieken voor tekstvoorbewerking en methoden voor het representeren van tekstdata te bestuderen. Doe praktische ervaring op met de tools die worden gebruikt om tekstuele informatie te reinigen, analyseren en interpreteren. Ontwikkel de vaardigheden die nodig zijn om ruwe taal om te zetten in gestructureerde inzichten, als solide basis voor geavanceerde toepassingen in kunstmatige intelligentie en machine learning.

Verdiep u in de grondbeginselen van tekstvoorbewerking om ruwe tekst voor analyse voor te bereiden. Leer hoe u tekst segmenteert, stopwoorden verwijdert en tokenisatie aanpast met reguliere expressies.

Ontdek hoe woorden kunnen worden teruggebracht tot hun basisvormen met behulp van stemming en lemmatizatie. Beheers het toekennen van woordsoorten om tekst te verrijken met grammaticale context en pas lemmatizatie toe met kennis van woordsoorten.

Leer hoe tekst kan worden weergegeven met getallen door middel van vectorruimte modellen. Doe praktische ervaring op met het implementeren en aanpassen van twee populaire vectorruimte modellen: bag-of-words en TF-IDF.

Verkrijg een grondig begrip van woordembeddings en hoe deze semantische betekenis vastleggen. Verken CBoW- en Skip-gram-architecturen die worden gebruikt in Word2Vec, en implementeer deze zelfstandig.

Uitdaging: Woordsoortherkenning

Oplossing

Uitdaging: Woordsoortherkenning

Oplossing