import unittest
import re
import nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag
import pandas as pd


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()


class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "word_tokenize"), 
            "`word_tokenize` is not declared"
        )
        _require_success(
            self,
            user_code.word_tokenize == word_tokenize,
            "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
        )
        _require_success(
            self, 
            hasattr(user_code, "pos_tag"), 
            "`pos_tag` is not declared"
        )
        _require_success(
            self,
            user_code.pos_tag == pos_tag,
            "`pos_tag` is declared, but it's not `nltk.pos_tag`"
        )
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        self._testMethodName = "imports are correct"

    # Test that required NLTK resources are downloaded
    def test_nltk_downloads(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "nltk"),
            "`nltk` is not declared"
        )
        _require_success(
            self,
            user_code.nltk == nltk,
            "`nltk` is declared, but it's not `nltk` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
                "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
            )
            _require_success(
                self,
                re.search(r"nltk *\. *download *\( *'averaged_perceptron_tagger_eng' *\)", user_code_text),
                "Code for downloading `'averaged_perceptron_tagger_eng'` is not provided"
            )
        self._testMethodName = "NLTK resources are downloaded"

    # Test original text
    def test_text(self):
        import user_code
        expected = "The central bank raised interest rates by 0.5 percentage points yesterday."
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _check_success(
            self,
            user_code.text == expected,
            "`text` contains correct value",
            "`text` shouldn't be modified"
        )

    # Test lowercase conversion
    def test_text_lower(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text"), 
            "`text` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text, str), 
            "`text` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _check_success(
            self,
            user_code.text_lower == user_code.text.lower(),
            "`text_lower` is computed correctly",
            "`text_lower` is not the lowercase of `text`"
        )

    # Test tokenization of text_lower
    def test_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "text_lower"), 
            "`text_lower` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.text_lower, str), 
            "`text_lower` is not a string"
        )
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tokens == word_tokenize(user_code.text_lower),
            "`tokens` is computed correctly",
            "`text_lower` is not correctly tokenized into words"
        )

    # Test pos tagging results
    def test_tagged_tokens(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "tokens"), 
            "`tokens` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.tokens, list), 
            "`tokens` is not a list"
        )
        _require_success(
            self, 
            hasattr(user_code, "tagged_tokens"), 
            "`tagged_tokens` is not declared"
        )
        _check_success(
            self,
            user_code.tagged_tokens == pos_tag(user_code.tokens),
            "`tagged_tokens` is computed correctly",
            "`tokens` are not correctly pos tagged"
        )

    # Test print statement
    def test_dataframe_print(self):
        import user_code
        _require_success(
            self,
            hasattr(user_code, "pd"),
            "`pd` is not declared"
        )
        _require_success(
            self,
            user_code.pd == pd,
            "`pd` is declared, but it's not `pandas` library"
        )
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *\( *pd *. *DataFrame *\( *tagged_tokens *, *columns *= *\[ *'Token', *'POS tag' *\] *\) *\)", user_code_text),
            "print statement is correct",
            "print statement is either incorrect, or original parts were modified"
        )


if __name__ == '__main__':
    nltk.download('punkt_tab')
    nltk.download('averaged_perceptron_tagger_eng')
    unittest.main()

test_main.py

Explore os fundamentos do Processamento de Linguagem Natural (PLN) aprendendo técnicas essenciais de pré-processamento de texto e métodos para representação de dados textuais. Adquira experiência prática com ferramentas utilizadas para limpar, analisar e interpretar informações textuais. Desenvolva as habilidades necessárias para transformar linguagem bruta em insights estruturados, estabelecendo uma base sólida para aplicações avançadas em inteligência artificial e aprendizado de máquina.

Aprofunde-se nos fundamentos do pré-processamento de texto para preparar textos brutos para análise. Aprenda a tokenizar textos, filtrar stop words e personalizar a tokenização com expressões regulares.

Descubra como as palavras podem ser reduzidas às suas formas básicas utilizando stemming e lematização. Domine a marcação de classe gramatical para enriquecer o texto com contexto gramatical e aplique lematização sensível à classe gramatical.

Aprenda como textos podem ser representados por números utilizando modelos de espaço vetorial. Experimente na prática implementando e customizando dois modelos vetoriais populares: bag of words e TF-IDF.

Obtenha uma compreensão sólida sobre embeddings de palavras e como eles capturam significados semânticos. Explore as arquiteturas CBoW e Skip-gram utilizadas no Word2Vec e implemente-as por conta própria.

Desafio: Marcação de Classe Gramatical

Solução

Awesome!

Desafio: Marcação de Classe Gramatical

Solução