import unittest
import re
import nltk
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
from scipy.sparse import csr_matrix


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()

class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "pd"), 
            "`pd` is not declared"
        )
        _require_success(
            self,
            user_code.pd == pd,
            "`pd` is declared, but it's not `pandas` library"
        )
        _require_success(
            self, 
            hasattr(user_code, "TfidfVectorizer"), 
            "`TfidfVectorizer` is not declared"
        )
        _require_success(
            self,
            user_code.TfidfVectorizer == TfidfVectorizer,
            "`TfidfVectorizer` is declared, but it's not `sklearn.feature_extraction.text.TfidfVectorizer`"
        )
        self._testMethodName = "imports are correct"

    # Test original corpus
    def test_corpus(self):
        import user_code
        expected = pd.read_csv(
    'https://content-media-cdn.codefinity.com/courses/c68c1f2e-2c90-4d5d-8db9-1e97ca89d15e/section_3/chapter_4/example_corpus.csv')
        _require_success(
            self, 
            hasattr(user_code, "corpus"), 
            "`corpus` is not declared"
        )
        _check_success(
            self,
            expected.equals(user_code.corpus),
            "`corpus` contains correct value",
            "`corpus` shouldn't be modified"
        )

    # Test that tfidf_vectorizer is declared correctly
    def test_tfidf_vectorizer(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "tfidf_vectorizer"), 
            "`tfidf_vectorizer` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.tfidf_vectorizer, TfidfVectorizer), 
            "`tfidf_vectorizer` is not an instance of `TfidfVectorizer`"
        )
        _require_success(
            self, 
            user_code.tfidf_vectorizer.ngram_range == (1, 3), 
            "`tfidf_vectorizer` should work with and only with unigrams, bigrams, and trigrams"
        )
        self._testMethodName = "`tfidf_vectorizer` is defined correctly"

    # Test tfidf_matrix computation
    def test_tfidf_matrix(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "corpus"), 
            "`corpus` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.corpus, pd.DataFrame), 
            "`corpus` is not a DataFrame"
        )
        _require_success(
            self, 
            "Document" in user_code.corpus.columns, 
            "`corpus` doesn't contain `'Document'` column"
        )
        _require_success(
            self, 
            hasattr(user_code, "tfidf_matrix"), 
            "`tfidf_matrix` is not declared"
        )
        tfidf_vectorizer = TfidfVectorizer(ngram_range=(1, 3))
        tfidf_matrix = tfidf_vectorizer.fit_transform(user_code.corpus['Document'])
        _check_success(
            self,
            user_code.tfidf_matrix.shape == tfidf_matrix.shape and (user_code.tfidf_matrix - tfidf_matrix).nnz == 0,
            "`tfidf_matrix` is computed correctly",
            "`tfidf_matrix` is not computed correctly"
        )

    # Test tfidf_matrix_df conversion
    def test_tfidf_matrix_df(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "tfidf_vectorizer"), 
            "`tfidf_vectorizer` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.tfidf_vectorizer, TfidfVectorizer), 
            "`tfidf_vectorizer` is not an instance of `TfidfVectorizer`"
        )
        _require_success(
            self, 
            hasattr(user_code, "tfidf_matrix"), 
            "`tfidf_matrix` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.tfidf_matrix, csr_matrix), 
            "`tfidf_matrix` is not a `csr_matrix`"
        )
        _require_success(
            self, 
            hasattr(user_code, "tfidf_matrix_df"), 
            "`tfidf_matrix_df` is not declared"
        )
        tfidf_matrix_df = pd.DataFrame(user_code.tfidf_matrix.toarray(), columns=user_code.tfidf_vectorizer.get_feature_names_out())
        _check_success(
            self,
            tfidf_matrix_df.equals(user_code.tfidf_matrix_df),
            "`tfidf_matrix_df` is computed correctly",
            "`tfidf_matrix_df` is not computed correctly"
        )

    # Test print statement
    def test_print(self):
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *\( *tfidf_matrix_df *\[ *(?P<quote>[\\"'])medical(?P=quote) *\] *\. *values *\)", user_code_text),
            "print statement is correct",
            "print statement is not correct"
        )


if __name__ == '__main__':
    unittest.main()

test_main.py

Explore os fundamentos do Processamento de Linguagem Natural (PLN) aprendendo técnicas essenciais de pré-processamento de texto e métodos para representação de dados textuais. Adquira experiência prática com ferramentas utilizadas para limpar, analisar e interpretar informações textuais. Desenvolva as habilidades necessárias para transformar linguagem bruta em insights estruturados, estabelecendo uma base sólida para aplicações avançadas em inteligência artificial e aprendizado de máquina.

Aprofunde-se nos fundamentos do pré-processamento de texto para preparar textos brutos para análise. Aprenda a tokenizar textos, filtrar stop words e personalizar a tokenização com expressões regulares.

Descubra como as palavras podem ser reduzidas às suas formas básicas utilizando stemming e lematização. Domine a marcação de classe gramatical para enriquecer o texto com contexto gramatical e aplique lematização sensível à classe gramatical.

Aprenda como textos podem ser representados por números utilizando modelos de espaço vetorial. Experimente na prática implementando e customizando dois modelos vetoriais populares: bag of words e TF-IDF.

Obtenha uma compreensão sólida sobre embeddings de palavras e como eles capturam significados semânticos. Explore as arquiteturas CBoW e Skip-gram utilizadas no Word2Vec e implemente-as por conta própria.

Desafio: TF-IDF

Solução

Awesome!

Desafio: TF-IDF

Solução