import unittest
import re
import nltk
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
from scipy.sparse import csr_matrix


# Helper functions
def _require_success(test_case, condition, failure_message):
    if not condition:
        test_case._testMethodName = failure_message
        test_case.fail()

def _check_success(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
    else:
        test_case._testMethodName = failure_message
        test_case.fail()

class TestUserCode(unittest.TestCase):
    # Test that everything is imported correctly
    def test_imports(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "pd"), 
            "`pd` is not declared"
        )
        _require_success(
            self,
            user_code.pd == pd,
            "`pd` is declared, but it's not `pandas` library"
        )
        _require_success(
            self, 
            hasattr(user_code, "CountVectorizer"), 
            "`CountVectorizer` is not declared"
        )
        _require_success(
            self,
            user_code.CountVectorizer == CountVectorizer,
            "`CountVectorizer` is declared, but it's not `sklearn.feature_extraction.text.CountVectorizer`"
        )
        self._testMethodName = "imports are correct"

    # Test original corpus
    def test_corpus(self):
        import user_code
        expected = pd.read_csv(
    'https://content-media-cdn.codefinity.com/courses/c68c1f2e-2c90-4d5d-8db9-1e97ca89d15e/section_3/chapter_4/example_corpus.csv')
        _require_success(
            self, 
            hasattr(user_code, "corpus"), 
            "`corpus` is not declared"
        )
        _check_success(
            self,
            expected.equals(user_code.corpus),
            "`corpus` contains correct value",
            "`corpus` shouldn't be modified"
        )

    # Test that count_vectorizer is declared correctly
    def test_count_vectorizer(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "count_vectorizer"), 
            "`count_vectorizer` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.count_vectorizer, CountVectorizer), 
            "`count_vectorizer` is not an instance of `CountVectorizer`"
        )
        _require_success(
            self, 
            user_code.count_vectorizer.binary == False, 
            "`count_vectorizer` should be a frequency-based BoW"
        )
        _require_success(
            self, 
            user_code.count_vectorizer.ngram_range == (1, 2), 
            "`count_vectorizer` should work with and only with unigrams and bigrams"
        )
        self._testMethodName = "`count_vectorizer` is defined correctly"

    # Test bow_matrix computation
    def test_bow_matrix(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "corpus"), 
            "`corpus` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.corpus, pd.DataFrame), 
            "`corpus` is not a DataFrame"
        )
        _require_success(
            self, 
            "Document" in user_code.corpus.columns, 
            "`corpus` doesn't contain `'Document'` column"
        )
        _require_success(
            self, 
            hasattr(user_code, "bow_matrix"), 
            "`bow_matrix` is not declared"
        )
        count_vectorizer = CountVectorizer(ngram_range=(1, 2))
        bow_matrix = count_vectorizer.fit_transform(user_code.corpus['Document'])
        _check_success(
            self,
            user_code.bow_matrix.shape == bow_matrix.shape and (user_code.bow_matrix - bow_matrix).nnz == 0,
            "`bow_matrix` is computed correctly",
            "`bow_matrix` is not computed correctly"
        )

    # Test bow_df conversion
    def test_bow_df(self):
        import user_code
        _require_success(
            self, 
            hasattr(user_code, "count_vectorizer"), 
            "`count_vectorizer` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.count_vectorizer, CountVectorizer), 
            "`count_vectorizer` is not an instance of `CountVectorizer`"
        )
        _require_success(
            self, 
            hasattr(user_code, "bow_matrix"), 
            "`bow_matrix` is not declared"
        )
        _require_success(
            self, 
            isinstance(user_code.bow_matrix, csr_matrix), 
            "`bow_matrix` is not a `csr_matrix`"
        )
        _require_success(
            self, 
            hasattr(user_code, "bow_df"), 
            "`bow_df` is not declared"
        )
        bow_df = pd.DataFrame(user_code.bow_matrix.toarray(), columns=user_code.count_vectorizer.get_feature_names_out())
        _check_success(
            self,
            bow_df.equals(user_code.bow_df),
            "`bow_df` is computed correctly",
            "`bow_df` is not computed correctly"
        )

    # Test print statement
    def test_print(self):
        with open("user_code.py", "r") as f:
            user_code_text = f.read()
        _check_success(
            self,
            re.search(r"print *\( *bow_df *\[ *(?P<quote>[\\"'])graphic design(?P=quote) *\] *\. *values *\)", user_code_text),
            "print statement is correct",
            "print statement is not correct"
        )


if __name__ == '__main__':
    unittest.main()

test_main.py

Master practical NLP pipelines using TensorFlow and NLTK. Focus on advanced text preprocessing, tokenization, and the implementation of word embeddings, including Word2Vec. This course is tailored for deep learning engineers seeking hands-on experience in building robust NLP foundations for AI applications.

Challenge: Bag of Words

Solution