import unittest
import importlib
import re
import types
from nltk.tokenize import sent_tokenize, word_tokenize
import nltk


# Helper function to require a successful test
def _require_success(test_case, condition, failure_message):
  if not condition:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


# Helper function to check if test is successful
def _check_success(test_case, condition, success_message, failure_message):
  if condition:
    test_case._testMethodName = success_message  # Rename the test for better readability in test reports
  else:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


class TestUserCode(unittest.TestCase):
  # Test that everything is imported correctly
  def test_imports(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sent_tokenize"),
      "`sent_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.sent_tokenize == sent_tokenize,
      "`sent_tokenize` is declared, but it's not `nltk.tokenize.sent_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "word_tokenize"),
      "`word_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.word_tokenize == word_tokenize,
      "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    self._testMethodName = "imports are correct"

  # Test that 'punk_tab' is downloaded
  def test_nltk_download_punkt(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _require_success(
        self,
        re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
        "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
      )
    self._testMethodName = "`'punkt_tab'` tokenizer is downloaded"

  # Test that text was not modified
  @unittest.skip("doesn't work properly now")
  def test_text(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    expected = 'Sherlock Holmes picked them up one by one, and laid them along the edge of the table. Then he reseated himself in the chair, and looked over with a gleam of satisfaction in his eyes. "These," said he, "are all that I have left to remind me of âThe Adventure of the Musgrave Ritualâ."'
    _check_success(
      self,
      user_code.text == expected,
      "`text` contains correct value",
      "`text` shouldn't be modified"
    )

  # Test that text was correctly tokenized into sentences
  def test_sent_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.text, str),
      "`text` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _check_success(
      self,
      sent_tokenize(user_code.text) == user_code.sentences,
      "`sentences` is computed correctly",
      "`text` is incorrectly tokenized into sentences"
    )

  # Test that last sentence was saved and transformed into lowercase
  def test_last_sentence(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.sentences, list),
      "`sentences` is not a list"
    )
    _require_success(
      self,
      isinstance(user_code.sentences[-1], str),
      "last element of `sentences` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      user_code.last_sentence.lower() == user_code.sentences[-1].lower(),
      "`last_sentence` does not contain the last sentence"
    )
    _require_success(
      self,
      user_code.last_sentence == user_code.sentences[-1].lower(),
      "`last_sentence` is not in lowercase"
    )
    _testMethodName = "`last_sentence` is computed correctly"

  # Test that last sentence was tokenized into words
  def test_word_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.last_sentence, str),
      "`last_sentence` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "tokens"),
      "`tokens` is not declared"
    )
    _check_success(
      self,
      word_tokenize(user_code.last_sentence) == user_code.tokens,
      "`tokens` is computed correctly",
      "`last_sentence` is incorrectly tokenized into words"
    )

  # Test that print is not modified
  def test_print(self):
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _check_success(
        self,
        re.search(r"print *\( *tokens *\)", user_code_text),
        "print statement is correct",
        "print statement shouldn't be modified"
      )


# Run the tests when executing the script
if __name__ == '__main__':
  nltk.download('punkt_tab')
  unittest.main()

test_main.py

Explore os fundamentos do Processamento de Linguagem Natural (PLN) aprendendo técnicas essenciais de pré-processamento de texto e métodos para representação de dados textuais. Adquira experiência prática com ferramentas utilizadas para limpar, analisar e interpretar informações textuais. Desenvolva as habilidades necessárias para transformar linguagem bruta em insights estruturados, estabelecendo uma base sólida para aplicações avançadas em inteligência artificial e aprendizado de máquina.

Aprofunde-se nos fundamentos do pré-processamento de texto para preparar textos brutos para análise. Aprenda a tokenizar textos, filtrar stop words e personalizar a tokenização com expressões regulares.

Descubra como as palavras podem ser reduzidas às suas formas básicas utilizando stemming e lematização. Domine a marcação de classe gramatical para enriquecer o texto com contexto gramatical e aplique lematização sensível à classe gramatical.

Aprenda como textos podem ser representados por números utilizando modelos de espaço vetorial. Experimente na prática implementando e customizando dois modelos vetoriais populares: bag of words e TF-IDF.

Obtenha uma compreensão sólida sobre embeddings de palavras e como eles capturam significados semânticos. Explore as arquiteturas CBoW e Skip-gram utilizadas no Word2Vec e implemente-as por conta própria.

Desafio: Tokenização

Solução

Awesome!

Desafio: Tokenização

Solução