import unittest
import importlib
import re
import types
from nltk.tokenize import sent_tokenize, word_tokenize
import nltk


# Helper function to require a successful test
def _require_success(test_case, condition, failure_message):
  if not condition:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


# Helper function to check if test is successful
def _check_success(test_case, condition, success_message, failure_message):
  if condition:
    test_case._testMethodName = success_message  # Rename the test for better readability in test reports
  else:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


class TestUserCode(unittest.TestCase):
  # Test that everything is imported correctly
  def test_imports(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sent_tokenize"),
      "`sent_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.sent_tokenize == sent_tokenize,
      "`sent_tokenize` is declared, but it's not `nltk.tokenize.sent_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "word_tokenize"),
      "`word_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.word_tokenize == word_tokenize,
      "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    self._testMethodName = "imports are correct"

  # Test that 'punk_tab' is downloaded
  def test_nltk_download_punkt(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _require_success(
        self,
        re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
        "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
      )
    self._testMethodName = "`'punkt_tab'` tokenizer is downloaded"

  # Test that text was not modified
  @unittest.skip("doesn't work properly now")
  def test_text(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    expected = 'Sherlock Holmes picked them up one by one, and laid them along the edge of the table. Then he reseated himself in the chair, and looked over with a gleam of satisfaction in his eyes. "These," said he, "are all that I have left to remind me of âThe Adventure of the Musgrave Ritualâ."'
    _check_success(
      self,
      user_code.text == expected,
      "`text` contains correct value",
      "`text` shouldn't be modified"
    )

  # Test that text was correctly tokenized into sentences
  def test_sent_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.text, str),
      "`text` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _check_success(
      self,
      sent_tokenize(user_code.text) == user_code.sentences,
      "`sentences` is computed correctly",
      "`text` is incorrectly tokenized into sentences"
    )

  # Test that last sentence was saved and transformed into lowercase
  def test_last_sentence(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.sentences, list),
      "`sentences` is not a list"
    )
    _require_success(
      self,
      isinstance(user_code.sentences[-1], str),
      "last element of `sentences` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      user_code.last_sentence.lower() == user_code.sentences[-1].lower(),
      "`last_sentence` does not contain the last sentence"
    )
    _require_success(
      self,
      user_code.last_sentence == user_code.sentences[-1].lower(),
      "`last_sentence` is not in lowercase"
    )
    _testMethodName = "`last_sentence` is computed correctly"

  # Test that last sentence was tokenized into words
  def test_word_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.last_sentence, str),
      "`last_sentence` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "tokens"),
      "`tokens` is not declared"
    )
    _check_success(
      self,
      word_tokenize(user_code.last_sentence) == user_code.tokens,
      "`tokens` is computed correctly",
      "`last_sentence` is incorrectly tokenized into words"
    )

  # Test that print is not modified
  def test_print(self):
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _check_success(
        self,
        re.search(r"print *\( *tokens *\)", user_code_text),
        "print statement is correct",
        "print statement shouldn't be modified"
      )


# Run the tests when executing the script
if __name__ == '__main__':
  nltk.download('punkt_tab')
  unittest.main()

test_main.py

Maîtrise des pipelines pratiques de PNL avec TensorFlow et NLTK. Accent sur le prétraitement avancé des textes, la tokenisation et la mise en œuvre des plongements de mots, y compris Word2Vec. Ce cours est conçu pour les ingénieurs en apprentissage profond souhaitant acquérir une expérience pratique dans la construction de bases solides en PNL pour des applications d'IA.

Défi : Tokenisation

Solution