import unittest
import importlib
import re
import types
from nltk.tokenize import sent_tokenize, word_tokenize
import nltk


# Helper function to require a successful test
def _require_success(test_case, condition, failure_message):
  if not condition:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


# Helper function to check if test is successful
def _check_success(test_case, condition, success_message, failure_message):
  if condition:
    test_case._testMethodName = success_message  # Rename the test for better readability in test reports
  else:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


class TestUserCode(unittest.TestCase):
  # Test that everything is imported correctly
  def test_imports(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sent_tokenize"),
      "`sent_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.sent_tokenize == sent_tokenize,
      "`sent_tokenize` is declared, but it's not `nltk.tokenize.sent_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "word_tokenize"),
      "`word_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.word_tokenize == word_tokenize,
      "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    self._testMethodName = "imports are correct"

  # Test that 'punk_tab' is downloaded
  def test_nltk_download_punkt(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _require_success(
        self,
        re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
        "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
      )
    self._testMethodName = "`'punkt_tab'` tokenizer is downloaded"

  # Test that text was not modified
  @unittest.skip("doesn't work properly now")
  def test_text(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    expected = 'Sherlock Holmes picked them up one by one, and laid them along the edge of the table. Then he reseated himself in the chair, and looked over with a gleam of satisfaction in his eyes. "These," said he, "are all that I have left to remind me of âThe Adventure of the Musgrave Ritualâ."'
    _check_success(
      self,
      user_code.text == expected,
      "`text` contains correct value",
      "`text` shouldn't be modified"
    )

  # Test that text was correctly tokenized into sentences
  def test_sent_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.text, str),
      "`text` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _check_success(
      self,
      sent_tokenize(user_code.text) == user_code.sentences,
      "`sentences` is computed correctly",
      "`text` is incorrectly tokenized into sentences"
    )

  # Test that last sentence was saved and transformed into lowercase
  def test_last_sentence(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.sentences, list),
      "`sentences` is not a list"
    )
    _require_success(
      self,
      isinstance(user_code.sentences[-1], str),
      "last element of `sentences` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      user_code.last_sentence.lower() == user_code.sentences[-1].lower(),
      "`last_sentence` does not contain the last sentence"
    )
    _require_success(
      self,
      user_code.last_sentence == user_code.sentences[-1].lower(),
      "`last_sentence` is not in lowercase"
    )
    _testMethodName = "`last_sentence` is computed correctly"

  # Test that last sentence was tokenized into words
  def test_word_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.last_sentence, str),
      "`last_sentence` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "tokens"),
      "`tokens` is not declared"
    )
    _check_success(
      self,
      word_tokenize(user_code.last_sentence) == user_code.tokens,
      "`tokens` is computed correctly",
      "`last_sentence` is incorrectly tokenized into words"
    )

  # Test that print is not modified
  def test_print(self):
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _check_success(
        self,
        re.search(r"print *\( *tokens *\)", user_code_text),
        "print statement is correct",
        "print statement shouldn't be modified"
      )


# Run the tests when executing the script
if __name__ == '__main__':
  nltk.download('punkt_tab')
  unittest.main()

test_main.py

Explorez les principes fondamentaux du traitement automatique du langage naturel (TALN) en apprenant les techniques essentielles de prétraitement du texte et les méthodes de représentation des données textuelles. Acquérez une expérience pratique avec les outils utilisés pour nettoyer, analyser et interpréter les informations textuelles. Développez les compétences nécessaires pour transformer le langage brut en informations structurées, établissant ainsi une base solide pour des applications avancées en intelligence artificielle et en apprentissage automatique.

Explorez les fondamentaux du prétraitement de texte afin de préparer le texte brut pour l'analyse. Découvrez la tokenisation du texte, l'élimination des mots vides et la personnalisation de la tokenisation à l'aide d'expressions régulières.

Découvrez comment les mots peuvent être réduits à leur forme de base grâce à la racinisation et à la lemmatisation.
Maîtrise de l'étiquetage des parties du discours pour enrichir le texte avec un contexte grammatical et application de la lemmatisation sensible au POS.

Découvrez comment le texte peut être représenté par des nombres à l'aide de modèles d'espace vectoriel. Expérimentez concrètement en implémentant et en personnalisant deux modèles d'espace vectoriel populaires : sac de mots et TF-IDF.

Acquérir une compréhension approfondie des représentations vectorielles de mots et de leur capacité à capturer la signification sémantique. Explorer les architectures CBoW et Skip-gram utilisées dans Word2Vec, et les mettre en œuvre soi-même.

Défi : Tokenisation

Solution

Awesome!

Défi : Tokenisation

Solution