import unittest
import importlib
import re
import types
from nltk.tokenize import sent_tokenize, word_tokenize
import nltk


# Helper function to require a successful test
def _require_success(test_case, condition, failure_message):
  if not condition:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


# Helper function to check if test is successful
def _check_success(test_case, condition, success_message, failure_message):
  if condition:
    test_case._testMethodName = success_message  # Rename the test for better readability in test reports
  else:
    test_case._testMethodName = failure_message  # Rename the test with a failure description
    test_case.fail()  # Fail the test with a failure message


class TestUserCode(unittest.TestCase):
  # Test that everything is imported correctly
  def test_imports(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sent_tokenize"),
      "`sent_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.sent_tokenize == sent_tokenize,
      "`sent_tokenize` is declared, but it's not `nltk.tokenize.sent_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "word_tokenize"),
      "`word_tokenize` is not declared"
    )
    _require_success(
      self,
      user_code.word_tokenize == word_tokenize,
      "`word_tokenize` is declared, but it's not `nltk.tokenize.word_tokenize`"
    )
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    self._testMethodName = "imports are correct"

  # Test that 'punk_tab' is downloaded
  def test_nltk_download_punkt(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "nltk"),
      "`nltk` is not declared"
    )
    _require_success(
      self,
      user_code.nltk == nltk,
      "`nltk` is declared, but it's not `nltk` library"
    )
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _require_success(
        self,
        re.search(r"nltk *\. *download *\( *'punkt_tab' *\)", user_code_text),
        "Code for downloading `'punkt_tab'` tokenizer shouldn't be modified"
      )
    self._testMethodName = "`'punkt_tab'` tokenizer is downloaded"

  # Test that text was not modified
  @unittest.skip("doesn't work properly now")
  def test_text(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    expected = 'Sherlock Holmes picked them up one by one, and laid them along the edge of the table. Then he reseated himself in the chair, and looked over with a gleam of satisfaction in his eyes. "These," said he, "are all that I have left to remind me of âThe Adventure of the Musgrave Ritualâ."'
    _check_success(
      self,
      user_code.text == expected,
      "`text` contains correct value",
      "`text` shouldn't be modified"
    )

  # Test that text was correctly tokenized into sentences
  def test_sent_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "text"),
      "`text` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.text, str),
      "`text` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _check_success(
      self,
      sent_tokenize(user_code.text) == user_code.sentences,
      "`sentences` is computed correctly",
      "`text` is incorrectly tokenized into sentences"
    )

  # Test that last sentence was saved and transformed into lowercase
  def test_last_sentence(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "sentences"),
      "`sentences` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.sentences, list),
      "`sentences` is not a list"
    )
    _require_success(
      self,
      isinstance(user_code.sentences[-1], str),
      "last element of `sentences` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      user_code.last_sentence.lower() == user_code.sentences[-1].lower(),
      "`last_sentence` does not contain the last sentence"
    )
    _require_success(
      self,
      user_code.last_sentence == user_code.sentences[-1].lower(),
      "`last_sentence` is not in lowercase"
    )
    _testMethodName = "`last_sentence` is computed correctly"

  # Test that last sentence was tokenized into words
  def test_word_tokenize(self):
    import user_code
    _require_success(
      self,
      hasattr(user_code, "last_sentence"),
      "`last_sentence` is not declared"
    )
    _require_success(
      self,
      isinstance(user_code.last_sentence, str),
      "`last_sentence` is not a string"
    )
    _require_success(
      self,
      hasattr(user_code, "tokens"),
      "`tokens` is not declared"
    )
    _check_success(
      self,
      word_tokenize(user_code.last_sentence) == user_code.tokens,
      "`tokens` is computed correctly",
      "`last_sentence` is incorrectly tokenized into words"
    )

  # Test that print is not modified
  def test_print(self):
    with open("user_code.py", "r") as f:
      user_code_text = f.read()
      _check_success(
        self,
        re.search(r"print *\( *tokens *\)", user_code_text),
        "print statement is correct",
        "print statement shouldn't be modified"
      )


# Run the tests when executing the script
if __name__ == '__main__':
  nltk.download('punkt_tab')
  unittest.main()

test_main.py

Erkunden Sie die Grundlagen der Verarbeitung natürlicher Sprache (NLP), indem Sie wesentliche Techniken zur Textvorverarbeitung und Methoden zur Repräsentation von Textdaten erlernen. Sammeln Sie praktische Erfahrungen mit Werkzeugen zur Bereinigung, Analyse und Interpretation von Textinformationen. Entwickeln Sie die Fähigkeiten, rohe Sprache in strukturierte Erkenntnisse umzuwandeln und schaffen Sie so eine solide Grundlage für fortgeschrittene Anwendungen in künstlicher Intelligenz und maschinellem Lernen.

Tauchen Sie in die Grundlagen der Textvorverarbeitung ein, um Rohtext für die Analyse vorzubereiten. Erfahren Sie, wie Text tokenisiert, Stoppwörter entfernt und die Tokenisierung mit regulären Ausdrücken angepasst wird.

Erfahren Sie, wie Wörter mithilfe von Stemming und Lemmatisierung auf ihre Grundformen reduziert werden können. Beherrschen Sie die Part-of-Speech-Tagging-Technik, um Texte mit grammatischem Kontext anzureichern, und wenden Sie POS-basierte Lemmatisierung an.

Erfahren Sie, wie Text mithilfe von Vektorraum-Modellen in Zahlen dargestellt werden kann. Sammeln Sie praktische Erfahrungen durch die Implementierung und Anpassung zweier gängiger Vektorraum-Modelle: Bag-of-Words und TF-IDF.

Fundiertes Verständnis von Wort-Einbettungen und deren Erfassung semantischer Bedeutungen. Untersuchung der CBoW- und Skip-gram-Architekturen, die in Word2Vec verwendet werden, sowie deren eigenständige Implementierung.

Herausforderung: Tokenisierung

Lösung

Awesome!

Herausforderung: Tokenisierung

Lösung