When working with real-world data, you often encounter categorical values that are meant to represent the same thing but are written in different ways. For example, a survey might record responses such as `Yes`, `yes`, and `YES` in the same column. These inconsistencies can cause problems when you try to analyze or summarize your data, since Python and pandas treat these as distinct values. Standardizing these entries is essential to ensure your data is consistent and your results are accurate.

import pandas as pd

data = {
    "Response": ["Yes", "no", "YES", "No", "yes", "NO", "nO", "YeS"]
}
df = pd.DataFrame(data)
print(df)


import unittest
import user_code
import ast
import re   
import unittest
import pandas as pd
import importlib
import user_code

class TestTask(unittest.TestCase):
    def setUp(self):
        self.data = {
            "Response": ["Yes", "no", "YES", "No", "yes", "NO", "nO", "YeS"],
            "Other": [1,2,3,4,5,6,7,8]
        }
        self.df = pd.DataFrame(self.data)

    def test_all_values_lowercase(self):
        importlib.reload(user_code)
        result = user_code.standardize_column_case(self.df, "Response")
        expected = [x.lower() for x in self.data["Response"]]
        actual = list(result["Response"])
        _dynamic_test(
            self,
            actual == expected,
            "All values in the specified column are converted to lowercase.",
            f"Expected lowercase values {expected}, got {actual}.",
        )

    def test_other_columns_unchanged(self):
        importlib.reload(user_code)
        result = user_code.standardize_column_case(self.df, "Response")
        actual = list(result["Other"])
        expected = self.data["Other"]
        _dynamic_test(
            self,
            actual == expected,
            "Other columns are unchanged.",
            f"Other column changed: expected {expected}, got {actual}.",
        )

    def test_shape_unchanged(self):
        importlib.reload(user_code)
        result = user_code.standardize_column_case(self.df, "Response")
        _dynamic_test(
            self,
            result.shape == self.df.shape,
            "Returned DataFrame has the same shape as input.",
            f"Expected shape {self.df.shape}, got {result.shape}.",
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

def normalize_text(text):
    text = text.lower()
    text = re.sub(r"\s{2,}", " ", text)
    text = re.sub(r"\s*([,:?])\s*", r"\1 ", text)
    return text.strip()

def change_var(code: str, var_name: str, value: str) -> str:
    tree = ast.parse(code)
    lines = code.splitlines()
    for i, node in enumerate(tree.body):
        if isinstance(node, ast.Assign):
            for target in node.targets:
                if isinstance(target, ast.Name) and target.id == var_name:
                    start_line = node.lineno - 1
                    line = lines[start_line]
                    indent = ' ' * (len(line) - len(line.lstrip()))
                    lines[start_line] = f"{indent}{var_name} = {value}"
                    next_line = len(lines)
                    for next_node in tree.body[i+1:]:
                        if hasattr(next_node, 'lineno'):
                            next_line = next_node.lineno - 1
                            break
                    if next_line > start_line + 1:
                        lines[start_line+1:next_line] = []
                    
                    return '\\n'.join(lines)
    return code

if __name__ == "__main__":
    unittest.main()


test_main.py

Master essential data cleaning techniques in Python using powerful libraries and practical tasks. This course is designed for learners with intermediate Python skills who want to efficiently prepare and clean data for analysis and machine learning.

Explore the core concepts of data cleaning, why it matters, and the essential tools and techniques in Python.

Delve deeper into techniques for managing missing and duplicate data using pandas and numpy.

Focus on techniques for making data consistent, correcting errors, and detecting outliers.

Challenge: Standardize Categorical Values

Solution

Awesome!

Challenge: Standardize Categorical Values

Solution