When working with categorical data in a DataFrame, extra whitespace at the beginning or end of string values can cause serious inconsistencies. For example, the values `"apple"`, `" apple"`, and `"apple "` may look the same to you, but Python treats them as different strings. This can lead to problems when grouping, filtering, or comparing data, and may result in incorrect analysis or missed patterns. Cleaning up these inconsistencies by stripping whitespace is a crucial first step in preparing your data for analysis.

import pandas as pd

data = {
    "Fruit": [" apple", "banana ", "  cherry ", "date"],
    "Color": [" red", "yellow ", " red ", "brown"],
    "Count": [10, 5, 7, 3]
}

df = pd.DataFrame(data)
print(df)

You can use `select_dtypes(include="object")` to select only the columns in a DataFrame that contain string data. This makes it easy to apply string operations, such as `str.strip()`, only to columns storing text and not to columns with numbers or other types.

Note


import unittest
import user_code
import ast
import re   
import unittest
import pandas as pd
import importlib

class TestTask(unittest.TestCase):
    def setUp(self):
        self.data = {
            "Fruit": [" apple", "banana ", "  cherry ", "date"],
            "Color": [" red", "yellow ", " red ", "brown"],
            "Count": [10, 5, 7, 3]
        }
        self.df = pd.DataFrame(self.data)
        self.expected_data = {
            "Fruit": ["apple", "banana", "cherry", "date"],
            "Color": ["red", "yellow", "red", "brown"],
            "Count": [10, 5, 7, 3]
        }
        self.expected_df = pd.DataFrame(self.expected_data)

    def test_whitespace_removed(self):
        import user_code
        importlib.reload(user_code)
        result = user_code.strip_whitespace(self.df)
        # Check all string columns have whitespace removed
        for col in ["Fruit", "Color"]:
            actual = list(result[col])
            expected = list(self.expected_df[col])
            _dynamic_test(
                self,
                actual == expected,
                f"Whitespace successfully removed from column '{col}'",
                f"Expected: {expected} in column '{col}', got: {actual}",
            )

    def test_non_string_columns_unchanged(self):
        import user_code
        importlib.reload(user_code)
        result = user_code.strip_whitespace(self.df)
        actual = list(result["Count"])
        expected = list(self.df["Count"])
        _dynamic_test(
            self,
            actual == expected,
            "Non-string columns remain unchanged",
            f"Expected: {expected} in 'Count', got: {actual}",
        )

    def test_shape_and_column_order(self):
        import user_code
        importlib.reload(user_code)
        result = user_code.strip_whitespace(self.df)
        _dynamic_test(
            self,
            result.shape == self.df.shape,
            "Shape of DataFrame is unchanged",
            f"Expected shape {self.df.shape}, got {result.shape}",
        )
        _dynamic_test(
            self,
            list(result.columns) == list(self.df.columns),
            "Column order is unchanged",
            f"Expected columns {list(self.df.columns)}, got {list(result.columns)}",
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

def normalize_text(text):
    text = text.lower()
    text = re.sub(r"\s{2,}", " ", text)
    text = re.sub(r"\s*([,:?])\s*", r"\1 ", text)
    return text.strip()

def change_var(code: str, var_name: str, value: str) -> str:
    tree = ast.parse(code)
    lines = code.splitlines()
    for i, node in enumerate(tree.body):
        if isinstance(node, ast.Assign):
            for target in node.targets:
                if isinstance(target, ast.Name) and target.id == var_name:
                    start_line = node.lineno - 1
                    line = lines[start_line]
                    indent = ' ' * (len(line) - len(line.lstrip()))
                    lines[start_line] = f"{indent}{var_name} = {value}"
                    next_line = len(lines)
                    for next_node in tree.body[i+1:]:
                        if hasattr(next_node, 'lineno'):
                            next_line = next_node.lineno - 1
                            break
                    if next_line > start_line + 1:
                        lines[start_line+1:next_line] = []
                    
                    return '\\n'.join(lines)
    return code

if __name__ == "__main__":
    unittest.main()


test_main.py

Master essential data cleaning techniques in Python using powerful libraries and practical tasks. This course is designed for learners with intermediate Python skills who want to efficiently prepare and clean data for analysis and machine learning.

Explore the core concepts of data cleaning, why it matters, and the essential tools and techniques in Python.

Delve deeper into techniques for managing missing and duplicate data using pandas and numpy.

Focus on techniques for making data consistent, correcting errors, and detecting outliers.

Challenge: Remove Whitespace from Strings

Рішення