In many real-world data cleaning scenarios, you may want to flag duplicate entries rather than remove them right away. Flagging gives you the flexibility to review duplicates, analyze their patterns, and make informed decisions about which ones to keep or discard. For instance, in customer databases, you may want to investigate why duplicates occur before deletion, or in transactional data, you might need to audit the records before any removal. By marking duplicates, you can also generate reports, track data quality issues, and collaborate with others on resolution strategies without losing potentially valuable information.

import pandas as pd

data = {
    "name": ["Alice", "Bob", "Alice", "Charlie", "Bob"],
    "age": [25, 30, 25, 35, 30]
}

df = pd.DataFrame(data)
print(df)


import unittest
import user_code
import ast
import re   
import unittest
import pandas as pd
import importlib
import io
from contextlib import redirect_stdout

class TestTask(unittest.TestCase):
    def setUp(self):
        self.data = {
            "name": ["Alice", "Bob", "Alice", "Charlie", "Bob"],
            "age": [25, 30, 25, 35, 30]
        }
        self.df = pd.DataFrame(self.data)

    def test_column_exists(self):
        import user_code
        importlib.reload(user_code)
        flagged = user_code.flag_duplicates(self.df)
        _dynamic_test(
            self,
            "is_duplicate" in flagged.columns,
            "DataFrame contains 'is_duplicate' column.",
            "DataFrame does not contain 'is_duplicate' column."
        )

    def test_first_occurrence_false(self):
        import user_code
        importlib.reload(user_code)
        flagged = user_code.flag_duplicates(self.df)
        # The first 'Alice', first 'Bob', and 'Charlie' should be False
        _dynamic_test(
            self,
            not flagged.loc[0, "is_duplicate"],
            "First occurrence of 'Alice' is not flagged as duplicate.",
            "First occurrence of 'Alice' should not be flagged as duplicate."
        )
        _dynamic_test(
            self,
            not flagged.loc[1, "is_duplicate"],
            "First occurrence of 'Bob' is not flagged as duplicate.",
            "First occurrence of 'Bob' should not be flagged as duplicate."
        )
        _dynamic_test(
            self,
            not flagged.loc[3, "is_duplicate"],
            "'Charlie' is not flagged as duplicate.",
            "First occurrence of 'Charlie' should not be flagged as duplicate."
        )

    def test_duplicates_true(self):
        import user_code
        importlib.reload(user_code)
        flagged = user_code.flag_duplicates(self.df)
        # The second 'Alice' (index 2) and second 'Bob' (index 4) should be True
        _dynamic_test(
            self,
            flagged.loc[2, "is_duplicate"],
            "Second occurrence of 'Alice' is flagged as duplicate.",
            "Second occurrence of 'Alice' should be flagged as duplicate."
        )
        _dynamic_test(
            self,
            flagged.loc[4, "is_duplicate"],
            "Second occurrence of 'Bob' is flagged as duplicate.",
            "Second occurrence of 'Bob' should be flagged as duplicate."
        )

    def test_shape(self):
        import user_code
        importlib.reload(user_code)
        flagged = user_code.flag_duplicates(self.df)
        expected_shape = (self.df.shape[0], self.df.shape[1] + 1)
        _dynamic_test(
            self,
            flagged.shape == expected_shape,
            "Shape is correct after adding 'is_duplicate' column.",
            f"Expected shape {expected_shape}, got {flagged.shape}."
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

def normalize_text(text):
    text = text.lower()
    text = re.sub(r"\s{2,}", " ", text)
    text = re.sub(r"\s*([,:?])\s*", r"\1 ", text)
    return text.strip()

def change_var(code: str, var_name: str, value: str) -> str:
    tree = ast.parse(code)
    lines = code.splitlines()
    for i, node in enumerate(tree.body):
        if isinstance(node, ast.Assign):
            for target in node.targets:
                if isinstance(target, ast.Name) and target.id == var_name:
                    start_line = node.lineno - 1
                    line = lines[start_line]
                    indent = ' ' * (len(line) - len(line.lstrip()))
                    lines[start_line] = f"{indent}{var_name} = {value}"
                    next_line = len(lines)
                    for next_node in tree.body[i+1:]:
                        if hasattr(next_node, 'lineno'):
                            next_line = next_node.lineno - 1
                            break
                    if next_line > start_line + 1:
                        lines[start_line+1:next_line] = []
                    
                    return '\\n'.join(lines)
    return code

if __name__ == "__main__":
    unittest.main()


test_main.py

Master essential data cleaning techniques in Python using powerful libraries and practical tasks. This course is designed for learners with intermediate Python skills who want to efficiently prepare and clean data for analysis and machine learning.

Explore the core concepts of data cleaning, why it matters, and the essential tools and techniques in Python.

Delve deeper into techniques for managing missing and duplicate data using pandas and numpy.

Focus on techniques for making data consistent, correcting errors, and detecting outliers.

Challenge: Flag Duplicate Entries

Solution

Awesome!

Challenge: Flag Duplicate Entries

Solution