データに**一意のレコード**のみが含まれていることを保証することは、正確な分析のために重要です。重複した行は統計を歪め、誤解を招く結果をもたらし、結論の信頼性を損なう可能性があります。重複を削除することで、各観測値が一度だけカウントされ、データセットの整合性が維持されます。

import pandas as pd

data = {
    "Name": ["Alice", "Bob", "Alice", "Charlie", "Bob"],
    "Age": [25, 30, 25, 35, 30],
    "City": ["New York", "Paris", "New York", "London", "Paris"]
}

df = pd.DataFrame(data)
print(df)


import unittest
import user_code
import ast
import re   
import unittest
import pandas as pd
import importlib
import io
from contextlib import redirect_stdout

class TestTask(unittest.TestCase):
    def setUp(self):
        import user_code
        importlib.reload(user_code)
        self.func = getattr(user_code, 'remove_duplicates', None)

    def test_removes_duplicate_rows(self):
        import user_code
        data = {
            "Name": ["Alice", "Bob", "Alice", "Charlie", "Bob"],
            "Age": [25, 30, 25, 35, 30],
            "City": ["New York", "Paris", "New York", "London", "Paris"]
        }
        df = pd.DataFrame(data)
        result = self.func(df)
        expected = pd.DataFrame({
            "Name": ["Alice", "Bob", "Charlie"],
            "Age": [25, 30, 35],
            "City": ["New York", "Paris", "London"]
        }, index=[0,1,3])
        cond = (
            isinstance(result, pd.DataFrame) and
            result.reset_index(drop=True).equals(expected.reset_index(drop=True)) and
            list(result.columns) == list(df.columns) and
            all(result.dtypes == df.dtypes)
        )
        _dynamic_test(
            self,
            cond,
            "Duplicate rows are removed, only first occurrence is kept, column order and dtypes preserved.",
            f"Expected DataFrame:\\n{expected}\\nGot:\\n{result}"
        )

    def test_no_duplicates(self):
        import user_code
        data = {
            "A": [1, 2, 3],
            "B": ["x", "y", "z"]
        }
        df = pd.DataFrame(data)
        result = self.func(df)
        cond = (
            isinstance(result, pd.DataFrame) and
            result.equals(df) and
            list(result.columns) == list(df.columns) and
            all(result.dtypes == df.dtypes)
        )
        _dynamic_test(
            self,
            cond,
            "No change for DataFrame with no duplicates.",
            f"Expected unchanged DataFrame, got:\\n{result}"
        )

    def test_all_rows_duplicated(self):
        import user_code
        data = {
            "A": [1, 1, 1],
            "B": [2, 2, 2]
        }
        df = pd.DataFrame(data)
        result = self.func(df)
        expected = pd.DataFrame({"A": [1], "B": [2]})
        cond = (
            isinstance(result, pd.DataFrame) and
            result.reset_index(drop=True).equals(expected.reset_index(drop=True)) and
            list(result.columns) == list(df.columns) and
            all(result.dtypes == df.dtypes)
        )
        _dynamic_test(
            self,
            cond,
            "All duplicate rows removed, only first kept.",
            f"Expected DataFrame:\\n{expected}\\nGot:\\n{result}"
        )

    def test_preserves_column_order_and_types(self):
        import user_code
        data = {
            "first": [1, 2, 1],
            "second": [3.0, 4.5, 3.0],
            "third": [True, False, True]
        }
        df = pd.DataFrame(data)
        result = self.func(df)
        cond = (
            isinstance(result, pd.DataFrame) and
            list(result.columns) == list(df.columns) and
            all(result.dtypes == df.dtypes)
        )
        _dynamic_test(
            self,
            cond,
            "Column order and types are preserved after duplicate removal.",
            f"Column order or types changed. Columns: {list(result.columns)}, Types: {result.dtypes}"
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

def normalize_text(text):
    text = text.lower()
    text = re.sub(r"\s{2,}", " ", text)
    text = re.sub(r"\s*([,:?])\s*", r"\1 ", text)
    return text.strip()

def change_var(code: str, var_name: str, value: str) -> str:
    tree = ast.parse(code)
    lines = code.splitlines()
    for i, node in enumerate(tree.body):
        if isinstance(node, ast.Assign):
            for target in node.targets:
                if isinstance(target, ast.Name) and target.id == var_name:
                    start_line = node.lineno - 1
                    line = lines[start_line]
                    indent = ' ' * (len(line) - len(line.lstrip()))
                    lines[start_line] = f"{indent}{var_name} = {value}"
                    next_line = len(lines)
                    for next_node in tree.body[i+1:]:
                        if hasattr(next_node, 'lineno'):
                            next_line = next_node.lineno - 1
                            break
                    if next_line > start_line + 1:
                        lines[start_line+1:next_line] = []
                    
                    return '\\n'.join(lines)
    return code

if __name__ == "__main__":
    unittest.main()


test_main.py

強力なライブラリと実践的な課題を用いて、Pythonでのデータクリーニングの基本技術を習得します。本コースは、中級レベルのPythonスキルを持ち、分析や機械学習のために効率的にデータを準備・クリーニングしたい学習者向けに設計されています。

データクリーニングの基本概念、その重要性、そしてPythonにおける主要なツールと手法を学びます。

pandasとnumpyを使用した欠損データおよび重複データの管理手法についてさらに深く学びます。

データの一貫性を確保し、エラーを修正し、外れ値を検出するための手法に焦点を当てます。

チャレンジ：重複行を削除する

解答