import unittest
import importlib
import numpy as np
import pandas as pd


def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)


class TestUserCode(unittest.TestCase):

    def test_age_column_has_no_missing_values(self):
        import user_code

        variable = 'data'
        actual_df = getattr(user_code, variable, None)
        condition = False
        if actual_df is None:
            failure_message = f"The `{variable}` variable is not declared."
        elif not isinstance(actual_df, pd.DataFrame):
            failure_message = f"`{variable}` is not a `DataFrame`."
        elif actual_df.empty:
            failure_message = f"`{variable}` is empty."
        elif 'Age' not in actual_df.columns:
            failure_message = f"`{variable}` has no 'Age' column."
        elif not actual_df['Age'].isnull().any():
            condition = True
            failure_message = None
        else:
            null_count = actual_df['Age'].isnull().sum()
            failure_message = f"Expected the 'Age' column to contain no missing values, but got `{null_count}`."

        _dynamic_test(
            self,
            condition,
            "The 'Age' column contains no missing values.",
            failure_message
        )

    def test_missing_values_are_replaced_correctly(self):
        import user_code

        expected_value = pd.Series([25, 30, 30, 35, 30]).astype(float)

        variable = 'data'
        actual_df = getattr(user_code, variable, None)
        condition = False
        if actual_df is None:
            failure_message = f"The `{variable}` variable is not declared."
        elif not isinstance(actual_df, pd.DataFrame):
            failure_message = f"`{variable}` is not a `DataFrame`."
        elif actual_df.empty:
            failure_message = f"`{variable}` is empty."
        elif 'Age' not in actual_df.columns:
            failure_message = f"`{variable}` has no 'Age' column."
        elif actual_df['Age'].equals(expected_value):
            condition = True
            failure_message = None
        else:
            failure_message = f"Expected `{variable}['Age']` to contain `{expected_value.values}`, but got `{actual_df['Age'].values}`."

        _dynamic_test(
            self,
            condition,
            "The missing values in the 'Age' column are correctly replaced with the mean.",
            failure_message
        )

    def test_city_encoder_is_declared(self):
        import user_code
        _dynamic_test(
            self,
            hasattr(user_code, 'city_encoder'),
            "The `city_encoder` variable is declared.",
            "Expected `city_encoder` to be declared."
        )

    def test_city_encoder_is_onehotencoder(self):
        import user_code
        from sklearn.preprocessing import OneHotEncoder

        try:
            condition = isinstance(user_code.city_encoder, OneHotEncoder)
            failure_message = f"Expected `city_encoder` to be an instance of `OneHotEncoder`, but got `{type(user_code.city_encoder).__name__}`."
        except AttributeError:
            condition = False
            failure_message = "The `city_encoder` variable is not declared."

        _dynamic_test(
            self,
            condition,
            "The `city_encoder` is an instance of `OneHotEncoder`.",
            failure_message
        )

    def test_city_encoded_is_declared(self):
        import user_code
        _dynamic_test(
            self,
            hasattr(user_code, 'city_encoded'),
            "The `city_encoded` variable is declared.",
            "Expected `city_encoded` to be declared."
        )

    def test_city_encoded_is_correct(self):
        import user_code
        from sklearn.preprocessing import OneHotEncoder
        from scipy.sparse import csr_matrix

        data = pd.DataFrame({
            'Age': [25, np.nan, 30, 35, np.nan],
            'City': ['New York', 'London', 'Paris', 'Berlin', 'London'],
            'Income': ['Low', 'Middle', 'High', 'High', 'Middle']
        })

        city_encoder = OneHotEncoder(drop='first')
        expected_value = city_encoder.fit_transform(data[['City']]).toarray()

        variable = 'city_encoded'
        actual_value = getattr(user_code, variable, None)
        condition = False
        if actual_value is None:
            failure_message = f"The `{variable}` variable is not declared."
        elif not isinstance(actual_value, csr_matrix):
            failure_message = f"`{variable}` is not a `scipy.sparse.csr_matrix`."
        elif not actual_value.size:
            failure_message = f"`{variable}` is empty."
        elif np.array_equal(actual_value.toarray(), expected_value):
            condition = True
            failure_message = None
        else:
            failure_message = f"Expected `{variable}` to contain `{expected_value}`, but got `{actual_value.toarray()}`."

        _dynamic_test(
            self,
            condition,
            f"The 'City' column is encoded correctly.",
            failure_message
        )

    def test_income_encoder_is_declared(self):
        import user_code
        _dynamic_test(
            self,
            hasattr(user_code, 'income_encoder'),
            "The `income_encoder` variable is declared.",
            "Expected `income_encoder` to be declared."
        )

    def test_income_encoder_is_ordinalencoder(self):
        import user_code
        from sklearn.preprocessing import OrdinalEncoder

        try:
            condition = isinstance(user_code.income_encoder, OrdinalEncoder)
            failure_message = f"Expected `income_encoder` to be an instance of `OrdinalEncoder`, but got `{type(user_code.income_encoder).__name__}`."
        except AttributeError:
            condition = False
            failure_message = "The `income_encoder` variable is not declared."

        _dynamic_test(
            self,
            condition,
            "The `income_encoder` is an instance of `OrdinalEncoder`.",
            failure_message
        )

    def test_income_column_is_encoded_correctly(self):
        import user_code

        expected_value = pd.Series([0, 1, 2, 2, 1]).astype(float)

        variable = 'data'
        actual_df = getattr(user_code, variable, None)
        condition = False
        if actual_df is None:
            failure_message = f"The `{variable}` variable is not declared."
        elif not isinstance(actual_df, pd.DataFrame):
            failure_message = f"`{variable}` is not a `DataFrame`."
        elif actual_df.empty:
            failure_message = f"`{variable}` is empty."
        elif 'Income' not in actual_df.columns:
            failure_message = f"`{variable}` has no 'Income' column."
        elif actual_df['Income'].equals(expected_value):
            condition = True
            failure_message = None
        else:
            failure_message = f"Expected `{variable}['Income']` to contain `{expected_value.values}`, but got `{actual_df['Income'].values}`."

        _dynamic_test(
            self,
            condition,
            "The 'Income' column is encoded correctly.",
            failure_message
        )


if __name__ == '__main__':
    unittest.main()


test_main.py

Gain a solid understanding of cluster analysis, a key unsupervised learning technique for uncovering patterns in unlabeled data. Explore the essentials of K-Means, Hierarchical Clustering, DBSCAN, and GMMs, and get hands-on experience with real datasets to build confidence in applying clustering to real-world problems.

Dive into the fundamentals of clustering and discover how it differs from classification. Explore essential algorithms, tools, and libraries that power this unsupervised learning technique to uncover hidden patterns in data.

Gain a solid understanding of key preprocessing techniques that ensure effective clustering. Learn how to handle missing values, encode categorical features, normalize data, and choose appropriate distance measures and linkages to boost clustering accuracy.

Master the skills needed to apply K-Means clustering effectively. Learn how the algorithm works, determine the optimal number of clusters, and gain hands-on experience by implementing K-Means on both synthetic and real-world datasets.

Explore the essentials of hierarchical clustering and learn how to group data into meaningful clusters using dendrograms. Build confidence in identifying the optimal number of clusters and implementing the technique on both synthetic and real-world datasets.

Discover how DBSCAN excels at detecting clusters of varying shapes and handling noise in data. Learn the mechanics behind this density-based algorithm, how to assign points to clusters, and apply it to both synthetic and real datasets with confidence.

Gain a solid understanding of Gaussian Mixture Models and how they use probability to model complex cluster shapes. Learn the principles of Gaussian distribution, explore how GMMs work, and build confidence by applying them to both dummy and real-world data.

Challenge: Preprocessing the Dataset

Solution