Imagine you are working with a massive dataset containing millions of records, and you need to quickly extract only those rows where a specific column's value exceeds a certain threshold. Using the **polars** library, you can perform this type of filtering efficiently and at scale. This is a common requirement in **data science workflows**, especially when you need to focus your analysis on just a subset of the data that meets certain criteria.

import unittest
import user_code
import ast
import re   
import importlib
import polars as pl

class TestTask(unittest.TestCase):
    def test_filter_basic(self):
        importlib.reload(user_code)
        df = pl.DataFrame({
            "customer_id": [1, 2, 3, 4],
            "amount": [10, 20, 30, 40]
        })
        result = user_code.filter_above_threshold(df, "amount", 20)
        expected = pl.DataFrame({"customer_id": [3, 4], "amount": [30, 40]})
        
        # FIX: Changed .frame_equal to .equals
        _dynamic_test(
            self,
            isinstance(result, pl.DataFrame) and result.equals(expected),
            "Returns rows where column value is greater than threshold.",
            f"Expected filtered DataFrame {expected}, got {result}.",
        )

    def test_does_not_modify_original(self):
        importlib.reload(user_code)
        df = pl.DataFrame({
            "a": [1, 2, 3, 4],
            "b": [10, 20, 30, 40]
        })
        df_copy = df.clone()
        _ = user_code.filter_above_threshold(df, "b", 25)
        
        # FIX: Changed .frame_equal to .equals
        _dynamic_test(
            self,
            df.equals(df_copy),
            "Original DataFrame is not modified.",
            "Original DataFrame was modified in-place.",
        )

    def test_empty_result(self):
        importlib.reload(user_code)
        df = pl.DataFrame({
            "x": [1, 2, 3],
            "y": [4, 5, 6]
        })
        result = user_code.filter_above_threshold(df, "x", 10)
        _dynamic_test(
            self,
            isinstance(result, pl.DataFrame) and result.height == 0,
            "Returns empty DataFrame if no rows match.",
            f"Expected empty DataFrame, got {result}.",
        )

    def test_any_numeric_column(self):
        importlib.reload(user_code)
        df = pl.DataFrame({
            "foo": [0.1, 0.2, 0.3, 0.4],
            "bar": [5, 10, 15, 20]
        })
        result = user_code.filter_above_threshold(df, "foo", 0.25)
        expected = pl.DataFrame({"foo": [0.3, 0.4], "bar": [15, 20]})
        
        # FIX: Changed .frame_equal to .equals
        _dynamic_test(
            self,
            isinstance(result, pl.DataFrame) and result.equals(expected),
            "Works with any valid numeric column.",
            f"Expected {expected}, got {result}.",
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

if __name__ == "__main__":
    unittest.main()

test_main.py

A practical, hands-on course for aspiring data scientists ready to tackle real-world large data challenges. Learn to efficiently process, sample, and analyze massive datasets using Python and essential libraries. Each section features engaging video explanations and interactive challenges to build your expertise.

Learn foundational strategies for handling datasets too large to fit in memory, including chunking and streaming techniques.

Explore methods to balance and sample large datasets, including oversampling and undersampling.

Learn to use the polars library for fast, memory-efficient data processing.

Challenge: Filtering with Polars

Solution