When you conduct an A/B test, your goal is to determine whether a new variant (B) is truly different from the control (A) based on the data you collect. However, your conclusion can be incorrect for two main reasons: you might detect a difference when there is none (**Type I error**, or "false positive"), or you might miss a real difference (**Type II error**, or "false negative"). To identify which error - if any - has occurred, you must compare the real-world truth (whether a true effect exists) with the outcome of your statistical test (whether you declared a significant effect).

If your test finds a significant result when no real effect exists, you have made a **Type I error**. If your test fails to find a significant result when a real effect does exist, you have made a **Type II error**. If your conclusion matches the reality (either correctly detecting a real effect or correctly concluding there is none), you have made a **correct decision**. Understanding these scenarios is crucial for interpreting the practical implications of your tests and for making informed business decisions based on your findings.


import unittest
import user_code
import ast
import re   
import importlib
import csv
import unittest
import importlib

class TestTask(unittest.TestCase):
    def test_correct_decision_true_effect_significant(self):
        import user_code
        importlib.reload(user_code)
        result = user_code.classify_ab_test_outcome(True, True)
        _dynamic_test(
            self,
            result == "Correct Decision",
            'Returns "Correct Decision" for (True, True)',
            f'Expected "Correct Decision" for (True, True), got {result}'
        )

    def test_correct_decision_no_effect_not_significant(self):
        import user_code
        importlib.reload(user_code)
        result = user_code.classify_ab_test_outcome(False, False)
        _dynamic_test(
            self,
            result == "Correct Decision",
            'Returns "Correct Decision" for (False, False)',
            f'Expected "Correct Decision" for (False, False), got {result}'
        )

    def test_type_i_error(self):
        import user_code
        importlib.reload(user_code)
        result = user_code.classify_ab_test_outcome(False, True)
        _dynamic_test(
            self,
            result == "Type I Error",
            'Returns "Type I Error" for (False, True)',
            f'Expected "Type I Error" for (False, True), got {result}'
        )

    def test_type_ii_error(self):
        import user_code
        importlib.reload(user_code)
        result = user_code.classify_ab_test_outcome(True, False)
        _dynamic_test(
            self,
            result == "Type II Error",
            'Returns "Type II Error" for (True, False)',
            f'Expected "Type II Error" for (True, False), got {result}'
        )

def _dynamic_test(test_case, condition, success_message, failure_message):
    if condition:
        test_case._testMethodName = success_message
        test_case.assertTrue(True, success_message)
    else:
        test_case._testMethodName = failure_message
        test_case.fail(failure_message)

def normalize_text(text):
    text = text.lower()
    text = re.sub(r"\\s{2,}", " ", text)
    text = re.sub(r"\\s*([,:?])\\s*", r"\\1 ", text)
    return text.strip()

def change_var(code: str, var_name: str, value: str) -> str:
    tree = ast.parse(code)
    lines = code.splitlines()
    changed = False
    # Collect all assignment nodes to modify
    assign_nodes = [
        (i, node)
        for i, node in enumerate(tree.body)
        if isinstance(node, ast.Assign)
        and any(isinstance(target, ast.Name) and target.id == var_name for target in node.targets)
    ]

    # If nothing to change, return unmodified code
    if not assign_nodes:
        return code

    # Perform replacements for all matching assignments (from last to first to not break line offsets)
    for i, node in reversed(assign_nodes):
        start_line = node.lineno - 1
        line = lines[start_line]
        indent = ' ' * (len(line) - len(line.lstrip()))
        lines[start_line] = f"{indent}{var_name} = {value}"
        next_line = len(lines)
        for next_node in tree.body[i+1:]:
            if hasattr(next_node, 'lineno'):
                next_line = next_node.lineno - 1
                break
        if next_line > start_line + 1:
            lines[start_line+1:next_line] = []
        changed = True

    return '\\n'.join(lines) if changed else code

if __name__ == "__main__":
    unittest.main()


test_main.py

Master the end-to-end workflow of data-driven experimentation with this comprehensive course on A/B Testing with Python. Designed to bridge the gap between statistical theory and practical product decision-making, you will learn how to design, execute, and analyze trustworthy experiments.

Challenge: Error Type Identification

Challenge: Error Type Identification

Solução