
library(testthat)

source("user_code.R")
test_that("remove_outliers sets outliers in income to NA using IQR", {
    survey_test <- data.frame(income = c(1, 2, 3, 4, 1000, NA))
    out <- remove_outliers(survey_test$income)
    # 1000 should be NA, others not
    expect_true(is.na(out[5]), info = "Outlier value not set to NA as expected")
    expect_false(any(is.na(out[1:4])), info = "Non-outlier values incorrectly set to NA")
})

test_that("fix_gender standardizes gender values and sets unknowns to NA", {
    genders <- c("M", "female", "F", "MALE", "Woman", "man", "FEMALE", "m", "unknown", NA)
    fixed <- fix_gender(genders)
    expect_equal(fixed[1], "Male", info = "'M' not converted to 'Male'")
    expect_equal(fixed[2], "Female", info = "'female' not converted to 'Female'")
    expect_equal(fixed[5], "Female", info = "'Woman' not converted to 'Female'")
    # expect_equal(fixed[9], NA, info = "Unknown gender not set to NA")
    # expect_equal(fixed[10], NA, info = "NA input should remain NA")
})

test_that("ages less than 0 or greater than 120 are set to NA", {
    test_ages <- c(25, 130, -5, 50)
    df <- data.frame(age = test_ages)
    out <- df %>% mutate(age = ifelse(age < 0 | age > 120, NA, age))
    expect_true(is.na(out$age[2]), info = "Age > 120 not set to NA")
    expect_true(is.na(out$age[3]), info = "Negative age not set to NA")
    expect_equal(out$age[1], 25, info = "Valid age incorrectly modified")
})

test_that("rows with missing or invalid age or gender are removed", {
    survey <- data.frame(
      id = 1:4,
      age = c(25, NA, 30, 150),
      income = c(40000, 50000, 60000, 70000),
      gender = c("M", "female", NA, "unknown"),
      stringsAsFactors = FALSE
    )
    cleaned <- survey %>%
      mutate(age = ifelse(age < 0 | age > 120, NA, age)) %>%
      mutate(gender = fix_gender(gender)) %>%
      filter(!is.na(age) & !is.na(gender))
    expect_equal(nrow(cleaned), 1, info = "Rows with invalid/missing age or gender not removed correctly")
    expect_equal(cleaned$id, 1, info = "Incorrect row kept after filtering")
})

test_that("cleaned_survey contains only valid, cleaned rows for age, income, gender", {
    # Use the full pipeline and check the result
    survey <- data.frame(
      id = 1:10,
      age = c(25, 30, NA, 22, 150, 29, 27, 24, -5, 28),
      income = c(40000, 50000, 60000, NA, 70000, 80000, 90000, 1000000, 55000, 65000),
      gender = c("M", "female", "F", "MALE", "Woman", "man", "FEMALE", "m", "unknown", NA),
      stringsAsFactors = FALSE
    )
    cleaned <- survey %>%
      mutate(age = ifelse(age < 0 | age > 120, NA, age)) %>%
      mutate(income = remove_outliers(income)) %>%
      mutate(gender = fix_gender(gender)) %>%
      filter(!is.na(age) & !is.na(gender))
    # Only rows 1, 2, 4, 6, 7, 8, 10 are valid after cleaning (but 10 has NA gender)
    expect_true(all(cleaned$age > 0 & cleaned$age <= 120), info = "Invalid ages present after cleaning")
    expect_true(all(cleaned$gender %in% c("Male", "Female")), info = "Invalid genders present after cleaning")
    expect_false(any(is.na(cleaned$age)), info = "NA ages present after filtering")
    expect_false(any(is.na(cleaned$gender)), info = "NA genders present after filtering")
    # Check that income outlier (1000000) is NA
    expect_true(is.na(cleaned$income[cleaned$id == 8]), info = "Income outlier not set to NA")
})

test_main.R

Master the essentials of handling text, dates, and files in R. This course guides you through string manipulation, regular expressions, date-time operations, and file I/O, with engaging explanations and practical challenges for beginners.

Learn to work with text data in R, from basic string operations to cleaning messy data using regular expressions.

Master handling dates and times in R, from parsing and formatting to performing calculations and managing time zones.

Explore the foundations of data cleaning and quality assurance in R. Each chapter introduces a core concept, followed by a hands-on challenge to reinforce your learning.

Challenge: Build a Cleaning Pipeline for Survey Data

Solution

Challenge: Build a Cleaning Pipeline for Survey Data

Solution