# ============================================================================
#  04-01  Survey Data Organizer & Codebook
#  Chapters 01-01, 01-02
#
#  Run from the project folder:   Rscript r/analysis.R
# ============================================================================

raw <- read.csv("data/survey_raw.csv", stringsAsFactors = FALSE,
                na.strings = c("", "NA"))

cat("\n===== 1. QUALITY REPORT (before any changes) =====\n")

cat("Rows:                ", nrow(raw), "\n")
cat("Columns:             ", ncol(raw), "\n")
cat("Duplicate IDs:       ", sum(duplicated(raw$student_id)), "\n")
if (any(duplicated(raw$student_id)))
  cat("  -> ", paste(raw$student_id[duplicated(raw$student_id)], collapse = ", "), "\n")

cat("\nMissing values per column:\n")
print(colSums(is.na(raw)))

cat("\nstudy_hours stored as text (non-numeric after coercion):\n")
print(sum(is.na(suppressWarnings(as.numeric(raw$study_hours))) & !is.na(raw$study_hours)))

cat("\nDistinct raw categories:\n")
print(table(raw$gender, useNA = "ifany"))
print(table(raw$major,  useNA = "ifany"))

cat("\nRange checks:\n")
cat("  age     ", min(raw$age), "to", max(raw$age), "\n")
cat("  midterm ", min(raw$midterm, na.rm = TRUE), "to",
    max(raw$midterm, na.rm = TRUE),
    ifelse(max(raw$midterm, na.rm = TRUE) > 100, "  <-- IMPOSSIBLE", ""), "\n")

# ============================================================================
cat("\n===== 2. CLEAN =====\n")

clean <- raw

# ── the duplicated key: keep the FIRST occurrence, and say so ───────────────
dupes <- duplicated(clean$student_id)
if (any(dupes)) {
  cat("Removing", sum(dupes), "duplicate row(s):",
      paste(clean$student_id[dupes], collapse = ", "), "\n")
  clean <- clean[!dupes, ]
}

# ── gender: 10 raw spellings -> exactly F / M / X ───────────────────────────
g <- toupper(trimws(clean$gender))
g <- substr(g, 1, 1)
clean$gender <- factor(ifelse(g %in% c("F", "M"), g, "X"),
                       levels = c("F", "M", "X"))

# ── major: trim, then title-case ────────────────────────────────────────────
title_case <- function(x) {
  x <- tolower(trimws(x))
  paste0(toupper(substring(x, 1, 1)), substring(x, 2))
}
clean$major <- factor(title_case(clean$major))

# ── study_hours: strip " hrs" and any stray spaces, then coerce ─────────────
clean$study_hours <- suppressWarnings(
  as.numeric(gsub("[^0-9.]", "", clean$study_hours))
)

# ── midterm: void the impossible value rather than guessing at it ───────────
bad <- !is.na(clean$midterm) & (clean$midterm > 100 | clean$midterm < 0)
if (any(bad)) {
  cat("Voiding", sum(bad), "impossible midterm value(s):",
      paste(clean$midterm[bad], collapse = ", "), "\n")
  clean$midterm[bad] <- NA
}

# ── satisfaction: ORDINAL, never numeric ────────────────────────────────────
clean$satisfaction <- factor(clean$satisfaction, levels = 1:5, ordered = TRUE)

# ============================================================================
cat("\n===== 3. DERIVE =====\n")

clean$female    <- as.integer(clean$gender == "F")
clean$age_group <- cut(clean$age,
                       breaks = c(-Inf, 25, 40, Inf),
                       labels = c("Under 25", "25-39", "40+"),
                       right  = FALSE)

# k levels -> k-1 dummies; the first level is the reference
major_dummies <- model.matrix(~ major, data = clean)[, -1, drop = FALSE]
cat("Major dummies created (reference =", levels(clean$major)[1], "):",
    paste(colnames(major_dummies), collapse = ", "), "\n")

str(clean)

# ============================================================================
cat("\n===== 4. SUMMARY (respecting the measurement levels) =====\n")

cat("\nQuantitative columns — mean is legal:\n")
print(round(sapply(clean[, c("age", "study_hours", "midterm")],
                   function(x) c(n      = sum(!is.na(x)),
                                 mean   = mean(x, na.rm = TRUE),
                                 median = median(x, na.rm = TRUE),
                                 sd     = sd(x, na.rm = TRUE),
                                 min    = min(x, na.rm = TRUE),
                                 max    = max(x, na.rm = TRUE))), 2))

cat("\nNominal columns — frequencies only:\n")
print(table(clean$gender))
print(table(clean$major))

cat("\nOrdinal column — median and frequencies, NOT a mean:\n")
print(table(clean$satisfaction))
cat("median satisfaction:", as.character(median(clean$satisfaction)), "\n")

cat("\nStudy hours by major:\n")
print(round(tapply(clean$study_hours, clean$major, mean, na.rm = TRUE), 2))

# ============================================================================
cat("\n===== 5. CODEBOOK =====\n")

codebook <- data.frame(
  variable = c("student_id", "gender", "major", "age",
               "study_hours", "midterm", "satisfaction"),
  type     = c("text", "factor", "factor", "integer",
               "numeric", "integer", "ordered factor"),
  level    = c("nominal", "nominal", "nominal", "ratio",
               "ratio", "ratio", "ORDINAL"),
  allowed  = c("S001-S060", "F, M, X",
               paste(levels(clean$major), collapse = ", "),
               "15-100", "0-40", "0-100", "1-5"),
  notes    = c("unique key; 1 duplicate removed",
               "normalized from 10 raw spellings",
               "trimmed and title-cased",
               "-", "text suffixes stripped",
               "blank = absent; >100 voided",
               "DO NOT AVERAGE - report the median"),
  stringsAsFactors = FALSE
)
print(codebook, row.names = FALSE)

# write.csv(clean,    "data/survey_clean.csv", row.names = FALSE)
# write.csv(codebook, "data/codebook.csv",     row.names = FALSE)
cat("\nDone. Uncomment the write.csv lines to save the cleaned outputs.\n")