processing.R

# ==============================================================================
# Author: Elijah Appelson
# Date: June 24th, 2026
# Summary: Loads and cleans the Louisiana Department of Corrections (DOC) data from
# multiple sources including incarceration snapshots, admissions, releases, 
# probation/parole, deaths, and monthly facility census reports. Raw files are 
# downloaded directly from the DOC website, combined across time periods, and 
# cleaned into analysis-ready dataframes. Lookup tables from a codebook are 
# used to decode race, offense, parish, supervision type, and facility location
# codes.
# ==============================================================================

# ----------------------- LOADING LIBRARIES AND DATA ---------------------------
# Loading libraries
library(tidyverse)
library(janitor)
library(readxl)

# Codebook url
codebook_url <- "https://doc.la.gov/wp-content/uploads/2026/04/Codebook-for-DPSC-Datasets.xlsx"

# Snapshot urls
incarceration_snapshot <- c(
  snapshot_mar_2026 = "https://doc.la.gov/wp-content/uploads/2026/05/IncSnapshot_2026.03-.csv",
  snapshot_feb_2026 = "https://doc.la.gov/wp-content/uploads/2026/04/IncSnapshot_02.2026.csv",
  snapshot_jan_2026 = "https://doc.la.gov/wp-content/uploads/2026/04/IncSnapshot_01.2026.csv",
  snapshot_dec_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/IncSnapshot_2025.12.csv"
)

# Admits urls
incarceration_admits <- c(
  admit_2025 = "https://doc.la.gov/wp-content/uploads/2026/04/IncAdmits_2025.csv",
  admit_2024 = "https://doc.la.gov/wp-content/uploads/2026/05/IncAdmits_2024.csv",
  admit_2023 = "https://doc.la.gov/wp-content/uploads/2026/05/IncAdmits_2023.csv",
  admit_2022 = "https://doc.la.gov/wp-content/uploads/2026/05/IncAdmits_2022.csv",
  admit_2021 = "https://doc.la.gov/wp-content/uploads/2026/05/IncAdmits_2021.csv",
  admit_2020 = "https://doc.la.gov/wp-content/uploads/2026/05/IncAdmits_2020.csv",
  admit_2019 = "https://doc.la.gov/wp-content/uploads/2026/05/IncAdmits_2019.csv"
)

# Release urls
incarceration_release <- c(
  release_2025 = "https://doc.la.gov/wp-content/uploads/2026/04/IncReleases2025-1.csv",
  release_2024 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2024.csv",
  release_2023 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2023.csv",
  release_2022 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2022.csv",
  release_2021 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2021.csv",
  release_2020 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2020.csv",
  release_2019 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2019.csv",
  release_2018 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2018.csv",
  release_2017 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2017.csv",
  release_2016 = "https://doc.la.gov/wp-content/uploads/2026/05/IncReleases2016.csv"
)

# PP url
pp_snapshot <- c(
  pp_feb_2026 = "https://doc.la.gov/wp-content/uploads/2026/04/PPSnapshot_02.2026.csv"
)

# Death url
deaths <- c(
  death_2025 = "https://doc.la.gov/wp-content/uploads/2026/04/Deaths2025.csv",
  death_2024 = "https://doc.la.gov/wp-content/uploads/2026/06/Death2024.csv",
  death_2023 = "https://doc.la.gov/wp-content/uploads/2026/05/Death2023.csv",
  death_2022 = "https://doc.la.gov/wp-content/uploads/2026/05/Death2022.csv",
  death_2021 = "https://doc.la.gov/wp-content/uploads/2026/05/Death2021.csv",
  death_2020 = "https://doc.la.gov/wp-content/uploads/2026/05/Deaths2020.csv",
  death_2019 = "https://doc.la.gov/wp-content/uploads/2026/05/Deaths2019.csv",
  death_2018 = "https://doc.la.gov/wp-content/uploads/2026/05/Deaths2018.csv"
)

# State census url
state_census <- c(
  census_mar_2026 = "https://doc.la.gov/wp-content/uploads/2026/05/census-3-31-26.xlsx",
  census_feb_2026 = "https://doc.la.gov/wp-content/uploads/2026/04/census-2-25-26.xlsx",
  census_jan_2026 = "https://doc.la.gov/wp-content/uploads/2026/05/census-1-28-26.xlsx",
  census_dec_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-12-30-25.xlsx",
  census_nov_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-11-25-25.xlsx",
  census_oct_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-10-29-25.xlsx",
  census_sep_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-9-24-25.xlsx",
  census_aug_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-8-27-25.xlsx",
  census_jul_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-7-30-25.xlsx",
  census_jun_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-6-25-25.xlsx",
  census_may_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-5-28-25.xlsx",
  census_apr_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-4-30-25.xlsx",
  census_mar_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-3-26-25.xlsx",
  census_feb_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-2-26-25.xlsx",
  census_jan_2025 = "https://doc.la.gov/wp-content/uploads/2026/05/census-1-29-25.xlsx"
)

# Function to read excel from files
read_excel_url <- function(url, sheet = NULL, all_sheets = F) {
  
  # Creating a temporary file and downloading it
  tmp <- tempfile(fileext = ".xlsx")
  download.file(url, tmp, mode = "wb", quiet = T)
  
  # Defining and setting names of sheets or just reading the first sheet
  if (all_sheets) {
    sheets <- excel_sheets(tmp)
    sheets %>%
      set_names() %>%
      map(~ read_excel(tmp, sheet = .x))
  } else {
    read_excel(tmp, sheet = sheet)
  }
}

# Reading CSVs and naming
read_to_list <- function(url_vec, read_fn = read_csv) {
  map(url_vec, \(url) read_fn(url))
}

# Defining data
snapshot <- read_to_list(incarceration_snapshot)
admit <- read_to_list(incarceration_admits)
release <- read_to_list(incarceration_release)
pp <- read_to_list(pp_snapshot)
death <- read_to_list(deaths)
census <- read_to_list(state_census, read_excel_url)
codebook <- read_excel_url(codebook_url, all_sheets = T)

# -------------------------- SAVING RAW DATA -----------------------------------

# Saving raw data
dir.create("raw", showWarnings = F)

saveRDS(snapshot, "raw/snapshot.rds")
saveRDS(admit, "raw/admit.rds")
saveRDS(release, "raw/release.rds")
saveRDS(pp, "raw/pp.rds")
saveRDS(death, "raw/death.rds")
saveRDS(census, "raw/census.rds")

# ----------------------- DEFINING LOOKUP TABLES -------------------------------

# Location
location_lookup <- codebook$AssignedLocation %>%
  clean_names() %>%
  rename(
    "location_code" = "assigned_location",
    "location_literal" = "assigned_location_literal"
  )

# Race
race_lookup <- codebook$Race %>%
  clean_names() %>%
  select(-race_comments)

# Parish
parish_lookup <- codebook$Parish %>%
  clean_names()

# Reason
reason_lookup <- codebook$Reason %>%
  clean_names()

# Supervision
supervision_lookup <- codebook$TypeSupervision %>%
  clean_names()

# Offense
offense_lookup <- codebook$OffenseCategory %>%
  clean_names() %>%
  filter(older_offense_categories_1 != "Code") %>%
  select(
    offense_code = older_offense_categories_1, 
    offense_literal = x2
  ) %>%
  bind_rows(
    codebook$OffenseCategory %>%
      clean_names() %>%
      filter(older_offense_categories_4 != "Code") %>%
      select(
        offense_code = older_offense_categories_4, 
        offense_literal = x5
      )
  )

# ------------------- COMBINING FILES (Besides Census) -------------------------

# Snapshot (These are combined for cleaning but need to be uncombined)
snapshots_combined <- snapshot %>%
  imap(~ clean_names(.x) %>% mutate(release_period = .y)) %>%
  map(~ rename(.x, any_of(c(age = "age_as_of_the_date")))) %>%
  map(~ mutate(.x, across(everything(), as.character))) %>%
  bind_rows() %>%
  filter(!if_all(-release_period, is.na))

# Admits
admits_combined <- admit %>%
  imap(~ clean_names(.x) %>% mutate(release_period = .y)) %>%
  map(~ mutate(.x, across(everything(), as.character))) %>%
  bind_rows() %>%
  filter(!if_all(-release_period, is.na))

# Release
release_combined <- release %>%
  imap(~ clean_names(.x) %>% mutate(release_period = .y)) %>%
  map(~ mutate(.x, across(everything(), as.character))) %>%
  bind_rows() %>%
  filter(!if_all(-release_period, is.na))

# PP (These are combined for cleaning but need to be uncombined)
pp_combined <- pp %>%
  imap(~ clean_names(.x) %>% mutate(release_period = .y)) %>%
  map(~ mutate(.x, across(everything(), as.character))) %>%
  bind_rows() %>%
  filter(!if_all(-release_period, is.na))

# Deaths (These are combined for cleaning but need to be uncombined)
deaths_combined <- death %>%
  imap(~ clean_names(.x) %>% mutate(release_period = .y)) %>%
  map(~ mutate(.x, across(everything(), as.character))) %>%
  bind_rows() %>%
  filter(!if_all(-release_period, is.na))

# --------------------- DEFINING CLEANING FUNCTIONS ----------------------------

# Function to clean dates
parse_mixed_date <- function(x) {
  case_when(
    str_detect(x, "^\\d{8}$") ~ ymd(x),
    str_detect(x, "^\\d{4}-\\d{2}-\\d{2}$") ~ ymd(x),
    str_detect(x, "^\\d{1,2}/\\d{1,2}/\\d{4}$") ~ mdy(x),
    T ~ as.Date(NA)
  )
}

# Function to parse sentence length
parse_sentence_length <- function(x) {
  padded <- str_pad(x, width = 7, side = "left", pad = "0")
  as.numeric(str_sub(padded, 1, 3)) +
    (as.numeric(str_sub(padded, 4, 5)) / 12) +
    (as.numeric(str_sub(padded, 6, 7)) / 365.25)
}

# Function to clean census's
clean_census <- function(df) {
  df %>%
    row_to_names(3) %>%
    clean_names() %>%
    slice(1:(which(institutions == "can hide section") - 1)) %>%
    mutate(
      prison = if_else(institutions %in% prison_names, institutions, NA),
      custody = if_else(institutions %in% custody_names, institutions, NA)
    ) %>%
    fill(prison, custody) %>%
    filter(
      !(institutions %in% prison_names),
      !(institutions %in% prison_subtotals),
      !(institutions %in% region_names),
      !(institutions %in% custody_names),
      !(institutions %in% total_names),
      !is.na(institutions),
      institutions != "INSTITUTIONS"
    ) %>%
    mutate(across(-c(institutions, prison, custody), ~ as.numeric(gsub(",", "", .x)))) %>%
    select(prison, custody_type = custody, facility = institutions, everything())
}

# --------------------------- CLEANING ADMITS ----------------------------------

# Admits dataset
admits_clean <- admits_combined %>%
  mutate(
    
    # Adding new columns
    admission_date = parse_mixed_date(admission_date),
    parole_date = parse_mixed_date(parole_date),
    good_time_parole_supervision_date = parse_mixed_date(good_time_parole_supervision_date),
    max_full_term_date = parse_mixed_date(max_full_term_date),
    years_to_parole = as.numeric(difftime(parole_date, admission_date, units = "days")) / 365.25,
    years_to_max = as.numeric(difftime(max_full_term_date, admission_date, units = "days")) / 365.25,
    years_to_good_time = as.numeric(difftime(good_time_parole_supervision_date, admission_date, units = "days")) / 365.25,
    total_length_padded  = str_pad(total_length, width = 7, side = "left", pad = "0"),
    total_length_numeric = as.numeric(str_sub(total_length_padded, 1, 3)) +
      (as.numeric(str_sub(total_length_padded, 4, 5)) / 12) +
      (as.numeric(str_sub(total_length_padded, 6, 7)) / 365.25),
    sex_literal = case_when(
      sex == "M" ~ "Male",
      sex == "F" ~ "Female",
      T ~ NA
    ),
    sex_flag = ifelse(sex_flag == "X" & !is.na(sex_flag), T, F),
    habitual_flag = ifelse(habitual_flag == "X" & !is.na(habitual_flag), T, F),
    violent_flag = ifelse(violent_flag == "X" & !is.na(violent_flag), T, F),
    parish_of_conviction = as.character(as.numeric(parish_of_conviction)),
    sentence_category = case_when(
      total_length == "DEATH" ~ "Death",
      total_length == "LIFE" ~ "Life",
      T ~ "Years"
    )
  ) %>%
  
  # Joining with other data
  left_join(supervision_lookup, by = c("admission_type" = "supervision_code")) %>%
  left_join(reason_lookup %>% mutate(reason_code = as.character(as.numeric(reason_code))), by = c("admission_reason" = "reason_code")) %>%
  left_join(race_lookup, by = c("race" = "race_code")) %>%
  left_join(offense_lookup, by = c("longest_offense_category" = "offense_code"), suffix = c("", "_longest")) %>%
  rename(longest_offense_literal = offense_literal) %>%
  left_join(offense_lookup, by = c("most_serious_offense_category" = "offense_code"), suffix = c("", "_most_serious")) %>%
  rename(most_serious_offense_literal = offense_literal) %>%
  left_join(
    parish_lookup %>% mutate(parish_code = as.character(as.numeric(parish_code))),
    by = c("parish_of_conviction" = "parish_code")
  ) %>%
  
  # Selecting important columns
  select(
    unique_id,
    parish = parish_literal,
    admission_date,
    race = race_literal,
    sex = sex_literal,
    admission_age = age_at_admit,
    parole_date,
    good_time_date = good_time_parole_supervision_date,
    max_date = max_full_term_date,
    years_to_parole,
    years_to_max,
    years_to_good_time,
    sentence_category,
    sex_flag,
    violent_flag,
    habitual_flag,
    longest_offense_literal,
    most_serious_offense_literal,
    supervision_literal,
    reason_literal,
    release_period
  ) %>%
  mutate(
    across(where(is.character), str_to_title)
  )

# --------------------------- CLEANING RELEASE ---------------------------------

# Release dataset
release_clean <- release_combined %>%
  mutate(
    
    # Adding columns
    admission_date = parse_mixed_date(admission_date),
    release_date = parse_mixed_date(release_date),
    total_length_padded = str_pad(total_length, width = 7, side = "left", pad = "0"),
    years_served = as.numeric(difftime(release_date, admission_date, units = "days")) / 365.25,
    sex_literal = case_when(
      gender == "M" ~ "Male",
      gender == "F" ~ "Female",
      T ~ NA
    ),
    sex_flag      = ifelse(sex_offender_flag == "X" & !is.na(sex_offender_flag), T, F),
    violent_flag  = ifelse(violent_offender_flag == "X" & !is.na(violent_offender_flag), T, F),
    habitual_flag = ifelse(habitual_offender_flag == "X" & !is.na(habitual_offender_flag), T, F),
    release_location = coalesce(release_location, x10),
    parish_of_conviction = as.character(as.numeric(parish_of_conviction)),
    sentence_category = case_when(
      total_length == "DEATH" ~ "Death",
      total_length == "LIFE" ~ "Life",
      T ~ "Years"
    ),
    admission_age = as.numeric(age) - years_served,
  ) %>%
  
  # Adding extra datasets
  left_join(supervision_lookup, by = c("admission_type" = "supervision_code")) %>%
  left_join(reason_lookup %>% mutate(reason_code = as.character(as.numeric(reason_code))), by = c("admission_reason" = "reason_code")) %>%
  rename(admission_reason_literal = reason_literal) %>%
  left_join(race_lookup, by = c("race" = "race_code")) %>%
  left_join(offense_lookup, by = c("longest_offense_category" = "offense_code"), suffix = c("", "_longest")) %>%
  rename(longest_offense_literal = offense_literal) %>%
  left_join(offense_lookup, by = c("most_serious_offense_category" = "offense_code"), suffix = c("", "_most_serious")) %>%
  rename(most_serious_offense_literal = offense_literal) %>%
  left_join(
    parish_lookup %>% mutate(parish_code = as.character(as.numeric(parish_code))),
    by = c("parish_of_conviction" = "parish_code")
  ) %>%
  
  # Selecting important variables
  select(
    unique_id,
    parish = parish_literal,
    admission_date,
    release_date,
    release_location,
    race = race_literal,
    sex = sex_literal,
    age,
    admission_age,
    years_served,
    sentence_category,
    sex_flag,
    violent_flag,
    habitual_flag,
    longest_offense_literal,
    most_serious_offense_literal,
    supervision_literal,
    admission_reason_literal,
    release_reason,
    release_period
  ) %>%
  mutate(
    across(where(is.character), str_to_title)
  )

# -------------------------- CLEANING SNAPSHOTS --------------------------------

# Snapshot data
snapshots_clean <- snapshots_combined %>%
  mutate(
    
    # Adding columns
    admission_date = parse_mixed_date(admission_date),
    as_of_date = parse_mixed_date(as_of_date),
    total_length_numeric = if_else(
      total_length %in% c("LIFE", "DEATH"),
      NA,
      parse_sentence_length(total_length)
    ),
    sentence_category = case_when(
      total_length == "DEATH" ~ "Death",
      total_length == "LIFE" ~ "Life",
      T ~ "Years"
    ),
    years_served = as.numeric(difftime(as_of_date, admission_date, units = "days")) / 365.25,
    sex_offender_flag = ifelse(sex_offender_flag == "X" & !is.na(sex_offender_flag), T, F),
    violent_offender_flag = ifelse(violent_offender_flag == "X" & !is.na(violent_offender_flag), T, F),
    habitual_offender_flag = ifelse(habitual_offender_flag == "X" & !is.na(habitual_offender_flag), T, F),
    sex_literal = case_when(
      sex == "M" ~ "Male",
      sex == "F" ~ "Female",
      T ~ NA
    ),
    age = as.numeric(coalesce(age, age_as_of_as_of_date)),
    admission_age = age - years_served,
    parish_of_conviction = as.character(as.numeric(parish_of_conviction)),
    current_assigned_location = as.character(as.numeric(current_assigned_location))
  ) %>%
  
  # Adding datasets
  left_join(location_lookup, by = c("current_assigned_location" = "location_code")) %>%
  left_join(race_lookup, by = c("race" = "race_code")) %>%
  left_join(offense_lookup, by = c("longest_offense_category" = "offense_code"), suffix = c("", "_longest")) %>%
  rename(longest_offense_literal = offense_literal) %>%
  left_join(offense_lookup, by = c("most_serious_offense_category" = "offense_code"), suffix = c("", "_most_serious")) %>%
  rename(most_serious_offense_literal = offense_literal) %>%
  left_join(
    parish_lookup %>% mutate(parish_code = as.character(as.numeric(parish_code))),
    by = c("parish_of_conviction" = "parish_code")
  ) %>%
  
  # Selecting variables of importance
  select(
    unique_id,
    admission_date,
    as_of_date,
    location = location_literal,
    parish = parish_literal,
    race = race_literal,
    sex = sex_literal,
    age,
    admission_age,
    total_length,
    total_length_numeric,
    sentence_category,
    years_served,
    sex_offender_flag,
    violent_offender_flag,
    habitual_offender_flag,
    longest_offense_literal,
    most_serious_offense_literal,
    release_period
  ) %>%
  mutate(across(where(is.character), str_to_title))

# -------------------------- CLEANING PROBATION --------------------------------

# Probation and Parole clean
pp_clean <- pp_combined %>%
  mutate(
    admission_date = parse_mixed_date(admission_date),
    supervision_end_date = parse_mixed_date(supervision_end_date),
    as_of_date = parse_mixed_date(as_of_date),
    current_years_on_supervision = as.numeric(difftime(as_of_date, admission_date, units = "days")) / 365.25,
    total_length_supervision = as.numeric(difftime(supervision_end_date, admission_date, units = "days")) / 365.25,
    sentence_category = case_when(
      total_length == "DEATH" ~ "Death", 
      total_length == "LIFE" ~ "Life",
      T ~ "Years"
    ),
    probate_length_category = case_when(
      str_to_lower(probate_length) %in% c("indefin", "indefinite") ~ "Indefinite",
      str_to_lower(probate_length) == "life" ~ "Life",
      T ~ "Years"
    ),
    sex_offender_flag = ifelse(sex_offender_flag == "X" & !is.na(sex_offender_flag), T, F),
    violent_flag = ifelse(violent_flag == "X" & !is.na(violent_flag), T, F),
    habitual_offender_flag = ifelse(habitual_offender_flag == "X" & !is.na(habitual_offender_flag), T, F),
    sex_literal = case_when(
      sex == "M" ~ "Male",
      sex == "F" ~ "Female",
      T ~ NA
    ),
    age = as.numeric(age)
  ) %>%
  
  # Join lookups
  left_join(supervision_lookup, by = c("type_of_supervision" = "supervision_code")) %>%
  left_join(race_lookup, by = c("race" = "race_code")) %>%
  left_join(
    offense_lookup,
    by = c("longest_offense_category" = "offense_code"),
    suffix = c("", "_longest")
  ) %>%
  rename(longest_offense_literal = offense_literal) %>%
  left_join(
    offense_lookup,
    by = c("most_serious_offense_category" = "offense_code"),
    suffix = c("", "_most_serious")
  ) %>%
  rename(most_serious_offense_literal = offense_literal) %>%
  
  # Select and rename final columns
  select(
    unique_id,
    admission_date,
    as_of_date,
    supervision_end_date,
    supervision_type = supervision_literal,
    race = race_literal,
    sex = sex_literal,
    age,
    current_years_on_supervision,
    total_length_supervision,
    sentence_category,
    probate_length_category,
    sex_offender_flag,
    violent_flag,
    habitual_offender_flag,
    longest_offense_literal,
    most_serious_offense_literal,
    release_period
  ) %>%
  mutate(across(where(is.character), str_to_title))

# --------------------------- CLEANING DEATHS ----------------------------------

# Deaths clean
deaths_cleaned <- deaths_combined %>%
  mutate(
    death_date = mdy(date_of_death),
    admission_date = mdy(first_admission_date_to_doc),
    years_until_death = as.numeric(difftime(death_date, admission_date, units = "days")) / 365.25,
    admission_age = as.numeric(age) - years_until_death,
    race = str_trim(str_remove(race, "^\\d+\\.\\s*")),
    ethnicity = str_trim(str_remove(ethnicity, "^\\d+\\.\\s*")),
    gender = str_trim(str_remove(gender, "^\\d+\\.\\s*")),
    reason_death = case_when(
      manner_of_death_doc == "Cancer" ~ "Cancer",
      manner_of_death_doc == "Heart -any type of heart related death including cardiopulmonary arrest" ~ "Heart",
      manner_of_death_doc == "Vascular- any type of bleed to include Gi, Strokes, brain" ~ "Vascular",
      manner_of_death_doc == "Respiratory" ~ "Respiratory",
      manner_of_death_doc == "Sepsis" ~ "Sepsis",
      manner_of_death_doc == "Liver- any liver related death" ~ "Liver",
      manner_of_death_doc == "Renal" ~ "Renal",
      manner_of_death_doc == "Overdose" ~ "Overdose",
      manner_of_death_doc == "Suicide" ~ "Suicide",
      manner_of_death_doc == "Execution" ~ "Execution",
      manner_of_death_doc == "Accidental- based off the determination of the autopsy report; car accident" ~ "Accidental Car Accident",
      manner_of_death_doc == "Homicide- based off of the determination of the autopsy report" ~ "Homicide",
      manner_of_death_doc == "Pending- waiting on autopsy results" ~ "Pending",
      manner_of_death_doc == "Other- Multiple System Organ Failure- Death by natural causes due to age" ~ "Other - MSOF",
      manner_of_death_doc == "Other- Anoxic Brain Injury" ~ "No Oxygen Brain Injury",
      manner_of_death_doc == "Other - MSOF" ~ "Multiple Systems Organ Failure",
      manner_of_death_doc == "Other- Undetermined- based off the determination of the autopsy report" ~ "Other - Undetermined",
      is.na(manner_of_death_doc) ~ NA,
      T ~ manner_of_death_doc
    )
  ) %>%
  select(
    unique_id,
    admission_date,
    death_date,
    location,
    race,
    ethnicity,
    gender,
    death_age = age,
    admission_age,
    years_until_death,
    reason_death,
    release_period
  )

# --------------------------- CLEANING CENSUS ----------------------------------

# Defining variables to remove
prison_names <- c("ALC", "DCI", "LSP", "RLCC", "DWCC","EHCC", "LCIW", "RCC", "STATE POLICE")
prison_subtotals <- paste("SUBTOTAL", prison_names)
region_names <- c("REGION 1", "REGION 2")
custody_names <- c("Medium-Minimum Custody", "Maximum Custody")
total_names <- c("REGION 1 TOTAL MEDIUM-MINIMUM", "REGION 1 TOTAL MAXIMUM", "REGION 1 TOTAL", 
                 "REGION 2 TOTAL MEDIUM-MINIMUM", "REGION 2 TOTAL MAXIMUM", "REGION 2 TOTAL",
                 "TOTAL MEDIUM-MINIMUM", "TOTAL MAXIMUM", "TOTAL INSTITUTIONS", "GRAND TOTAL")

# Cleaning the census data
census_clean <- map_dfr(census, clean_census, .id = "release_period")


# ------------------------ SAVING PROCESSED DATA -------------------------------

# Saving processed data
dir.create("processed", showWarnings = F)

saveRDS(snapshots_clean, "processed/snapshots_clean.rds")
saveRDS(admits_clean, "processed/admits_clean.rds")
saveRDS(release_clean, "processed/release_clean.rds")
saveRDS(pp_clean, "processed/pp_clean.rds")
saveRDS(deaths_cleaned, "processed/deaths_clean.rds")
saveRDS(census_clean, "processed/census_clean.rds")