DD_DC_EDA.Rmd

---
title: "EDA"
author: "Dmitrii Zhakota"
date: "`r Sys.Date()`"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
library(readxl)
# library(openxlsx)
library(dlookr)
# library(dplyr)
library(flextable)
library(ggstatsplot)

library(tidyverse)
```

# Load modified data v2
```{r}
# Переименованы колонки. Добавлен префикс Before_ для группы Präoperativ After_ для группы Postoperativ FUEcho_ для FUEcho.
# В excel сопоставлены признаки STERBEDAT, LAST_FU и FUEcho_DATUM. 
# Заменены в STERBEDAT unbekannt на пропуски, т.к. в признаке VERSTORBEN они все закодированы как 1. 
# Заменены в STERBEDAT значения в формате YYYY на формат даты 31.12.YYYY. Item с такими изменениями сохранены во вкладке mod. Оставшиеся unbekannt заменены на пропуски.
# Переназначены ячейки столбцов, в которых числя записаны как текст
# Заменены в числах запятые на точки.


df_read_excel <- read_excel("data/raw/Epic_Supra_V_1_mod_2.xlsx", 
    col_types = c("numeric", "date", "numeric", 
        "numeric", "date", "numeric", "date", 
        "text", "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "text", "date", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "text", "numeric", "numeric", 
        "numeric", "text", "text", "numeric", 
        "text", "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "text", "numeric", "text", 
        "date", "numeric", "text", "date", "numeric", 
        "text", "date", "numeric", "text", "date", 
        "numeric", "text", "date", "numeric", "text", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", "numeric", 
        "numeric", "numeric", "numeric", "numeric", 
        "numeric", "date"))

Epic_Supra_dates_v2 <- read_excel("data/raw/Epic_Supra_dates_v2.xlsx")

```

# Replace date columns
```{r}
# Copy and replace columns from Epic_Supra_dates_v2 to df_read_excel
df <- df_read_excel
df$LAST_FU <- Epic_Supra_dates_v2$LAST_FU
df$FUEcho_DATUM <- Epic_Supra_dates_v2$FUEcho_DATUM
df$STERBEDAT <- Epic_Supra_dates_v2$STERBEDAT
# df$STERBEDAT_unbekant <- Epic_Supra_dates_v2$STERBEDAT_unbekant
# df$STERBEDAT_only_year <- Epic_Supra_dates_v2$STERBEDAT_only_year
# df$LAST_FU_only_year <- Epic_Supra_dates_v2$LAST_FU_only_year

```


# Rename and Recode
```{r}
rename <- read.table(file = 'data/raw/Column_Renamer.tsv', sep = '\t', header = TRUE)

# Convert to vector column English_Column_Name_Short for rename
English_Column_Name_Short <- as.vector(rename$English_Column_Name_Short)

# Rename columns names in df_read_excel with vector English_Column_Name_Short
df_en <- df %>% 
  rename_with(~English_Column_Name_Short, everything())
  
  
```

# Data preparation
```{r}
# Сопоставляем колонки с датами, которые принимаются за конечную точку. 
# df_followup <- df_read_excel %>% 
#   select(Item, VERSTORBEN, STERBEDAT, LAST_FU, FUEcho_DATUM)

df_followup <- df_en %>% 
  select(Case_Num, OS_Event, OS_Date, Last_FU, FUEcho_Date)

# Common rename df.
df <- df_en

# Перекодировать значения в колонке GESCHLECHT в 1 и 2. 1 - M, 2 - W. Будет 1 - male 2 - female
df <- df %>%
  mutate(Gender = ifelse(Gender == "M", 1, 2))

# Round all numeric columns
df <- df %>%
  mutate_if(is.numeric, round, digits = 2)

# !!!! Потребуется если работать с первичным файлом. В файле mod_2 уже исправлено в excel !!!
# if column is numeric then replace , to . 
# df <- df %>% 
#   mutate_if(is.numeric, ~gsub(",", ".", .x)) %>% 
#   mutate_if(is.numeric, as.numeric)

  

# Удаляем нулевые значения
# df <- df %>% 
#   filter(BMI != 0)

# Перекодируем даты из YYYY-MM-DD в DD.MM.YYYY. Проблема - превращает в формат character.
# df <- df %>% 
#   mutate(GEBURT = format(as.Date(GEBURT, format = "%Y-%m-%d"), "%d.%m.%Y"))
# 
# df <- df %>% 
#   mutate(STERBEDAT = format(as.Date(STERBEDAT, format = "%Y-%m-%d"), "%d.%m.%Y"))
# 
# df <- df %>% 
#   mutate(OP_DATUM = format(as.Date(OP_DATUM, format = "%Y-%m-%d"), "%d.%m.%Y"))
# 
# df <- df %>% 
#   mutate(FUEcho_DATUM = format(as.Date(FUEcho_DATUM, format = "%Y-%m-%d"), "%d.%m.%Y"))

```


# Diagnosis Data
```{r}
df %>% 
  diagnose() %>% 
  flextable()

df %>% 
  plot_na_pareto()


df %>% 
  summary()

# Удаляем лишние датафреймы
# rm("df_read_excel")
```

# Data clearning
```{r}
# Привести к факторам или числам
factors_auto <- df %>% select_if(function(x) {all(unique(x) %in% c(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NA))}) %>% colnames()
factors_list <- c("Case_Num", "Gender")

df %>%  mutate(across(
  c(all_of(factors_auto), all_of(factors_list)), as.factor
)) -> df1

# Convert columns if class is POSIXct or POSIXt to date
# df1 <- df1 %>% 
#   mutate(across(c(GEBURT, STERBEDAT, LAST_FU, OP_DATUM, REOP_DAT, REOP_DAT2, REOP_DAT3, REOP_DAT4, REOP_DAT5, FUEcho_DATUM), ~ as.Date(. )))

df1 <- df1 %>% 
  mutate(across(c(Birth_Date, OS_Date, Last_FU, Sx_Date, FUEcho_Date), ~ as.Date(. )))

class(df1$Birth_Date)


#Remove columns where date class
# df1_no_date <- df1 %>%
#   select(-c(GEBURT, STERBEDAT, LAST_FU, OP_DATUM, REOP_DAT, REOP_DAT2, REOP_DAT3, REOP_DAT4, REOP_DAT5, FUEcho_DATUM))

df1_no_date <- df1 %>%
  select(-c(Birth_Date, OS_Date, Last_FU, Sx_Date, FUEcho_Date))


# Delete columns if more than 50% of values are NA
df1_no_na_col <- df1_no_date %>% select_if(~sum(!is.na(.))/length(.) >= 0.5)

```

# Save data frames to interim folder
```{r}
# Сохраняем датафреймы
write.csv(df_read_excel, "data/interim/df_read_excel.csv", row.names = FALSE) # Без изменений после загрузки из excel
write.csv(df_en, "data/interim/df_en.csv", row.names = FALSE) # Переимнование колонок в English_Column_Name_Short
write.csv(df, "data/interim/df.csv", row.names = FALSE) # Удаление лишних колонок и перекодировка пола
write.csv(df1, "data/interim/df1.csv", row.names = FALSE) # Назначены классы признакам (фактор, дата, количественный)

write.csv(df1_no_date, "data/interim/df1_no_date.csv", row.names = FALSE) # Удалены все даты
write.csv(df1_no_na_col, "data/interim/df1_no_na_col.csv", row.names = FALSE) # Удалены все даты и колонки с NA > 50%


# Удаляем полностью среду
rm(list=ls())
```



#Pre-report EDA in dlookr
```{r eval=FALSE, include=FALSE}
read.csv("data/interim/df1_no_na_col.csv") -> df1_no_na_col


factors_auto <- df1_no_na_col %>% select_if(function(x) {all(unique(x) %in% c(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NA))}) %>% colnames()
factors_list <- c("Case_Num", "Gender")

df1_no_na_col %>%  mutate(across(
  c(all_of(factors_auto), all_of(factors_list)), as.factor
)) -> df1_no_na_col

read.csv("data/interim/df1_no_date.csv") -> df1_no_date

factors_auto <- df1_no_date %>% select_if(function(x) {all(unique(x) %in% c(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NA))}) %>% colnames()
factors_list <- c("Case_Num", "Gender")

df1_no_date %>%  mutate(across(
  c(all_of(factors_auto), all_of(factors_list)), as.factor
)) -> df1_no_date

# Отчёт по датафрейму со всеми признаками
diagnose_web_report(df1_no_date,
                    output_file = "Diagn_df1.html",  
                    title = "Data Diagnosis Web",
                    subtitle = deparse(substitute(.data))
                    )

eda_web_report(df1_no_date,
               output_file = "EDA_df1.html",
               title = "Exploratory Data Analysis",
               subtitle = deparse(substitute(.data))
               )

#######################################################################################################################
# Отчёт по датафрейму после удаления признаков, в которых больше 50% пропущенных значений ######
#######################################################################################################################

diagnose_web_report(df1_no_na_col,
                    output_file = "Diagn_df1_no_na_col.html",  
                    title = "Data Diagnosis",
                    subtitle = deparse(substitute(.data))
                    )

eda_web_report(df1_no_na_col,
               output_file = "EDA_df1_no_na_col.html",
               title = "Exploratory Data Analysis",
               subtitle = deparse(substitute(.data))
               )

```

# Static report in dlookr !!! OPTION !!!
```{r eval=FALSE, include=FALSE}
# Статичтные отчёты Diagos Data
diagnose_paged_report(df1_no_date,
                      output_format = "html",
                      output_file = "Diagn_df1.html",
                      output_dir = "."
                      )

diagnose_paged_report(df1_no_na_col,
                      output_file = "Diagn_df1_no_na_col.html",
                      output_format = "html",
                      output_dir = "."
                      )

# Статичные отчёты EDA
eda_paged_report(df1_no_date,
                 output_file = "EDA_df1.html",
                 output_format = "html",
                 output_dir = "."
                      )

eda_paged_report(df1_no_na_col,
                 output_file = "EDA_df1_no_na_col.html", 
                 output_format = "html",
                 output_dir = "."
                      )

# diagnose_paged_report(df1_no_na_col,
#                       output_dir = "./data/interim/",
#                       output_format = "html")
# 
# diagnose_paged_report(df1_no_na_col,
#                       output_format = "pdf")


```


# Exploratory Data Analysis
## Descriptive statistics
```{r}
rm(list=ls())

df <- read.csv("data/interim/df1.csv")

# Привести к факторам или числам
factors_auto <- df %>% select_if(function(x) {all(unique(x) %in% c(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NA))}) %>% colnames()
factors_list <- c("Case_Num", "Gender")

df %>%  mutate(across(
  c(all_of(factors_auto), all_of(factors_list)), as.factor
)) -> df

# Convert columns if class is POSIXct or POSIXt to date
# df1 <- df1 %>% 
#   mutate(across(c(GEBURT, STERBEDAT, LAST_FU, OP_DATUM, REOP_DAT, REOP_DAT2, REOP_DAT3, REOP_DAT4, REOP_DAT5, FUEcho_DATUM), ~ as.Date(. )))

df <- df %>% 
  mutate(across(c(Birth_Date, OS_Date, Last_FU, Sx_Date, FUEcho_Date), ~ as.Date(. )))

# df1 <- df %>% 
#   dplyr::filter(!is.na(Last_FU)) 

df1 <- df %>% 
  mutate(Gender = recode(Gender, '1' = "Male", '2' = "Female")) %>% 
  mutate(Upsizing = recode(Upsizing, '0' = "Without upsizing", '1' = "With upsizing"))

#####################################################################################################
# Count of follow-ups by year
ggplot(df1, aes(x = Last_FU)) +
  geom_histogram(binwidth = 365) +
  scale_x_date(date_breaks = "1 year", date_labels = "%Y") +
  theme_bw() +
  theme(axis.text.x = element_text(angle = 90, hjust = 1))

# Count genger by upsizing
ggbarstats(data = df1,
               x = Gender,
               y = Upsizing
)

# Count Gender by STS
ggbetweenstats(data = df1,
               x = Gender,
               y = STS
)


grouped_ggbetweenstats(data = df1,
               x = Gender,
               y = STS,
               grouping.var = Upsizing
)

# Count Gender by Age
ggbetweenstats(data = df1,
               x = Gender,
               y = Age
)


grouped_ggbetweenstats(data = df1,
               x = Gender,
               y = Age,
               grouping.var = Upsizing
)


  ggbarstats(data = df1,
               x = Upsizing,
               y = Sx_Urgency
)


grouped_ggbarstats(data = df1,
               x = Gender,
               y = Sx_Urgency,
               grouping.var = Upsizing
)


grouped_ggbetweenstats(data = df1,
                       x = Upsizing,
                       y = Sx_Aort_Clamp_t,
                       grouping.var = Gender,
                       ylab = "Aortic clamp time (min)",
)



```



## Clustering
```{r}
rm(list = ls())
df <- read.csv("data/interim/df1_no_date.csv")

# Привести к факторам или числам
factors_auto <- df %>% select_if(function(x) {all(unique(x) %in% c(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NA))}) %>% colnames()
factors_list <- c("Gender")

df %>%  mutate(across(
  c(all_of(factors_auto), all_of(factors_list)), as.factor
)) -> df

str(df) # Обязательно проверить присвоение классов. Иногда не срабатывает


df_new <- df %>% 
  select(where(is.numeric)) %>% 
  select(-c(Case_Num)) %>%  # Удаляем Item, так как он не несёт смысловой нагрузки
  mutate_all(round, 1)  
  # stats::na.omit()



library(pheatmap)
df_new_scaled <- scale(df_new)


df_new_dist <- dist(df_new_scaled, 
                        method = "euclidean"
                        )

as.matrix(df_new_dist)[1:42,1:42]

pheatmap(df_new_scaled, 
         scale = "row", 
         clastering_distance_rows = df_new_dist,
         claster_method = "ward.D2",
         cutree_rows = 5, # группируем (условно)
         cutree_cols = length(colnames(df_new_scaled))
)


```


## PCA
<!-- Проводим PCA анализ, что бы снизить размерность данных. Сравнить с тем, что порекомендует Охотин объединить. В первую очередь это касается ЭхоКГ. -->

```{r}
# Удаляем полностью среду
rm(list=ls())

# Датафрейм  только с удалёнными датами содержит много пропуcков. В итоге удаляются все строки.
# read.csv("data/interim/df1_no_date.csv") -> df1_no_date
# 
# # # Привести к факторам или числам
# factors_auto <- df1_no_date %>% select_if(function(x) {all(unique(x) %in% c(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NA))}) %>% colnames()
# factors_list <- c("Item", "GESCHLECHT")
# 
# df1_no_date %>%  mutate(across(
#   c(all_of(factors_auto), all_of(factors_list)), as.factor
# )) -> df1_no_date
# 
# df1_pca <- df1_no_date %>%
#   select(where(is.numeric)) %>%
#   # select(-c(MORTALITAT_TAGE)
#   mutate_all(round, 1) %>%
#   stats::na.omit()


# Датафрейм без дат и без колонок с количеством пропущенных значений больше 50% годится в работу.
read.csv("data/interim/df1_no_na_col.csv") -> df1_no_na_col

# Привести к факторам или числам
factors_auto <- df1_no_na_col %>% select_if(function(x) {all(unique(x) %in% c(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NA))}) %>% colnames()
factors_list <- c("Gender")

df1_no_na_col %>%  mutate(across(
  c(all_of(factors_auto), all_of(factors_list)), as.factor
)) -> df1_no_na_col

str(df1_no_na_col) # Обязательно проверить присвоение классов. Иногда не срабатывает. Если не сработало - презапустить сессию и запустить этот чанк отдельно.


df1_pca <- df1_no_na_col %>% 
  select(where(is.numeric)) %>% 
  # select(-c(MORTALITAT_TAGE)
  mutate_all(round, 1) %>% 
  stats::na.omit()



library(FactoMineR)

df_full.pca <- prcomp(df1_pca, 
                        scale = TRUE)

summary(df_full.pca)


library(factoextra)

fviz_eig(df_full.pca, addlabels = TRUE)

fviz_pca_var(df_full.pca, col.var = "contrib")

fviz_pca_var(df_full.pca, 
             select.var = list(contrib = 3), # Задаём число здесь 
             col.var = "contrib")

fviz_contrib(df_full.pca, choice = "var", axes = 1, top = 24) # 1
fviz_contrib(df_full.pca, choice = "var", axes = 2, top = 24) # 2
fviz_contrib(df_full.pca, choice = "var", axes = 3, top = 24) # 3


# PCA с группировкой по UPSIZING
library(ggbiplot)

df0 <- df1_no_na_col %>%
  select(c(Case_Num, Upsizing))

df1_pca2 <- left_join(df1_pca, df0, "Case_Num" )


ggbiplot(df_full.pca, 
         scale=0, 
         groups = as.factor(df1_pca2$Upsizing), 
         ellipse = TRUE,
         # circle = TRUE,
         alpha = 0.2) +
  theme_minimal()


```