Architecture_analysis.Rmd

---
title: "Architectures results analysis"
author: "Katarzyna Sidorczuk"
date: "10/11/2021"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = FALSE, fig.width = 12)

library(dplyr)
library(ggplot2)
library(tidyr)
library(ggrepel)
library(targets)

tar_load(c(architectures_performance, mean_architecture_performance, model_variants))

change_variant_names <- function(df, colname) {
  df[[colname]] <- sapply(df[[colname]], function(i) case_when(grepl("v1", i) ~ gsub("v1", "NMmc", i),
                                                               grepl("v2", i) ~ gsub("v2", "NMmc_S", i),
                                                               grepl("v3", i) ~ gsub("v3", "NMmc_OMS", i),
                                                               grepl("v4", i) ~ gsub("v4", "NMbin", i),
                                                               grepl("v5", i) ~ gsub("v5", "NMbin_S", i),
                                                               grepl("v6", i) ~ gsub("v6", "NMbin_OMS", i),
                                                               grepl("v7", i) ~ gsub("v7", "NMmc_PS_NS", i),
                                                               grepl("v8", i) ~ gsub("v8", "NMbin_PS_NS", i)),
                          USE.NAMES = FALSE)
  df
} 
arch_dat <- mutate(mean_architecture_performance,
                   arch_variant = sapply(mean_architecture_performance[["model"]], function(i) gsub("-Sec|-Tat", "", strsplit(i, "_")[[1]][2])),
                   hierarchical = grepl(pattern = "Filtering", x = model, ignore.case = FALSE),
                   algorithm = sapply(mean_architecture_performance[["model"]], function(i) last(strsplit(i, "_")[[1]])),
                   sec = !grepl(pattern = "-Sec", x = model, ignore.case = FALSE),
                   tat = !grepl(pattern = "-Tat", x = model, ignore.case = FALSE),
                   smote = !grepl(pattern = "0-1", x = model),
                   smote_variant = sapply(mean_architecture_performance[["model"]], function(i) strsplit(i, "_")[[1]][3])) %>% 
  pivot_longer(cols = !c(model, arch_variant, hierarchical, algorithm, sec, tat, smote, smote_variant), names_to = "measure") 

```

## Architecture variants

```{r}
data.frame(arch_variant = names(model_variants)[1:8], 
           shortcut = c("NMmc", "NMmc_S", "NMmc_OMS", "NMbin", "NMbin_S", "NMbin_OMS", "NMmc_PS_NS", "NMbin_PS_NS"),
           models = sapply(model_variants[1:8], function(i) paste0(i, collapse = ", "))) %>% 
  knitr::kable(., row.names = FALSE)
```

## Overall performance with hierarchical or not hierarchical architecture

```{r fig.height=6}
arch_dat <- change_variant_names(arch_dat, "arch_variant")

filter(arch_dat, measure %in% c("mean_AU1U", "mean_kappa")) %>% 
  ggplot(aes(x = arch_variant, y = value, color = hierarchical)) +
  geom_boxplot() +
  facet_wrap(~measure, scales = "free_y", nrow = 2) + 
  theme_bw()

filter(arch_dat, measure %in% c("mean_AU1U", "mean_kappa")) %>% 
  ggplot(aes(x = arch_variant, y = value, color = hierarchical)) +
  geom_boxplot() +
  facet_grid(measure~algorithm, scales = "free_y") + 
  theme_bw() +
  theme(axis.text.x = element_text(angle = 90))
```

## Prediction of all classes with hierarchical or not hierarchical architecture

```{r fig.height=15}
filter(arch_dat, !(measure %in% c("mean_AU1U", "mean_kappa"))) %>% 
  ggplot(aes(x = arch_variant, y = value, color = measure)) +
  geom_boxplot() +
  facet_grid(measure~hierarchical, scales = "free_y") + 
  theme_bw() +
  theme(legend.position = "none",
        axis.text.x = element_text(angle = 90))

filter(arch_dat, !(measure %in% c("mean_AU1U", "mean_kappa"))) %>% 
  ggplot(aes(x = arch_variant, y = value, color = measure)) +
  geom_boxplot() +
  facet_grid(measure~algorithm+hierarchical, scales = "free_y") + 
  theme_bw() +
  theme(legend.position = "none",
        axis.text.x = element_text(angle = 90))
```

## Prediction of the most difficult classes with hierarchical or not hierarchical architecture
```{r fig.height=6}
filter(arch_dat, measure %in% c("mean_N_IM_sensitivity", "mean_N_OM_sensitivity", "mean_N_TM_sensitivity", "mean_N_TL_SEC_sensitivity")) %>% 
  ggplot(aes(x = arch_variant, y = value, color = measure)) +
  geom_boxplot() +
  facet_wrap(~hierarchical) + 
  theme_bw() +
  theme(legend.position = "bottom",
        axis.text.x = element_text(angle = 90))
```
```{r fig.height = 10}
filter(arch_dat, measure %in% c("mean_N_IM_sensitivity", "mean_N_OM_sensitivity", "mean_N_TM_sensitivity", "mean_N_TL_SEC_sensitivity")) %>% 
  ggplot(aes(x = arch_variant, y = value, color = measure)) +
  geom_boxplot() +
  facet_grid(algorithm~hierarchical) + 
  theme_bw() +
  theme(legend.position = "bottom",
        axis.text.x = element_text(angle = 90))
```

## Prediction of the most difficult classes depending on the number of models that use SMOTE

```{r fig.height=14}
filter(arch_dat, measure %in% c("mean_N_IM_sensitivity", "mean_N_OM_sensitivity", "mean_N_TM_sensitivity", "mean_N_TL_SEC_sensitivity")) %>% 
  mutate(n_smote_models = sapply(strsplit(smote_variant, "-"), function(i) i[1])) %>% 
  ggplot(aes(x = n_smote_models, y = value, color = hierarchical)) +
  geom_boxplot() +
  facet_grid(measure~arch_variant, scales = "free") +
  theme_bw() +
  theme(legend.position = "bottom")
```

## Overall performance depending on presence of Sec and Tat models in the architecture

```{r fig.height=6}
filter(arch_dat, measure %in% c("mean_AU1U", "mean_kappa")) %>% 
  mutate(models = case_when(sec == TRUE & tat == TRUE ~ "with Sec and Tat",
                            sec == FALSE & tat == TRUE ~ "without Sec",
                            sec == TRUE & tat == FALSE ~ "without Tat")) %>% 
  ggplot(aes(x = models, y = value, color = measure)) +
  geom_boxplot() +
  facet_grid(measure~arch_variant, scales = "free_y") + 
  theme_bw() +
  theme(legend.position = "none",
        axis.text.x = element_text(angle = 90))

filter(arch_dat, measure %in% c("mean_AU1U", "mean_kappa") & algorithm == "GLM") %>% 
  mutate(models = case_when(sec == TRUE & tat == TRUE ~ "with Sec and Tat",
                            sec == FALSE & tat == TRUE ~ "without Sec",
                            sec == TRUE & tat == FALSE ~ "without Tat")) %>% 
  ggplot(aes(x = models, y = value, color = measure)) +
  geom_boxplot() +
  facet_grid(measure~algorithm+arch_variant, scales = "free_y") + 
  theme_bw() +
  theme(legend.position = "none",
        axis.text.x = element_text(angle = 90))
filter(arch_dat, measure %in% c("mean_AU1U", "mean_kappa") & algorithm == "RF") %>% 
  mutate(models = case_when(sec == TRUE & tat == TRUE ~ "with Sec and Tat",
                            sec == FALSE & tat == TRUE ~ "without Sec",
                            sec == TRUE & tat == FALSE ~ "without Tat")) %>% 
  ggplot(aes(x = models, y = value, color = measure)) +
  geom_boxplot() +
  facet_grid(measure~algorithm+arch_variant, scales = "free_y") + 
  theme_bw() +
  theme(legend.position = "none",
        axis.text.x = element_text(angle = 90))
```

## Performance on HMM-dependent classes depending on presence of Sec and Tat models in the architecture

```{r fig.height=11}
filter(arch_dat, measure %in% c("mean_N_TL_SEC_sensitivity", "mean_N_TL_TAT_sensitivity", "mean_N_S_sensitivity")) %>% 
  mutate(models = case_when(sec == TRUE & tat == TRUE ~ "with Sec and Tat",
                            sec == FALSE & tat == TRUE ~ "without Sec",
                            sec == TRUE & tat == FALSE ~ "without Tat")) %>% 
  ggplot(aes(x = models, y = value, color = measure)) +
  geom_boxplot() +
  facet_grid(measure~arch_variant, scales = "free_y") + 
  theme_bw() +
  theme(legend.position = "bottom",
        axis.text.x = element_text(angle = 90))
```

## Performance on HMM models-dependent classes depending on presence of Sec and Tat models and hierarchy in the architecture

```{r fig.height=11}
filter(arch_dat, measure %in% c("mean_N_TL_SEC_sensitivity", "mean_N_TL_TAT_sensitivity", "mean_N_S_sensitivity")) %>% 
  mutate(models = case_when(sec == TRUE & tat == TRUE ~ "with Sec and Tat",
                            sec == FALSE & tat == TRUE ~ "without Sec",
                            sec == TRUE & tat == FALSE ~ "without Tat")) %>% 
  ggplot(aes(x = models, y = value, color = hierarchical)) +
  geom_boxplot() +
  facet_grid(measure~arch_variant, scales = "free_y") + 
  theme_bw()+
  theme(legend.position = "bottom",
        axis.text.x = element_text(angle = 90))

filter(arch_dat, measure %in% c("mean_N_TL_SEC_sensitivity", "mean_N_TL_TAT_sensitivity", "mean_N_S_sensitivity") & algorithm == "GLM") %>% 
  mutate(models = case_when(sec == TRUE & tat == TRUE ~ "with Sec and Tat",
                            sec == FALSE & tat == TRUE ~ "without Sec",
                            sec == TRUE & tat == FALSE ~ "without Tat")) %>% 
  ggplot(aes(x = models, y = value, color = hierarchical)) +
  geom_boxplot() +
  facet_grid(measure~algorithm+arch_variant, scales = "free_y") + 
  theme_bw()+
  theme(legend.position = "bottom",
        axis.text.x = element_text(angle = 90))
filter(arch_dat, measure %in% c("mean_N_TL_SEC_sensitivity", "mean_N_TL_TAT_sensitivity", "mean_N_S_sensitivity") & algorithm == "RF") %>% 
  mutate(models = case_when(sec == TRUE & tat == TRUE ~ "with Sec and Tat",
                            sec == FALSE & tat == TRUE ~ "without Sec",
                            sec == TRUE & tat == FALSE ~ "without Tat")) %>% 
  ggplot(aes(x = models, y = value, color = hierarchical)) +
  geom_boxplot() +
  facet_grid(measure~algorithm+arch_variant, scales = "free_y") + 
  theme_bw()+
  theme(legend.position = "bottom",
        axis.text.x = element_text(angle = 90))
```


## Performance of the six best architectures (according to mean kappa value) in 10-fold CV for GLM models
```{r fig.height=12}
best_6_glm <- mean_architecture_performance %>% 
  filter(grepl("GLM", model)) %>% 
  change_variant_names("model") %>% 
  arrange(desc(mean_kappa)) %>% 
  head()
change_variant_names(architectures_performance, "model") %>% 
  filter(model %in% best_6_glm[["model"]])  %>% 
  mutate(model = factor(model, levels = rev(best_6_glm[["model"]]))) %>% 
  pivot_longer(cols = !c(model, fold, rep)) %>% 
  ggplot(aes(y = model, x = value)) +
  geom_boxplot() +
  facet_wrap(~name, scales = "free_x") +
  theme_bw()
```


## Performance of the six best architectures (according to mean kappa value) in 10-fold CV for RF models
```{r fig.height=12}
best_6_rf <- mean_architecture_performance %>% 
  filter(grepl("RF", model)) %>% 
  change_variant_names("model") %>% 
  arrange(desc(mean_kappa)) %>% 
  head()
change_variant_names(architectures_performance, "model") %>% 
  filter(model %in% best_6_rf[["model"]])  %>% 
  mutate(model = factor(model, levels = rev(best_6_rf[["model"]]))) %>% 
  pivot_longer(cols = !c(model, fold, rep)) %>% 
  ggplot(aes(y = model, x = value)) +
  geom_boxplot() +
  facet_wrap(~name, scales = "free_x") +
  theme_bw()
```