Hybrid Apps Security Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "76dadf66-3e83-403d-aa54-8f0ddec5ff1d",
   "metadata": {},
   "source": [
    "# Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a2d9869-6272-4cae-94ef-6dc7429214de",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import missingno\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score, classification_report, confusion_matrix, f1_score, \n",
    "    precision_recall_curve, precision_score, recall_score, roc_auc_score\n",
    ")\n",
    "from mlxtend.plotting import plot_confusion_matrix\n",
    "from scikitplot.metrics import plot_precision_recall, plot_roc_curve\n",
    "\n",
    "import warnings\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "902ff419-569f-46f4-bd52-9dc37ad77ad5",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "918463fe-7648-4724-be94-36eab9722086",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/mnt/hdd/Datasets/HybridAppsDataset.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01b3c6d3-d9c3-45a8-a9b8-ec689fd8bba5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop([\"Unnamed: 0\", \"app_hash\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0b2822b-3561-47ce-bdfb-85551b87f361",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8d7bf5d-5349-4e2c-a475-c82c59488bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6d667b5-2ca3-4408-b066-cd1fe9c99b9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "missingno.bar(df, color=\"red\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fec645be-28c1-4953-bcca-8fc09f93ce2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50512e88-6b8b-402d-9b86-5090965e9ecb",
   "metadata": {},
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45d21c73-6f0d-48d0-afa1-5c15926f84e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"webview_tab\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"webview_tab\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b6614ed-77c0-4043-9bc2-51ad918c029e",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"js_enabled\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"js_enabled\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c042f89e-827b-44dc-ac8e-caa4bd511431",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"js_inf_defined\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"js_inf_defined\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2daf5659-7a80-41f8-a29d-a1cbb82be2f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"acc_sys_call\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"acc_sys_call\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e24ef54a-cf8e-4e65-ab7d-44bbf74a7ce5",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"obf_js_permit\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"obf_js_permit\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f02c0463-5309-445f-b51b-43b3f3ad24ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"inf_droid_Code_obf\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"inf_droid_Code_obf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c17a093-abb1-44b4-9c5c-f29a3a0d09d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"out_url\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"out_url\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "945cd441-b4cf-4025-9fc4-1f9980e6d8bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"gsafe_brow\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"gsafe_brow\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2532587-1ac2-4615-b971-93f693a4fb56",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"https\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"https\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "faae159c-7cce-4c60-bfa3-fb8434abd24b",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"js_input_val\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"js_input_val\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec34805b-2891-414c-aaf5-8907aadddd72",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "ax = sns.countplot(data=df, x=\"web_redirect\", hue=\"label\")\n",
    "for container in ax.containers:\n",
    "    ax.bar_label(container)\n",
    "plt.title(\"web_redirect\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ad0b00f-6340-48fb-bcd1-7cdc579b3df5",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(14, 5))\n",
    "sns.histplot(data=df, x=\"js_inf_len\", kde=True)\n",
    "mean = df[\"js_inf_len\"].mean()\n",
    "median = df[\"js_inf_len\"].median()\n",
    "plt.axvline(mean, color=\"r\", linestyle=\"--\", label=\"Mean\")\n",
    "plt.axvline(median, color=\"g\", linestyle=\"-\", label=\"Median\")\n",
    "plt.xlabel(\"js_inf_len\")\n",
    "plt.ylabel(\"count\")\n",
    "plt.legend()\n",
    "plt.title(\"js_inf_len\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fabbc5a9-71e8-402b-b8dc-4ac34e0b37c5",
   "metadata": {},
   "source": [
    "# Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09d5b9d7-1515-4fd0-a246-c8b85e749eb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a1865e8-1f18-4d54-91bb-d1714b15c051",
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in df.columns:\n",
    "    if df[col].dtype == np.bool_:\n",
    "        df[col] = df[col].map({True: 1, False: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90747397-b41f-4e97-b858-7d3c43f5c02a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eda46a42-1096-4feb-be0b-5b04fd8e4a94",
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_encoder(column):\n",
    "    le = LabelEncoder().fit(column)\n",
    "    print(column.name, le.classes_)\n",
    "    return le.transform(column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e129a9d-5436-466b-a5fb-4684a9ed0026",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"webview_tab\"] = label_encoder(df[\"webview_tab\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94ae3fe4-ddd5-47dd-a8c6-285a86eb50b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"label\"] = label_encoder(df[\"label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1d5d542-cc67-4dfb-a711-dc8222f2fa5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names = [\"benign\", \"malicious\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74dd602a-4b1c-4264-ad0d-be77114c3af5",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(\"label\", axis=1)\n",
    "y = df[\"label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8839b1b1-bde5-4517-93f8-e3aafde98607",
   "metadata": {},
   "outputs": [],
   "source": [
    "smote = SMOTE(random_state=42)\n",
    "X_resampled, y_resampled = smote.fit_resample(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60d91aa4-e537-4a98-abd8-28f26924a3d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_scaled = ss.fit_transform(X_resampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe4a9243-23e7-4f59-be7d-c5eb519d8bda",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(ss, open(\"ss.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0184d4ff-d48c-4a7e-b47f-d74f2078de8c",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5261b74-cc53-49ee-a24e-5c627fed78c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y_resampled, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "483c944e-29b8-44d3-b80d-21882f954eb4",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b912eb0-c4be-4dab-91bc-b9fd44a68a86",
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression()\n",
    "start = time.time()\n",
    "logreg.fit(X_train, y_train)\n",
    "end = time.time()\n",
    "logreg_time = end - start\n",
    "print(\"Logistic Regression Train Time:\", logreg_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f521fed8-ac04-4483-9c2f-ea750e23a38a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(logreg, open(\"logreg.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43fc6f8f-f90e-486e-87fe-86872ada33b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg_scores = cross_val_score(LogisticRegression(), X_train, y_train, cv=3)\n",
    "print(\"Logistic Regression Cross-Validation Scores:\", logreg_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d62776b0-113a-4edb-b29c-4711bdff0519",
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg_pred_train = logreg.predict(X_train)\n",
    "logreg_pred_test = logreg.predict(X_test)\n",
    "logreg_test_proba = logreg.predict_proba(X_test)\n",
    "\n",
    "logreg_train_score = accuracy_score(logreg_pred_train, y_train)\n",
    "logreg_test_score = accuracy_score(logreg_pred_test, y_test)\n",
    "print(\"Logistic Regression Train Score:\", logreg_train_score)\n",
    "print(\"Logistic Regression Test Score:\", logreg_test_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa12b77d-f0b2-4795-a116-e3fa673eaa95",
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg_precision_score = precision_score(y_test, logreg_pred_test)\n",
    "logreg_f1_score = f1_score(y_test, logreg_pred_test)\n",
    "logreg_recall_score = recall_score(y_test, logreg_pred_test)\n",
    "logreg_accuracy_score = accuracy_score(y_test, logreg_pred_test)\n",
    "\n",
    "print(\"Logistic Regression Precision Score:\", logreg_precision_score)\n",
    "print(\"Logistic Regression F1 Score:\", logreg_f1_score)\n",
    "print(\"Logistic Regression Recall Score:\", logreg_recall_score)\n",
    "print(\"Logistic Regression Accuracy Score:\", logreg_accuracy_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc058dac-eff5-49d5-8e60-374dd4cd38c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test, logreg_pred_test, target_names=class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4a7218a-e73f-4f46-823c-4e64d9b8493d",
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg_cm = confusion_matrix(y_test, logreg_pred_test)\n",
    "fig, ax = plot_confusion_matrix(conf_mat=logreg_cm, show_absolute=True, show_normed=True, colorbar=True, class_names=class_names)\n",
    "plt.title(\"Logistic Regression Confusion Matrix\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26cec737-672e-4b47-a197-30d2b2f981b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_precision_recall(y_test, logreg_test_proba)\n",
    "plt.title(\"Logistic Regression Precision-Recall Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c97a68f6-9cc4-49ab-a9ad-83ab7a42958c",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_roc_curve(y_test, logreg_test_proba)\n",
    "plt.title(\"Logistic Regression ROC Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71b5c500-7102-4776-b130-40e4285e5b04",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3bc2b01-707c-452c-9fbb-51ffa0d91edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier()\n",
    "start = time.time()\n",
    "rf.fit(X_train, y_train)\n",
    "end = time.time()\n",
    "rf_time = end - start\n",
    "print(\"Random Forest Train Time:\", rf_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b4c42cc-3311-4a32-9244-41c31e1f8d3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(rf, open(\"rf.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd4f05ad-5a97-418f-b231-95308a369815",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_scores = cross_val_score(RandomForestClassifier(), X_train, y_train, cv=3)\n",
    "print(\"Random Forest Cross-Validation Scores:\", rf_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66a0045c-bee8-4410-a7ac-084043515414",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_pred_train = rf.predict(X_train)\n",
    "rf_pred_test = rf.predict(X_test)\n",
    "rf_test_proba = rf.predict_proba(X_test)\n",
    "\n",
    "rf_train_score = accuracy_score(rf_pred_train, y_train)\n",
    "rf_test_score = accuracy_score(rf_pred_test, y_test)\n",
    "print(\"Random Forest Train Score:\", rf_train_score)\n",
    "print(\"Random Forest Test Score:\", rf_test_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0f57576-9261-4dde-a749-5bf07f6a6d20",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_precision_score = precision_score(y_test, rf_pred_test)\n",
    "rf_f1_score = f1_score(y_test, rf_pred_test)\n",
    "rf_recall_score = recall_score(y_test, rf_pred_test)\n",
    "rf_accuracy_score = accuracy_score(y_test, rf_pred_test)\n",
    "\n",
    "print(\"Random Forest Precision Score:\", rf_precision_score)\n",
    "print(\"Random Forest F1 Score:\", rf_f1_score)\n",
    "print(\"Random Forest Recall Score:\", rf_recall_score)\n",
    "print(\"Random Forest Accuracy Score:\", rf_accuracy_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06c70f96-e61a-4284-a5b1-1665a83cf22a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test, rf_pred_test, target_names=class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fbcddcc-f1a5-40f2-a203-b48bb5230add",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_cm = confusion_matrix(y_test, rf_pred_test)\n",
    "fig, ax = plot_confusion_matrix(conf_mat=rf_cm, show_absolute=True, show_normed=True, colorbar=True, class_names=class_names)\n",
    "plt.title(\"Random Forest Confusion Matrix\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6e513a1-9ff3-48d4-8d64-c4e681eed428",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_precision_recall(y_test, rf_test_proba)\n",
    "plt.title(\"Random Forest Precision-Recall Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e1e0d55-e52f-4ece-a751-3db4de4d6bbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_roc_curve(y_test, rf_test_proba)\n",
    "plt.title(\"Random Forest ROC Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3266b1d-d76f-4a5f-86bb-fa18cc80404e",
   "metadata": {},
   "source": [
    "# AdaBoost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccd44d54-fe0e-4bf6-a83a-87e9590e8e19",
   "metadata": {},
   "outputs": [],
   "source": [
    "ada = AdaBoostClassifier()\n",
    "start = time.time()\n",
    "ada.fit(X_train, y_train)\n",
    "end = time.time()\n",
    "ada_time = end - start\n",
    "print(\"AdaBoost Train Time:\", ada_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f92dbfcf-d8ec-4888-a7db-5810fab2cd6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(ada, open(\"ada.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dd30c7f-27f6-4ab8-bad6-ccbe656bf719",
   "metadata": {},
   "outputs": [],
   "source": [
    "ada_scores = cross_val_score(AdaBoostClassifier(), X_train, y_train, cv=3)\n",
    "print(\"AdaBoost Cross-Validation Scores:\", ada_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64a12eb2-e046-4bfc-aa82-d8ec14cf9635",
   "metadata": {},
   "outputs": [],
   "source": [
    "ada_pred_train = ada.predict(X_train)\n",
    "ada_pred_test = ada.predict(X_test)\n",
    "ada_test_proba = ada.predict_proba(X_test)\n",
    "\n",
    "ada_train_score = accuracy_score(ada_pred_train, y_train)\n",
    "ada_test_score = accuracy_score(ada_pred_test, y_test)\n",
    "print(\"AdaBoost Train Score:\", ada_train_score)\n",
    "print(\"AdaBoost Test Score:\", ada_test_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a5d2970-144b-447f-895f-12b8bf16076c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ada_precision_score = precision_score(y_test, ada_pred_test)\n",
    "ada_f1_score = f1_score(y_test, ada_pred_test)\n",
    "ada_recall_score = recall_score(y_test, ada_pred_test)\n",
    "ada_accuracy_score = accuracy_score(y_test, ada_pred_test)\n",
    "\n",
    "print(\"AdaBoost Precision Score:\", ada_precision_score)\n",
    "print(\"AdaBoost F1 Score:\", ada_f1_score)\n",
    "print(\"AdaBoost Recall Score:\", ada_recall_score)\n",
    "print(\"AdaBoost Accuracy Score:\", ada_accuracy_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbf4269c-b16b-4583-8849-b36e9250ff3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test, ada_pred_test, target_names=class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f189846f-a015-4e94-b7f6-cccdd53f01f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "ada_cm = confusion_matrix(y_test, ada_pred_test)\n",
    "fig, ax = plot_confusion_matrix(conf_mat=ada_cm, show_absolute=True, show_normed=True, colorbar=True, class_names=class_names)\n",
    "plt.title(\"AdaBoost Confusion Matrix\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "487c00e0-cd3d-4ee6-96cc-237339217bf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_precision_recall(y_test, ada_test_proba)\n",
    "plt.title(\"AdaBoost Precision-Recall Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6552b86f-61f1-443a-a41d-335a7154da0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_roc_curve(y_test, ada_test_proba)\n",
    "plt.title(\"AdaBoost ROC Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6accdddc-9374-4be6-b6dc-6bb5faf3cff4",
   "metadata": {},
   "source": [
    "# Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ced39f13-5399-4949-bd94-c89cd845e92f",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb = GradientBoostingClassifier()\n",
    "start = time.time()\n",
    "gb.fit(X_train, y_train)\n",
    "end = time.time()\n",
    "gb_time = end - start\n",
    "print(\"Gradient Boosting Train Time:\", gb_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82fb48b1-8429-43bb-9c31-892fd04ccd5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(gb, open(\"gb.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4da24399-762d-45fe-8b5d-1284134a8119",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_scores = cross_val_score(GradientBoostingClassifier(), X_train, y_train, cv=3)\n",
    "print(\"Gradient Boosting Cross-Validation Scores:\", gb_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36e12f0c-7242-4db5-b440-95dbb8abc27b",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_pred_train = gb.predict(X_train)\n",
    "gb_pred_test = gb.predict(X_test)\n",
    "gb_test_proba = gb.predict_proba(X_test)\n",
    "\n",
    "gb_train_score = accuracy_score(gb_pred_train, y_train)\n",
    "gb_test_score = accuracy_score(gb_pred_test, y_test)\n",
    "print(\"Gradient Boosting Train Score:\", gb_train_score)\n",
    "print(\"Gradient Boosting Test Score:\", gb_test_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1061781-d774-4c85-ad07-d8b7d19cd6ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_precision_score = precision_score(y_test, gb_pred_test)\n",
    "gb_f1_score = f1_score(y_test, gb_pred_test)\n",
    "gb_recall_score = recall_score(y_test, gb_pred_test)\n",
    "gb_accuracy_score = accuracy_score(y_test, gb_pred_test)\n",
    "\n",
    "print(\"Gradient Boosting Precision Score:\", gb_precision_score)\n",
    "print(\"Gradient Boosting F1 Score:\", gb_f1_score)\n",
    "print(\"Gradient Boosting Recall Score:\", gb_recall_score)\n",
    "print(\"Gradient Boosting Accuracy Score:\", gb_accuracy_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd756fbc-262c-4e28-b4fc-f6c5b30edc72",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test, gb_pred_test, target_names=class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bde4d3df-8d79-4966-bea6-0dcaf5dc5fc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_cm = confusion_matrix(y_test, gb_pred_test)\n",
    "fig, ax = plot_confusion_matrix(conf_mat=gb_cm, show_absolute=True, show_normed=True, colorbar=True, class_names=class_names)\n",
    "plt.title(\"Gradient Boosting Confusion Matrix\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9f2f57b-3e0b-4bc4-bec9-aae02e4525e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_precision_recall(y_test, gb_test_proba)\n",
    "plt.title(\"Gradient Boosting Precision-Recall Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18a1b009-aefb-432a-a82f-cf603516cfb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_roc_curve(y_test, gb_test_proba)\n",
    "plt.title(\"Gradient Boosting ROC Curve\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a34b7d7a-8947-4816-91d5-cd1541956a07",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [\"LR\", \"ADA\", \"RF\", \"GB\"]\n",
    "scores = [logreg_test_score, ada_test_score, rf_test_score, gb_test_score]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01772787-5377-48d0-af17-edc836970a1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_model_plot(labels, scores):\n",
    "    plt.figure()\n",
    "    ax = sns.barplot(x=labels, y=scores)\n",
    "    ax.set_title(\"Trained Models Accuracy\")\n",
    "    for container in ax.containers:\n",
    "        ax.bar_label(container)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff08cbee-b183-4233-a453-53673e4c35a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_model_plot(labels, scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43bfd898-0018-4427-b1bd-1059ad7be9d8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}