3 Feature Selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1c3219be-8cee-46f7-bc21-8f805e20ce0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e64cb9a6-bd9e-43fc-b0e1-47da0187fb3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read a pickle file and load the data\n",
    "import pickle\n",
    "with open('final_df.pkl', 'rb') as file:\n",
    "    final_df = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bda6605e-efc1-4811-a706-fdf0d9b1ff07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>radius_mean</th>\n",
       "      <th>texture_mean</th>\n",
       "      <th>perimeter_mean</th>\n",
       "      <th>area_mean</th>\n",
       "      <th>smoothness_mean</th>\n",
       "      <th>compactness_mean</th>\n",
       "      <th>concavity_mean</th>\n",
       "      <th>concave points_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>radius_worst</th>\n",
       "      <th>texture_worst</th>\n",
       "      <th>perimeter_worst</th>\n",
       "      <th>area_worst</th>\n",
       "      <th>smoothness_worst</th>\n",
       "      <th>compactness_worst</th>\n",
       "      <th>concavity_worst</th>\n",
       "      <th>concave points_worst</th>\n",
       "      <th>symmetry_worst</th>\n",
       "      <th>fractal_dimension_worst</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>842302.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.30010</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>...</td>\n",
       "      <td>25.380</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.16220</td>\n",
       "      <td>0.66560</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>842517.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.08690</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>...</td>\n",
       "      <td>24.990</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.12380</td>\n",
       "      <td>0.18660</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>84300903.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.19740</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>...</td>\n",
       "      <td>23.570</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.14440</td>\n",
       "      <td>0.42450</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>84348301.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.24140</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>...</td>\n",
       "      <td>14.910</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.20980</td>\n",
       "      <td>0.86630</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84358402.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.19800</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>...</td>\n",
       "      <td>22.540</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.13740</td>\n",
       "      <td>0.20500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>926424.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>21.56</td>\n",
       "      <td>22.39</td>\n",
       "      <td>142.00</td>\n",
       "      <td>1479.0</td>\n",
       "      <td>0.11100</td>\n",
       "      <td>0.11590</td>\n",
       "      <td>0.24390</td>\n",
       "      <td>0.13890</td>\n",
       "      <td>...</td>\n",
       "      <td>25.450</td>\n",
       "      <td>26.40</td>\n",
       "      <td>166.10</td>\n",
       "      <td>2027.0</td>\n",
       "      <td>0.14100</td>\n",
       "      <td>0.21130</td>\n",
       "      <td>0.4107</td>\n",
       "      <td>0.2216</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.07115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>926682.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.13</td>\n",
       "      <td>28.25</td>\n",
       "      <td>131.20</td>\n",
       "      <td>1261.0</td>\n",
       "      <td>0.09780</td>\n",
       "      <td>0.10340</td>\n",
       "      <td>0.14400</td>\n",
       "      <td>0.09791</td>\n",
       "      <td>...</td>\n",
       "      <td>23.690</td>\n",
       "      <td>38.25</td>\n",
       "      <td>155.00</td>\n",
       "      <td>1731.0</td>\n",
       "      <td>0.11660</td>\n",
       "      <td>0.19220</td>\n",
       "      <td>0.3215</td>\n",
       "      <td>0.1628</td>\n",
       "      <td>0.2572</td>\n",
       "      <td>0.06637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>926954.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>16.60</td>\n",
       "      <td>28.08</td>\n",
       "      <td>108.30</td>\n",
       "      <td>858.1</td>\n",
       "      <td>0.08455</td>\n",
       "      <td>0.10230</td>\n",
       "      <td>0.09251</td>\n",
       "      <td>0.05302</td>\n",
       "      <td>...</td>\n",
       "      <td>18.980</td>\n",
       "      <td>34.12</td>\n",
       "      <td>126.70</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>0.11390</td>\n",
       "      <td>0.30940</td>\n",
       "      <td>0.3403</td>\n",
       "      <td>0.1418</td>\n",
       "      <td>0.2218</td>\n",
       "      <td>0.07820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567</th>\n",
       "      <td>927241.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.60</td>\n",
       "      <td>29.33</td>\n",
       "      <td>140.10</td>\n",
       "      <td>1265.0</td>\n",
       "      <td>0.11780</td>\n",
       "      <td>0.27700</td>\n",
       "      <td>0.35140</td>\n",
       "      <td>0.15200</td>\n",
       "      <td>...</td>\n",
       "      <td>25.740</td>\n",
       "      <td>39.42</td>\n",
       "      <td>184.60</td>\n",
       "      <td>1821.0</td>\n",
       "      <td>0.16500</td>\n",
       "      <td>0.86810</td>\n",
       "      <td>0.9387</td>\n",
       "      <td>0.2650</td>\n",
       "      <td>0.4087</td>\n",
       "      <td>0.12400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>568</th>\n",
       "      <td>92751.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.76</td>\n",
       "      <td>24.54</td>\n",
       "      <td>47.92</td>\n",
       "      <td>181.0</td>\n",
       "      <td>0.05263</td>\n",
       "      <td>0.04362</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>9.456</td>\n",
       "      <td>30.37</td>\n",
       "      <td>59.16</td>\n",
       "      <td>268.6</td>\n",
       "      <td>0.08996</td>\n",
       "      <td>0.06444</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2871</td>\n",
       "      <td>0.07039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>569 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  diagnosis  radius_mean  texture_mean  perimeter_mean  \\\n",
       "0      842302.0        1.0        17.99         10.38          122.80   \n",
       "1      842517.0        1.0        20.57         17.77          132.90   \n",
       "2    84300903.0        1.0        19.69         21.25          130.00   \n",
       "3    84348301.0        1.0        11.42         20.38           77.58   \n",
       "4    84358402.0        1.0        20.29         14.34          135.10   \n",
       "..          ...        ...          ...           ...             ...   \n",
       "564    926424.0        1.0        21.56         22.39          142.00   \n",
       "565    926682.0        1.0        20.13         28.25          131.20   \n",
       "566    926954.0        1.0        16.60         28.08          108.30   \n",
       "567    927241.0        1.0        20.60         29.33          140.10   \n",
       "568     92751.0        0.0         7.76         24.54           47.92   \n",
       "\n",
       "     area_mean  smoothness_mean  compactness_mean  concavity_mean  \\\n",
       "0       1001.0          0.11840           0.27760         0.30010   \n",
       "1       1326.0          0.08474           0.07864         0.08690   \n",
       "2       1203.0          0.10960           0.15990         0.19740   \n",
       "3        386.1          0.14250           0.28390         0.24140   \n",
       "4       1297.0          0.10030           0.13280         0.19800   \n",
       "..         ...              ...               ...             ...   \n",
       "564     1479.0          0.11100           0.11590         0.24390   \n",
       "565     1261.0          0.09780           0.10340         0.14400   \n",
       "566      858.1          0.08455           0.10230         0.09251   \n",
       "567     1265.0          0.11780           0.27700         0.35140   \n",
       "568      181.0          0.05263           0.04362         0.00000   \n",
       "\n",
       "     concave points_mean  ...  radius_worst  texture_worst  perimeter_worst  \\\n",
       "0                0.14710  ...        25.380          17.33           184.60   \n",
       "1                0.07017  ...        24.990          23.41           158.80   \n",
       "2                0.12790  ...        23.570          25.53           152.50   \n",
       "3                0.10520  ...        14.910          26.50            98.87   \n",
       "4                0.10430  ...        22.540          16.67           152.20   \n",
       "..                   ...  ...           ...            ...              ...   \n",
       "564              0.13890  ...        25.450          26.40           166.10   \n",
       "565              0.09791  ...        23.690          38.25           155.00   \n",
       "566              0.05302  ...        18.980          34.12           126.70   \n",
       "567              0.15200  ...        25.740          39.42           184.60   \n",
       "568              0.00000  ...         9.456          30.37            59.16   \n",
       "\n",
       "     area_worst  smoothness_worst  compactness_worst  concavity_worst  \\\n",
       "0        2019.0           0.16220            0.66560           0.7119   \n",
       "1        1956.0           0.12380            0.18660           0.2416   \n",
       "2        1709.0           0.14440            0.42450           0.4504   \n",
       "3         567.7           0.20980            0.86630           0.6869   \n",
       "4        1575.0           0.13740            0.20500           0.4000   \n",
       "..          ...               ...                ...              ...   \n",
       "564      2027.0           0.14100            0.21130           0.4107   \n",
       "565      1731.0           0.11660            0.19220           0.3215   \n",
       "566      1124.0           0.11390            0.30940           0.3403   \n",
       "567      1821.0           0.16500            0.86810           0.9387   \n",
       "568       268.6           0.08996            0.06444           0.0000   \n",
       "\n",
       "     concave points_worst  symmetry_worst  fractal_dimension_worst  \n",
       "0                  0.2654          0.4601                  0.11890  \n",
       "1                  0.1860          0.2750                  0.08902  \n",
       "2                  0.2430          0.3613                  0.08758  \n",
       "3                  0.2575          0.6638                  0.17300  \n",
       "4                  0.1625          0.2364                  0.07678  \n",
       "..                    ...             ...                      ...  \n",
       "564                0.2216          0.2060                  0.07115  \n",
       "565                0.1628          0.2572                  0.06637  \n",
       "566                0.1418          0.2218                  0.07820  \n",
       "567                0.2650          0.4087                  0.12400  \n",
       "568                0.0000          0.2871                  0.07039  \n",
       "\n",
       "[569 rows x 32 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = final_df\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cbc7057b-005e-418a-96a1-ad64699d73e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 569 entries, 0 to 568\n",
      "Data columns (total 32 columns):\n",
      " #   Column                   Non-Null Count  Dtype  \n",
      "---  ------                   --------------  -----  \n",
      " 0   id                       569 non-null    float64\n",
      " 1   diagnosis                569 non-null    float64\n",
      " 2   radius_mean              569 non-null    float64\n",
      " 3   texture_mean             569 non-null    float64\n",
      " 4   perimeter_mean           569 non-null    float64\n",
      " 5   area_mean                569 non-null    float64\n",
      " 6   smoothness_mean          569 non-null    float64\n",
      " 7   compactness_mean         569 non-null    float64\n",
      " 8   concavity_mean           569 non-null    float64\n",
      " 9   concave points_mean      569 non-null    float64\n",
      " 10  symmetry_mean            569 non-null    float64\n",
      " 11  fractal_dimension_mean   569 non-null    float64\n",
      " 12  radius_se                569 non-null    float64\n",
      " 13  texture_se               569 non-null    float64\n",
      " 14  perimeter_se             569 non-null    float64\n",
      " 15  area_se                  569 non-null    float64\n",
      " 16  smoothness_se            569 non-null    float64\n",
      " 17  compactness_se           569 non-null    float64\n",
      " 18  concavity_se             569 non-null    float64\n",
      " 19  concave points_se        569 non-null    float64\n",
      " 20  symmetry_se              569 non-null    float64\n",
      " 21  fractal_dimension_se     569 non-null    float64\n",
      " 22  radius_worst             569 non-null    float64\n",
      " 23  texture_worst            569 non-null    float64\n",
      " 24  perimeter_worst          569 non-null    float64\n",
      " 25  area_worst               569 non-null    float64\n",
      " 26  smoothness_worst         569 non-null    float64\n",
      " 27  compactness_worst        569 non-null    float64\n",
      " 28  concavity_worst          569 non-null    float64\n",
      " 29  concave points_worst     569 non-null    float64\n",
      " 30  symmetry_worst           569 non-null    float64\n",
      " 31  fractal_dimension_worst  569 non-null    float64\n",
      "dtypes: float64(32)\n",
      "memory usage: 142.4 KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "83979c6d-c9c9-4c8c-ac98-2fb14450aafb",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df['diagnosis']\n",
    "X = df.drop(columns=['diagnosis'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e31162c-c0cc-4215-842a-7084bc13065e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17c48683-51af-4cf7-aff5-5d1319cde206",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Multivariable Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "08e45668-25ce-498f-94f4-b170321f0dd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a3f199b7-a74a-4f69-b66a-b225a55bfeab",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Summarization and Selection of Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "828628ac-cd35-40a0-b2d8-55a9958037b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                    Feature  Lasso  SVM  GradientBoost  RandomForest  Sum\n",
      "0                        id      1    1              1             1    4\n",
      "1               radius_mean      0    1              1             1    3\n",
      "2              texture_mean      1    1              1             1    4\n",
      "3            perimeter_mean      0    1              1             1    3\n",
      "4                 area_mean      1    1              1             1    4\n",
      "5           smoothness_mean      0    0              1             1    2\n",
      "6          compactness_mean      0    0              1             1    2\n",
      "7            concavity_mean      0    0              1             1    2\n",
      "8       concave points_mean      0    0              1             1    2\n",
      "9             symmetry_mean      0    0              1             1    2\n",
      "10   fractal_dimension_mean      0    0              1             1    2\n",
      "11                radius_se      0    0              1             1    2\n",
      "12               texture_se      0    0              1             1    2\n",
      "13             perimeter_se      0    0              1             1    2\n",
      "14                  area_se      1    1              1             1    4\n",
      "15            smoothness_se      0    0              1             1    2\n",
      "16           compactness_se      0    0              1             1    2\n",
      "17             concavity_se      0    0              1             1    2\n",
      "18        concave points_se      0    0              1             1    2\n",
      "19              symmetry_se      0    0              1             1    2\n",
      "20     fractal_dimension_se      0    0              1             1    2\n",
      "21             radius_worst      1    1              1             1    4\n",
      "22            texture_worst      1    1              1             1    4\n",
      "23          perimeter_worst      1    1              1             1    4\n",
      "24               area_worst      1    1              1             1    4\n",
      "25         smoothness_worst      0    0              1             1    2\n",
      "26        compactness_worst      0    0              1             1    2\n",
      "27          concavity_worst      1    0              1             1    3\n",
      "28     concave points_worst      0    0              1             1    2\n",
      "29           symmetry_worst      0    0              1             1    2\n",
      "30  fractal_dimension_worst      0    0              1             1    2\n"
     ]
    }
   ],
   "source": [
    "# Fit models and determine if a feature is selected (1) or not (0)\n",
    "lasso = Lasso(alpha=0.01).fit(X, y)\n",
    "lasso_selected = (np.abs(lasso.coef_) > 0).astype(int)\n",
    "\n",
    "svm = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit(X, y)\n",
    "svm_selected = (np.abs(svm.coef_[0]) > 0).astype(int)\n",
    "\n",
    "gb = GradientBoostingClassifier().fit(X, y)\n",
    "gb_selected = (gb.feature_importances_ > 0).astype(int)\n",
    "\n",
    "rf = RandomForestClassifier().fit(X, y)\n",
    "rf_selected = (rf.feature_importances_ > 0).astype(int)\n",
    "\n",
    "# Create a DataFrame to store results\n",
    "selection_df = pd.DataFrame({\n",
    "    'Feature': X.columns,\n",
    "    'Lasso': lasso_selected,\n",
    "    'SVM': svm_selected,\n",
    "    'GradientBoost': gb_selected,\n",
    "    'RandomForest': rf_selected\n",
    "})\n",
    "\n",
    "# Sum the number of selections for each feature\n",
    "selection_df['Sum'] = selection_df[['Lasso', 'SVM', 'GradientBoost', 'RandomForest']].sum(axis=1)\n",
    "\n",
    "# Output the results\n",
    "print(selection_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fffae7d-27f9-4cd2-a2e1-1a49d5489ccc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating DataFrame with most valuable variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c63028b-b07b-4ab8-8871-601122c33116",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Selected variables - recommended by 3 or more models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "59d0828e-ed33-4d43-9762-60cbe8b0e007",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 569 entries, 0 to 568\n",
      "Data columns (total 12 columns):\n",
      " #   Column           Non-Null Count  Dtype  \n",
      "---  ------           --------------  -----  \n",
      " 0   id               569 non-null    float64\n",
      " 1   radius_mean      569 non-null    float64\n",
      " 2   texture_mean     569 non-null    float64\n",
      " 3   perimeter_mean   569 non-null    float64\n",
      " 4   area_mean        569 non-null    float64\n",
      " 5   area_se          569 non-null    float64\n",
      " 6   radius_worst     569 non-null    float64\n",
      " 7   texture_worst    569 non-null    float64\n",
      " 8   perimeter_worst  569 non-null    float64\n",
      " 9   area_worst       569 non-null    float64\n",
      " 10  concavity_worst  569 non-null    float64\n",
      " 11  diagnosis        569 non-null    float64\n",
      "dtypes: float64(12)\n",
      "memory usage: 53.5 KB\n"
     ]
    }
   ],
   "source": [
    " #Selecting variables with a sum of selections >= 3\n",
    "final_var = selection_df[selection_df['Sum'] >= 3]['Feature'].tolist()\n",
    "df_model = df[final_var].copy()\n",
    "df_model['diagnosis'] = df['diagnosis'].copy()\n",
    "\n",
    "# Output the result to verify\n",
    "df_model.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7fc6698e-b7d3-474f-904d-9ee58eee2994",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_model.to_csv('df_model.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c8928e8-15a2-4b6c-a78f-ca8077ab2ea2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature Selection Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25366953-b54a-4f8c-9262-633ca08cb4fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Univariable Analysis \n",
    "# Anova, t-test, Chi- square"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e2337fa-ee6e-4124-8996-facb43aa42f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using Anova"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "907603aa-179e-423c-94a6-0b16f25b461d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "from sklearn.datasets import load_iris\n",
    "import pandas as pd\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "34c3de08-eb3a-4255-b310-9b79b28a2f17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>radius_mean</th>\n",
       "      <th>texture_mean</th>\n",
       "      <th>perimeter_mean</th>\n",
       "      <th>area_mean</th>\n",
       "      <th>smoothness_mean</th>\n",
       "      <th>compactness_mean</th>\n",
       "      <th>concavity_mean</th>\n",
       "      <th>concave points_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>radius_worst</th>\n",
       "      <th>texture_worst</th>\n",
       "      <th>perimeter_worst</th>\n",
       "      <th>area_worst</th>\n",
       "      <th>smoothness_worst</th>\n",
       "      <th>compactness_worst</th>\n",
       "      <th>concavity_worst</th>\n",
       "      <th>concave points_worst</th>\n",
       "      <th>symmetry_worst</th>\n",
       "      <th>fractal_dimension_worst</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>842302.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.30010</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>...</td>\n",
       "      <td>25.380</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.16220</td>\n",
       "      <td>0.66560</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>842517.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.08690</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>...</td>\n",
       "      <td>24.990</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.12380</td>\n",
       "      <td>0.18660</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>84300903.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.19740</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>...</td>\n",
       "      <td>23.570</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.14440</td>\n",
       "      <td>0.42450</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>84348301.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.24140</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>...</td>\n",
       "      <td>14.910</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.20980</td>\n",
       "      <td>0.86630</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84358402.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.19800</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>...</td>\n",
       "      <td>22.540</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.13740</td>\n",
       "      <td>0.20500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>926424.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>21.56</td>\n",
       "      <td>22.39</td>\n",
       "      <td>142.00</td>\n",
       "      <td>1479.0</td>\n",
       "      <td>0.11100</td>\n",
       "      <td>0.11590</td>\n",
       "      <td>0.24390</td>\n",
       "      <td>0.13890</td>\n",
       "      <td>...</td>\n",
       "      <td>25.450</td>\n",
       "      <td>26.40</td>\n",
       "      <td>166.10</td>\n",
       "      <td>2027.0</td>\n",
       "      <td>0.14100</td>\n",
       "      <td>0.21130</td>\n",
       "      <td>0.4107</td>\n",
       "      <td>0.2216</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.07115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>926682.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.13</td>\n",
       "      <td>28.25</td>\n",
       "      <td>131.20</td>\n",
       "      <td>1261.0</td>\n",
       "      <td>0.09780</td>\n",
       "      <td>0.10340</td>\n",
       "      <td>0.14400</td>\n",
       "      <td>0.09791</td>\n",
       "      <td>...</td>\n",
       "      <td>23.690</td>\n",
       "      <td>38.25</td>\n",
       "      <td>155.00</td>\n",
       "      <td>1731.0</td>\n",
       "      <td>0.11660</td>\n",
       "      <td>0.19220</td>\n",
       "      <td>0.3215</td>\n",
       "      <td>0.1628</td>\n",
       "      <td>0.2572</td>\n",
       "      <td>0.06637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>926954.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>16.60</td>\n",
       "      <td>28.08</td>\n",
       "      <td>108.30</td>\n",
       "      <td>858.1</td>\n",
       "      <td>0.08455</td>\n",
       "      <td>0.10230</td>\n",
       "      <td>0.09251</td>\n",
       "      <td>0.05302</td>\n",
       "      <td>...</td>\n",
       "      <td>18.980</td>\n",
       "      <td>34.12</td>\n",
       "      <td>126.70</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>0.11390</td>\n",
       "      <td>0.30940</td>\n",
       "      <td>0.3403</td>\n",
       "      <td>0.1418</td>\n",
       "      <td>0.2218</td>\n",
       "      <td>0.07820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567</th>\n",
       "      <td>927241.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.60</td>\n",
       "      <td>29.33</td>\n",
       "      <td>140.10</td>\n",
       "      <td>1265.0</td>\n",
       "      <td>0.11780</td>\n",
       "      <td>0.27700</td>\n",
       "      <td>0.35140</td>\n",
       "      <td>0.15200</td>\n",
       "      <td>...</td>\n",
       "      <td>25.740</td>\n",
       "      <td>39.42</td>\n",
       "      <td>184.60</td>\n",
       "      <td>1821.0</td>\n",
       "      <td>0.16500</td>\n",
       "      <td>0.86810</td>\n",
       "      <td>0.9387</td>\n",
       "      <td>0.2650</td>\n",
       "      <td>0.4087</td>\n",
       "      <td>0.12400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>568</th>\n",
       "      <td>92751.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.76</td>\n",
       "      <td>24.54</td>\n",
       "      <td>47.92</td>\n",
       "      <td>181.0</td>\n",
       "      <td>0.05263</td>\n",
       "      <td>0.04362</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>9.456</td>\n",
       "      <td>30.37</td>\n",
       "      <td>59.16</td>\n",
       "      <td>268.6</td>\n",
       "      <td>0.08996</td>\n",
       "      <td>0.06444</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2871</td>\n",
       "      <td>0.07039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>569 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  diagnosis  radius_mean  texture_mean  perimeter_mean  \\\n",
       "0      842302.0        1.0        17.99         10.38          122.80   \n",
       "1      842517.0        1.0        20.57         17.77          132.90   \n",
       "2    84300903.0        1.0        19.69         21.25          130.00   \n",
       "3    84348301.0        1.0        11.42         20.38           77.58   \n",
       "4    84358402.0        1.0        20.29         14.34          135.10   \n",
       "..          ...        ...          ...           ...             ...   \n",
       "564    926424.0        1.0        21.56         22.39          142.00   \n",
       "565    926682.0        1.0        20.13         28.25          131.20   \n",
       "566    926954.0        1.0        16.60         28.08          108.30   \n",
       "567    927241.0        1.0        20.60         29.33          140.10   \n",
       "568     92751.0        0.0         7.76         24.54           47.92   \n",
       "\n",
       "     area_mean  smoothness_mean  compactness_mean  concavity_mean  \\\n",
       "0       1001.0          0.11840           0.27760         0.30010   \n",
       "1       1326.0          0.08474           0.07864         0.08690   \n",
       "2       1203.0          0.10960           0.15990         0.19740   \n",
       "3        386.1          0.14250           0.28390         0.24140   \n",
       "4       1297.0          0.10030           0.13280         0.19800   \n",
       "..         ...              ...               ...             ...   \n",
       "564     1479.0          0.11100           0.11590         0.24390   \n",
       "565     1261.0          0.09780           0.10340         0.14400   \n",
       "566      858.1          0.08455           0.10230         0.09251   \n",
       "567     1265.0          0.11780           0.27700         0.35140   \n",
       "568      181.0          0.05263           0.04362         0.00000   \n",
       "\n",
       "     concave points_mean  ...  radius_worst  texture_worst  perimeter_worst  \\\n",
       "0                0.14710  ...        25.380          17.33           184.60   \n",
       "1                0.07017  ...        24.990          23.41           158.80   \n",
       "2                0.12790  ...        23.570          25.53           152.50   \n",
       "3                0.10520  ...        14.910          26.50            98.87   \n",
       "4                0.10430  ...        22.540          16.67           152.20   \n",
       "..                   ...  ...           ...            ...              ...   \n",
       "564              0.13890  ...        25.450          26.40           166.10   \n",
       "565              0.09791  ...        23.690          38.25           155.00   \n",
       "566              0.05302  ...        18.980          34.12           126.70   \n",
       "567              0.15200  ...        25.740          39.42           184.60   \n",
       "568              0.00000  ...         9.456          30.37            59.16   \n",
       "\n",
       "     area_worst  smoothness_worst  compactness_worst  concavity_worst  \\\n",
       "0        2019.0           0.16220            0.66560           0.7119   \n",
       "1        1956.0           0.12380            0.18660           0.2416   \n",
       "2        1709.0           0.14440            0.42450           0.4504   \n",
       "3         567.7           0.20980            0.86630           0.6869   \n",
       "4        1575.0           0.13740            0.20500           0.4000   \n",
       "..          ...               ...                ...              ...   \n",
       "564      2027.0           0.14100            0.21130           0.4107   \n",
       "565      1731.0           0.11660            0.19220           0.3215   \n",
       "566      1124.0           0.11390            0.30940           0.3403   \n",
       "567      1821.0           0.16500            0.86810           0.9387   \n",
       "568       268.6           0.08996            0.06444           0.0000   \n",
       "\n",
       "     concave points_worst  symmetry_worst  fractal_dimension_worst  \n",
       "0                  0.2654          0.4601                  0.11890  \n",
       "1                  0.1860          0.2750                  0.08902  \n",
       "2                  0.2430          0.3613                  0.08758  \n",
       "3                  0.2575          0.6638                  0.17300  \n",
       "4                  0.1625          0.2364                  0.07678  \n",
       "..                    ...             ...                      ...  \n",
       "564                0.2216          0.2060                  0.07115  \n",
       "565                0.1628          0.2572                  0.06637  \n",
       "566                0.1418          0.2218                  0.07820  \n",
       "567                0.2650          0.4087                  0.12400  \n",
       "568                0.0000          0.2871                  0.07039  \n",
       "\n",
       "[569 rows x 32 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load data\n",
    "#Read a pickle file and load the data\n",
    "import pickle\n",
    "with open('final_df.pkl', 'rb') as file:\n",
    "    final_df = pickle.load(file)\n",
    "\n",
    "df = final_df\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fdc6e988-d400-4383-a706-fafe32792ff7",
   "metadata": {},
   "outputs": [],
   "source": [
    "y=df['diagnosis']\n",
    "X = df.drop(columns=['diagnosis'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6ec0bd80-9254-401d-857c-0bc3dff5243b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                    Feature     F-Value        P-Value\n",
      "0                        id    0.898150   3.436819e-01\n",
      "1               radius_mean  646.981021   8.465941e-96\n",
      "2              texture_mean  118.096059   4.058636e-25\n",
      "3            perimeter_mean  697.235272  8.436251e-101\n",
      "4                 area_mean  573.060747   4.734564e-88\n",
      "5           smoothness_mean   83.651123   1.051850e-18\n",
      "6          compactness_mean  313.233079   3.938263e-56\n",
      "7            concavity_mean  533.793126   9.966556e-84\n",
      "8       concave points_mean  861.676020  7.101150e-116\n",
      "9             symmetry_mean   69.527444   5.733384e-16\n",
      "10   fractal_dimension_mean    0.093459   7.599368e-01\n",
      "11                radius_se  268.840327   9.738949e-50\n",
      "12               texture_se    0.039095   8.433320e-01\n",
      "13             perimeter_se  253.897392   1.651905e-47\n",
      "14                  area_se  243.651586   5.895521e-46\n",
      "15            smoothness_se    2.557968   1.102966e-01\n",
      "16           compactness_se   53.247339   9.975995e-13\n",
      "17             concavity_se   39.014482   8.260176e-10\n",
      "18        concave points_se  113.262760   3.072309e-24\n",
      "19              symmetry_se    0.024117   8.766418e-01\n",
      "20     fractal_dimension_se    3.468275   6.307355e-02\n",
      "21             radius_worst  860.781707  8.482292e-116\n",
      "22            texture_worst  149.596905   1.078057e-30\n",
      "23          perimeter_worst  897.944219  5.771397e-119\n",
      "24               area_worst  661.600206   2.828848e-97\n",
      "25         smoothness_worst  122.472880   6.575144e-26\n",
      "26        compactness_worst  304.341063   7.069816e-55\n",
      "27          concavity_worst  436.691939   2.464664e-72\n",
      "28     concave points_worst  964.385393  1.969100e-124\n",
      "29           symmetry_worst  118.860232   2.951121e-25\n",
      "30  fractal_dimension_worst   66.443961   2.316432e-15\n"
     ]
    }
   ],
   "source": [
    "# Calculate F-values and p-values\n",
    "f_values, p_values = f_classif(X, y)\n",
    "\n",
    "# Create a DataFrame to display F-values and p-values for each feature\n",
    "feature_scores = pd.DataFrame({'Feature': X.columns, 'F-Value': f_values, 'P-Value': p_values})\n",
    "\n",
    "# Print the DataFrame\n",
    "print(feature_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cd4eebb0-26dd-48ca-86e2-52c98b4077f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Feature     F-Value        P-Value\n",
      "28  concave points_worst  964.385393  1.969100e-124\n",
      "23       perimeter_worst  897.944219  5.771397e-119\n"
     ]
    }
   ],
   "source": [
    "# Select the top 2 features based on F-value\n",
    "top_features = feature_scores.nlargest(2, 'F-Value')\n",
    "\n",
    "# Print the 2 best features\n",
    "print(top_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "515812aa-ca61-40ae-99fd-b3f0e32e56d5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}