Amex_Hackathon_FinalCode.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "#importing necessary librairies\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt  # Matlab-style plotting\n",
    "import seaborn as sns\n",
    "color = sns.color_palette()\n",
    "sns.set_style('darkgrid')\n",
    "import warnings\n",
    "def ignore_warn(*args, **kwargs):\n",
    "    pass\n",
    "warnings.warn = ignore_warn #ignore annoying warning (from sklearn and seaborn)\n",
    "\n",
    "\n",
    "from scipy import stats\n",
    "from scipy.stats import norm, skew #for some statistics\n",
    "\n",
    "\n",
    "pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x)) #Limiting floats output to 3 decimal points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.read_csv('D/Amex/train.csv')\n",
    "test=pd.read_csv('D/Amex/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The train data size before dropping Id feature is : (32820, 43) \n",
      "The test data size before dropping Id feature is : (14067, 43) \n",
      "\n",
      "The train data size after dropping Id feature is : (32820, 42) \n",
      "The test data size after dropping Id feature is : (14067, 42) \n"
     ]
    }
   ],
   "source": [
    "#check the numbers of samples and features\n",
    "print(\"The train data size before dropping Id feature is : {} \".format(train.shape))\n",
    "print(\"The test data size before dropping Id feature is : {} \".format(test.shape))\n",
    "\n",
    "#Save the 'Id' column\n",
    "train_ID = train['id']\n",
    "test_ID = test['id']\n",
    "\n",
    "#Now drop the  'Id' colum since it's unnecessary for  the prediction process.\n",
    "train.drop(\"id\", axis = 1, inplace = True)\n",
    "test.drop(\"id\", axis = 1, inplace = True)\n",
    "\n",
    "#check again the data size after dropping the 'Id' variable\n",
    "print(\"\\nThe train data size after dropping Id feature is : {} \".format(train.shape)) \n",
    "print(\"The test data size after dropping Id feature is : {} \".format(test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.distplot(train['cc_cons'] , fit=norm);\n",
    "\n",
    "# Get the fitted parameters used by the function\n",
    "(mu, sigma) = norm.fit(train['cc_cons'])\n",
    "print( '\\n mu = {:.2f} and sigma = {:.2f}\\n'.format(mu, sigma))\n",
    "\n",
    "#Now plot the distribution\n",
    "plt.legend(['Normal dist. ($\\mu=$ {:.2f} and $\\sigma=$ {:.2f} )'.format(mu, sigma)],\n",
    "            loc='best')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('CC Consumption distribution')\n",
    "\n",
    "#Get also the QQ-plot\n",
    "fig = plt.figure()\n",
    "res = stats.probplot(train['cc_cons'], plot=plt)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We use the numpy fuction log1p which  applies log(1+x) to all elements of the column\n",
    "train[\"cc_cons\"] = np.log1p(train[\"cc_cons\"])\n",
    "\n",
    "#Check the new distribution \n",
    "sns.distplot(train['cc_cons'] , fit=norm);\n",
    "\n",
    "# Get the fitted parameters used by the function\n",
    "(mu, sigma) = norm.fit(train['cc_cons'])\n",
    "print( '\\n mu = {:.2f} and sigma = {:.2f}\\n'.format(mu, sigma))\n",
    "\n",
    "#Now plot the distribution\n",
    "plt.legend(['Normal dist. ($\\mu=$ {:.2f} and $\\sigma=$ {:.2f} )'.format(mu, sigma)],\n",
    "            loc='best')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('cc consumption distribution')\n",
    "\n",
    "#Get also the QQ-plot\n",
    "fig = plt.figure()\n",
    "res = stats.probplot(train['cc_cons'], plot=plt)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Features engineering\n",
    "#let's first concatenate the train and test data in the same dataframe\n",
    "ntrain = train.shape[0]\n",
    "ntest = test.shape[0]\n",
    "y_train = train.cc_cons.values\n",
    "all_data = pd.concat((train, test)).reset_index(drop=True)\n",
    "all_data.drop(['cc_cons'], axis=1, inplace=True)\n",
    "print(\"all_data size is : {}\".format(all_data.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Missing data\n",
    "all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\n",
    "all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)[:30]\n",
    "missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\n",
    "missing_data.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imputing missing values\n",
    "#replacing missing values with 0\n",
    "for col in ('investment_4', 'debit_count_jun', 'credit_count_jun', 'cc_count_may', 'card_lim', 'loan_enq', 'vehicle_loan_active', 'investment_3', 'investment_1', 'vehicle_loan_closed', 'investment_2', 'personal_loan_active', 'personal_loan_closed', 'dc_cons_jun', 'dc_count_apr', 'dc_cons_apr', 'dc_count_may', 'dc_cons_may', 'dc_count_jun', 'credit_amount_may', 'max_credit_amount_may', 'credit_amount_apr', 'max_credit_amount_apr', 'cc_count_apr', 'debit_amount_may', 'debit_amount_jun', 'credit_count_may', 'debit_count_may', 'debit_amount_apr', 'credit_count_apr', 'debit_count_apr', 'cc_count_jun', 'credit_amount_jun', 'max_credit_amount_jun'):\n",
    "    all_data[col] = all_data[col].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Check remaining missing values if any \n",
    "all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\n",
    "all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)\n",
    "missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\n",
    "missing_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transforming some numerical variables that are really categorical\n",
    "all_data['account_type'] = pd.Categorical(all_data.account_type)\n",
    "all_data['gender'] = pd.Categorical(all_data.gender)\n",
    "all_data['loan_enq'] = pd.Categorical(all_data.loan_enq)\n",
    "all_data['personal_loan_active'] = pd.Categorical(all_data.personal_loan_active)\n",
    "all_data['personal_loan_closed'] = pd.Categorical(all_data.personal_loan_closed)\n",
    "all_data['region_code'] = pd.Categorical(all_data.region_code)\n",
    "all_data['vehicle_loan_active'] = pd.Categorical(all_data.vehicle_loan_active)\n",
    "all_data['vehicle_loan_closed'] = pd.Categorical(all_data.vehicle_loan_closed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Label Encoding some categorical variables that may contain information in their ordering set\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "cols = ('account_type', 'gender', 'loan_enq', 'personal_loan_active', 'personal_loan_closed', 'region_code', 'vehicle_loan_active', 'vehicle_loan_closed')\n",
    "# process columns, apply LabelEncoder to categorical features\n",
    "for c in cols:\n",
    "    lbl = LabelEncoder() \n",
    "    lbl.fit(list(all_data[c].values)) \n",
    "    all_data[c] = lbl.transform(list(all_data[c].values))\n",
    "\n",
    "# shape        \n",
    "print('Shape all_data: {}'.format(all_data.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Check remaining missing values if any \n",
    "all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\n",
    "all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)\n",
    "missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\n",
    "missing_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.any(np.isnan(all_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numeric_feats = all_data.dtypes[all_data.dtypes != \"object\"].index\n",
    "\n",
    "# Check the skew of all numerical features\n",
    "skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)\n",
    "print(\"\\nSkew in numerical features: \\n\")\n",
    "skewness = pd.DataFrame({'Skew' :skewed_feats})\n",
    "skewness.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skewness = skewness[abs(skewness) > 0.75]\n",
    "print(\"There are {} skewed numerical features to Box Cox transform\".format(skewness.shape[0]))\n",
    "\n",
    "from scipy.special import boxcox1p\n",
    "skewed_features = skewness.index\n",
    "lam = 0.15\n",
    "for feat in skewed_features:\n",
    "    #all_data[feat] += 1\n",
    "    all_data[feat] = boxcox1p(all_data[feat], lam)\n",
    "    \n",
    "#all_data[skewed_features] = np.log1p(all_data[skewed_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = pd.get_dummies(all_data)\n",
    "print(all_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.any(np.isnan(all_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data['investment_4'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.any(np.isnan(all_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = all_data[:ntrain]\n",
    "test = all_data[ntrain:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.model_selection import KFold, cross_val_score, train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Validation function\n",
    "n_folds = 5\n",
    "\n",
    "def rmsle_cv(model):\n",
    "    kf = KFold(n_folds, shuffle=True, random_state=42).get_n_splits(train.values)\n",
    "    rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring=\"neg_mean_squared_error\", cv = kf))\n",
    "    return(rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,\n",
    "                                   max_depth=4, max_features='sqrt',\n",
    "                                   min_samples_leaf=15, min_samples_split=10, \n",
    "                                   loss='huber', random_state =5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468, \n",
    "                             learning_rate=0.05, max_depth=3, \n",
    "                             min_child_weight=1.7817, n_estimators=2200,\n",
    "                             reg_alpha=0.4640, reg_lambda=0.8571,\n",
    "                             subsample=0.5213, silent=1,\n",
    "                             random_state =7, nthread = -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,\n",
    "                              learning_rate=0.05, n_estimators=720,\n",
    "                              max_bin = 55, bagging_fraction = 0.8,\n",
    "                              bagging_freq = 5, feature_fraction = 0.2319,\n",
    "                              feature_fraction_seed=9, bagging_seed=9,\n",
    "                              min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Lasso score: 1.1745 (0.0101)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "score = rmsle_cv(lasso)\n",
    "print(\"\\nLasso score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet score: 1.1745 (0.0101)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "score = rmsle_cv(ENet)\n",
    "print(\"ElasticNet score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kernel Ridge score: 1.1788 (0.0095)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "score = rmsle_cv(KRR)\n",
    "print(\"Kernel Ridge score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting score: 1.1784 (0.0066)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "score = rmsle_cv(GBoost)\n",
    "print(\"Gradient Boosting score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))\n",
    "#Done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Xgboost score: 1.1748 (0.0078)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "score = rmsle_cv(model_xgb)\n",
    "print(\"Xgboost score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LGBM score: 1.1645 (0.0083)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "score = rmsle_cv(model_lgb)\n",
    "print(\"LGBM score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):\n",
    "    def __init__(self, models):\n",
    "        self.models = models\n",
    "        \n",
    "    # we define clones of the original models to fit the data in\n",
    "    def fit(self, X, y):\n",
    "        self.models_ = [clone(x) for x in self.models]\n",
    "        \n",
    "        # Train cloned base models\n",
    "        for model in self.models_:\n",
    "            model.fit(X, y)\n",
    "\n",
    "        return self\n",
    "    \n",
    "    #Now we do the predictions for cloned models and average them\n",
    "    def predict(self, X):\n",
    "        predictions = np.column_stack([\n",
    "            model.predict(X) for model in self.models_\n",
    "        ])\n",
    "        return np.mean(predictions, axis=1)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Averaged base models score: 1.1642 (0.0093)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "averaged_models = AveragingModels(models = (ENet, GBoost, KRR, lasso))\n",
    "\n",
    "score = rmsle_cv(averaged_models)\n",
    "print(\" Averaged base models score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):\n",
    "    def __init__(self, base_models, meta_model, n_folds=5):\n",
    "        self.base_models = base_models\n",
    "        self.meta_model = meta_model\n",
    "        self.n_folds = n_folds\n",
    "   \n",
    "    # We again fit the data on clones of the original models\n",
    "    def fit(self, X, y):\n",
    "        self.base_models_ = [list() for x in self.base_models]\n",
    "        self.meta_model_ = clone(self.meta_model)\n",
    "        kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)\n",
    "        \n",
    "        # Train cloned base models then create out-of-fold predictions\n",
    "        # that are needed to train the cloned meta-model\n",
    "        out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))\n",
    "        for i, model in enumerate(self.base_models):\n",
    "            for train_index, holdout_index in kfold.split(X, y):\n",
    "                instance = clone(model)\n",
    "                self.base_models_[i].append(instance)\n",
    "                instance.fit(X[train_index], y[train_index])\n",
    "                y_pred = instance.predict(X[holdout_index])\n",
    "                out_of_fold_predictions[holdout_index, i] = y_pred\n",
    "                \n",
    "        # Now train the cloned  meta-model using the out-of-fold predictions as new feature\n",
    "        self.meta_model_.fit(out_of_fold_predictions, y)\n",
    "        return self\n",
    "   \n",
    "    #Do the predictions of all base models on the test data and use the averaged predictions as \n",
    "    #meta-features for the final prediction which is done by the meta-model\n",
    "    def predict(self, X):\n",
    "        meta_features = np.column_stack([\n",
    "            np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)\n",
    "            for base_models in self.base_models_ ])\n",
    "        return self.meta_model_.predict(meta_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stacking Averaged models score: 1.1644 (0.0097)\n"
     ]
    }
   ],
   "source": [
    "stacked_averaged_models = StackingAveragedModels(base_models = (ENet, GBoost, KRR),\n",
    "                                                 meta_model = lasso)\n",
    "\n",
    "score = rmsle_cv(stacked_averaged_models)\n",
    "print(\"Stacking Averaged models score: {:.4f} ({:.4f})\".format(score.mean(), score.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmsle(y, y_pred):\n",
    "    return np.sqrt(mean_squared_error(y, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0828048600264746\n"
     ]
    }
   ],
   "source": [
    "stacked_averaged_models.fit(train.values, y_train)\n",
    "stacked_train_pred = stacked_averaged_models.predict(train.values)\n",
    "stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))\n",
    "print(rmsle(y_train, stacked_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.030817035261632\n"
     ]
    }
   ],
   "source": [
    "model_xgb.fit(train, y_train)\n",
    "xgb_train_pred = model_xgb.predict(train)\n",
    "xgb_pred = np.expm1(model_xgb.predict(test))\n",
    "print(rmsle(y_train, xgb_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.1342452289861564\n"
     ]
    }
   ],
   "source": [
    "model_lgb.fit(train, y_train)\n",
    "lgb_train_pred = model_lgb.predict(train)\n",
    "lgb_pred = np.expm1(model_lgb.predict(test.values))\n",
    "print(rmsle(y_train, lgb_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSLE score on train data:\n",
      "1.0796020319726762\n"
     ]
    }
   ],
   "source": [
    "'''RMSE on the entire Train data when averaging'''\n",
    "\n",
    "print('RMSLE score on train data:')\n",
    "print(rmsle(y_train,stacked_train_pred*0.70 +\n",
    "               xgb_train_pred*0.15 + lgb_train_pred*0.15 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.DataFrame()\n",
    "sub['id'] = test_ID\n",
    "sub['cc_cons'] = ensemble\n",
    "sub.to_csv('submission.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href=submission.csv>Download CSV file</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import HTML\n",
    "\n",
    "sub = pd.DataFrame()\n",
    "sub['id'] = test_ID\n",
    "sub['cc_cons'] = ensemble\n",
    "sub.to_csv('submission.csv',index=False)\n",
    "\n",
    "\n",
    "def create_download_link(title = \"Download CSV file\", filename = \"data.csv\"):  \n",
    "    html = '<a href={filename}>{title}</a>'\n",
    "    html = html.format(title=title,filename=filename)\n",
    "    return HTML(html)\n",
    "\n",
    "# create a link to download the dataframe which was saved with .to_csv method\n",
    "create_download_link(filename='submission.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}