Comprehensive Guide to build a Recommendation Engine from scratch (in Python).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Case study in Python using the MovieLens Dataset\n",
    "\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# pass in column names for each CSV as the column name is not given in the file and read them using pandas.\n",
    "# You can check the column names from the readme file\n",
    "\n",
    "#Reading users file:\n",
    "u_cols = ['user_id', 'age', 'sex', 'occupation', 'zip_code']\n",
    "users = pd.read_csv('ml-100k/u.user', sep='|', names=u_cols,encoding='latin-1')\n",
    "\n",
    "#Reading ratings file:\n",
    "r_cols = ['user_id', 'movie_id', 'rating', 'unix_timestamp']\n",
    "ratings = pd.read_csv('ml-100k/u.data', sep='\\t', names=r_cols,encoding='latin-1')\n",
    "\n",
    "#Reading items file:\n",
    "i_cols = ['movie id', 'movie title' ,'release date','video release date', 'IMDb URL', 'unknown', 'Action', 'Adventure',\n",
    "'Animation', 'Children\\'s', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy',\n",
    "'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western']\n",
    "items = pd.read_csv('ml-100k/u.item', sep='|', names=i_cols,\n",
    "encoding='latin-1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(users.shape)\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(ratings.shape)\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(items.shape)\n",
    "items.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_cols = ['user_id', 'movie_id', 'rating', 'unix_timestamp']\n",
    "ratings_train = pd.read_csv('ml-100k/ua.base', sep='\\t', names=r_cols, encoding='latin-1')\n",
    "ratings_test = pd.read_csv('ml-100k/ua.test', sep='\\t', names=r_cols, encoding='latin-1')\n",
    "ratings_train.shape, ratings_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building collaborative filtering model from scratch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_users = ratings.user_id.unique().shape[0]\n",
    "n_items = ratings.movie_id.unique().shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_matrix = np.zeros((n_users, n_items))\n",
    "for line in ratings.itertuples():\n",
    "    data_matrix[line[1]-1, line[2]-1] = line[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import pairwise_distances \n",
    "user_similarity = pairwise_distances(data_matrix, metric='cosine')\n",
    "item_similarity = pairwise_distances(data_matrix.T, metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(ratings, similarity, type='user'):\n",
    "    if type == 'user':\n",
    "        mean_user_rating = ratings.mean(axis=1)\n",
    "        #We use np.newaxis so that mean_user_rating has same format as ratings\n",
    "        ratings_diff = (ratings - mean_user_rating[:, np.newaxis])\n",
    "        pred = mean_user_rating[:, np.newaxis] + similarity.dot(ratings_diff) / np.array([np.abs(similarity).sum(axis=1)]).T\n",
    "    elif type == 'item':\n",
    "        pred = ratings.dot(similarity) / np.array([np.abs(similarity).sum(axis=1)])\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_prediction = predict(data_matrix, user_similarity, type='user')\n",
    "item_prediction = predict(data_matrix, item_similarity, type='item')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building a simple popularity and collaborative filtering model using Turicreate "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import turicreate\n",
    "train_data = turicreate.SFrame(ratings_train)\n",
    "test_data = turicreate.Sframe(ratings_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "popularity_model = turicreate.popularity_recommender.create(train_data, user_id='user_id', item_id='movie_id', target='rating')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "popularity_recomm = popularity_model.recommend(users=[1,2,3,4,5],k=5)\n",
    "popularity_recomm.print_rows(num_rows=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the model\n",
    "item_sim_model = turicreate.item_similarity_recommender.create(train_data, user_id='user_id', item_id='movie_id', target='rating', similarity_type='cosine')\n",
    "\n",
    "#Making recommendations\n",
    "item_sim_recomm = item_sim_model.recommend(users=[1,2,3,4,5],k=5)\n",
    "item_sim_recomm.print_rows(num_rows=25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building a recommendation engine using matrix factorization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MF():\n",
    "\n",
    "    # Initializing the user-movie rating matrix, no. of latent features, alpha and beta.\n",
    "    def __init__(self, R, K, alpha, beta, iterations):\n",
    "        self.R = R\n",
    "        self.num_users, self.num_items = R.shape\n",
    "        self.K = K\n",
    "        self.alpha = alpha\n",
    "        self.beta = beta\n",
    "        self.iterations = iterations\n",
    "\n",
    "    # Initializing user-feature and movie-feature matrix \n",
    "    def train(self):\n",
    "        self.P = np.random.normal(scale=1./self.K, size=(self.num_users, self.K))\n",
    "        self.Q = np.random.normal(scale=1./self.K, size=(self.num_items, self.K))\n",
    "\n",
    "        # Initializing the bias terms\n",
    "        self.b_u = np.zeros(self.num_users)\n",
    "        self.b_i = np.zeros(self.num_items)\n",
    "        self.b = np.mean(self.R[np.where(self.R != 0)])\n",
    "\n",
    "        # List of training samples\n",
    "        self.samples = [\n",
    "        (i, j, self.R[i, j])\n",
    "        for i in range(self.num_users)\n",
    "        for j in range(self.num_items)\n",
    "        if self.R[i, j] > 0\n",
    "        ]\n",
    "\n",
    "        # Stochastic gradient descent for given number of iterations\n",
    "        training_process = []\n",
    "        for i in range(self.iterations):\n",
    "        np.random.shuffle(self.samples)\n",
    "        self.sgd()\n",
    "        mse = self.mse()\n",
    "        training_process.append((i, mse))\n",
    "        if (i+1) % 20 == 0:\n",
    "            print(\"Iteration: %d ; error = %.4f\" % (i+1, mse))\n",
    "\n",
    "        return training_process\n",
    "\n",
    "    # Computing total mean squared error\n",
    "    def mse(self):\n",
    "        xs, ys = self.R.nonzero()\n",
    "        predicted = self.full_matrix()\n",
    "        error = 0\n",
    "        for x, y in zip(xs, ys):\n",
    "            error += pow(self.R[x, y] - predicted[x, y], 2)\n",
    "        return np.sqrt(error)\n",
    "\n",
    "    # Stochastic gradient descent to get optimized P and Q matrix\n",
    "    def sgd(self):\n",
    "        for i, j, r in self.samples:\n",
    "            prediction = self.get_rating(i, j)\n",
    "            e = (r - prediction)\n",
    "\n",
    "            self.b_u[i] += self.alpha * (e - self.beta * self.b_u[i])\n",
    "            self.b_i[j] += self.alpha * (e - self.beta * self.b_i[j])\n",
    "\n",
    "            self.P[i, :] += self.alpha * (e * self.Q[j, :] - self.beta * self.P[i,:])\n",
    "            self.Q[j, :] += self.alpha * (e * self.P[i, :] - self.beta * self.Q[j,:])\n",
    "\n",
    "    # Ratings for user i and moive j\n",
    "    def get_rating(self, i, j):\n",
    "        prediction = self.b + self.b_u[i] + self.b_i[j] + self.P[i, :].dot(self.Q[j, :].T)\n",
    "        return prediction\n",
    "\n",
    "    # Full user-movie rating matrix\n",
    "    def full_matrix(self):\n",
    "        return mf.b + mf.b_u[:,np.newaxis] + mf.b_i[np.newaxis:,] + mf.P.dot(mf.Q.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "R= np.array(ratings.pivot(index = 'user_id', columns ='movie_id', values = 'rating').fillna(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mf = MF(R, K=20, alpha=0.001, beta=0.01, iterations=100)\n",
    "training_process = mf.train()\n",
    "print()\n",
    "print(\"P x Q:\")\n",
    "print(mf.full_matrix())\n",
    "print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}