ml_21-Recomendation systems.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d65be1e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division, print_function\n",
    "import math, random\n",
    "from collections import defaultdict, Counter\n",
    "from scipy import spatial\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from operator import itemgetter, attrgetter\n",
    "from pprint import pprint\n",
    "from collections import OrderedDict as od\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "84eaad4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Big Data', 'C++', 'Cassandra', 'HBase', 'Hadoop', 'Haskell', 'Java', 'Mahout', 'MapReduce', 'MongoDB', 'MySQL', 'NoSQL', 'Postgres', 'Python', 'R', 'Spark', 'Storm', 'artificial intelligence', 'databases', 'decision trees', 'deep learning', 'libsvm', 'machine learning', 'mathematics', 'neural networks', 'numpy', 'pandas', 'probability', 'programming languages', 'regression', 'scikit-learn', 'scipy', 'statistics', 'statsmodels', 'support vector machines', 'theory']\n",
      "36\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# темы повторяющиеся у разных пользователей\n",
    "users_interests = [\n",
    "    [\"Hadoop\", \"Big Data\", \"HBase\", \"Java\", \"Spark\", \"Storm\", \"Cassandra\"],\n",
    "    [\"NoSQL\", \"MongoDB\", \"Cassandra\", \"HBase\", \"Postgres\"],\n",
    "    [\"Python\", \"scikit-learn\", \"scipy\", \"numpy\", \"statsmodels\", \"pandas\"],\n",
    "    [\"R\", \"Python\", \"statistics\", \"regression\", \"probability\"],\n",
    "    [\"machine learning\", \"regression\", \"decision trees\", \"libsvm\"],\n",
    "    [\"Python\", \"R\", \"Java\", \"C++\", \"Haskell\", \"programming languages\"],\n",
    "    [\"statistics\", \"probability\", \"mathematics\", \"theory\"],\n",
    "    [\"machine learning\", \"scikit-learn\", \"Mahout\", \"neural networks\"],\n",
    "    [\"neural networks\", \"deep learning\", \"Big Data\", \"artificial intelligence\"],\n",
    "    [\"Hadoop\", \"Java\", \"MapReduce\", \"Big Data\"],\n",
    "    [\"statistics\", \"R\", \"statsmodels\"],\n",
    "    [\"C++\", \"deep learning\", \"artificial intelligence\", \"probability\"],\n",
    "    [\"pandas\", \"R\", \"Python\"],\n",
    "    [\"databases\", \"HBase\", \"Postgres\", \"MySQL\", \"MongoDB\"],\n",
    "    [\"libsvm\", \"regression\", \"support vector machines\"]\n",
    "]\n",
    "\n",
    "unique_interests = sorted({interest for users_interests in users_interests for interest in users_interests})\n",
    "\n",
    "print(unique_interests)\n",
    "print(len(unique_interests))\n",
    "print(len(users_interests))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90c1c394",
   "metadata": {},
   "source": [
    "# Популярность темы\n",
    "Простой подход  - рекомендуем пользователю самые популярные темы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "41e86486",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Popular Interests\n",
      "[('Python', 4), ('R', 4), ('Big Data', 3), ('HBase', 3), ('Java', 3), ('statistics', 3), ('regression', 3), ('probability', 3), ('Hadoop', 2), ('Cassandra', 2), ('MongoDB', 2), ('Postgres', 2), ('scikit-learn', 2), ('statsmodels', 2), ('pandas', 2), ('machine learning', 2), ('libsvm', 2), ('C++', 2), ('neural networks', 2), ('deep learning', 2), ('artificial intelligence', 2), ('Spark', 1), ('Storm', 1), ('NoSQL', 1), ('scipy', 1), ('numpy', 1), ('decision trees', 1), ('Haskell', 1), ('programming languages', 1), ('mathematics', 1), ('theory', 1), ('Mahout', 1), ('MapReduce', 1), ('databases', 1), ('MySQL', 1), ('support vector machines', 1)]\n"
     ]
    }
   ],
   "source": [
    "popular_interests = Counter(\n",
    "    interest for user_interests in users_interests for interest in user_interests\n",
    ").most_common()\n",
    "print('Popular Interests')\n",
    "print(popular_interests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "96d27097",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most popular New Interests\n",
      "already like: ['NoSQL', 'MongoDB', 'Cassandra', 'HBase', 'Postgres']\n",
      "[('Python', 4), ('R', 4), ('Big Data', 3), ('Java', 3), ('statistics', 3)]\n"
     ]
    }
   ],
   "source": [
    "def most_popular_new_interests(user_interests, max_results=5):\n",
    "    suggestions = [(interest, frequency)\n",
    "                   for interest, frequency in popular_interests\n",
    "                   if interest not in user_interests\n",
    "                  ]\n",
    "    return suggestions[:max_results]\n",
    "\n",
    "print('Most popular New Interests')\n",
    "print('already like:', ['NoSQL', 'MongoDB', 'Cassandra', 'HBase', 'Postgres'])\n",
    "print(most_popular_new_interests(['NoSQL', 'MongoDB', 'Cassandra', 'HBase', 'Postgres']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a467b00",
   "metadata": {},
   "source": [
    "Такой подход применим для пользователей, о которых ничего не известно. Так называемый \"холодный старт\". Почему он\n",
    "плохо работает для всех?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4c33bd85",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 1., 1., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
       "        1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 1., 1., 0., 0., 0., 0., 0., 1., 0., 1., 1., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 1.,\n",
       "        0., 1., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0.,\n",
       "        1., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 1., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 1., 1., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "        1., 0., 0., 1.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 1., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 1., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        1., 1., 0., 0.],\n",
       "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 1., 0., 0., 0.,\n",
       "        0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
       "        0., 0., 1., 0.]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# составим матрицу интересов пользователей\n",
    "\n",
    "v = DictVectorizer(sparse=False)\n",
    "\n",
    "d = [{item: 1 for item in ui} for ui in users_interests]\n",
    "user_interest_matrix = v.fit_transform(d)\n",
    "\n",
    "user_interest_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fec99db8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        , 0.3380617 , 0.        , 0.        , 0.        ,\n",
       "       0.15430335, 0.        , 0.        , 0.18898224, 0.56694671,\n",
       "       0.        , 0.        , 0.        , 0.16903085, 0.        ])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# User-based  Collaborative Filtering\n",
    "#\n",
    "#\n",
    "user_similarities = cosine_similarity(user_interest_matrix, user_interest_matrix)\n",
    "\n",
    "# схожесть первого пользователя\n",
    "user_similarities[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "33b7841f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User based similarity\n",
      "most similar to 0\n",
      "['Hadoop', 'Big Data', 'HBase', 'Java', 'Spark', 'Storm', 'Cassandra']\n",
      "['Hadoop', 'Java', 'MapReduce', 'Big Data']\n",
      "[(9, 0.5669467095138407), (1, 0.3380617018914066), (8, 0.1889822365046136), (13, 0.1690308509457033), (5, 0.1543033499620919)]\n",
      "Suggestions for 0\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "too many values to unpack (expected 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-177-2ccc310e51bd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Suggestions for 0'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muser_based_suggestions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-177-2ccc310e51bd>\u001b[0m in \u001b[0;36muser_based_suggestions\u001b[1;34m(user_id, include_current_interests)\u001b[0m\n\u001b[0;32m     23\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0msuggestions\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m         return [(suggestions, weight)\n\u001b[0m\u001b[0;32m     26\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0msuggestion\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweight\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msuggestions\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0msuggestion\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0munique_interests\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0muser_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-177-2ccc310e51bd>\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m     23\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0msuggestions\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m         return [(suggestions, weight)\n\u001b[0m\u001b[0;32m     26\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0msuggestion\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweight\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msuggestions\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0msuggestion\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0munique_interests\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0muser_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: too many values to unpack (expected 2)"
     ]
    }
   ],
   "source": [
    "def most_similar_users_to(user_id):\n",
    "    user = user_similarities[user_id]\n",
    "    \n",
    "    pairs = [(other_user_id, similarity)\n",
    "             for other_user_id, similarity in enumerate(user)\n",
    "             if user_id != other_user_id and similarity > 0]\n",
    "    \n",
    "    return sorted(pairs, key=lambda x: x[1], reverse=True)\n",
    "\n",
    "def user_based_suggestions(user_id, include_current_interests=False):\n",
    "    suggestions = defaultdict(float)\n",
    "    for other_user_id, similarity in most_similar_users_to(user_id):\n",
    "         for interest in users_interests[other_user_id]:\n",
    "            suggestions[interest] += similarity\n",
    "#     print(suggestions.items())\n",
    "    suggestions = sorted(suggestions.items(),\n",
    "                        key=lambda weight: weight,\n",
    "                         reverse=True\n",
    "                        )\n",
    "    suggestions = od(suggestions)\n",
    "    \n",
    "    if include_current_interests:\n",
    "        return suggestions\n",
    "    else:\n",
    "        return [(suggestions, weight)\n",
    "                for suggestion, weight in suggestions\n",
    "                if suggestion not in unique_interests[user_id]\n",
    "               ]\n",
    "\n",
    "print('User based similarity')\n",
    "print('most similar to 0')\n",
    "print(users_interests[0])\n",
    "print(users_interests[9])\n",
    "print(most_similar_users_to(0))\n",
    "\n",
    "print('Suggestions for 0')\n",
    "print(user_based_suggestions(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fadd19e7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}