mongo_pulls.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Document Preprocessing and Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Package Import and Mongo Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import credentials\n",
    "import time\n",
    "import string\n",
    "import config as cfg\n",
    "\n",
    "import spacy\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "from spacy.lang.en import English\n",
    "#!python -m spacy download en_core_web_lg\n",
    "\n",
    "from sklearn.decomposition import NMF, LatentDirichletAllocation, TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "#!pip install pyLDAvis\n",
    "import pyLDAvis.sklearn\n",
    "import pyLDAvis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Pulldown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = MongoClient(**cfg.config)\n",
    "\n",
    "db = client.youtube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_caption_block(videoID):\n",
    "    try:\n",
    "        return db.captions.find_one({'videoID' : videoID},{'_id' : 0, 'caption_block' : 1})['caption_block']\n",
    "    except:\n",
    "        return None\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rec_vids(collection,filter_dict={}):\n",
    "    pipeline = [\n",
    "        {'$unwind': '$videos'}, \n",
    "        {'$match': filter_dict},\n",
    "        {'$project': {'_id': 0, 'query' : 1, 'order' : '$videos.order', 'videoID' : '$videos.videoID'}}\n",
    "    ]\n",
    "    \n",
    "    vid_df = pd.DataFrame(list(collection.aggregate(pipeline)))\n",
    "    vid_df['caption'] = vid_df['videoID'].apply(get_caption_block)\n",
    "    caption_series = vid_df['caption']\n",
    "    caption_series.index = list(vid_df['videoID'])\n",
    "    caption_series = caption_series.dropna().drop_duplicates()\n",
    "\n",
    "    return vid_df, caption_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = get_rec_vids(db.recommendations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "vGPU5SWV1DE    half of Britain said that breaks it's made it ...\n",
       "6afu04-KO90    Greta turn Berg the teenage climate change act...\n",
       "FcFUa9SC8JA    you've er decided to join the Liberal Democrat...\n",
       "Z612WQhdOQ8    it was like an invitation to a dinner party no...\n",
       "q3W678l8bok    hello there so the Prime Minister Boris Johnso...\n",
       "xnEr_sNghTg    first of all I have to tell to mr. Hank oh tha...\n",
       "l0H-BykoktI    I think that if this is part of the conservati...\n",
       "JD7Ol0gz11k    so where will you be marching this in the next...\n",
       "O8ClR86bSMg    I absolutely condemn any attempt to blow up th...\n",
       "JISpI1h8xys    why is it remotely patronizing for a man to op...\n",
       "-iKcR4tb62Y    and I think women we've always been under trem...\n",
       "PludY6bjSIU    after recent holiday the journalist Ansel Epst...\n",
       "uHGPVsQlsFI    how good a friend was your your bridesmaid my ...\n",
       "RTXqaLEdK5k    [Music] my name is Tam Thompson I'm here today...\n",
       "VM05BvXgXmM    it is do-or-die in the sense that is my one bi...\n",
       "AKYvMAVguAM    now Banksy might be the only graffiti artist t...\n",
       "zMJ7K7UPvL0    all entrepreneurs entering the den believe the...\n",
       "CNkVGiSgOM4    This video is sponsored by dashlane As a youtu...\n",
       "zz6v6OfoQvs    Deep in western Russia, the frigid desert\\ncon...\n",
       "2z35_1e1MtI    It's a story straight out of a Bond film. On S...\n",
       "0ZfS8dr3jbc    [Music] there's a bunch of weird places on ear...\n",
       "gnwoZk6dr0I    newsmakers Sunday with foxtons john hook guys ...\n",
       "-xc61kv4aH0    April 16 1972 Apollo 16 man's 5th lunar landin...\n",
       "W7zh-Ro2y9w    Narrator: NASA's MISSION\\nIS TO EXPLORE THE UN...\n",
       "uwP7UT4ewJA    Narrator: NASA's MISSION\\nIS TO EXPLORE THE UN...\n",
       "T481m7mqzq0    >> Narrator: A TEMPLE OF\\nGEOMETRIC PRECISION ...\n",
       "Qyrrdoqspdg    the most famous death in history is the death ...\n",
       "Pw_0cgO2JKE    [Music] he was just dad in in a way he's alway...\n",
       "qO_IkNfT3Iw    it was the longest war in US history spanning ...\n",
       "ofL55tEcM8I    [Music] the Renaissance had reached its greate...\n",
       "                                     ...                        \n",
       "G9a2pFWTgzs    now reaction was swift and it was harsh you kn...\n",
       "pMurBQuBU2w    >>ELCOME TO  EAREEYOU.  TOU HAVING ME.  T ADSH...\n",
       "_RDu0YDeqNk    LETTER TO LAUD THEIR RESILIENCE AND THEIR RESO...\n",
       "sT6j6byOXbI    so the FISA memo that has been talked about so...\n",
       "R9h1Lw-uQpM    THERE IS NO CONSIDERATION ABOUT HOUSE INTELLIG...\n",
       "_qJR-IAtAe8    [Music] [Music] just heard Patricia speak and ...\n",
       "S1Dcw2tJczI    it's so good to be here with all of you I I ha...\n",
       "npYkF0FMWl0    today that I witnessed the ultrasound-guided a...\n",
       "-rgcXAnzz-o    joining me now is the woman who I know who I t...\n",
       "uwyau49NVjI    well good morning it's not very good morning m...\n",
       "GHL9lXVGPEQ    now comes the actually the moment that I think...\n",
       "TLDVVN21A2o    today on facing life head on then sometimes th...\n",
       "vdm-i62hRdc    we just Oh Melissa Odin always knew she was ad...\n",
       "Z9j4gAeWqoY    I am a mother whose daughter has identified as...\n",
       "rYtGPLpW-g8    thank you thank you Ryan thank you Elaine than...\n",
       "FbzPthhj6vI    [Applause] good afternoon everyone my name is ...\n",
       "VovGbtShGvk    I'm gonna talk about what happens when you do ...\n",
       "pvRYamafT0c    my name is Leo cell and my pronouns are they t...\n",
       "nS9W-wlJHPA    We've all found ourselves in conversation and ...\n",
       "hsQLksbfDSo    We often think that getting respect and being ...\n",
       "EOh1Gflu7Uw    How much responsibility do you feel you have p...\n",
       "dmMleBjkn2U    here's the form that you were asked to I said ...\n",
       "_yGHv6fCZbA    wearing short skirts all your just inviting yo...\n",
       "a0HCWuDJY4k    now 20% of you have said that you consider fli...\n",
       "uDtCGOcIack    if there was a dad who got the shock of his li...\n",
       "nKJCR-90Y44    the shocking news last night Prince Philip inv...\n",
       "9n-JrWVuzXs    now dialing out sleep sleep of dr. sleep let's...\n",
       "83hOWOZBKi4    now once again allow me to introduce you to Am...\n",
       "DtCafBVCHfM    HER, WHAT THEY'RE DOING. SO I'M NOT SURPRISED ...\n",
       "Tth-CSN21_A    >> Sandra: BREAKING NEWS RIGHT NOW AS WE LEARN...\n",
       "Name: caption, Length: 1650, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = English()\n",
    "punctuations = string.punctuation\n",
    "stopwords = list(STOP_WORDS)\n",
    "\n",
    "def spacy_tokenizer(sentence):\n",
    "    mytokens = parser(sentence)\n",
    "    mytokens = [ str(word).lower().strip() if word.lemma_ != \"-PRON-\" else str(word).lower() for word in mytokens ]\n",
    "    mytokens = [ word for word in mytokens if word not in stopwords and word not in punctuations ]\n",
    "    mytokens = \" \".join([i for i in mytokens])\n",
    "    return mytokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_tokens(videoID,collection):\n",
    "    filter_req = { \"tokenized\": {'$exists': False,},'videoID' : videoID}\n",
    "    \n",
    "    query = collection.find_one(filter_req, {'_id' : 0, 'caption_block' : 1})\n",
    "    \n",
    "    if query:\n",
    "        collection.update_one(filter_req,{'$set' : {\"tokenized\" : spacy_tokenizer(query['caption_block'])}})\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "update_tokens('McRYTC56DC4',db.captions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for elem in db.captions.find({},{'_id' : 0, 'videoID' : 1}):\n",
    "    update_tokens(elem['videoID'],db.captions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_vector_list = [elem['tokenized'] for elem in db.captions.find({},{'_id' : 0, 'tokenized' : 1})]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<input>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "<ipython-input-13-0a830703afc0>:3: DeprecationWarning: invalid escape sequence \\-\n",
      "  token_pattern='[a-zA-Z\\-][a-zA-Z\\-]{2,}')\n"
     ]
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(min_df=5, max_df=0.9, \n",
    "                             stop_words='english', lowercase=True, \n",
    "                             token_pattern='[a-zA-Z\\-][a-zA-Z\\-]{2,}')\n",
    "\n",
    "\n",
    "data_vectorized = vectorizer.fit_transform(to_vector_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1650x20107 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 916623 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_vectorized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>requestID</th>\n",
       "      <th>query</th>\n",
       "      <th>max_results</th>\n",
       "      <th>request_status</th>\n",
       "      <th>datetime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>da23078c-bf88-4024-91e9-d6cc6df4353a</td>\n",
       "      <td>global warming</td>\n",
       "      <td>50</td>\n",
       "      <td>202</td>\n",
       "      <td>Sat, 17 Aug 2019 18:25:00 GMT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f211c3a5-9ffc-4cd1-a700-6448aae3f589</td>\n",
       "      <td>gun rights</td>\n",
       "      <td>50</td>\n",
       "      <td>202</td>\n",
       "      <td>Sat, 17 Aug 2019 18:25:03 GMT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fd4189b0-8c3b-4821-a5cc-59fee424c73e</td>\n",
       "      <td>donald trump rally</td>\n",
       "      <td>50</td>\n",
       "      <td>202</td>\n",
       "      <td>Sat, 17 Aug 2019 18:25:49 GMT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f3af736f-91ea-46a6-88ff-7e44f206e5d2</td>\n",
       "      <td>abortion pro life</td>\n",
       "      <td>50</td>\n",
       "      <td>202</td>\n",
       "      <td>Sat, 17 Aug 2019 18:26:16 GMT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              requestID               query  max_results  \\\n",
       "0  da23078c-bf88-4024-91e9-d6cc6df4353a      global warming           50   \n",
       "1  f211c3a5-9ffc-4cd1-a700-6448aae3f589          gun rights           50   \n",
       "2  fd4189b0-8c3b-4821-a5cc-59fee424c73e  donald trump rally           50   \n",
       "3  f3af736f-91ea-46a6-88ff-7e44f206e5d2   abortion pro life           50   \n",
       "\n",
       "   request_status                       datetime  \n",
       "0             202  Sat, 17 Aug 2019 18:25:00 GMT  \n",
       "1             202  Sat, 17 Aug 2019 18:25:03 GMT  \n",
       "2             202  Sat, 17 Aug 2019 18:25:49 GMT  \n",
       "3             202  Sat, 17 Aug 2019 18:26:16 GMT  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NUM_TOPICS = 4\n",
    "pd.read_csv('request_log.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "class nlp_preprocessor:\n",
    "            \n",
    "    def dummy(doc):\n",
    "        return doc\n",
    "   \n",
    "    def __init__(self, vectorizer=None, tokenizer=None, cleaning_function=None, \n",
    "                 stemmer=None, model=None):\n",
    "        \"\"\"\n",
    "        A class for pipelining our data in NLP problems. The user provides a series of \n",
    "        tools, and this class manages all of the training, transforming, and modification\n",
    "        of the text data.\n",
    "        ---\n",
    "        Inputs:\n",
    "        vectorizer: the model to use for vectorization of text data\n",
    "        tokenizer: The tokenizer to use, if none defaults to split on spaces\n",
    "        cleaning_function: how to clean the data, if None, defaults to the in built class\n",
    "        \"\"\"\n",
    "        if not tokenizer:\n",
    "            tokenizer = self.splitter\n",
    "        if not cleaning_function:\n",
    "            cleaning_function = self.default_clean\n",
    "        if not vectorizer:\n",
    "            vectorizer = CountVectorizer(tokenizer=self.dummy,preprocessor=self.dummy)\n",
    "        else:\n",
    "            vectorizer.tokenizer = self.dummy\n",
    "            vectorizer.preprocessor = self.dummy\n",
    "        self.stemmer = stemmer\n",
    "        self.tokenizer = tokenizer\n",
    "        self.model = model\n",
    "        self.cleaning_function = cleaning_function\n",
    "        self.vectorizer = vectorizer\n",
    "        self._is_fit = False\n",
    "\n",
    "        \n",
    "    def splitter(self, text):\n",
    "        \"\"\"\n",
    "        Default tokenizer that splits on spaces naively\n",
    "        \"\"\"\n",
    "        return text.split(' ')\n",
    "        \n",
    "    def preprocess(self, docs, tokenizer,stemmer,cleaning_function):\n",
    "        \"\"\"\n",
    "        A naive function to lowercase all works can clean them quickly.\n",
    "        This is the default behavior if no other cleaning function is specified\n",
    "        \"\"\"\n",
    "        cleaned_docs = []\n",
    "        for doc in docs:\n",
    "            cleaned_tokens = []\n",
    "            for token in tokenizer(doc):\n",
    "                token_word_list = []\n",
    "                for word in token:\n",
    "                    clean_word = cleaning_function(word)\n",
    "                    if stemmer:\n",
    "                        clean_word = stemmer.stem(clean_word)\n",
    "                    token_word_list.append(clean_word)\n",
    "                cleaned_tokens.append(tuple(token_word_list))\n",
    "            cleaned_docs.append(cleaned_tokens)\n",
    "        return cleaned_docs\n",
    "    \n",
    "    def default_clean(self, word):\n",
    "        return word.lower()\n",
    "    \n",
    "    def clean_text_old(self, text, tokenizer, stemmer):\n",
    "        \"\"\"\n",
    "        A naive function to lowercase all works can clean them quickly.\n",
    "        This is the default behavior if no other cleaning function is specified\n",
    "        \"\"\"\n",
    "        cleaned_text = []\n",
    "        for post in text:\n",
    "            cleaned_words = []\n",
    "            for word in tokenizer(post):\n",
    "                low_word = word.lower()\n",
    "                if stemmer:\n",
    "                    low_word = stemmer.stem(low_word)\n",
    "                cleaned_words.append(low_word)\n",
    "            cleaned_text.append(' '.join(cleaned_words))\n",
    "        return cleaned_text\n",
    "    \n",
    "    def fit(self, docs):\n",
    "        \"\"\"\n",
    "        Cleans the data and then fits the vectorizer with\n",
    "        the user provided text\n",
    "        \"\"\"\n",
    "        clean_text = self.preprocess(docs, self.tokenizer, self.stemmer,self.cleaning_function)\n",
    "        self.vectorizer.fit(clean_text)\n",
    "        self._is_fit = True\n",
    "        \n",
    "    def transform(self, docs):\n",
    "        \"\"\"\n",
    "        Cleans any provided data and then transforms the data into\n",
    "        a vectorized format based on the fit function. Returns the\n",
    "        vectorized form of the data.\n",
    "        \"\"\"\n",
    "        if not self._is_fit:\n",
    "            raise ValueError(\"Must fit the models before transforming!\")\n",
    "        clean_text = self.preprocess(docs, self.tokenizer, self.stemmer,self.cleaning_function)\n",
    "        return self.vectorizer.transform(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "\n",
    "\n",
    "def lower_clean(word):\n",
    "    return word.lower()\n",
    "\n",
    "\n",
    "nlp = nlp_preprocessor(vectorizer=CountVectorizer(min_df=0.1,max_df=0.9), tokenizer=TreebankWordTokenizer().tokenize,\n",
    "                       cleaning_function=lower_clean,stemmer=PorterStemmer())\n",
    "\n",
    "nlp.fit(test_captions)\n",
    "\n",
    "nlp_dict['untrained'] = nlp.transform(test_captions)\n",
    "\n",
    "NUM_TOPICS = 8\n",
    "lda = LatentDirichletAllocation(n_components=NUM_TOPICS, max_iter=100, learning_method='online',verbose=True)\n",
    "data_lda = lda.fit_transform(data_vectorized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration: 1 of max_iter: 10\n",
      "iteration: 2 of max_iter: 10\n",
      "iteration: 3 of max_iter: 10\n",
      "iteration: 4 of max_iter: 10\n",
      "iteration: 5 of max_iter: 10\n",
      "iteration: 6 of max_iter: 10\n",
      "iteration: 7 of max_iter: 10\n",
      "iteration: 8 of max_iter: 10\n",
      "iteration: 9 of max_iter: 10\n",
      "iteration: 10 of max_iter: 10\n"
     ]
    }
   ],
   "source": [
    "# Latent Dirichlet Allocation Model\n",
    "lda = LatentDirichletAllocation(n_components=NUM_TOPICS, max_iter=10, learning_method='online',verbose=True)\n",
    "data_lda = lda.fit_transform(data_vectorized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Non-Negative Matrix Factorization Model\n",
    "nmf = NMF(n_components=NUM_TOPICS)\n",
    "data_nmf = nmf.fit_transform(data_vectorized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Latent Semantic Indexing Model using Truncated SVD\n",
    "lsi = TruncatedSVD(n_components=NUM_TOPICS)\n",
    "data_lsi = lsi.fit_transform(data_vectorized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selected_topics(model, vectorizer, top_n=10):\n",
    "    for idx, topic in enumerate(model.components_):\n",
    "        print(\"Topic %d:\" % (idx))\n",
    "        print([(vectorizer.get_feature_names()[i], topic[i])\n",
    "                        for i in topic.argsort()[:-top_n - 1:-1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LDA Model:\n",
      "Topic 0:\n",
      "[('said', 2578.3349613407854), ('president', 1829.2725649900908), ('trump', 1668.255733195031), ('man', 1406.2749027625916), ('police', 1310.512568545276), ('time', 1211.620649788353), ('know', 1204.2478343235764), ('investigation', 1157.7639157382434), ('told', 1058.5315236249278), ('narrator', 1054.3975877744986)]\n",
      "Topic 1:\n",
      "[('like', 3360.10924096578), ('time', 2847.8738477943502), ('years', 2617.6421670461955), ('world', 2251.2347494744545), ('actually', 2249.933691764769), ('going', 2180.2481229553687), ('universe', 1963.384951700923), ('know', 1959.9447214779436), ('way', 1898.0842075820867), ('earth', 1873.796735026446)]\n",
      "Topic 2:\n",
      "[('know', 23309.36600420221), ('like', 20074.56493911058), ('people', 16491.143995979863), ('think', 15678.916932181362), ('right', 11834.655172505105), ('going', 10273.167131650069), ('want', 9659.48656141703), ('said', 7840.977369803221), ('got', 7388.494272398069), ('yeah', 7370.634176962241)]\n",
      "Topic 3:\n",
      "[('music', 3464.755551638535), ('god', 2876.1429422813717), ('life', 2179.6019869720167), ('jesus', 2112.432843557805), ('let', 1861.6218514585503), ('lord', 1368.9098628378022), ('people', 986.2549896478089), ('come', 979.4997893972189), ('church', 816.1831394688053), ('body', 796.6317356495397)]\n"
     ]
    }
   ],
   "source": [
    "# Keywords for topics clustered by Latent Dirichlet Allocation\n",
    "print(\"LDA Model:\")\n",
    "selected_topics(lda, vectorizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NMF Model:\n",
      "Topic 0:\n",
      "[('know', 24.480051410555156), ('like', 20.745264214677352), ('think', 17.36288756168784), ('people', 15.251921734391466), ('right', 11.615074980067009), ('going', 9.04977696754732), ('time', 7.43873519068487), ('said', 7.138734844930159), ('want', 7.098055782909104), ('got', 6.753350003973424)]\n",
      "Topic 1:\n",
      "[('jesus', 18.769902864378846), ('let', 17.650392746490855), ('lord', 10.945765255874706), ('life', 10.742220082627039), ('god', 7.433188805700665), ('bind', 5.676348454463307), ('spirits', 5.664028252713735), ('rebuke', 5.170496173956417), ('break', 4.714986927322923), ('spirit', 4.655893311322159)]\n",
      "Topic 2:\n",
      "[('president', 16.228213831888713), ('trump', 10.379482562695602), ('said', 7.539526174382537), ('report', 7.526470945694014), ('investigation', 6.112563269978902), ('correct', 5.192586843476637), ('campaign', 4.290874036893819), ('thank', 4.043887113616778), ('going', 3.8877712850612185), ('time', 3.482087263819365)]\n",
      "Topic 3:\n",
      "[('yeah', 12.172360817797033), ('love', 11.987062727049812), ('wow', 9.505787805441438), ('like', 9.457244765563969), ('know', 7.787780629767229), ('sounds', 7.3452043772371916), ('going', 7.316545723969415), ('yes', 7.2895805608582815), ('great', 6.485735322300037), ('good', 5.518717468874318)]\n"
     ]
    }
   ],
   "source": [
    "# Keywords for topics clustered by Latent Semantic Indexing\n",
    "print(\"NMF Model:\")\n",
    "selected_topics(nmf, vectorizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSI Model:\n",
      "Topic 0:\n",
      "[('know', 0.40317086896863435), ('like', 0.34618583522321994), ('think', 0.29120266806416806), ('people', 0.25109131595978224), ('right', 0.1954859987900009), ('going', 0.1842328623973778), ('said', 0.1542009751415395), ('yeah', 0.13905865289808), ('want', 0.13788717589118574), ('time', 0.13731336689264173)]\n",
      "Topic 1:\n",
      "[('jesus', 0.4937346631289153), ('let', 0.4443042590974779), ('lord', 0.2879261167444852), ('life', 0.27140106176918766), ('god', 0.19022663423800842), ('bind', 0.15000471468584173), ('spirits', 0.149553165558465), ('rebuke', 0.1367159378133451), ('break', 0.12192753637972531), ('spirit', 0.12139475748774703)]\n",
      "Topic 2:\n",
      "[('like', 0.29852414480405187), ('yeah', 0.26954569907590226), ('love', 0.18293434474216894), ('wow', 0.1596601815582499), ('know', 0.15633516573345912), ('sounds', 0.1214458476527412), ('theater', 0.075606963837201), ('idea', 0.07028613575188701), ('mean', 0.06987826536085501), ('movie', 0.06971449128868645)]\n",
      "Topic 3:\n",
      "[('love', 0.29585808085493265), ('president', 0.2740323636262114), ('wow', 0.23950033655866773), ('yes', 0.20700028961808584), ('sounds', 0.18547174642499237), ('report', 0.18479861215213414), ('yeah', 0.182753772207702), ('trump', 0.16156725327854596), ('great', 0.15308535638924428), ('investigation', 0.1280625134171775)]\n"
     ]
    }
   ],
   "source": [
    "# Keywords for topics clustered by Non-Negative Matrix Factorization\n",
    "print(\"LSI Model:\")\n",
    "selected_topics(lsi, vectorizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/alexanderparker/anaconda3/envs/metis/lib/python3.6/site-packages/pyLDAvis/_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  return pd.concat([default_term_info] + list(topic_dfs))\n"
     ]
    }
   ],
   "source": [
    "dash = pyLDAvis.sklearn.prepare(lda, data_vectorized, vectorizer, mds='tsne')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1703x20421 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 901265 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_vectorized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyLDAvis.save_html(dash,'lda.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:metis]",
   "language": "python",
   "name": "conda-env-metis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}