topic_modeling_for_greek_mooc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import timeit\n",
    "import re\n",
    "import unicodedata\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "#conda install -c conda-forge spacy\n",
    "#python -m spacy download el_core_news_sm\n",
    "import spacy\n",
    "import el_core_news_lg\n",
    "from nltk.tokenize.toktok import ToktokTokenizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation, NMF\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import warnings\n",
    "import random\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "#nltk.download('stopwords')\n",
    "\n",
    "nlp = el_core_news_lg.load()\n",
    "tokenizer = ToktokTokenizer()\n",
    "\n",
    "#Nltk Stopwords\n",
    "#stopwords = nltk.corpus.stopwords.words('greek')\n",
    "\n",
    "\n",
    "#Custom Stopwords\n",
    "stopwords = []\n",
    "with open('stopwords_gr.txt',mode = \"r\",encoding='utf-8') as f:\n",
    "    stopwords = f.read().splitlines()\n",
    "\n",
    "\n",
    "custom_stopwords = ['λυση','κατερινας','nikou','νικο','νικου','νικος','κατερινα','κατερίνα','ελληνικη','λεξη','ελληνικες','λεξει','katerina','λεξικο', 'λιστα', 'λεξικου','katerina','katerinas', 'ειναι','είναι']\n",
    "\n",
    "stopwords = stopwords + custom_stopwords\n",
    "\n",
    "# get bag of words features in sparse format\n",
    "#cv = CountVectorizer(min_df=0., max_df=20.,ngram_range=(1,2))\n",
    "#cv = CountVectorizer(ngram_range=(3,3))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_accents(input_str):\n",
    "    nfkd_form = unicodedata.normalize('NFKD', input_str)\n",
    "    return u\"\".join([c for c in nfkd_form if not unicodedata.combining(c)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_text(text):\n",
    "    text = nlp(text)\n",
    "    text = ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text])\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_special_characters(text, remove_digits=False):\n",
    "    pattern = r'[^a-zA-zα-ωΑ-Ωάέίόώήύ0-9\\s]' if not remove_digits else r'[^a-zA-zα-ωΑ-Ωάέίόώήύ\\s]'\n",
    "    text = re.sub(pattern, '', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    \n",
    "    tokens = tokenizer.tokenize(text)\n",
    "    tokens = [token.strip() for token in tokens]\n",
    "    filtered_tokens = [token for token in tokens if token not in stopwords]\n",
    "    filtered_text = ' '.join(filtered_tokens)    \n",
    "    return filtered_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_mooc(corpus, accented_char_removal=True, text_lower_case=True, \n",
    "                     text_lemmatization=True, special_char_removal=True, \n",
    "                     stopword_removal=True, remove_digits=True):\n",
    "    normalized_corpus = []\n",
    "    # normalize each document in the corpus\n",
    "    for doc in corpus:\n",
    "        # remove accented characters\n",
    "        if accented_char_removal:\n",
    "            doc = remove_accents(doc)\n",
    "        # lowercase the text    \n",
    "        if text_lower_case:\n",
    "            doc = doc.lower()\n",
    "        # remove extra newlines\n",
    "        doc = re.sub(r'[\\r|\\n|\\r\\n]+', ' ',doc)\n",
    "        # remove words with up to 2 letters\n",
    "        doc = re.sub(r'\\b\\w{1,2}\\b', ' ', doc)\n",
    "        # lemmatize text\n",
    "        if text_lemmatization:\n",
    "            doc = lemmatize_text(doc)\n",
    "        # remove special characters and\\or digits    \n",
    "        if special_char_removal:\n",
    "            # insert spaces between special characters to isolate them    \n",
    "            special_char_pattern = re.compile(r'([{.(-)!}])')\n",
    "            doc = special_char_pattern.sub(\" \\\\1 \", doc)\n",
    "            doc = remove_special_characters(doc, remove_digits=remove_digits)  \n",
    "        \n",
    "        doc = re.sub(r'\\]',' ', doc)\n",
    "        doc = re.sub(r'\\[',' ', doc)\n",
    "        # remove extra whitespace\n",
    "        doc = re.sub(' +', ' ', doc)\n",
    "\n",
    "        # remove stopwords\n",
    "        if stopword_removal:\n",
    "            doc = remove_stopwords(doc)\n",
    "        \n",
    "      #  print(doc)\n",
    "        normalized_corpus.append(doc)\n",
    "        \n",
    "    return normalized_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_topics(model, vectorizer, n_top_words):\n",
    "    words = vectorizer.get_feature_names()\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        print(\"\\nTopic #%d:\" % (topic_idx+1))\n",
    "        print(\", \".join([words[i]\n",
    "                        for i in topic.argsort()[:-n_top_words - 1:-1]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Get Only Final Answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Answers found:  106\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "mooc_list = ['SPOC Chat Log.xlsx']\n",
    "#mooc_list = ['MOOC Chat Log.xlsx']\n",
    "\n",
    "\n",
    "answers = []\n",
    "for m_file in mooc_list:\n",
    "    mooc = pd.read_excel(m_file,\n",
    "    header=0,\n",
    "    index_col=False,\n",
    "    keep_default_na=True\n",
    "    )\n",
    "    \n",
    "    for index, row in mooc.iterrows():      \n",
    "        if(row['message_type'] == 0 and int(row['message_length']) > 0):\n",
    "            answers.append(row['the_message'])  \n",
    "\n",
    "\n",
    "print(\"Final Answers found: \",len(answers))\n",
    "random.shuffle(answers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Normilize Corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pre-processing Steps\n",
    "\n",
    "1. Remove Accents\n",
    "2. Turn all to lower case\n",
    "3. Lemmatize text\n",
    "4. Remove special characters and empty lines\n",
    "5. Remove numbers\n",
    "6. Remove stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized = normalize_mooc(answers)\n",
    "#print(normalized)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LDA & NMF\n",
    "\n",
    "Firstly we need to generate a document-term matrix with a bag-of-word model.\n",
    "We use CountVectorizer for LDA and TfidfVectorizer for NMF."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting tf-idf features for NMF...\n",
      "done in 0.026s.\n",
      "Extracting tf features for LDA...\n",
      "done in 0.007s.\n",
      "\n",
      "\n",
      "\n",
      " Fitting LDA models with tf features, n_samples=2000 and n_features=1000...\n",
      "done in 10.307s.\n",
      "\n",
      "Topic #1:\n",
      "μεταφρασει αγγλικα, αγγλικων λεξεο, εκτελεση προγραμματο, αγγλικα πλεονεκτημο, ταχυτηα εκτελεση\n",
      "\n",
      "Topic #2:\n",
      "χρονο εκτελεση, χρηση λεξικων, αντιστοιχη τιμη, τιμη κλειδιο, αυξανομαι πληθο\n",
      "\n",
      "Topic #3:\n",
      "κλειδι τιμες, τιμη αγγλικη, χρονο αναζητηση, αντιστοιχες αγγλικα, αντιστοιχες αγγλικες\n",
      "\n",
      "Topic #4:\n",
      "αντιστοιχη αγγλικη, χωρο μνημη, αντιστοιχες αγγλικες, χρονο εκτελεση, τιμες αντιστοιχες\n",
      "\n",
      "Topic #5:\n",
      "χρονο χρειαζομαι, κλειδια τιμες, χρειαζομαι υπολογιστης, περισσοτερος χρονο, τιμη κλειδιο\n",
      "\n",
      "\n",
      " Fitting the NMF model (generalized Kullback-Leibler divergence) with tf-idf features, n_samples=2000 and n_features=1000...\n",
      "done in 0.185s.\n",
      "\n",
      "Topic #1:\n",
      "αντιστοιχες αγγλικες, χρονο εκτελεση, τιμες αντιστοιχες, αντιστοιχες αγγλικα, υπαρχω κλειδι\n",
      "\n",
      "Topic #2:\n",
      "χωρο μνημη, λιγοτερος χρονο, μνημη υπολογιστη, χρονο εκτελεση, λιγοτερη μνημη\n",
      "\n",
      "Topic #3:\n",
      "αντιστοιχη αγγλικη, συνδυασμο λιστας, ελληνικης γλωσσα, ελληνικα αγγλικες, αντιστοιχες αγγλικης\n",
      "\n",
      "Topic #4:\n",
      "μεταφρασει αγγλικα, δομη δεδομενα, χρονο προσπελαση, κλειδι αντιστοιχω, χρηση δυο\n",
      "\n",
      "Topic #5:\n",
      "χρηση λεξικων, αντιστοιχη τιμη, καλυτερος τροπο, προσπελαση δεδομενος, καταλληλη βιβλιοθηκη\n"
     ]
    }
   ],
   "source": [
    "#based on https://scikit-learn.org/stable/auto_examples/applications/plot_topics_extraction_with_nmf_lda.html\n",
    "\n",
    "from time import time\n",
    "\n",
    "t0 = time()\n",
    "n_samples = 2000\n",
    "n_features = 1000\n",
    "n_components_a = 5\n",
    "n_top_words = 20\n",
    "n_iterations = 800\n",
    "\n",
    "# Use tf-idf features for NMF.\n",
    "print(\"Extracting tf-idf features for NMF...\")\n",
    "tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,\n",
    "                                   #max_features=n_features,\n",
    "                                   ngram_range=(2, 2)\n",
    "                                )\n",
    "t0 = time()\n",
    "tfidf = tfidf_vectorizer.fit_transform(normalized)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))\n",
    "\n",
    "# Use tf (raw term count) features for LDA.\n",
    "print(\"Extracting tf features for LDA...\")\n",
    "tf_vectorizer = CountVectorizer(min_df=2, max_df=0.95,ngram_range=(2, 2))\n",
    "t0 = time()\n",
    "tf = tf_vectorizer.fit_transform(normalized)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))\n",
    "print()\n",
    "\n",
    "\n",
    "# Fit the LDA model\n",
    "print('\\n' * 2, \"Fitting LDA models with tf features, \"\n",
    "      \"n_samples=%d and n_features=%d...\"\n",
    "      % (n_samples, n_features))\n",
    "lda = LatentDirichletAllocation(n_components=n_components_a, max_iter=n_iterations,\n",
    "                                learning_method='online',\n",
    "                                learning_offset=50.,\n",
    "                                random_state=0,\n",
    "                               n_jobs = -1)\n",
    "t0 = time()\n",
    "lda.fit(tf)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))\n",
    "\n",
    "#tf_feature_names = tf_vectorizer.get_feature_names()\n",
    "#plot_top_words(lda, tf_feature_names, n_top_words, 'Topics in LDA model')\n",
    "print_topics(lda, tf_vectorizer, 5)\n",
    "\n",
    "##### ####\n",
    "\n",
    "# Fit the NMF model\n",
    "print('\\n' * 2, \"Fitting the NMF model (generalized Kullback-Leibler \"\n",
    "      \"divergence) with tf-idf features, n_samples=%d and n_features=%d...\"\n",
    "      % (n_samples, n_features))\n",
    "t0 = time()\n",
    "nmf = NMF(n_components=n_components_a, random_state=1,\n",
    "          beta_loss='kullback-leibler', solver='mu', max_iter=n_iterations, alpha=.01,\n",
    "          l1_ratio=.5).fit(tfidf)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))\n",
    "\n",
    "#tfidf_feature_names = tfidf_vectorizer.get_feature_names()\n",
    "#plot_top_words(nmf, tfidf_feature_names, n_top_words, 'Topics in NMF model (generalized Kullback-Leibler divergence)')\n",
    "print_topics(nmf, tfidf_vectorizer, 5)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_top_words(model, feature_names, n_top_words, title):\n",
    "    fig, axes = plt.subplots(2, 5, figsize=(30, 15), sharex=True)\n",
    "    axes = axes.flatten()\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        top_features_ind = topic.argsort()[:-n_top_words - 1:-1]\n",
    "        top_features = [feature_names[i] for i in top_features_ind]\n",
    "        weights = topic[top_features_ind]\n",
    "\n",
    "        ax = axes[topic_idx]\n",
    "        ax.barh(top_features, weights, height=0.7)\n",
    "        ax.set_title(f'Topic {topic_idx +1}',\n",
    "                     fontdict={'fontsize': 30})\n",
    "        ax.invert_yaxis()\n",
    "        ax.tick_params(axis='both', which='major', labelsize=20)\n",
    "        for i in 'top right left'.split():\n",
    "            ax.spines[i].set_visible(False)\n",
    "        fig.suptitle(title, fontsize=40)\n",
    "\n",
    "    plt.subplots_adjust(top=0.90, bottom=0.05, wspace=0.90, hspace=0.3)\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SeaNMF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Εκτέλεση SeaNMF\n",
    "\n",
    "GitHub Repository https://github.com/tshi04/SeaNMF\n",
    "\n",
    "Μόλις δημιουργήσουμε το αρχείο με τα προ-επεξεργασμένα έγγραφα, θα χρησιμοποιήσουμε αυτές τις εντολές στη γραμμή εντολών (CMD)\n",
    "1. python3 data_process.py\n",
    "2. python3 train.py --n_topics 10\n",
    "3. python3 vis_topic.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SeaNMF bigram\n",
    "docs = []\n",
    "for i in normalized:\n",
    "    bigrams = nltk.ngrams(i.split(), 2)\n",
    "    w = []\n",
    "    for grams in bigrams:\n",
    "        string_bigram = '_'.join(grams)\n",
    "        w.append(string_bigram)\n",
    "    doc = ' '.join(w)\n",
    "    docs.append(doc)\n",
    "#print(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('normalized_documents.txt', 'w', encoding='utf-8') as f:\n",
    "    for line in docs:\n",
    "        print(line, file=f)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GSDMM\n",
    "\n",
    "GitHub Repository: https://github.com/rwalk/gsdmm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uses the bigram docs generated in seanmf\n",
    "bigrams = []\n",
    "for doc in docs:\n",
    "    listt = doc.split(\" \")\n",
    "    bigrams.append(listt)\n",
    "#print(bigrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Voc size: 3014\n",
      "Number of documents: 106\n",
      "In stage 0: transferred 69 clusters with 5 clusters populated\n",
      "In stage 1: transferred 27 clusters with 5 clusters populated\n",
      "In stage 2: transferred 23 clusters with 5 clusters populated\n",
      "In stage 3: transferred 27 clusters with 5 clusters populated\n",
      "In stage 4: transferred 25 clusters with 5 clusters populated\n",
      "In stage 5: transferred 18 clusters with 5 clusters populated\n",
      "In stage 6: transferred 17 clusters with 5 clusters populated\n",
      "In stage 7: transferred 15 clusters with 5 clusters populated\n",
      "In stage 8: transferred 14 clusters with 5 clusters populated\n",
      "In stage 9: transferred 20 clusters with 5 clusters populated\n",
      "In stage 10: transferred 14 clusters with 5 clusters populated\n",
      "In stage 11: transferred 19 clusters with 5 clusters populated\n",
      "In stage 12: transferred 30 clusters with 5 clusters populated\n",
      "In stage 13: transferred 22 clusters with 5 clusters populated\n",
      "In stage 14: transferred 20 clusters with 5 clusters populated\n",
      "In stage 15: transferred 21 clusters with 5 clusters populated\n",
      "In stage 16: transferred 14 clusters with 5 clusters populated\n",
      "In stage 17: transferred 21 clusters with 5 clusters populated\n",
      "In stage 18: transferred 28 clusters with 5 clusters populated\n",
      "In stage 19: transferred 21 clusters with 5 clusters populated\n",
      "In stage 20: transferred 26 clusters with 5 clusters populated\n",
      "In stage 21: transferred 24 clusters with 5 clusters populated\n",
      "In stage 22: transferred 24 clusters with 5 clusters populated\n",
      "In stage 23: transferred 24 clusters with 5 clusters populated\n",
      "In stage 24: transferred 21 clusters with 5 clusters populated\n",
      "In stage 25: transferred 24 clusters with 5 clusters populated\n",
      "In stage 26: transferred 19 clusters with 5 clusters populated\n",
      "In stage 27: transferred 22 clusters with 5 clusters populated\n",
      "In stage 28: transferred 25 clusters with 5 clusters populated\n",
      "In stage 29: transferred 23 clusters with 5 clusters populated\n",
      "In stage 30: transferred 20 clusters with 5 clusters populated\n",
      "In stage 31: transferred 15 clusters with 5 clusters populated\n",
      "In stage 32: transferred 18 clusters with 5 clusters populated\n",
      "In stage 33: transferred 22 clusters with 5 clusters populated\n",
      "In stage 34: transferred 21 clusters with 5 clusters populated\n",
      "In stage 35: transferred 24 clusters with 5 clusters populated\n",
      "In stage 36: transferred 25 clusters with 5 clusters populated\n",
      "In stage 37: transferred 24 clusters with 5 clusters populated\n",
      "In stage 38: transferred 23 clusters with 5 clusters populated\n",
      "In stage 39: transferred 25 clusters with 5 clusters populated\n",
      "In stage 40: transferred 25 clusters with 5 clusters populated\n",
      "In stage 41: transferred 20 clusters with 5 clusters populated\n",
      "In stage 42: transferred 19 clusters with 5 clusters populated\n",
      "In stage 43: transferred 9 clusters with 5 clusters populated\n",
      "In stage 44: transferred 26 clusters with 5 clusters populated\n",
      "In stage 45: transferred 23 clusters with 5 clusters populated\n",
      "In stage 46: transferred 19 clusters with 5 clusters populated\n",
      "In stage 47: transferred 15 clusters with 5 clusters populated\n",
      "In stage 48: transferred 25 clusters with 5 clusters populated\n",
      "In stage 49: transferred 18 clusters with 5 clusters populated\n",
      "In stage 50: transferred 23 clusters with 5 clusters populated\n",
      "In stage 51: transferred 22 clusters with 5 clusters populated\n",
      "In stage 52: transferred 22 clusters with 5 clusters populated\n",
      "In stage 53: transferred 14 clusters with 5 clusters populated\n",
      "In stage 54: transferred 22 clusters with 5 clusters populated\n",
      "In stage 55: transferred 17 clusters with 5 clusters populated\n",
      "In stage 56: transferred 11 clusters with 5 clusters populated\n",
      "In stage 57: transferred 21 clusters with 5 clusters populated\n",
      "In stage 58: transferred 27 clusters with 5 clusters populated\n",
      "In stage 59: transferred 30 clusters with 5 clusters populated\n",
      "In stage 60: transferred 24 clusters with 5 clusters populated\n",
      "In stage 61: transferred 27 clusters with 5 clusters populated\n",
      "In stage 62: transferred 21 clusters with 5 clusters populated\n",
      "In stage 63: transferred 21 clusters with 5 clusters populated\n",
      "In stage 64: transferred 24 clusters with 5 clusters populated\n",
      "In stage 65: transferred 23 clusters with 5 clusters populated\n",
      "In stage 66: transferred 22 clusters with 5 clusters populated\n",
      "In stage 67: transferred 19 clusters with 5 clusters populated\n",
      "In stage 68: transferred 19 clusters with 5 clusters populated\n",
      "In stage 69: transferred 21 clusters with 5 clusters populated\n",
      "In stage 70: transferred 21 clusters with 5 clusters populated\n",
      "In stage 71: transferred 22 clusters with 5 clusters populated\n",
      "In stage 72: transferred 19 clusters with 5 clusters populated\n",
      "In stage 73: transferred 18 clusters with 5 clusters populated\n",
      "In stage 74: transferred 25 clusters with 5 clusters populated\n",
      "In stage 75: transferred 25 clusters with 5 clusters populated\n",
      "In stage 76: transferred 19 clusters with 5 clusters populated\n",
      "In stage 77: transferred 20 clusters with 5 clusters populated\n",
      "In stage 78: transferred 22 clusters with 5 clusters populated\n",
      "In stage 79: transferred 20 clusters with 5 clusters populated\n",
      "In stage 80: transferred 20 clusters with 5 clusters populated\n",
      "In stage 81: transferred 20 clusters with 5 clusters populated\n",
      "In stage 82: transferred 15 clusters with 5 clusters populated\n",
      "In stage 83: transferred 16 clusters with 5 clusters populated\n",
      "In stage 84: transferred 24 clusters with 5 clusters populated\n",
      "In stage 85: transferred 23 clusters with 5 clusters populated\n",
      "In stage 86: transferred 21 clusters with 5 clusters populated\n",
      "In stage 87: transferred 13 clusters with 5 clusters populated\n",
      "In stage 88: transferred 18 clusters with 5 clusters populated\n",
      "In stage 89: transferred 22 clusters with 5 clusters populated\n",
      "In stage 90: transferred 22 clusters with 5 clusters populated\n",
      "In stage 91: transferred 17 clusters with 5 clusters populated\n",
      "In stage 92: transferred 19 clusters with 5 clusters populated\n",
      "In stage 93: transferred 22 clusters with 5 clusters populated\n",
      "In stage 94: transferred 23 clusters with 5 clusters populated\n",
      "In stage 95: transferred 18 clusters with 5 clusters populated\n",
      "In stage 96: transferred 19 clusters with 5 clusters populated\n",
      "In stage 97: transferred 15 clusters with 5 clusters populated\n",
      "In stage 98: transferred 15 clusters with 5 clusters populated\n",
      "In stage 99: transferred 20 clusters with 5 clusters populated\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from gsdmm import MovieGroupProcess\n",
    "\n",
    "vocab = set(x for doc in bigrams for x in doc)\n",
    "#print(vocab)\n",
    "n_terms = len(vocab)\n",
    "print(\"Voc size:\", n_terms)\n",
    "print(\"Number of documents:\", len(bigrams))\n",
    "\n",
    "mgp = MovieGroupProcess(K=5, alpha=0.1, beta=0.1, n_iters=100)\n",
    "\n",
    "vocab = set(x for doc in bigrams for x in doc)\n",
    "n_terms = len(vocab)\n",
    "n_docs = len(bigrams)\n",
    "\n",
    "# Fit the model on the data given the chosen seeds\n",
    "y = mgp.fit(bigrams, n_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_words(cluster_word_distribution, top_cluster, values):\n",
    "    for cluster in top_cluster:\n",
    "        sort_dicts =sorted(cluster_word_distribution[cluster].items(), key=lambda k: k[1], reverse=True)[:values]\n",
    "        print('Topic %s : %s'%(cluster,sort_dicts))\n",
    "        print(' — — — — — — — — — ')\n",
    "        print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents per topics : [24 19 27 17 19]\n",
      "********************\n",
      "Most important clusters (by number of docs inside): [2 0 4 1 3]\n",
      "********************\n",
      "Topic 2 : [('αντιστοιχη_αγγλικη', 5), ('χρονο_χρειαζομαι', 4), ('εκτελεση_προγραμματο', 3), ('κλειδια_τιμες', 3), ('τιμες_αντιστοιχη', 3)]\n",
      " — — — — — — — — — \n",
      "\n",
      "Topic 0 : [('αντιστοιχες_αγγλικες', 8), ('αντιστοιχη_αγγλικη', 5), ('συνδυασμο_λιστας', 5), ('τιμες_αντιστοιχες', 5), ('κλειδιος_τιμες', 4)]\n",
      " — — — — — — — — — \n",
      "\n",
      "Topic 4 : [('αντιστοιχη_τιμη', 4), ('αγγλικα_πλεονεκτημο', 3), ('αγγλικες_τιμες', 2), ('ελληνικος_λεξεο', 2), ('βρισκω_αντιστοιχη', 2)]\n",
      " — — — — — — — — — \n",
      "\n",
      "Topic 1 : [('χωρο_μνημη', 9), ('λιγοτερος_χρονο', 6), ('χρονο_εκτελεση', 4), ('μνημη_υπολογιστη', 3), ('λιγοτερος_χωρο', 3)]\n",
      " — — — — — — — — — \n",
      "\n",
      "Topic 3 : [('χρονο_εντοπισμο', 5), ('χρονο_προσπελαση', 5), ('αυξανομαι_χρονο', 5), ('αντιστοιχες_αγγλικες', 4), ('χρηση_λιστας', 4)]\n",
      " — — — — — — — — — \n",
      "\n"
     ]
    }
   ],
   "source": [
    "doc_count = np.array(mgp.cluster_doc_count)\n",
    "print('Number of documents per topics :', doc_count)\n",
    "print('*'*20)\n",
    "\n",
    "# Topics sorted by document inside\n",
    "top_index = doc_count.argsort()[-10:][::-1]\n",
    "print('Most important clusters (by number of docs inside):', top_index)\n",
    "print('*'*20)\n",
    "\n",
    "\n",
    "# Show the top 5 words by cluster, it helps to make the topic_dict below\n",
    "top_words(mgp.cluster_word_distribution, top_index, 5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}