baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:29:20.494976Z",
     "start_time": "2018-11-17T19:29:20.485322Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "import gensim\n",
    "import nltk\n",
    "import functools as ft\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.metrics import accuracy_score,precision_recall_fscore_support as score\n",
    "script_path = os.path.dirname(os.path.abspath('__file__'))\n",
    "stopwords = set(\n",
    "    ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves',\n",
    "     'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their',\n",
    "     'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was',\n",
    "     'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the',\n",
    "     'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against',\n",
    "     'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in',\n",
    "     'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why',\n",
    "     'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only',\n",
    "     'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now'])\n",
    "STOP=stopwords\n",
    "PATH_TO_WORD2VEC =\"GoogleNews-vectors-negative300.bin\"\n",
    "PATH_TO_GLOVE = \"glove.840B.300d.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:30:01.838393Z",
     "start_time": "2018-11-17T19:29:20.498504Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading word2vec model\n",
      "Done. 3000000  words loaded!\n"
     ]
    }
   ],
   "source": [
    "print(\"Loading word2vec model\")\n",
    "word2vec = gensim.models.KeyedVectors.load_word2vec_format(PATH_TO_WORD2VEC, binary=True)\n",
    "print(\"Done.\",len(word2vec.vocab),\" words loaded!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:34.495011Z",
     "start_time": "2018-11-17T19:30:01.841724Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading glove model\n",
      "Done. 1176051  words loaded!\n"
     ]
    }
   ],
   "source": [
    "print(\"Loading glove model\")\n",
    "df = pd.read_csv('glove.840B.300d.txt', sep=\" \", quoting=3, header=None, index_col=0)\n",
    "glove = {key: val.values for key, val in df.T.items()}\n",
    "print(\"Done.\",len(glove),\" words loaded!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.055709Z",
     "start_time": "2018-11-17T19:31:34.498408Z"
    }
   },
   "outputs": [],
   "source": [
    "PATH_TO_FREQUENCIES_FILE = \"frequencies.tsv\"\n",
    "PATH_TO_DOC_FREQUENCIES_FILE = \"doc_frequencies.tsv\"\n",
    "##Sentence class where we keep both the raw sentence and the tokenized sentence\n",
    "class Sentence:  \n",
    "    def __init__(self, sentence):\n",
    "        self.raw = sentence\n",
    "        normalized_sentence = sentence.replace(\"‘\", \"'\").replace(\"’\", \"'\")\n",
    "        self.tokens = [t.lower() for t in nltk.word_tokenize(normalized_sentence)]\n",
    "        self.tokens_without_stop = [t for t in self.tokens if t not in STOP]\n",
    "        \n",
    "## tsv file reader\n",
    "def read_tsv(f):\n",
    "    frequencies = {}\n",
    "    with open(f) as tsv:\n",
    "        tsv_reader = csv.reader(tsv, delimiter=\"\\t\")\n",
    "        for row in tsv_reader: \n",
    "            frequencies[row[0]] = int(row[1])\n",
    "        \n",
    "    return frequencies\n",
    "\n",
    "## In order to compute weighted averages of word embeddings later, we are going to load a file \n",
    "## with word frequencies. These word frequencies have been collected from Wikipedia and saved \n",
    "## in a tab-separated file.\n",
    "\n",
    "frequencies = read_tsv(PATH_TO_FREQUENCIES_FILE)\n",
    "doc_frequencies = read_tsv(PATH_TO_DOC_FREQUENCIES_FILE)\n",
    "doc_frequencies[\"NUM_DOCS\"] = 1288431       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.068527Z",
     "start_time": "2018-11-17T19:31:41.057310Z"
    }
   },
   "outputs": [],
   "source": [
    "## Simplest way of computing sentence embeddings: just take the embeddings of the words \n",
    "## in the sentence (minus the stopwords), and compute their average, weighted by the sentence \n",
    "## frequency of each word. We then use the cosine similarity to calculate the similarity between \n",
    "## two sentence embeddings.\n",
    "def avg_embedding_sim(sentences1, sentences2, model=None, use_stoplist=False, doc_freqs=None): \n",
    "\n",
    "    if doc_freqs is not None:\n",
    "        N = doc_freqs[\"NUM_DOCS\"]\n",
    "    \n",
    "    sims = []\n",
    "    for (sent1, sent2) in zip(sentences1, sentences2):\n",
    "    \n",
    "        tokens1 = sent1.tokens_without_stop if use_stoplist else sent1.tokens\n",
    "        tokens2 = sent2.tokens_without_stop if use_stoplist else sent2.tokens\n",
    "\n",
    "        tokens1 = [token for token in tokens1 if token in model]\n",
    "        tokens2 = [token for token in tokens2 if token in model]\n",
    "        \n",
    "        if len(tokens1) == 0 or len(tokens2) == 0:\n",
    "            sims.append(0)\n",
    "            continue\n",
    "        \n",
    "        tokfreqs1 = Counter(tokens1)\n",
    "        tokfreqs2 = Counter(tokens2)\n",
    "        \n",
    "        weights1 = [tokfreqs1[token] * math.log(N/(doc_freqs.get(token, 0)+1)) \n",
    "                    for token in tokfreqs1] if doc_freqs else None\n",
    "        weights2 = [tokfreqs2[token] * math.log(N/(doc_freqs.get(token, 0)+1)) \n",
    "                    for token in tokfreqs2] if doc_freqs else None\n",
    "                \n",
    "        embedding1 = np.average([model[token] for token in tokfreqs1], axis=0, weights=weights1).reshape(1, -1)\n",
    "        embedding2 = np.average([model[token] for token in tokfreqs2], axis=0, weights=weights2).reshape(1, -1)\n",
    "\n",
    "        sim = cosine_similarity(embedding1, embedding2)[0][0]\n",
    "        sims.append(sim)\n",
    "\n",
    "    return sims\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.082952Z",
     "start_time": "2018-11-17T19:31:41.071613Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.40206002214242403\n",
      "0.8585334221718391\n"
     ]
    }
   ],
   "source": [
    "### Unit testing...\n",
    "methods = [(\"AVG-W2V-TFIDF-STOP\", ft.partial(avg_embedding_sim, model=word2vec, use_stoplist=True, doc_freqs=doc_frequencies)),\n",
    "              (\"AVG-GLOVE-TFIDF-STOP\", ft.partial(avg_embedding_sim, model=glove, use_stoplist=True, doc_freqs=doc_frequencies))]\n",
    "\n",
    "s1=\"He was right, of course, but his harsh words were like salt on a raw wound.\"\n",
    "s2=\"In truth, the raw information funneled to us was transmitted as received after passing through our office.\"\n",
    "sentences1 = [Sentence(s1)]\n",
    "sentences2 = [Sentence(s2)]\n",
    "for _,method in methods:\n",
    "    sim = method(sentences1, sentences2)[0]\n",
    "    print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.095049Z",
     "start_time": "2018-11-17T19:31:41.085109Z"
    }
   },
   "outputs": [],
   "source": [
    "def jaccard(t1, t2,stopwords=[]): \n",
    "    tokenize = lambda t: set([w for w in t.split() if (w not in stopwords)])\n",
    "    t1, t2 = tokenize(t1), tokenize(t2)\n",
    "    c = t1.intersection(t2)\n",
    "    return len(c) / (len(t1) + len(t2) - len(c))\n",
    "\n",
    "\n",
    "def dice(t1, t2, stopwords=[]):\n",
    "    tokenize = lambda t: set([w for w in t.split() if (w not in stopwords)])\n",
    "    t1, t2 = tokenize(t1), tokenize(t2)\n",
    "    return 2. * len(t1.intersection(t2)) / (len(t1) + len(t2))\n",
    "\n",
    "def fd(counts):\n",
    "    '''Given a list of occurrences (e.g., [1,1,1,2]), return a dictionary of frequencies (e.g., {1:3, 2:1}.)'''\n",
    "    d = {}\n",
    "    for i in counts: d[i] = d[i] + 1 if i in d else 1\n",
    "    return d\n",
    "\n",
    "\n",
    "freq_rank = lambda d: sorted(d, key=d.get, reverse=True)\n",
    "'''Given a map, return ranked the keys based on their values.'''\n",
    "\n",
    "\n",
    "def fd2(counts):\n",
    "    '''Given a list of 2-uplets (e.g., [(a,pos), (a,pos), (a,neg), ...]), form a dict of frequencies of specific items (e.g., {a:{pos:2, neg:1}, ...}).'''\n",
    "    d = {}\n",
    "    for i in counts:\n",
    "        # If the first element of the 2-uplet is not in the map, add it.\n",
    "        if i[0] in d:\n",
    "            if i[1] in d[i[0]]:\n",
    "                d[i[0]][i[1]] += 1\n",
    "            else:\n",
    "                d[i[0]][i[1]] = 1\n",
    "        else:\n",
    "            d[i[0]] = {i[1]: 1}\n",
    "    return d\n",
    "\n",
    "def replacewith(input_str, pattern, replaceWith): \n",
    "    return input_str.replace(pattern, replaceWith) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.107573Z",
     "start_time": "2018-11-17T19:31:41.100654Z"
    }
   },
   "outputs": [],
   "source": [
    "def extract_data(data_file,polarity_map,max_len=80):\n",
    "    data = open(data_file,'r').readlines()\n",
    "    corpus=[]\n",
    "    for i in range(0,len(data),3):\n",
    "        temp={}\n",
    "        if len(data[i].split(\"\\n\")[0])>=max_len:\n",
    "            temp[\"sentence\"] = data[i].split(\"\\n\")[0]\n",
    "            temp[\"aspect_term\"] = data[i+1].split(\"\\n\")[0]\n",
    "            temp[\"polarity\"] = polarity_map[data[i+2].split(\"\\n\")[0]]\n",
    "            corpus.append(temp)\n",
    "    return corpus \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.132451Z",
     "start_time": "2018-11-17T19:31:41.109788Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training sentences:  3699\n",
      "Number of testing sentences:  1134\n"
     ]
    }
   ],
   "source": [
    "polarity_map={\"1\":\"positive\",\"-1\":\"negative\",\"0\":\"neutral\"}\n",
    "train_data=script_path+\"/rest_2014_train.txt\"\n",
    "test_data=script_path+\"/rest_2014_test.txt\"\n",
    "train= extract_data(train_data,polarity_map,max_len=0)\n",
    "test = extract_data(test_data,polarity_map,max_len=0)\n",
    "print(\"Number of training sentences: \",len(train))\n",
    "print(\"Number of testing sentences: \",len(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.798064Z",
     "start_time": "2018-11-17T19:31:41.133759Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ASPECT EXTRACTION .....\n",
      "Accuracy=0.642857\n"
     ]
    }
   ],
   "source": [
    "correct=[i[\"aspect_term\"].lower() for i in test]\n",
    "predicted=[]\n",
    "train_aspect=[i[\"aspect_term\"].lower() for i in train]\n",
    "for te_sent in test:\n",
    "    temp=[]\n",
    "    replaced_test_sent = replacewith(te_sent[\"sentence\"],\"$T$\",te_sent[\"aspect_term\"])\n",
    "    for aspect in train_aspect:\n",
    "        if aspect in replaced_test_sent:\n",
    "            temp.append(aspect)\n",
    "    if not temp:\n",
    "        temp.append(\"$?$\") ##takes care of unknown aspects which are not present in train aspect list\n",
    "    predicted.append(temp)\n",
    "for x in range(len(predicted)):\n",
    "    for aspect in predicted[x]:\n",
    "        if aspect==correct[x]:\n",
    "            predicted[x]=aspect\n",
    "            break\n",
    "\n",
    "for x in range(len(predicted)):\n",
    "    if type(predicted[x])==list:\n",
    "        predicted[x] = \"$?$\"\n",
    "\n",
    "print(\"ASPECT EXTRACTION .....\")\n",
    "print(\"Accuracy={:.6f}\".format(accuracy_score(correct, predicted)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.805095Z",
     "start_time": "2018-11-17T19:31:41.799385Z"
    }
   },
   "outputs": [],
   "source": [
    "train_aspect_polarity_freq=fd2([(i[\"aspect_term\"],i[\"polarity\"]) for i in train])\n",
    "most_frequent_polarity = freq_rank(fd([i[\"polarity\"] for i in train]))[0]\n",
    "#print(most_frequent_polarity)\n",
    "#print(train_aspect_polarity_freq)\n",
    "params = [train_aspect_polarity_freq,most_frequent_polarity]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T19:31:41.821724Z",
     "start_time": "2018-11-17T19:31:41.807317Z"
    }
   },
   "outputs": [],
   "source": [
    "def retrieved_topk(tr_data,sentence,aspect,k,label,sim_measure):\n",
    "    if label==\"None\":\n",
    "        neighbors = dict([(index,sim_measure(sentence,replacewith(tr_sent[\"sentence\"],\"$T$\",tr_sent[\"aspect_term\"]),stopwords)) \n",
    "                      for index,tr_sent in enumerate(tr_data) if aspect.lower()==tr_sent[\"aspect_term\"].lower()])\n",
    "    else:\n",
    "        neighbors = dict([(index,sim_measure([Sentence(sentence)],[Sentence(replacewith(tr_sent[\"sentence\"],\"$T$\",tr_sent[\"aspect_term\"]))])[0]) \n",
    "                      for index,tr_sent in enumerate(tr_data) if aspect.lower()==tr_sent[\"aspect_term\"].lower()])\n",
    "    ranked = freq_rank(neighbors)\n",
    "    topk = [tr_data[i] for i in ranked[:k]]\n",
    "    return freq_rank(fd([i[\"polarity\"] for i in topk]))\n",
    "\n",
    "def polarity_determination(tr_data,params,sentence,aspect,label,sim_measure,k=5):\n",
    "    train_aspect_polarity_freq=params[0]\n",
    "    most_frequent_polarity=params[1]\n",
    "    train_aspect_polarity_freq=fd2([(i[\"aspect_term\"],i[\"polarity\"]) for i in train])\n",
    "    most_frequent_polarity = freq_rank(fd([i[\"polarity\"] for i in train]))[0]\n",
    "    if aspect not in train_aspect_polarity_freq:\n",
    "        return most_frequent_polarity\n",
    "    else:\n",
    "        polarities = retrieved_topk(tr_data,sentence,aspect,k,label,sim_measure)\n",
    "        if polarities:\n",
    "            return polarities[0]\n",
    "        else:\n",
    "            return most_frequent_polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T20:06:28.891637Z",
     "start_time": "2018-11-17T19:31:41.823854Z"
    }
   },
   "outputs": [],
   "source": [
    "correct_polarity=[i[\"polarity\"] for i in test]\n",
    "ks=[5,6,7,8,9,10,11,12,13,14,15,26,17,18,19,20]\n",
    "max_acc=0\n",
    "optimal_k=0\n",
    "optimal_results=None\n",
    "sim_measures=[(\"None\",jaccard),\n",
    "              (\"None\",dice),\n",
    "              (\"AVG-W2V-TFIDF-STOP\", ft.partial(avg_embedding_sim, model=word2vec, use_stoplist=True, doc_freqs=doc_frequencies)),\n",
    "              (\"AVG-GLOVE-TFIDF-STOP\", ft.partial(avg_embedding_sim, model=glove, use_stoplist=True, doc_freqs=doc_frequencies))]\n",
    "for label,sim_measure in sim_measures:    \n",
    "    for k in ks:    \n",
    "        predicted_polarity=[]\n",
    "        for te_sent in test:\n",
    "            sent = replacewith(te_sent[\"sentence\"],\"$T$\",te_sent[\"aspect_term\"])\n",
    "            aspect = te_sent[\"aspect_term\"]\n",
    "            predicted_polarity.append(polarity_determination(train,params,sent,aspect,label,sim_measure,k))\n",
    "        acc = accuracy_score(correct_polarity, predicted_polarity)\n",
    "        if acc>max_acc:\n",
    "            max_acc=acc\n",
    "            optimal_k=k\n",
    "            optimal_results=predicted_polarity\n",
    "predicted_polarity=optimal_results        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-17T20:06:28.904008Z",
     "start_time": "2018-11-17T20:06:28.893553Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ASPECT POLARITY DETECTION .....\n",
      "Optimal K:  12\n",
      "P_positive = 0.696162 ,P_negative = 0.481818 ,P_neutral = 0.430233\n",
      "R_positive = 0.896978 ,R_negative = 0.252381 ,R_neutral = 0.188776\n",
      "F_positive = 0.783914 ,F_negative = 0.331250 ,F_neutral = 0.262411\n",
      "Accuracy=0.655203\n"
     ]
    }
   ],
   "source": [
    "print(\"ASPECT POLARITY DETECTION .....\")\n",
    "print(\"Optimal K: \",optimal_k)\n",
    "p, r, f, _ = score(correct_polarity,predicted_polarity,average=None,labels=[\"positive\",\"negative\",\"neutral\"])\n",
    "print(\"P_positive = {:.6f} ,P_negative = {:.6f} ,P_neutral = {:.6f}\".format(p[0],p[1],p[2]))\n",
    "print(\"R_positive = {:.6f} ,R_negative = {:.6f} ,R_neutral = {:.6f}\".format(r[0],r[1],r[2]))\n",
    "print(\"F_positive = {:.6f} ,F_negative = {:.6f} ,F_neutral = {:.6f}\".format(f[0],f[1],f[2]))\n",
    "#print(\"Acc_positive = {:.6f} ,Acc_negative = {:.6f} ,Acc_neutral = {:.6f}\".format(acc[0],acc[1],acc[2]))\n",
    "print(\"Accuracy={:.6f}\".format(accuracy_score(correct_polarity, predicted_polarity)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "IPython (Python 3)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}