Pujaa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import urllib\n",
    "import json\n",
    "import pandas as pd\n",
    "import base64\n",
    "import numpy as np\n",
    "import random\n",
    "import re, nltk\n",
    "from sklearn.feature_extraction.text import CountVectorizer        \n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def json_numpy_obj_hook(dct):\n",
    "    \"\"\"Decodes a previously encoded numpy ndarray with proper shape and dtype.\n",
    "    :param dct: (dict) json encoded ndarray\n",
    "    :return: (ndarray) if input was an encoded ndarray\n",
    "    \"\"\"\n",
    "    if isinstance(dct, dict) and '__ndarray__' in dct:\n",
    "        data = base64.b64decode(dct['__ndarray__'])\n",
    "        return np.frombuffer(data, dct['dtype']).reshape(dct['shape'])\n",
    "    return dct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('new_transcripts_2.json') as data_file:    \n",
    "    data = json.load(data_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#JSON to list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "beer_text_sentiment = []\n",
    "for beer in data:\n",
    "    for review in data[beer]:\n",
    "            beer_text_sentiment.append((beer,review['text'],review['overall']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1586614"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(beer_text_sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def training_data(beer_input):\n",
    "    random.shuffle(beer_text_sentiment)\n",
    "    training_number = int(len(beer_text_sentiment)*.2)\n",
    "    training_text = beer_text_sentiment[:training_number]\n",
    "    train_data_df = pd.DataFrame(training_text).convert_objects(convert_numeric=True)\n",
    "    train_data_df.columns = [\"Beer\", \"Text\", \"Sentiment\"]\n",
    "    num = train_data_df._get_numeric_data()\n",
    "    num[num <= 3] = 0\n",
    "    num[num > 3] = 1\n",
    "    return train_data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testing_data(beer_input):\n",
    "    training_number = int(len(beer_text_sentiment)*.2)\n",
    "    testing_text = beer_text_sentiment[training_number:]\n",
    "    test_data_df = pd.DataFrame(testing_text)\n",
    "    test_data_df.columns = [\"Beer\", \"Text\", \"Sentiment\"]\n",
    "    test_data_df = test_data_df.drop('Sentiment', 1)\n",
    "    return test_data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Python/2.7/site-packages/ipykernel-4.2.2-py2.7.egg/ipykernel/__main__.py:5: FutureWarning: convert_objects is deprecated.  Use the data-type specific converters pd.to_datetime, pd.to_timedelta and pd.to_numeric.\n"
     ]
    }
   ],
   "source": [
    "train_data_df = training_data(beer_text_sentiment)\n",
    "test_data_df = testing_data(beer_text_sentiment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Preparing a corpus\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# based on http://www.cs.duke.edu/courses/spring14/compsci290/assignments/lab02.html\n",
    "\n",
    "stemmer = PorterStemmer()\n",
    "def stem_tokens(tokens, stemmer):\n",
    "    stemmed = []\n",
    "    for item in tokens:\n",
    "        stemmed.append(stemmer.stem(item))\n",
    "    return stemmed\n",
    "\n",
    "def tokenize(text):\n",
    "    # remove non letters\n",
    "    text = re.sub(\"[^a-zA-Z]\", \" \", text)\n",
    "    # tokenize\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    # stem\n",
    "    stems = stem_tokens(tokens, stemmer)\n",
    "    return stems\n",
    "\n",
    "vectorizer = CountVectorizer(\n",
    "    analyzer = 'word',\n",
    "    tokenizer = tokenize,\n",
    "    lowercase = True,\n",
    "    stop_words = 'english',\n",
    "    max_features = 85\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpus_data_features = vectorizer.fit_transform(train_data_df.Text.tolist() + test_data_df.Text.tolist())\n",
    "corpus_data_features_nd = corpus_data_features.toarray()\n",
    "vocab = vectorizer.get_feature_names()\n",
    "# Sum up the counts of each vocabulary word\n",
    "dist = np.sum(corpus_data_features_nd, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#A bag-of-words linear classifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bag_of_words_linear_classifier(training_data):\n",
    "    # remember that corpus_data_features_nd contains all of our \n",
    "    # original train and test data, so we need to exclude\n",
    "    # the unlabeled test entries\n",
    "    X_train, X_test, y_train, y_test  = train_test_split(\n",
    "            corpus_data_features_nd[0:len(train_data_df)], \n",
    "            train_data_df.Sentiment,\n",
    "            train_size=0.80, \n",
    "            random_state=1234)\n",
    "    #Now we are ready to train our classifier.\n",
    "    log_model = LogisticRegression()\n",
    "    log_model = log_model.fit(X=X_train, y=y_train)\n",
    "    #Now we use the classifier to label our evaluation set. \n",
    "    #We can use either predict for classes or predict_proba for probabilities.\n",
    "    y_pred = log_model.predict(X_test)\n",
    "    #Finally, we can re-train our model with all the training data and use it for sentiment \n",
    "    #classification with the original (unlabeled) test set.\n",
    "    # train classifier\n",
    "    log_model = LogisticRegression()\n",
    "    log_model = log_model.fit(X=corpus_data_features_nd[0:len(train_data_df)], y=train_data_df.Sentiment)\n",
    "    # get predictions\n",
    "    test_pred = log_model.predict(corpus_data_features_nd[len(train_data_df):])\n",
    "    return test_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_pred = bag_of_words_linear_classifier(train_data_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_pred[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Sample and Print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "beer_sentiment  = dict.fromkeys(test_data_df.Beer)\n",
    "for x in xrange(len(test_data_df.Beer)):\n",
    "    beer_sentiment[test_data_df.Beer[x]] = test_pred[x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "beer_sentiment[\"Keystone Light\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}