Comprehensive Hands on Guide to Twitter Sentiment Analysis with dataset and code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd \n",
    "import numpy as np \n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns\n",
    "import string\n",
    "import nltk\n",
    "import warnings \n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train  = pd.read_csv('train_E6oV3lV.csv')\n",
    "test = pd.read_csv('test_tweets_anuFYb8.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing Twitter Handles (@user) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combi = train.append(test, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_pattern(input_txt, pattern):\n",
    "    r = re.findall(pattern, input_txt)\n",
    "    for i in r:\n",
    "        input_txt = re.sub(i, '', input_txt)\n",
    "        \n",
    "    return input_txt    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove twitter handles (@user)\n",
    "combi['tidy_tweet'] = np.vectorize(remove_pattern)(combi['tweet'], \"@[\\w]*\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing Punctuations, Numbers, and Special Characters\n",
    "\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove special characters, numbers, punctuations\n",
    "combi['tidy_tweet'] = combi['tidy_tweet'].str.replace(\"[^a-zA-Z#]\", \" \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing Short Words "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combi['tidy_tweet'] = combi['tidy_tweet'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combi.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_tweet = combi['tidy_tweet'].apply(lambda x: x.split())\n",
    "tokenized_tweet.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stemming "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.porter import *\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "tokenized_tweet = tokenized_tweet.apply(lambda x: [stemmer.stem(i) for i in x]) # stemming\n",
    "tokenized_tweet.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(tokenized_tweet)):\n",
    "    tokenized_tweet[i] = ' '.join(tokenized_tweet[i])\n",
    "\n",
    "combi['tidy_tweet'] = tokenized_tweet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Story Generation and Visualization from Tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Understanding the common words used in the tweets: WordCloud "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words = ' '.join([text for text in combi['tidy_tweet']])\n",
    "from wordcloud import WordCloud\n",
    "wordcloud = WordCloud(width=800, height=500, random_state=21, max_font_size=110).generate(all_words)\n",
    "\n",
    "plt.figure(figsize=(10, 7))\n",
    "plt.imshow(wordcloud, interpolation=\"bilinear\")\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Words in non racist/sexist tweets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "normal_words =' '.join([text for text in combi['tidy_tweet'][combi['label'] == 0]])\n",
    "\n",
    "wordcloud = WordCloud(width=800, height=500, random_state=21, max_font_size=110).generate(normal_words)\n",
    "plt.figure(figsize=(10, 7))\n",
    "plt.imshow(wordcloud, interpolation=\"bilinear\")\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Racist/Sexist Tweets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "negative_words = ' '.join([text for text in combi['tidy_tweet'][combi['label'] == 1]])\n",
    "wordcloud = WordCloud(width=800, height=500,\n",
    "random_state=21, max_font_size=110).generate(negative_words)\n",
    "plt.figure(figsize=(10, 7))\n",
    "plt.imshow(wordcloud, interpolation=\"bilinear\")\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Understanding the impact of Hashtags on tweets sentiment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to collect hashtags\n",
    "def hashtag_extract(x):\n",
    "    hashtags = []\n",
    "    # Loop over the words in the tweet\n",
    "    for i in x:\n",
    "        ht = re.findall(r\"#(\\w+)\", i)\n",
    "        hashtags.append(ht)\n",
    "\n",
    "    return hashtags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extracting hashtags from non racist/sexist tweets\n",
    "\n",
    "HT_regular = hashtag_extract(combi['tidy_tweet'][combi['label'] == 0])\n",
    "\n",
    "# extracting hashtags from racist/sexist tweets\n",
    "HT_negative = hashtag_extract(combi['tidy_tweet'][combi['label'] == 1])\n",
    "\n",
    "# unnesting list\n",
    "HT_regular = sum(HT_regular,[])\n",
    "HT_negative = sum(HT_negative,[])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Non-Racist/Sexist Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = nltk.FreqDist(HT_regular)\n",
    "d = pd.DataFrame({'Hashtag': list(a.keys()),\n",
    "                  'Count': list(a.values())})\n",
    "# selecting top 10 most frequent hashtags     \n",
    "d = d.nlargest(columns=\"Count\", n = 10) \n",
    "plt.figure(figsize=(16,5))\n",
    "ax = sns.barplot(data=d, x= \"Hashtag\", y = \"Count\")\n",
    "ax.set(ylabel = 'Count')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Racist/Sexist Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = nltk.FreqDist(HT_negative)\n",
    "e = pd.DataFrame({'Hashtag': list(b.keys()), 'Count': list(b.values())})\n",
    "# selecting top 10 most frequent hashtags\n",
    "e = e.nlargest(columns=\"Count\", n = 10)   \n",
    "plt.figure(figsize=(16,5))\n",
    "ax = sns.barplot(data=e, x= \"Hashtag\", y = \"Count\")\n",
    "ax.set(ylabel = 'Count')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting Features from Cleaned Tweets "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bag-of-Words Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "bow_vectorizer = CountVectorizer(max_df=0.90, min_df=2, max_features=1000, stop_words='english')\n",
    "# bag-of-words feature matrix\n",
    "bow = bow_vectorizer.fit_transform(combi['tidy_tweet'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TF-IDF Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(max_df=0.90, min_df=2, max_features=1000, stop_words='english')\n",
    "# TF-IDF feature matrix\n",
    "tfidf = tfidf_vectorizer.fit_transform(combi['tidy_tweet'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Building: Sentiment Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Building model using Bag-of-Words features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "train_bow = bow[:31962,:]\n",
    "test_bow = bow[31962:,:]\n",
    "\n",
    "# splitting data into training and validation set\n",
    "xtrain_bow, xvalid_bow, ytrain, yvalid = train_test_split(train_bow, train['label'], random_state=42, test_size=0.3)\n",
    "\n",
    "lreg = LogisticRegression()\n",
    "lreg.fit(xtrain_bow, ytrain) # training the model\n",
    "\n",
    "prediction = lreg.predict_proba(xvalid_bow) # predicting on the validation set\n",
    "prediction_int = prediction[:,1] >= 0.3 # if prediction is greater than or equal to 0.3 than 1 else 0\n",
    "prediction_int = prediction_int.astype(np.int)\n",
    "\n",
    "f1_score(yvalid, prediction_int) # calculating f1 score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pred = lreg.predict_proba(test_bow)\n",
    "test_pred_int = test_pred[:,1] >= 0.3\n",
    "test_pred_int = test_pred_int.astype(np.int)\n",
    "test['label'] = test_pred_int\n",
    "submission = test[['id','label']]\n",
    "submission.to_csv('sub_lreg_bow.csv', index=False) # writing data to a CSV file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Building model using TF-IDF features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tfidf = tfidf[:31962,:]\n",
    "test_tfidf = tfidf[31962:,:]\n",
    "\n",
    "xtrain_tfidf = train_tfidf[ytrain.index]\n",
    "xvalid_tfidf = train_tfidf[yvalid.index]\n",
    "\n",
    "lreg.fit(xtrain_tfidf, ytrain)\n",
    "\n",
    "prediction = lreg.predict_proba(xvalid_tfidf)\n",
    "prediction_int = prediction[:,1] >= 0.3\n",
    "prediction_int = prediction_int.astype(np.int)\n",
    "\n",
    "f1_score(yvalid, prediction_int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}