scripts_&_analysis/2_load_json_to_df.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading `lyricsgenius` JSONs to a Dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This script compiles the `lyricsgenius` JSON output files from [`1_lyricsgenius_request.ipynb`](https://github.com/Data-Science-for-Linguists-2022/Outsider-Music-Linguistic-Analysis/blob/main/scripts_%26_analysis/1_lyricsgenius_requests.ipynb), which are kept in the `data/musicians` path, into one dataframe.  Data cleaning and analysis is done in the next script, [`3_data_cleaning_and_exploration`](https://github.com/Data-Science-for-Linguists-2022/Outsider-Music-Linguistic-Analysis/blob/main/scripts_%26_analysis/3_data_cleaning_and_exploration.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os, json\n",
    "from os import listdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set path  -  this directory is not in the GitHub repo. files were moved here manually after lyricsgenius_request.ipynb\n",
    "path = \"../data/musicianfiles/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load files into list and prepare empty objects for loading to the df\n",
    "files = [f for f in listdir(path)]\n",
    "rough_df = pd.DataFrame()\n",
    "unsuccessful = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loop through JSON files and add all songs from each file to the dataframe\n",
    "for f in files:\n",
    "    with open(path + f) as filename:\n",
    "        fileLoad = json.load(filename)\n",
    "        test_df = pd.DataFrame(fileLoad['songs'])\n",
    "        try:\n",
    "            test_df = test_df[['artist', 'title', 'lyrics']]\n",
    "            test_df\n",
    "            rough_df = pd.concat([rough_df, test_df], sort = True)\n",
    "        except:\n",
    "            unsuccessful.append(f)     # keep track of JSONs that weren't succesfully uploaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>lyrics</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>Bring It When You Come Lyrics\\nTerrible things...</td>\n",
       "      <td>Bring It When You Come</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>Cocaine Lyrics\\nFolk music is high-class music...</td>\n",
       "      <td>Cocaine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>Cocaine Blues LyricsCocaine, Cocaine\\nRunning ...</td>\n",
       "      <td>Cocaine Blues</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>Don’t Mess with Me Baby Lyrics\\nTerrible thing...</td>\n",
       "      <td>Don’t Mess with Me Baby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>I cried LyricsI, I cried\\nI cried in the ni-ig...</td>\n",
       "      <td>I cried</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>Together LyricsHe says Phoenix\\nPegasus\\nGreci...</td>\n",
       "      <td>Together</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>Twilight Zone Lyrics(Somewhere in a lonely hot...</td>\n",
       "      <td>Twilight Zone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>Walking on the Moon LyricsGiant steps are what...</td>\n",
       "      <td>Walking on the Moon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>What Have You Done LyricsShe was underwater\\nI...</td>\n",
       "      <td>What Have You Done</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>You’ll Have Time LyricsLive life, live life li...</td>\n",
       "      <td>You’ll Have Time</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3388 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             artist                                             lyrics  \\\n",
       "0         Abner Jay  Bring It When You Come Lyrics\\nTerrible things...   \n",
       "1         Abner Jay  Cocaine Lyrics\\nFolk music is high-class music...   \n",
       "2         Abner Jay  Cocaine Blues LyricsCocaine, Cocaine\\nRunning ...   \n",
       "3         Abner Jay  Don’t Mess with Me Baby Lyrics\\nTerrible thing...   \n",
       "4         Abner Jay  I cried LyricsI, I cried\\nI cried in the ni-ig...   \n",
       "..              ...                                                ...   \n",
       "45  William Shatner  Together LyricsHe says Phoenix\\nPegasus\\nGreci...   \n",
       "46  William Shatner  Twilight Zone Lyrics(Somewhere in a lonely hot...   \n",
       "47  William Shatner  Walking on the Moon LyricsGiant steps are what...   \n",
       "48  William Shatner  What Have You Done LyricsShe was underwater\\nI...   \n",
       "49  William Shatner  You’ll Have Time LyricsLive life, live life li...   \n",
       "\n",
       "                      title  \n",
       "0    Bring It When You Come  \n",
       "1                   Cocaine  \n",
       "2             Cocaine Blues  \n",
       "3   Don’t Mess with Me Baby  \n",
       "4                   I cried  \n",
       "..                      ...  \n",
       "45                 Together  \n",
       "46            Twilight Zone  \n",
       "47      Walking on the Moon  \n",
       "48       What Have You Done  \n",
       "49         You’ll Have Time  \n",
       "\n",
       "[3388 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rough_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(unsuccessful)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Lyrics_BrittanyAnjou.json',\n",
       " 'Lyrics_BrucePalmer.json',\n",
       " 'Lyrics_DavidLiebeHart.json',\n",
       " 'Lyrics_ExtraditionOrder.json',\n",
       " 'Lyrics_PortsmouthSinfonia.json',\n",
       " 'Lyrics_RobertGraettinger.json',\n",
       " 'Lyrics_SextonMing.json',\n",
       " 'Lyrics_SteveFarnie.json']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unsuccessful"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The 8 artists above don't have any lyrics in their JSON files.  I'm not entirely sure as to why this is the case.  Bruce Palmer, for example, does appear to have lyrics available on Genius.com.  However, because there are still 63 remaining artists, I'll settle for this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>lyrics</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>3388</td>\n",
       "      <td>3388</td>\n",
       "      <td>3388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>unique</td>\n",
       "      <td>63</td>\n",
       "      <td>3187</td>\n",
       "      <td>3359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>top</td>\n",
       "      <td>Wesley Willis</td>\n",
       "      <td></td>\n",
       "      <td>Lullaby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>freq</td>\n",
       "      <td>561</td>\n",
       "      <td>202</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               artist lyrics    title\n",
       "count            3388   3388     3388\n",
       "unique             63   3187     3359\n",
       "top     Wesley Willis         Lullaby\n",
       "freq              561    202        3"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rough_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>title</th>\n",
       "      <th>lyrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>Bring It When You Come</td>\n",
       "      <td>Bring It When You Come Lyrics\\nTerrible things...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>Cocaine</td>\n",
       "      <td>Cocaine Lyrics\\nFolk music is high-class music...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>Cocaine Blues</td>\n",
       "      <td>Cocaine Blues LyricsCocaine, Cocaine\\nRunning ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>Don’t Mess with Me Baby</td>\n",
       "      <td>Don’t Mess with Me Baby Lyrics\\nTerrible thing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Abner Jay</td>\n",
       "      <td>I cried</td>\n",
       "      <td>I cried LyricsI, I cried\\nI cried in the ni-ig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>Together</td>\n",
       "      <td>Together LyricsHe says Phoenix\\nPegasus\\nGreci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>Twilight Zone</td>\n",
       "      <td>Twilight Zone Lyrics(Somewhere in a lonely hot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>Walking on the Moon</td>\n",
       "      <td>Walking on the Moon LyricsGiant steps are what...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>What Have You Done</td>\n",
       "      <td>What Have You Done LyricsShe was underwater\\nI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>William Shatner</td>\n",
       "      <td>You’ll Have Time</td>\n",
       "      <td>You’ll Have Time LyricsLive life, live life li...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3388 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             artist                    title  \\\n",
       "0         Abner Jay   Bring It When You Come   \n",
       "1         Abner Jay                  Cocaine   \n",
       "2         Abner Jay            Cocaine Blues   \n",
       "3         Abner Jay  Don’t Mess with Me Baby   \n",
       "4         Abner Jay                  I cried   \n",
       "..              ...                      ...   \n",
       "45  William Shatner                 Together   \n",
       "46  William Shatner            Twilight Zone   \n",
       "47  William Shatner      Walking on the Moon   \n",
       "48  William Shatner       What Have You Done   \n",
       "49  William Shatner         You’ll Have Time   \n",
       "\n",
       "                                               lyrics  \n",
       "0   Bring It When You Come Lyrics\\nTerrible things...  \n",
       "1   Cocaine Lyrics\\nFolk music is high-class music...  \n",
       "2   Cocaine Blues LyricsCocaine, Cocaine\\nRunning ...  \n",
       "3   Don’t Mess with Me Baby Lyrics\\nTerrible thing...  \n",
       "4   I cried LyricsI, I cried\\nI cried in the ni-ig...  \n",
       "..                                                ...  \n",
       "45  Together LyricsHe says Phoenix\\nPegasus\\nGreci...  \n",
       "46  Twilight Zone Lyrics(Somewhere in a lonely hot...  \n",
       "47  Walking on the Moon LyricsGiant steps are what...  \n",
       "48  What Have You Done LyricsShe was underwater\\nI...  \n",
       "49  You’ll Have Time LyricsLive life, live life li...  \n",
       "\n",
       "[3388 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reorder the dataframe columns\n",
    "rough_df = rough_df[['artist', 'title', 'lyrics']]\n",
    "rough_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle the rough dataframe\n",
    "import pickle\n",
    "fname = open('../data/rough_df.pkl', 'wb')\n",
    "pickle.dump(rough_df, fname, -1)\n",
    "fname.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}