Document_Class.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configure the language model, data source, and options\n",
    "\n",
    "model = 'en_core_web_sm'\n",
    "manifest_dir = 'data'\n",
    "manifest_file = '2008_10_humanities_student_major_22_economics.json'\n",
    "options = {'merge_noun_chunks': False, 'merge_subtokens': False, 'collect_readability_scores': True}\n",
    "add_stopwords = [\"'nt'\"]\n",
    "remove_stopwords = []\n",
    "lemmatization_cases = {\n",
    "    \"humanities\": [{ORTH: u'humanities', LEMMA: u'humanities', POS: u'NOUN', TAG: u'NNS'}],\n",
    "    \"aren't\": [{ORTH: \"are\"}, {ORTH: \"n't\", LEMMA: \"not\"}],\n",
    "    \"isn't\": [{ORTH: \"is\"}, {ORTH: \"n't\", LEMMA: \"not\"}]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import nltk\n",
    "import pandas as pd\n",
    "import re\n",
    "import spacy\n",
    "import time\n",
    "import unicodedata\n",
    "from collections import Counter\n",
    "from ftfy import fix_text\n",
    "from nltk.stem.porter import *  \n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.util import ngrams\n",
    "from spacy.tokenizer import Tokenizer\n",
    "\n",
    "# Constants for the preprocessing functions\n",
    "LINEBREAK_REGEX = re.compile(r'((\\r\\n)|[\\n\\v])+')\n",
    "NONBREAKING_SPACE_REGEX = re.compile(r'(?!\\n)\\s+')\n",
    "PREFIX_RE = re.compile(r'''^[\\[\\]\\(\"'\\.,;:-]''')\n",
    "SUFFIX_RE = re.compile(r'''[\\[\\]\\)\"'\\.,;:-]$''')\n",
    "INFIX_RE = re.compile(r'''[~]''')\n",
    "SIMPLE_URL_RE = re.compile(r'''^https?://''')\n",
    "\n",
    "# Load the language model\n",
    "nlp = spacy.load(model, disable=['sentencizer'])\n",
    "\n",
    "# Add Custom Tokenizer\n",
    "def custom_tokenizer(nlp):\n",
    "    return Tokenizer(nlp.vocab, prefix_search=PREFIX_RE.search,\n",
    "                                suffix_search=SUFFIX_RE.search,\n",
    "                                infix_finditer=INFIX_RE.finditer,\n",
    "                                token_match=SIMPLE_URL_RE.match)\n",
    "nlp.tokenizer = custom_tokenizer(nlp)\n",
    "\n",
    "# Handle lemmatisation exceptions\n",
    "from spacy.symbols import ORTH, LEMMA, POS, TAG\n",
    "for k, v in lemmatization_cases.items():\n",
    "    nlp.tokenizer.add_special_case(k, v)\n",
    "\n",
    "# Add and remove custom stop words\n",
    "for word in add_stopwords:\n",
    "    nlp.vocab[word].is_stop = True\n",
    "for word in remove_stopwords:\n",
    "    nlp.vocab[word].is_stop = False\n",
    "\n",
    "# Test for the spacy-readability module\n",
    "try:\n",
    "    from spacy_readability import Readability\n",
    "    nlp.add_pipe(Readability())\n",
    "except:\n",
    "    msg = 'The spacy-readability module is not installed on your system, so readability scores will be unavailable unless you `pip install spacy-_readability`.'\n",
    "    print(msg)\n",
    "\n",
    "# The Document class\n",
    "class Document():\n",
    "    \"\"\"Model a document's features.\n",
    "\n",
    "    Parameters:\n",
    "    - manifest_dir: the path to the manifest directory\n",
    "    - manifest_file: the name of the manifest file.\n",
    "    - content_property: the name of the property from which to extract the content\n",
    "\n",
    "    Returns a dataframe.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, manifest_dir, manifest_file, content_property, **kwargs):\n",
    "        \"\"\"Initialize the object.\"\"\"\n",
    "        self.manifest_filepath = os.path.join(manifest_dir, manifest_file)\n",
    "        self.manifest_dict = self._read_manifest()\n",
    "        self.manifest_json = json.dumps(self.manifest_dict, indent=2)\n",
    "        self.doc_string = self.scrub(self._get_docstring(content_property))\n",
    "        self.content = nlp(self.doc_string)\n",
    "        self.options = kwargs['kwargs']\n",
    "        # Re-do this to deserialise a list of lists.\n",
    "        if 'features' in self.manifest_dict:\n",
    "            self.features = deserialize(self.manifest_dict['features'])\n",
    "        else:\n",
    "            self.features = self.get_features()\n",
    "\n",
    "    def _remove_accents(self, text, method='unicode'):\n",
    "        \"\"\"Remove accents from any accented unicode characters in a string.\n",
    "\n",
    "        Either transforms them into ascii equivalents or removes them entirely.\n",
    "\n",
    "        Parameters:\n",
    "        - text (str): raw text\n",
    "        - method ({'unicode', 'ascii'}): if 'unicode', remove accented\n",
    "            char for any unicode symbol with a direct ASCII equivalent; if 'ascii',\n",
    "            remove accented char for any unicode symbol.\n",
    "            NB: the 'ascii' method is notably faster but less effective than 'unicode'.\n",
    "        Returns:\n",
    "            str\n",
    "        Raises:\n",
    "            ValueError: if ``method`` is not in {'unicode', 'ascii'}\n",
    "        \"\"\"\n",
    "        if method == 'unicode':\n",
    "            return ''.join(\n",
    "                c\n",
    "                for c in unicodedata.normalize('NFKD', text)\n",
    "                if not unicodedata.combining(c)\n",
    "            )\n",
    "        elif method == 'ascii':\n",
    "            return (\n",
    "                unicodedata.normalize('NFKD', text)\n",
    "                .encode('ascii', errors='ignore')\n",
    "                .decode('ascii')\n",
    "            )\n",
    "        else:\n",
    "            msg = '`method` must be either \"unicode\" and \"ascii\", not {}'.format(method)\n",
    "            raise ValueError(msg)\n",
    "\n",
    "    def scrub(self, text, unicode_normalization='NFC', accent_removal_method='unicode'):\n",
    "        \"\"\"Normalize whitespace and and bad unicode, and remove accents.\n",
    "\n",
    "        Parameters:\n",
    "        - unicode_normalization: The ftfy.fix_text() `normalization` parameter.\n",
    "        - accent_removal_method: The Doc.remove_accents() `method` parameter.    \n",
    "        Returns str\n",
    "        \"\"\"\n",
    "        # Change multiple spaces to one and multiple line breaks to one.\n",
    "        # Also strip leading/trailing whitespace.\n",
    "        text = NONBREAKING_SPACE_REGEX.sub(' ', LINEBREAK_REGEX.sub(r'\\n', text)).strip()\n",
    "        # Combine characters and diacritics written using separate code points\n",
    "        text = fix_text(text, normalization=unicode_normalization)\n",
    "        text = self._remove_accents(text, method=accent_removal_method)\n",
    "        return text\n",
    "\n",
    "    def _read_manifest(self):\n",
    "        \"\"\"Read a JSON file and return a Python dict.\"\"\"\n",
    "        with open(self.manifest_filepath, 'r', encoding='utf-8') as f:\n",
    "            return json.loads(f.read())\n",
    "        \n",
    "    def _get_docstring(self, content_property):\n",
    "        \"\"\"Extract a document string from a manifest property.\"\"\"\n",
    "        return self.manifest_dict[content_property]\n",
    "\n",
    "    def get_features(self):\n",
    "        \"\"\"Process the document with the spaCy pipeline into a pandas dataframe.\n",
    "        \n",
    "        If `collect_readability_scores` is set, Flesch-Kincaid Readability,\n",
    "        Flesch-Kincaid Reading Ease and Dale-Chall formula scores are collected\n",
    "        in a tuple in that order. Other formulas are available (see \n",
    "        https://github.com/mholtzscher/spacy_readability).\n",
    "        \n",
    "        Parameters:\n",
    "        - as_list: Return the features as a list instead of a dataframe\n",
    "        \"\"\"\n",
    "        # Handle optional pipes\n",
    "        if 'merge_noun_chunks' in self.options and self.options['merge_noun_chunks'] == True:\n",
    "            merge_nps = nlp.create_pipe('merge_noun_chunks')\n",
    "            nlp.add_pipe(merge_nps)\n",
    "        if 'merge_subtokens' in self.options and self.options['merge_subtokens'] == True:\n",
    "            merge_subtok = nlp.create_pipe('merge_subtokens')\n",
    "            nlp.add_pipe(merge_subtok)\n",
    "        # Build the feature list\n",
    "        feature_list = []\n",
    "        columns = ['TOKEN', 'LEMMA', 'POS', 'TAG', 'STOPWORD', 'ENTITIES']\n",
    "        for token in self.content:\n",
    "            # Get named entity info (I=Inside, O=Outside, B=Begin)\n",
    "            ner = (token.ent_iob_, token.ent_type_)\n",
    "            t = [token.text, token.lemma_, token.pos_, token.tag_, str(token.is_stop), ner]\n",
    "            feature_list.append(tuple(t))\n",
    "        return pd.DataFrame(feature_list, columns=columns)\n",
    "\n",
    "    def filter(self, pattern=None, column='TOKEN', skip_punct=False, skip_stopwords=False, skip_linebreaks=False, case=True, flags=0, na=False, regex=True):\n",
    "        \"\"\"Return a new dataframe with filtered rows.\n",
    "\n",
    "        Parameters:\n",
    "        - pattern: The string or regex pattern on which to filter.\n",
    "        - column: The column where the string is to be searched.\n",
    "        - skip_punct: Do not include punctuation marks.\n",
    "        - skip_stopwords: Do not include stopwords.\n",
    "        - skip_linebreaks: Do not include linebreaks.\n",
    "        - case: Perform a case-sensitive match.\n",
    "        - flags: Regex flags.\n",
    "        - na: Filler for empty cells.\n",
    "        - regex: Set to True; otherwise absolute values will be matched.\n",
    "\n",
    "        The last four parameters are from `pandas.Series.str.contains`.\n",
    "        \"\"\"\n",
    "        # Filter based on column content\n",
    "        new_df = self.features\n",
    "        if pattern is not None:\n",
    "            new_df = new_df[new_df[column].str.contains(pattern, case=case, flags=flags, na=na, regex=regex)]\n",
    "        # Filter based on token type\n",
    "        if skip_punct == True:\n",
    "            new_df = new_df[~new_df['POS'].str.contains('PUNCT', case=True, flags=0, na=False, regex=True)]\n",
    "        if skip_stopwords == True:\n",
    "            new_df = new_df[~new_df['STOPWORD'].str.contains('TRUE', case=False, flags=0, na=False, regex=True)]\n",
    "        if skip_linebreaks == True:\n",
    "            new_df = new_df[~new_df['POS'].str.contains('SPACE', case=True, flags=0, na=False, regex=True)]\n",
    "        return new_df\n",
    "\n",
    "    def lemmas(self, as_list=False):\n",
    "        \"\"\"Return a dataframe containing just the lemmas.\"\"\"\n",
    "        if as_list == True:\n",
    "            return [token.lemma_ for token in self.content]\n",
    "        else:\n",
    "            return pd.DataFrame([token.lemma_ for token in self.content], columns=['LEMMA'])\n",
    "\n",
    "    def punctuation(self, as_list=False):\n",
    "        \"\"\"Return a dataframe containing just the punctuation marks.\"\"\"\n",
    "        if as_list == True:\n",
    "            return [token.text for token in self.content if token.is_punct]\n",
    "        else:\n",
    "            return pd.DataFrame([token.text for token in self.content if token.is_punct], columns=['PUNCTUATION'])\n",
    "\n",
    "    def pos(self, as_list=False):\n",
    "        \"\"\"Return a dataframe containing just the parts of speech.\"\"\"\n",
    "        if as_list == True:\n",
    "            return [token.pos_ for token in self.content]\n",
    "        else:\n",
    "            return pd.DataFrame([token.pos_ for token in self.content], columns=['POS'])\n",
    "\n",
    "    def tags(self, as_list=False):\n",
    "        \"\"\"Return a dataframe containing just the tags.\"\"\"\n",
    "        if as_list == True:\n",
    "            return [token.tag_ for token in self.content]\n",
    "        else:\n",
    "            return pd.DataFrame([token.tag_ for token in self.content], columns=['TAG'])\n",
    "\n",
    "    def entities(self, options=['text', 'label'], as_list=False):\n",
    "        \"\"\"Return a dataframe containing just the entities from the document.\n",
    "        \n",
    "        Parameters:\n",
    "        - options: a list of attributes ('text', 'start', 'end', 'label')\n",
    "        - as_list: return the entities as a list of tuples.\n",
    "        \"\"\"\n",
    "        ents = []\n",
    "        for ent in self.content.ents:\n",
    "            e = []\n",
    "            if 'text' in options:\n",
    "                e.append(ent.text)\n",
    "            if 'start' in options:\n",
    "                e.append(ent.start)\n",
    "            if 'end' in options:\n",
    "                e.append(ent.end)\n",
    "            if 'label' in options:\n",
    "                e.append(ent.label_)\n",
    "            ents.append(tuple(e))\n",
    "        if as_list == True:\n",
    "            return ents\n",
    "        else:\n",
    "            return pd.DataFrame(ents, columns=[option.title() for option in options])\n",
    "\n",
    "    def readability_scores(self, columns=['Flesch-Kincaid Readability',\n",
    "        'Flesch-Kincaid Reading Ease', 'Dale-Chall'], as_list=False):\n",
    "        \"\"\"Get a list of readability scores from the document.\n",
    "        \n",
    "        Parameters:\n",
    "        - columns: a list of labels for the score types\n",
    "        - as_df: return the list as a dataframe.\n",
    "        \"\"\"\n",
    "        fkr = self.content._.flesch_kincaid_reading_ease\n",
    "        fkg = self.content._.flesch_kincaid_grade_level\n",
    "        dc = self.content._.dale_chall\n",
    "        scores = [(fkr, fkg, dc)]\n",
    "        if as_list == True:\n",
    "            return scores\n",
    "        else:\n",
    "            return pd.DataFrame(scores, columns=columns)\n",
    "\n",
    "    def stems(self, stemmer='porter', as_list=False):\n",
    "        \"\"\"Convert the tokens in a spaCy document to stems.\n",
    "\n",
    "        Parameters:\n",
    "        - stemmer: the stemming algorithm ('porter' or 'snowball').\n",
    "        - as_list: return the dataframe as a list.\n",
    "        \"\"\"\n",
    "        if stemmer == 'snowball':\n",
    "            stemmer = SnowballStemmer(language='english')\n",
    "        else:\n",
    "            stemmer = PorterStemmer()\n",
    "        stems = [stemmer.stem(token.text) for token in self.content]\n",
    "        if as_list == True:\n",
    "            return stems\n",
    "        else:\n",
    "            return pd.DataFrame(stems, columns=['Stems'])\n",
    "\n",
    "    def ngrams(self, n=2, as_list=False):\n",
    "        \"\"\"Convert the tokens in a spaCy document to ngrams.\n",
    "\n",
    "        Parameters:\n",
    "        - n: The number of tokens in an ngram.\n",
    "        - as_list: return the dataframe as a list.\n",
    "        \"\"\"\n",
    "        ngram_tokens = list(ngrams([token.text for token in self.content], n))\n",
    "        if as_list == True:\n",
    "            return ngram_tokens\n",
    "        else:\n",
    "            prefix = str(n) + '-'\n",
    "            if n == 2:\n",
    "                prefix = 'Bi'\n",
    "            if n == 3:\n",
    "                prefix = 'Tri'\n",
    "            label = prefix + 'grams'\n",
    "            return pd.DataFrame({label: pd.Series(ngram_tokens)})\n",
    "\n",
    "    def remove_property(self, property):\n",
    "        \"\"\"Remove a property from the manifest.\n",
    "\n",
    "        Parameters:\n",
    "        - property: The property or a list of properties to be removed from the manifest.\n",
    "        \"\"\"\n",
    "        if isinstance(property, str):\n",
    "            property = list(property)\n",
    "        for prop in property:\n",
    "            del self.manifest_dict[prop]\n",
    "        # Write the json to the manifest file\n",
    "        # IMPORTANT: May not work if the manifest file has binary content\n",
    "        with open(self.manifest_filepath, 'w', encoding='utf-8') as f:\n",
    "            f.write(json.dumps(self.manifest_dict))\n",
    "            \n",
    "    def serialize(self, df, indent=None):\n",
    "        \"\"\"Serialize a dataframe as a list of lists with the column headers as the first element.\n",
    "\n",
    "        Parameters:\n",
    "        - indent: An integer indicating the number of spaces to indent the json string. Default is None.\n",
    "        \"\"\"\n",
    "        j = json.loads(pd.DataFrame.to_json(df, orient='values'))\n",
    "        j.insert(0, list(df.columns))\n",
    "        return json.dumps(j, indent=indent)\n",
    "\n",
    "    def deserialize(self, j):\n",
    "        \"\"\"Deserialize a list of lists to a dataframe using the first element as the headers.\"\"\"\n",
    "        df = pd.read_json(j, orient='values')\n",
    "        headers = df.iloc[0]\n",
    "        return pd.DataFrame(df.values[1:], columns=headers)\n",
    "\n",
    "\n",
    "    def save(self, property=None, series=None, sort=False):\n",
    "        \"\"\"Convert a series of values and save them to the manifest file.\n",
    "        \n",
    "        Overwrites the original manifest file, so not to be used lightly.\n",
    "        IMPORTANT: May not work if the manifest file has binary content.\n",
    "\n",
    "        Parameters:\n",
    "        - property: A string naming the JSON property to save to.\n",
    "        - series: The list or dataframe to save.\n",
    "        - sort: Alphabetically sort ther series to loes token order.\n",
    "        \"\"\"\n",
    "        with open(self.manifest_filepath, 'w', encoding='utf-8') as f:\n",
    "            if isinstance(series, dict) or isinstance(series, list):\n",
    "                self.manifest_dict[property] = series\n",
    "                f.write(json.dumps(self.manifest_dict))\n",
    "            else:\n",
    "                if sort == True:\n",
    "                    col = list(series.columns)[0]\n",
    "                    series.sort_values(by=[col], inplace=True)\n",
    "                json_str = self.serialize(series)\n",
    "                self.manifest_dict[property] = json.loads(json_str)\n",
    "                f.write(json_str)\n",
    "\n",
    "# Not part of the Document class for ease of access.\n",
    "# Create bags as separate dicts and then save them to the manifest.\n",
    "def bagify(series, as_counter=False):\n",
    "    \"\"\"Convert a list of values to a dict of value frequencies.\n",
    "    \n",
    "    Parameters:\n",
    "    - as_counter: If True, returns a Python Counter object enabling its most_common() method.\n",
    "    \"\"\"\n",
    "    # Make sure we are working with a list of values\n",
    "    if isinstance(series, pd.DataFrame):\n",
    "        print('Please select only one columns from the dataframe.')\n",
    "    if isinstance(series, pd.Series):\n",
    "        series = list(series.values)\n",
    "    if as_counter == True:\n",
    "        return Counter(series)\n",
    "    else:\n",
    "        return dict(Counter(series))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executed in 0.8055734634399414seconds.\n"
     ]
    }
   ],
   "source": [
    "# Initialise the feature table\n",
    "start = time.time()\n",
    "doc = Document(manifest_dir, manifest_file, 'content_scrubbed', kwargs=options)\n",
    "end = time.time()\n",
    "t = end - start\n",
    "print('Executed in ' + str(t) + ' seconds.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        TOKEN   LEMMA    POS      STEM   TAG STOPWORD  ENTITIES\n",
      "87         \\n      \\n  SPACE        \\n          False  (B, GPE)\n",
      "33         \\n      \\n  SPACE        \\n          False     (O, )\n",
      "489        \\n      \\n  SPACE        \\n          False     (O, )\n",
      "399        \\n      \\n  SPACE        \\n          False     (O, )\n",
      "350        \\n      \\n  SPACE        \\n          False     (O, )\n",
      "495         !       !  PUNCT         !     .    False     (O, )\n",
      "411         !       !  PUNCT         !     .    False     (O, )\n",
      "196         \"       \"  PUNCT         \"    ''    False     (O, )\n",
      "194         \"       \"  PUNCT         \"    ``    False     (O, )\n",
      "287         \"       \"  PUNCT         \"    ''    False     (O, )\n",
      "284         \"       \"  PUNCT         \"    ``    False     (O, )\n",
      "282         \"       \"  PUNCT         \"    ''    False     (O, )\n",
      "280         \"       \"  PUNCT         \"    ``    False     (O, )\n",
      "276         \"       \"  PUNCT         \"    ``    False     (O, )\n",
      "278         \"       \"  PUNCT         \"    ''    False     (O, )\n",
      "63         'd   would   VERB        'd    MD    False     (O, )\n",
      "299       'll    will   VERB       'll    MD    False     (O, )\n",
      "260         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "296         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "430         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "155         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "315         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "268         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "465         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "198         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "218         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "468         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "258         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "165         ,       ,  PUNCT         ,     ,    False     (O, )\n",
      "418         -       -  PUNCT         -  HYPH    False     (O, )\n",
      "..        ...     ...    ...       ...   ...      ...       ...\n",
      "266       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "237       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "393       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "110       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "84        you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "214       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "494       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "208       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "340       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "205       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "301       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "446       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "305       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "159       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "171       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "313       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "459       you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "39        you  -PRON-   PRON       you   PRP     True     (O, )\n",
      "319      your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "162      your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "176      your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "156      your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "390      your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "26       your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "428      your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "355      your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "485      your  -PRON-    ADJ      your  PRP$     True     (O, )\n",
      "30      yours  -PRON-   PRON      your   PRP     True     (O, )\n",
      "82   yourself  -PRON-   PRON  yourself   PRP     True     (O, )\n",
      "410  yourself  -PRON-   PRON  yourself   PRP     True     (O, )\n",
      "\n",
      "[496 rows x 7 columns]\n",
      "Executed in 0.10708236694335938 seconds.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "## Uncomment lines below to try various class methods.\n",
    "## See the method docstrings for options not shown in the examples below.\n",
    "# doc.doc_string\n",
    "# Get all the doc features in a dataframe\n",
    "# doc.get_features()\n",
    "\n",
    "# Count the number of feature unfiltered tokens\n",
    "# len(doc.get_features())\n",
    "\n",
    "# Filter the tokens\n",
    "# filtered = doc.filter(pattern='NOUN', column='POS', skip_punct=True, skip_stopwords=True, skip_linebreaks=True)\n",
    "# pd.DataFrame(filtered['TOKEN'])\n",
    "# len(filtered)\n",
    "\n",
    "# Get Various Features\n",
    "# doc.lemmas()\n",
    "# doc.punctuation(as_list=True)\n",
    "# doc.pos()\n",
    "# doc.tags()\n",
    "# doc.entities()\n",
    "# doc.readability_scores()\n",
    "# doc.stems(stemmer='porter')\n",
    "# doc.ngrams(n=3)\n",
    "\n",
    "# Insert a column using pandas\n",
    "# stems = doc.stems()\n",
    "# features = doc.get_features()\n",
    "# features.insert(3, 'STEM', stems)\n",
    "# print(features)\n",
    "\n",
    "# Sort the features using pandas\n",
    "features.sort_values(by=['TOKEN'], inplace=True)\n",
    "print(features)\n",
    "\n",
    "# Save a column to the manifest file. Overwrites the original file, so not to be used lightly.\n",
    "# stems = doc.stems()\n",
    "# doc.save('stems', stems)\n",
    "\n",
    "# Bagify the tokens, skipping punctuation and stop words\n",
    "# filtered = doc.filter(column='TOKEN', skip_punct=True, skip_stopwords=True, skip_linebreaks=True)\n",
    "# bag = bagify(filtered['TOKEN'])\n",
    "# bag\n",
    "# Save the results to the manifest\n",
    "\n",
    "# Get 10 most common nouns\n",
    "# nouns = doc.filter(pattern='NOUN', column='POS', skip_punct=True, skip_stopwords=True, skip_linebreaks=True)\n",
    "# most_common = bagify(nouns['TOKEN'], as_counter=True).most_common(10)\n",
    "# doc.save('bag', dict(most_common))\n",
    "# dict(most_common)\n",
    "end = time.time()\n",
    "t = end - start\n",
    "print('Executed in ' + str(t) + ' seconds.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To do:\n",
    "\n",
    "- Test bagification and saving more extensively.\n",
    "- Fine tune tokenisation and lemmatisation rules.\n",
    "- Call the actual methods we want for preprocessing (including saving token counts). Don't filter stop words.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Algorithm\n",
    "\n",
    "1. Get the feature table and sort it.\n",
    "2. Serialise the feature table and update self.manifest_dict in it.\n",
    "3. Count the feature table rows and update self.manifest_dict with the token count.\n",
    "4. Bagify the non-punctuation tokens and update self.manifest_dict.\n",
    "5. Get the readability scores and add them to self.manifest_dict.\n",
    "6. Save the manifest."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}