generate_hierarchical_feature_groups.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "\n",
    "features_dir = \"features\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate feature group containing all complexity-based metrics\n",
    "# CM = NC + SS + WF + WL\n",
    "\n",
    "node_count = os.path.join(features_dir, \"node_count.npy\")\n",
    "node_count = np.load(node_count)\n",
    "node_count = node_count.reshape((node_count.shape[0], -1))\n",
    "\n",
    "syntactic_surprisal = os.path.join(features_dir, \"syntactic_surprisal.npy\")\n",
    "syntactic_surprisal = np.load(syntactic_surprisal)\n",
    "syntactic_surprisal = syntactic_surprisal.reshape((syntactic_surprisal.shape[0], -1))\n",
    "\n",
    "word_frequency = os.path.join(features_dir, \"word_frequency.npy\")\n",
    "word_frequency = np.load(word_frequency)\n",
    "word_frequency = word_frequency.reshape((word_frequency.shape[0], -1))\n",
    "\n",
    "word_length = os.path.join(features_dir, \"word_length.npy\")\n",
    "word_length = np.load(word_length)\n",
    "word_length = word_length.reshape((word_length.shape[0], -1))\n",
    "\n",
    "all_complexity_metrics = np.hstack((node_count, syntactic_surprisal, word_frequency, word_length))\n",
    "np.save(os.path.join(features_dir, \"all_complexity_metrics.npy\"), all_complexity_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create CM + PU\n",
    "\n",
    "punct = os.path.join(features_dir, \"punct_final.npy\")\n",
    "punct = np.load(punct)\n",
    "\n",
    "all_complexity_metrics_punct = np.hstack((all_complexity_metrics, punct))\n",
    "np.save(os.path.join(features_dir, \"all_complexity_metrics_punct.npy\"), all_complexity_metrics_punct)\n",
    "\n",
    "node_count_punct = np.hstack((node_count, punct))\n",
    "syntactic_surprisal_punct = np.hstack((syntactic_surprisal, punct))\n",
    "word_frequency_punct = np.hstack((word_frequency, punct))\n",
    "word_length_punct = np.hstack((word_length, punct))\n",
    "np.save(os.path.join(features_dir, \"node_count_punct.npy\"), node_count_punct)\n",
    "np.save(os.path.join(features_dir, \"syntactic_surprisal_punct.npy\"), syntactic_surprisal_punct)\n",
    "np.save(os.path.join(features_dir, \"word_frequency_punct.npy\"), word_frequency_punct)\n",
    "np.save(os.path.join(features_dir, \"word_length_punct.npy\"), word_length_punct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create PD + CM + PU\n",
    "\n",
    "pos_tags = os.path.join(features_dir, \"pos_tags.npy\")\n",
    "dep_tags = os.path.join(features_dir, \"dep_tags.npy\")\n",
    "pos_tags = np.load(pos_tags)\n",
    "dep_tags = np.load(dep_tags)\n",
    "\n",
    "pos_dep_tags = np.hstack((pos_tags,dep_tags))\n",
    "np.save(os.path.join(features_dir, \"pos_dep_tags.npy\"), pos_dep_tags)\n",
    "pos_dep_tags_all_complexity_metrics = np.hstack((pos_dep_tags, all_complexity_metrics))\n",
    "np.save(os.path.join(features_dir, \"pos_dep_tags_all_complexity_metrics.npy\"), pos_dep_tags_all_complexity_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create hierarchical spaces for contrege_comp, contrege_incomp, incontrege and the bert embeddings\n",
    "\n",
    "num_contrege_sets_per_space = 5\n",
    "contrege_sets = [\"contrege_comp\", \"contrege_incomp\", \"incontrege\"]\n",
    "\n",
    "bert_pca = os.path.join(features_dir, \"incremental_bert_embeddings_layer12_PCA_dims_15.npy\")\n",
    "bert_pca = np.load(bert_pca)\n",
    "\n",
    "for set_name in contrege_sets:\n",
    "    for i in range(num_contrege_sets_per_space):\n",
    "        cs = np.load(os.path.join(features_dir, set_name + \"_set_{}.npy\".format(i)))\n",
    "        cs_pos_dep_tags_all_complexity_metrics = np.hstack((cs, pos_dep_tags_all_complexity_metrics))\n",
    "        np.save(os.path.join(features_dir, \"{}_set_{}_pos_dep_tags_all_complexity_metrics.npy\".format(set_name, i)), cs_pos_dep_tags_all_complexity_metrics)        \n",
    "        if set_name == \"contrege_incomp\":\n",
    "            bert_cs_pos_dep_tags_all_complexity_metrics = np.hstack((bert_pca, cs, pos_dep_tags_all_complexity_metrics))\n",
    "            np.save(os.path.join(features_dir, \"bert_PCA_dims_15_{}_set_{}_pos_dep_tags_all_complexity_metrics.npy\".format(set_name, i)), bert_cs_pos_dep_tags_all_complexity_metrics)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}