04_select_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import os\n",
    "import sys\n",
    "\n",
    "sys.modules['numpy._core'] = None\n",
    "import numpy as np\n",
    "sys.modules['numpy._core.multiarray'] = np.core.multiarray\n",
    "sys.modules['numpy._core.numeric'] = np.core.numeric\n",
    "\n",
    "\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "from tensorflow import keras\n",
    "os.environ[\"OMP_NUM_THREADS\"] = \"2\"\n",
    "tf.config.threading.set_intra_op_parallelism_threads(1)\n",
    "tf.config.threading.set_inter_op_parallelism_threads(1)\n",
    "\n",
    "DAY_SECONDS = 24 * 60 * 60\n",
    "\n",
    "from music_nextsim_tuning import read_and_scale,get_good_columns1, train_autoenc_predict, plot_rmse, plot_all_rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idir = './music_matrix/cfg01_m20'\n",
    "max_date = datetime(2007, 4, 12)\n",
    "max_std = 1.5\n",
    "rmse_std_factor = 2.5\n",
    "epochs = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_names, inp_rgps, inp_ftrs = read_and_scale(idir, max_date)\n",
    "good_columns1 = get_good_columns1(inp_rgps, max_std)\n",
    "print(idir.split('/')[-1], \"AVG/STD filtering\", len(good_columns1), good_columns1)\n",
    "tf0, tp0, rf0, rp0, hi = train_autoenc_predict(inp_ftrs, inp_rgps, inp_ftrs.columns, epochs, keras)\n",
    "rmse_n0, rmse_r0 = plot_rmse(tf0, tp0, rf0, rp0, hi)\n",
    "tf1, tp1, rf1, rp1, hi = train_autoenc_predict(inp_ftrs, inp_rgps, good_columns1, epochs, keras)\n",
    "rmse_n1, rmse_r1 = plot_rmse(tf1, tp1, rf1, rp1, hi)\n",
    "# features filtered only by RMSE\n",
    "good_columns2 = inp_ftrs.columns[(rmse_n0.to_numpy() < rmse_n0.std() * rmse_std_factor)]\n",
    "print(idir.split('/')[-1], \"RMSE filtering\", len(good_columns2), good_columns2)\n",
    "tf2, tp2, rf2, rp2, hi = train_autoenc_predict(inp_ftrs, inp_rgps, good_columns2, epochs, keras)\n",
    "rmse_n2, rmse_r2 = plot_rmse(tf2, tp2, rf2, rp2, hi)\n",
    "# features filtered by AVG/STD first and then by RMSE\n",
    "good_columns3 = good_columns1[(rmse_n1.to_numpy() < rmse_n0.std()*rmse_std_factor)]\n",
    "print(idir.split('/')[-1], \"AVG/STD + RMSE filtering\", len(good_columns3), good_columns3)\n",
    "# features filtered by AVG/STD first and then by RMSE\n",
    "good_columns3 = good_columns1[(rmse_n1.to_numpy() < rmse_n0.std()*rmse_std_factor)]\n",
    "print(idir.split('/')[-1], \"AVG/STD + RMSE filtering\", len(good_columns3), good_columns3)\n",
    "rmse_r_list = [rmse_n0, rmse_n1, rmse_n2]\n",
    "rmse_n_list = [rmse_r0, rmse_r1, rmse_r2]\n",
    "tf_list = [tf0, tf1, tf2]\n",
    "tp_list = [tp0, tp1, tp2]\n",
    "rf_list = [rf0, rf1, rf2]\n",
    "rp_list = [rp0, rp1, rp2]\n",
    "plot_all_rmse(rmse_r_list, rmse_n_list, tf_list, tp_list, rf_list, rp_list)\n",
    "np.savez(f'tuning_paper_figures/filter_autoencoder.npz',\n",
    "            rmse_n0=rmse_n0, rmse_r0=rmse_r0,\n",
    "            rmse_n1=rmse_n1, rmse_r1=rmse_r1, good_columns1=good_columns1,\n",
    "            rmse_n2=rmse_n2, rmse_r2=rmse_r2, good_columns2=good_columns2,\n",
    "        )\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ice",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}