Evaluate_power_vs_classification_analysed.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up the dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for reading and validating data\n",
    "import emeval.input.spec_details as eisd\n",
    "import emeval.input.phone_view as eipv\n",
    "import emeval.input.eval_view as eiev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualization helpers\n",
    "import emeval.viz.phone_view as ezpv\n",
    "import emeval.viz.eval_view as ezev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analytics results\n",
    "import emeval.metrics.baseline_segmentation as embs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For plots\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.collections import PatchCollection\n",
    "from matplotlib.patches import Rectangle\n",
    "%matplotlib inline\n",
    "\n",
    "import IPython.display as ipyd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For maps\n",
    "import folium\n",
    "import branca.element as bre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For easier debugging while working on modules\n",
    "import importlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.options.display.float_format = '{:.6f}'.format\n",
    "import arrow\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "THIRTY_MINUTES = 30 * 60\n",
    "TEN_MINUTES = 10 * 60"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The spec\n",
    "\n",
    "The spec defines what experiments were done, and over which time ranges. Once the experiment is complete, most of the structure is read back from the data, but we use the spec to validate that it all worked correctly. The spec also contains the ground truth for the legs. Here, we read the spec for the trip to UC Berkeley."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASTORE_LOC = \"bin/data\"\n",
    "AUTHOR_EMAIL = \"shankari@eecs.berkeley.edu\"\n",
    "sd_la = eisd.FileSpecDetails(DATASTORE_LOC, AUTHOR_EMAIL, \"unimodal_trip_car_bike_mtv_la\")\n",
    "sd_sj = eisd.FileSpecDetails(DATASTORE_LOC, AUTHOR_EMAIL, \"car_scooter_brex_san_jose\")\n",
    "sd_ucb = eisd.FileSpecDetails(DATASTORE_LOC, AUTHOR_EMAIL, \"train_bus_ebike_mtv_ucb\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The views\n",
    "\n",
    "There are two main views for the data - the phone view and the evaluation view. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Phone view\n",
    "\n",
    "In the phone view, the phone is primary, and then there is a tree that you can traverse to get the data that you want. Traversing that tree typically involves nested for loops; here's an example of loading the phone view and traversing it. You can replace the print statements with real code. When you are ready to check this in, please move the function to one of the python modules so that we can invoke it more generally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(eipv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pv_la = eipv.PhoneView(sd_la)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_sj = eipv.PhoneView(sd_sj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_ucb = eipv.PhoneView(sd_ucb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "range_0 = pv_la.map()[\"android\"][\"ucb-sdb-android-2\"][\"evaluation_ranges\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "arrow.get(range_0[\"start_ts\"]).to(\"America/Los_angeles\"), arrow.get(range_0[\"end_ts\"]).to(\"America/Los_angeles\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for t in range_0[\"evaluation_trip_ranges\"]:\n",
    "    print(arrow.get(t[\"start_ts\"]).to(\"America/Los_angeles\"), arrow.get(t[\"end_ts\"]).to(\"America/Los_angeles\"))\n",
    "    for s in t[\"evaluation_section_ranges\"]:\n",
    "        print(\"-------\", arrow.get(s[\"start_ts\"]).to(\"America/Los_angeles\"), arrow.get(s[\"end_ts\"]).to(\"America/Los_angeles\"))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import emeval.analysed.phone_view as eapv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(eapv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change this to the tag that we want to evaluate\n",
    "MASTER_DATA_LOC = \"bin/data/master_9b70c97\"\n",
    "eisd.FileSpecDetails(MASTER_DATA_LOC, AUTHOR_EMAIL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "av_la = eapv.create_analysed_view(pv_la, eisd.FileSpecDetails(\"bin/data/master_9b70c97\", AUTHOR_EMAIL), \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "range_0 = av_la.map()[\"ios\"][\"ucb-sdb-ios-3\"][\"evaluation_ranges\"][0]\n",
    "for t in range_0[\"sensed_trip_ranges\"]:\n",
    "    print(arrow.get(t[\"start_ts\"]).to(\"America/Los_angeles\"), arrow.get(t[\"end_ts\"]).to(\"America/Los_angeles\"))\n",
    "print(\"=======\")\n",
    "for s in range_0[\"sensed_section_ranges\"]:\n",
    "    print(arrow.get(s[\"start_ts\"]).to(\"America/Los_angeles\"), arrow.get(s[\"end_ts\"]).to(\"America/Los_angeles\"))\n",
    "print(\"=======\")\n",
    "\n",
    "for t in range_0[\"evaluation_trip_ranges\"]:\n",
    "    print(arrow.get(t[\"start_ts\"]).to(\"America/Los_angeles\"), arrow.get(t[\"end_ts\"]).to(\"America/Los_angeles\"))\n",
    "    for s in t[\"sensed_section_ranges\"]:\n",
    "        print(\"-------\", arrow.get(s[\"start_ts\"]).to(\"America/Los_angeles\"), arrow.get(s[\"end_ts\"]).to(\"America/Los_angeles\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# av_la = eapv.create_analysed_view(pv_la, \"http://localhost:8080\", \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\")\n",
    "av_sj = eapv.create_analysed_view(pv_sj, eisd.FileSpecDetails(\"bin/data/master_9b70c97\", AUTHOR_EMAIL), \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\")\n",
    "av_ucb = eapv.create_analysed_view(pv_ucb, eisd.FileSpecDetails(\"bin/data/master_9b70c97\", AUTHOR_EMAIL), \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\")\n",
    "# av_ucb_reroute = eapv.create_analysed_view(pv_ucb_reroute, \"http://localhost:8080\", \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ANDROID_MODE_MAP = {0: \"AUTOMOTIVE\", 1: \"CYCLING\", 2: \"WALKING\", 3: \"STATIONARY\"}\n",
    "ANDROID_MAP_FN = lambda t: ANDROID_MODE_MAP[t[\"zzbhB\"]]\n",
    "\n",
    "def IOS_MAP_FN(t):\n",
    "    t_series = pd.Series(t)\n",
    "    all_true = t_series[t_series == True].index.tolist()\n",
    "    if len(all_true) == 1:\n",
    "        return all_true[0].upper()\n",
    "    else:\n",
    "        # Do something more sophisticated here?\n",
    "        return \"INVALID\"\n",
    "\n",
    "MAP_FNS = {\"android\": ANDROID_MAP_FN, \"ios\": IOS_MAP_FN}\n",
    "TRANSITION_FNS = {\"android\": embs.get_transition_mask_android, \"ios\": embs.get_transition_mask_ios}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(embs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_MODE = {\"WALKING\": 1, \"BICYCLING\": 2, \"ESCOOTER\": 2, \"BUS\": 3, \"TRAIN\": 4, \"LIGHT_RAIL\": 4, \"SUBWAY\": 4, \"CAR\": 5}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tradeoff_entries(pv):\n",
    "    tradeoff_entry_list = []\n",
    "    for phone_os, phone_map in pv.map().items():\n",
    "        print(15 * \"=*\")\n",
    "        print(phone_os, phone_map.keys())\n",
    "        for phone_label, phone_detail_map in phone_map.items():\n",
    "            print(4 * ' ', 15 * \"-*\")\n",
    "            print(4 * ' ', phone_label, phone_detail_map.keys())\n",
    "            if \"control\" in phone_detail_map[\"role\"]:\n",
    "                print(\"Ignoring %s phone %s since they are always on\" % (phone_detail_map[\"role\"], phone_label))\n",
    "                continue\n",
    "            # this spec does not have any calibration ranges, but evaluation ranges are actually cooler\n",
    "            for r in phone_detail_map[\"evaluation_ranges\"]:\n",
    "                print(8 * ' ', 30 * \"=\")\n",
    "                print(8 * ' ',r.keys())\n",
    "                print(8 * ' ',r[\"trip_id\"], r[\"eval_common_trip_id\"], r[\"eval_role\"], len(r[\"evaluation_trip_ranges\"]))\n",
    "                bcs = r[\"battery_df\"][\"battery_level_pct\"]\n",
    "                delta_battery = bcs.iloc[0] - bcs.iloc[-1]\n",
    "                print(\"Battery starts at %d, ends at %d, drain = %d\" % (bcs.iloc[0], bcs.iloc[-1], delta_battery))\n",
    "\n",
    "                for tr in r[\"evaluation_trip_ranges\"]:\n",
    "                    matching_section_map = embs.find_matching_segments(tr[\"evaluation_section_ranges\"], \n",
    "                                                                          \"trip_id\", \n",
    "                                                                          tr[\"sensed_section_ranges\"])\n",
    "                    print(\"For trip %s, found matching ranges %s\" % (tr[\"trip_id\"], matching_section_map))\n",
    "                    for sid, msm in matching_section_map.items():\n",
    "                        for ms in msm[\"match\"]:\n",
    "                            ms[\"mode\"] = ms[\"sensed_mode\"]\n",
    "                    for section in tr[\"evaluation_section_ranges\"]:\n",
    "                        section_gt_leg = pv.spec_details.get_ground_truth_for_leg(tr[\"trip_id_base\"],\n",
    "                                                                                  section[\"trip_id_base\"],\n",
    "                                                                                  tr[\"start_ts\"], tr[\"end_ts\"])\n",
    "                        if section_gt_leg[\"type\"] == \"WAITING\":\n",
    "                            print(\"Skipping WAITING section %s %s with potential partway transitions\" %\n",
    "                                  (tr[\"trip_id\"], section[\"trip_id\"]))\n",
    "                            continue\n",
    "                        result = embs.get_mode_check_results(section, section_gt_leg, matching_section_map)\n",
    "                        tradeoff_entry = {\"phone_os\": phone_os, \"phone_label\": phone_label,\n",
    "                                      \"timeline\": pv.spec_details.curr_spec[\"id\"],\n",
    "                                      \"range_id\": r[\"trip_id\"],\n",
    "                                     \"run\": r[\"trip_run\"], \"duration\": r[\"duration\"],\n",
    "                                     \"role\": r[\"eval_role_base\"], \"battery_drain\": delta_battery,\n",
    "                                     \"section_count\": len(tr[\"sensed_section_ranges\"]),\n",
    "                                      \"trip_id\": tr[\"trip_id\"],\n",
    "                                      \"section_id\": section[\"trip_id\"],\n",
    "                                      \"section_type\": section_gt_leg[\"type\"]}\n",
    "                        tradeoff_entry.update(result)\n",
    "                        tradeoff_entry_list.append(tradeoff_entry)\n",
    "\n",
    "    return tradeoff_entry_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# We are not going to look at battery life at the evaluation trip level; we will end with evaluation range\n",
    "# since we want to capture the overall drain for the timeline\n",
    "tradeoff_entries_list = []\n",
    "tradeoff_entries_list.extend(get_tradeoff_entries(av_la))\n",
    "tradeoff_entries_list.extend(get_tradeoff_entries(av_sj))\n",
    "tradeoff_entries_list.extend(get_tradeoff_entries(av_ucb))\n",
    "# tradeoff_entries_list.extend(get_tradeoff_entries(av_ucb_reroute))\n",
    "tradeoff_df = pd.DataFrame(tradeoff_entries_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.role.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add in other entries to the dataframe to allow us to plot better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r2q_map = {\"power_control\": 0, \"MAMFDC\": 1, \"HAMFDC\": 2, \"MAHFDC\": 3, \"HAHFDC\": 4, \"accuracy_control\": 5}\n",
    "q2r_map = {0: \"power\", 1: \"MAMFDC\", 2: \"HAMFDC\", 3: \"MAHFDC\", 4: \"HAHFDC\", 5: \"accuracy\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a number so that can get the plots to come out in order\n",
    "tradeoff_df[\"quality\"] = tradeoff_df.role.apply(lambda r: r2q_map[r])\n",
    "tradeoff_df[\"gt_duration_mins\"] = tradeoff_df.gt_duration // 60"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Timeline + section count variations\n",
    "\n",
    "We should ideally have only one transition in every TRAVEL section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"timeline=='unimodal_trip_car_bike_mtv_la' & run == 1 & role == 'HAMFDC'\").section_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tradeoff_df_filtered = tradeoff_df.query(\"gt_duration > (20*60) & ((section_id != 'commuter_rail_with_tunnels_0' & section_id != 'inner_suburb_downtown_walk_0') | phone_os != 'android')\")\n",
    "tradeoff_df_filtered = tradeoff_df.query(\"((section_id != 'commuter_rail_with_tunnels_0' & section_id != 'inner_suburb_downtown_walk_0') | phone_os != 'android') & section_type == 'TRAVEL'\")\n",
    "tradeoff_df_filtered.section_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'tt' not in 'tt_city_escooter_city_bus_rapid_transit_0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ifig, ax_array = plt.subplots(nrows=2,ncols=3,figsize=(9,6), sharex=False, sharey=False)\n",
    "timeline_list = [\"train_bus_ebike_mtv_ucb\", \"car_scooter_brex_san_jose\", \"unimodal_trip_car_bike_mtv_la\"]\n",
    "for i, tl in enumerate(timeline_list):\n",
    "    print(len(tradeoff_df_filtered.query(\"timeline == @tl & phone_os == 'android'\")))\n",
    "    tradeoff_df_filtered.query(\"timeline == @tl & phone_os == 'android'\").boxplot(ax = ax_array[0][i], column=[\"matching_pct\"], by=[\"quality\"])\n",
    "    ax_array[0][i].set_title(tl)\n",
    "    print(len(tradeoff_df_filtered.query(\"timeline == @tl & phone_os == 'ios'\")))\n",
    "    tradeoff_df_filtered.query(\"timeline == @tl & phone_os == 'ios'\").boxplot(ax = ax_array[1][i], column=[\"matching_pct\"], by=[\"quality\"])\n",
    "    ax_array[1][i].set_title(\"\")\n",
    "    # tradeoff_df.query(\"timeline == @tl & phone_os == 'ios'\").boxplot(ax = ax_array[2][i], column=[\"visit_reports\"], by=[\"quality\"])\n",
    "    # ax_array[2][i].set_title(\"\")\n",
    "\n",
    "    # print(android_ax_returned.shape, ios_ax_returned.shape)\n",
    "\n",
    "for i, ax in enumerate(ax_array[0]):\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "for i, ax in enumerate(ax_array[1]):\n",
    "    ax.set_xticklabels([q2r_map[int(t.get_text())] for t in ax.get_xticklabels()])\n",
    "    ax.set_xlabel(\"\")\n",
    "\n",
    "# for ax in ax_array[1]:\n",
    "#     ax.set_xticklabels(q2r_ios_list[1:])\n",
    "#     ax.set_xlabel(\"\")\n",
    "\n",
    "# for ax in ax_array[2]:\n",
    "#     ax.set_xticklabels(q2r_ios_list[1:])\n",
    "#     ax.set_xlabel(\"\")\n",
    "\n",
    "ax_array[0][0].set_ylabel(\"% of correct mode (android)\")\n",
    "ax_array[1][0].set_ylabel(\"% of current mode (ios)\")\n",
    "# ax_array[2][0].set_ylabel(\"Difference in visit reports (ios)\")\n",
    "# ifig.suptitle(\"% of section that has the correct mode v/s configured quality over multiple timelines\")\n",
    "ifig.suptitle(\"\")\n",
    "# ifig.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df_filtered.plot(x=\"gt_duration\", y=\"matching_pct\", kind='scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df_filtered.query(\"matching_pct > 1\").plot(x=\"gt_duration\", y=\"matching_pct\", kind='scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df_filtered.query(\"matching_pct <= 1\").plot(x=\"gt_duration\", y=\"matching_pct\", kind='scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matching_pct_range_list = []\n",
    "for k, df in tradeoff_df_filtered.groupby(\"gt_duration_mins\"):\n",
    "    print (k, df.matching_pct.mean(), df.matching_pct.min(), df.matching_pct.max())\n",
    "    matching_pct_range_list.append({\"gt_duration_mins\": k, \"mean\": df.matching_pct.mean(), \"min\": df.matching_pct.min(), \"max\": df.matching_pct.max()})\n",
    "matching_pct_range_df = pd.DataFrame(matching_pct_range_list)\n",
    "ifig, ax = plt.subplots(1,1, figsize=(4,4), squeeze=True)\n",
    "ax.errorbar(matching_pct_range_df.gt_duration_mins, y=matching_pct_range_df[\"mean\"],  yerr = [matching_pct_range_df[\"mean\"] - matching_pct_range_df[\"min\"],\n",
    "                                                                                              matching_pct_range_df[\"max\"] - matching_pct_range_df[\"mean\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matching_pct_range_df_filtered = matching_pct_range_df.query('gt_duration_mins > 10')\n",
    "ifig, ax = plt.subplots(1,1, figsize=(4,4), squeeze=True)\n",
    "ax.errorbar(matching_pct_range_df_filtered.gt_duration_mins, y=matching_pct_range_df_filtered[\"mean\"],  yerr = [matching_pct_range_df_filtered[\"mean\"] - matching_pct_range_df_filtered[\"min\"],\n",
    "                                                                                              matching_pct_range_df_filtered[\"max\"] - matching_pct_range_df_filtered[\"mean\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.array(list(zip(np.repeat([1], 37), np.repeat([10], 37)))).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df_filtered.sort_values(by=\"matching_pct\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.matching_pct.min(), tradeoff_df.matching_pct.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_of_battery_phones = tradeoff_df.query(\"timeline=='train_bus_ebike_mtv_ucb' & role=='HAHFDC' & trip_id=='berkeley_to_mtv_SF_express_bus_0' & phone_os == 'android'\")\n",
    "for i in out_of_battery_phones.index:\n",
    "    tradeoff_df.loc[i,\"end_diff_mins\"] = float('nan')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Anomaly checks\n",
    "\n",
    "We can clearly see that there are several outliers with the start/end timestamps for the sections. Let us explore these in greater detail and see if we can find any patterns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fmt = lambda ts: arrow.get(ts).to(\"America/Los_Angeles\")\n",
    "\n",
    "\n",
    "def check_outlier(eval_range, trip_idx, section_id, base_mode):\n",
    "    eval_trip = eval_range[\"evaluation_trip_ranges\"][trip_idx]\n",
    "    eval_range[\"motion_activity_df\"][\"fmt_time\"] = eval_range[\"motion_activity_df\"].ts.apply(lambda ts: fmt(ts))\n",
    "    eval_trip[\"motion_activity_df\"][\"fmt_time\"] = eval_trip[\"motion_activity_df\"].ts.apply(lambda ts: fmt(ts))\n",
    "    eval_section = [s for s in eval_trip[\"evaluation_section_ranges\"] if s[\"trip_id\"] == section_id][0]\n",
    "    print(fmt(eval_section[\"start_ts\"]), \"->\", fmt(eval_section[\"end_ts\"]))\n",
    "    print([(fmt(ssr[\"start_ts\"]), fmt(ssr[\"end_ts\"]), ssr[\"mode\"]) for ssr in eval_trip[\"sensed_section_ranges\"]])\n",
    "    matching_section_map = embs.find_matching_segments(eval_trip[\"evaluation_section_ranges\"], \"trip_id\", eval_trip[\"sensed_section_ranges\"])\n",
    "    sensed_section_range = matching_section_map[section_id][\"match\"]\n",
    "    print([(fmt(cm[\"start_ts\"]), fmt(cm[\"end_ts\"]), cm[\"mode\"]) for cm in sensed_section_range])\n",
    "    matching_sections = [s for s in sensed_section_range if s[\"mode\"] == base_mode]\n",
    "    print(\"For %s (%s -> %s) %s, matching_sections = %s\" % \n",
    "        (eval_section[\"trip_id\"], eval_section[\"start_ts\"], eval_section[\"end_ts\"], base_mode,\n",
    "        matching_sections))\n",
    "    matching_ts = sum([(s[\"end_ts\"] - s[\"start_ts\"]) for s in matching_sections])\n",
    "    print(\"matching_ts = %s, ground_truth ts = %s\" % (matching_ts, (eval_section[\"end_ts\"] - eval_section[\"start_ts\"])))\n",
    "    matching_pct = matching_ts / (eval_section[\"end_ts\"] - eval_section[\"start_ts\"])\n",
    "    print(matching_pct)\n",
    "    print(\"section activity head\")\n",
    "    ipyd.display(eval_section[\"motion_activity_df\"].head(n=3))\n",
    "    print(\"section activity tail\")\n",
    "    ipyd.display(eval_section[\"motion_activity_df\"].tail(n=3))\n",
    "    section_end_ts = eval_section[\"end_ts\"]\n",
    "    print(\"post-section end activity head\")\n",
    "    ipyd.display(eval_range[\"motion_activity_df\"].query(\"@section_end_ts <= ts <= @section_end_ts + 30 * 60\").head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_outlier_expanded(eval_range, trip_idx, section_id, base_mode):\n",
    "    eval_trip = eval_range[\"evaluation_trip_ranges\"][trip_idx]\n",
    "    eval_range[\"motion_activity_df\"][\"fmt_time\"] = eval_range[\"motion_activity_df\"].ts.apply(lambda ts: fmt(ts))\n",
    "    eval_trip[\"motion_activity_df\"][\"fmt_time\"] = eval_trip[\"motion_activity_df\"].ts.apply(lambda ts: fmt(ts))\n",
    "    eval_section = [s for s in eval_trip[\"evaluation_section_ranges\"] if s[\"trip_id\"] == section_id][0]\n",
    "    print(fmt(eval_section[\"start_ts\"]), \"->\", fmt(eval_section[\"end_ts\"]))\n",
    "    print([(fmt(ssr[\"start_ts\"]), fmt(ssr[\"end_ts\"]), ssr[\"mode\"])\n",
    "           for ssr in eval_trip[\"sensed_section_ranges\"]])\n",
    "    trip_ma_df = eval_trip[\"motion_activity_df\"]\n",
    "    # we may get some transitions after the trip ends \n",
    "    # let's expand the activity range to account for that\n",
    "    trip_end_ts = eval_trip[\"end_ts\"]\n",
    "    extended_ma_df = eval_range[\"motion_activity_df\"].query(\"@trip_end_ts <= ts <= @trip_end_ts + 30 * 60\")\n",
    "    ma_df = pd.concat([trip_ma_df, extended_ma_df],\n",
    "            axis=\"index\")\n",
    "\n",
    "    curr_trip_section_transitions = embs.find_section_transitions(\n",
    "        ma_df.query(embs.VALID_QUERIES_NO_STILL[\"android\"]), embs.TRANSITION_FNS[\"android\"])\n",
    "    \n",
    "    ipyd.display(curr_trip_section_transitions)\n",
    "    \n",
    "    last_section = eval_trip[\"evaluation_section_ranges\"][-1]\n",
    "    last_section_gt = pv_ucb.spec_details.get_ground_truth_for_leg(eval_trip[\"trip_id_base\"], last_section[\"trip_id_base\"], eval_trip[\"start_ts\"], eval_trip[\"end_ts\"])\n",
    "    if last_section_gt[\"mode\"] == \"WALKING\":\n",
    "        # For trip that end in walking, we need to include still transitions as valid\n",
    "        # otherwise, there is no end transition from walking to a valid mode\n",
    "        if len(curr_trip_section_transitions) > 0:\n",
    "            curr_last_transition_ts = curr_trip_section_transitions.iloc[-1].ts\n",
    "        else:\n",
    "            curr_last_transition_ts = 0\n",
    "        print(\"Trip ending in walking found, checking for any final still transitions > %s\" % curr_last_transition_ts)\n",
    "        still_section_transitions = extended_ma_df.query(\"ts > @curr_last_transition_ts\").query(embs.STILL_ENTRIES[\"android\"])\n",
    "        if len(still_section_transitions) > 0:\n",
    "            curr_trip_section_transitions = curr_trip_section_transitions.append(still_section_transitions.iloc[0])\n",
    "            \n",
    "    ipyd.display(curr_trip_section_transitions)\n",
    "\n",
    "    matching_section_map = embs.find_matching_segments(eval_trip[\"evaluation_section_ranges\"], \"trip_id\", eval_trip[\"sensed_section_ranges\"])\n",
    "    sensed_section_range = matching_section_map[section_id][\"match\"]\n",
    "    print([(fmt(cm[\"start_ts\"]), fmt(cm[\"end_ts\"]), cm[\"mode\"]) for cm in sensed_section_range])\n",
    "    matching_sections = [s for s in sensed_section_range if s[\"mode\"] == base_mode]\n",
    "    print(\"For %s (%s -> %s) %s, matching_sections = %s\" % \n",
    "        (eval_section[\"trip_id\"], eval_section[\"start_ts\"], eval_section[\"end_ts\"], base_mode,\n",
    "        matching_sections))\n",
    "    matching_ts = sum([(s[\"end_ts\"] - s[\"start_ts\"]) for s in matching_sections])\n",
    "    print(\"matching_ts = %s, ground_truth ts = %s\" % (matching_ts, (eval_section[\"end_ts\"] - eval_section[\"start_ts\"])))\n",
    "    matching_pct = matching_ts / (eval_section[\"end_ts\"] - eval_section[\"start_ts\"])\n",
    "    print(matching_pct)\n",
    "    print(\"section activity head\")\n",
    "    ipyd.display(eval_section[\"motion_activity_df\"].head(n=3))\n",
    "    print(\"section activity tail\")\n",
    "    ipyd.display(eval_section[\"motion_activity_df\"].tail(n=3))\n",
    "    section_end_ts = eval_section[\"end_ts\"]\n",
    "    print(\"post-section end activity head\")\n",
    "    ipyd.display(eval_range[\"motion_activity_df\"].query(\"@section_end_ts <= ts <= @section_end_ts + 30 * 60\").head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### sections which have matching pct > 1\n",
    "\n",
    "This is mainly caused by \n",
    "\n",
    "- most of the highest values are from `walk_start` and `walk_end`. This is because we end up matching them with sections that correspond to the entire trip and not just the transitions. For e.g. `walk_end` is from `19:20:31 -> 19:20:57` but it matches the section from `19:01:53 -> 19:27:21` because it is all WALKING.\n",
    "\n",
    "- looking at longer sections, the first \"real\" section is `walk to the bikeshare location_0`. Again, it was from `16:37:07 -> 2019-07-24T16:41:54` but we matched the entire `WALKING` range of `16:38:36 -> 17:21:13`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"matching_pct > 0\").sort_values(by=\"matching_pct\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"phone_os == 'ios' & phone_label == 'ucb-sdb-ios-3' & timeline == 'unimodal_trip_car_bike_mtv_la'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "av_la.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][0][\"evaluation_trip_ranges\"][0][\"sensed_section_ranges\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(av_la.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][0], 1, \"walk_end_0\", \"WALKING\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(av_ucb.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][0], 2, \"walk to the bikeshare location_0\", \"WALKING\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### sections which have 0 matching_pct\n",
    "\n",
    "- suburb_city_driving_weekend_0: matches a walking trip, no motion activity until tracking actually stops. GT trip end for the `walk_start` section is `17:40:03`, first point in the motion activity df for the **range** is at `17:46:39`. The AUTOMOTIVE range GT end is `17:52:26`; the sensed range is from `18:33:45 -> 19:41:13`.\n",
    "- similarly for `city_escooter`\n",
    "\n",
    "   ```\n",
    "   Ground truth = 16:18:07 -> 16:38:14\n",
    "   (<Arrow [2019-07-22T16:11:09.955601-07:00]>, <Arrow [2019-07-22T16:59:30.826229-07:00]>, 'WALKING'\n",
    "   (<Arrow [2019-07-22T16:59:30.826229-07:00]>, <Arrow [2019-07-22T17:01:30.321116-07:00]>, 'AUTOMOTIVE'\n",
    "   (<Arrow [2019-07-22T17:01:30.321116-07:00]>, <Arrow [2019-07-22T17:02:54.217346-07:00]>, 'WALKING'\n",
    "   (<Arrow [2019-07-22T17:02:54.217346-07:00]>, <Arrow [2019-07-22T17:34:33.386226-07:00]>, 'AUTOMOTIVE'\n",
    "   (<Arrow [2019-07-22T17:34:33.386226-07:00]>, <Arrow [2019-07-22T17:46:59.568747-07:00]>, 'WALKING')\n",
    "   ```\n",
    "\n",
    "- for `commuter_rail_with_tunnels_0`\n",
    "\n",
    "Phone ran out during this section. Need to exclude\n",
    "\n",
    "- similarly for `inner_suburb_downtown_walk_0`\n",
    "\n",
    "- for `suburb_city_driving_weekend_0`, classified as `CYCLING`\n",
    "\n",
    "\n",
    "```\n",
    "\n",
    "2019-07-27T17:40:03.318182-07:00 -> 2019-07-27T17:52:26.823849-07:00\n",
    "[(<Arrow [2019-07-27T17:43:45.507000-07:00]>, <Arrow [2019-07-27T17:51:10.151000-07:00]>, 'CYCLING'\n",
    "(<Arrow [2019-07-27T17:51:10.151000-07:00]>, <Arrow [2019-07-27T17:53:44.761000-07:00]>, 'AUTOMOTIVE')]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"matching_pct == 0\").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(av_la.map()['android']['ucb-sdb-android-3'][\"evaluation_ranges\"][0], 0, \"walk_start_0\", \"WALKING\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tradeoff_df.query(\"matching_pct == 0 & section_id != 'walk_start_0' and section_id != 'walk_end_0' & ((section_id != 'commuter_rail_with_tunnels_0' & section_id != 'inner_suburb_downtown_walk_0') | phone_os != 'android')\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(av_sj.map()['ios']['ucb-sdb-ios-3'][\"evaluation_ranges\"][0], 1, \"city_escooter_0\", \"CYCLING\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier_expanded(av_ucb.map()['android']['ucb-sdb-android-2'][\"evaluation_ranges\"][0], 2, \"commuter_rail_with_tunnels_0\", \"AUTOMOTIVE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_outlier(av_la.map()['android']['ucb-sdb-android-2'][\"evaluation_ranges\"][0], 0, \"suburb_city_driving_weekend_0\", \"AUTOMOTIVE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}