evaluation/00-analysis-android-hw-support.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "452905da",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "plt.rc('pdf', fonttype=42) # Avoid type 3 fonts\n",
    "\n",
    "sns.set_theme(style=\"white\")\n",
    "sns.set_palette(sns.color_palette(\"tab10\"))\n",
    "plt.rcParams.update({\"xtick.bottom\" : True, \"ytick.left\" : True}) # enables minor ticks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d23e4d7",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "BASE_DIR = '../android'\n",
    "RESULTS_DIR = os.path.join(BASE_DIR, '../results/CHANGE-ME/')\n",
    "RELEASE_LOOKUP_JSON = os.path.join(BASE_DIR, 'release_years.json')\n",
    "\n",
    "\n",
    "def read_device_info(path):\n",
    "    with open(path) as f:\n",
    "        x = json.load(f)\n",
    "        \n",
    "    name = x['name'].replace('\"', '')\n",
    "\n",
    "    os = x['os']\n",
    "    if '.' in os:\n",
    "        os = os[:os.find('.')]\n",
    "    \n",
    "    with open(RELEASE_LOOKUP_JSON) as f:\n",
    "        release_year = json.load(f)[name]\n",
    "    \n",
    "    return {\n",
    "        'name': name,\n",
    "        'os': int(os),\n",
    "        'release': release_year,\n",
    "        'type': \"Tablet\" if x['formFactor'] == 'TABLET' else 'Phone',\n",
    "    }\n",
    "\n",
    "\n",
    "def read_test_results(dir_path):\n",
    "    def read_log(test_name, index=1):\n",
    "        filename = test_name + '.json'\n",
    "        must_contain = test_name # the main result line includes the test name\n",
    "        \n",
    "        with open(os.path.join(dir_path, filename)) as f:\n",
    "            x = json.load(f)\n",
    "        x = [\n",
    "            m['data']\n",
    "            for m in x\n",
    "            if m['tag'] == \"HardwareKeySupportTest\" and must_contain in m['data']\n",
    "        ]\n",
    "        \n",
    "        if len(x) != 1: print(f\"bad len: {x}\")\n",
    "        return x[0].split(' ')[index] == \"true\"\n",
    "    \n",
    "    return {\n",
    "        # TEE checks\n",
    "        'hw_backed_ctr': read_log('testIsDefaultKeyGenerationHardwareBackedCtr'),\n",
    "        'hw_backed_gcm': read_log('testIsDefaultKeyGenerationHardwareBackedGcm'),\n",
    "        'hw_backed_hmac_short': read_log('testIsDefaultKeyGenerationHardwareBackedHmacShort'),\n",
    "        'hw_backed_hmac_long': read_log('testIsDefaultKeyGenerationHardwareBackedHmacLong'),\n",
    "        \n",
    "        # SE checks\n",
    "        'strongbox_ctr': read_log('testCanCreateStrongBoxKeyCtr'),\n",
    "        'strongbox_gcm': read_log('testCanCreateStrongBoxKeyGcm'),\n",
    "        'strongbox_hmac_short': read_log('testCanCreateStrongBoxKeyHmacShort'),\n",
    "        'strongbox_hmac_long': read_log('testCanCreateStrongBoxKeyHmacLong'),\n",
    "        \n",
    "        # extra checks\n",
    "        'system_feature_strongbox': read_log('testIsStrongBoxSupportSystemFeaturePresent'),\n",
    "        'system_feature_single_use_key': read_log('testIsSingleUseKeySupportSystemFeaturePresent'),\n",
    "        'system_feature_limited_use_key': read_log('testIsLimitedUseKeySupportSystemFeaturePresent'),\n",
    "        'strongbox_hmac_long_with_limited': read_log('testCanCreateStrongBoxKeyHmacLongWithLimitedUse'),\n",
    "    }\n",
    "    \n",
    "\n",
    "def read_job_result(job_name, job_path):\n",
    "    device_info = read_device_info(os.path.join(job_path, \"device.json\"))\n",
    "    raw_test_results = read_test_results(job_path)\n",
    "    \n",
    "    support = 'n/a'\n",
    "    if raw_test_results['hw_backed_ctr']:\n",
    "        support = 'TEE'\n",
    "    if raw_test_results['system_feature_strongbox'] and raw_test_results['strongbox_ctr']:\n",
    "        support = 'SE'\n",
    "        \n",
    "    test_results = {}\n",
    "    test_results['support'] = support\n",
    "    test_results['hmacLong'] = raw_test_results['strongbox_hmac_long']\n",
    "    test_results['limitedUse'] = raw_test_results['system_feature_limited_use_key']\n",
    "    test_results['hmacLongLimitedUse'] = raw_test_results['strongbox_hmac_long_with_limited']\n",
    "    \n",
    "    return {**device_info, **test_results}\n",
    "    \n",
    "def read_all_jobs():\n",
    "    all_data = []\n",
    "    for job_name in sorted(os.listdir(RESULTS_DIR)):\n",
    "        job_path = os.path.join(RESULTS_DIR, job_name)\n",
    "        if not os.path.isdir(job_path):\n",
    "            continue\n",
    "        all_data.append(read_job_result(job_name, job_path))\n",
    "    return pd.json_normalize(all_data)\n",
    "        \n",
    "df = read_all_jobs()\n",
    "\n",
    "df['os'] = df['os'].astype(pd.api.types.CategoricalDtype(list(range(6, 14)), ordered=True))\n",
    "df['release'] = df['release'].astype(pd.api.types.CategoricalDtype(list(range(2013, 2024)), ordered=True))\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3711512",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "fig.set_size_inches((5.5, 2.5))\n",
    "\n",
    "sns.swarmplot(\n",
    "    data=df,\n",
    "    x='release',\n",
    "    y='os',\n",
    "    hue='support',\n",
    "    ax=ax,\n",
    "    legend=False,\n",
    "    size=6,\n",
    ")\n",
    "\n",
    "ax.set_xlabel(\"Release Year\")\n",
    "\n",
    "ax.set_ylabel(\"Android OS Version\")\n",
    "ax.set_yticks([6, 7, 8, 9, 10, 11, 12, 13])\n",
    "ax.invert_yaxis()\n",
    "\n",
    "ax.grid(ls='--')\n",
    "\n",
    "fig.tight_layout()\n",
    "fig.savefig(\"output/android-se-survey-year-and-os.pdf\", dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3eae91d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Survey details table\n",
    "#\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    is_tee = \"\\\\checkmark\" if row['support'] in ('TEE', 'SE') else \"\"\n",
    "    is_se = \"\\\\checkmark\" if row['support'] in ('SE') else \"\"\n",
    "    print(f\"{row['name']} & {row['type']} & {row['os']} & {row['release']} & {is_tee} & {is_se} \\\\\\\\\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c502b7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Calculate estimate based on OS version distribution\n",
    "#\n",
    "\n",
    "API_DISTRIBUTIONS = {}\n",
    "\n",
    "API_DISTRIBUTIONS[2022] = {\n",
    "    9: 77.3 - 62.8,\n",
    "    10: 62.8 - 40.5,\n",
    "    11: 40.5 - 13.5,\n",
    "    12: 13.5,\n",
    "}\n",
    "\n",
    "API_DISTRIBUTIONS[2023] = {\n",
    "    9: 81.2 - 68.0,\n",
    "    10: 68.0 - 48.5,\n",
    "    11: 48.5 - 24.1,\n",
    "    12: 24.1,\n",
    "}\n",
    "\n",
    "PREVALANCE = {}\n",
    "prev_string = \"\"\n",
    "for k in API_DISTRIBUTIONS[2022].keys():\n",
    "    df_for_api = df[df.os == k]\n",
    "    PREVALANCE[k] = len(df_for_api[df_for_api.support == 'SE']) / len(df_for_api) if len(df_for_api) else 0\n",
    "    prev_string += f\"API {k} ({PREVALANCE[k]*100:.1f}\\\\%) \"\n",
    "\n",
    "prev_string = prev_string[:-1] + \".\"\n",
    "print(prev_string)\n",
    "\n",
    "for year in (2023, 2022,):\n",
    "    print(year)\n",
    "    calculation = \"$\"\n",
    "    total = 0.0\n",
    "    for k, v in API_DISTRIBUTIONS[year].items():\n",
    "        presence = PREVALANCE[k]\n",
    "        total += presence * v\n",
    "        calculation += f\"{v:.1f}\\\\% \\\\times {100*presence:.1f}\\\\% +\"\n",
    "\n",
    "    calculation = calculation[:-1]\n",
    "    calculation += f\"\\\\approx \\\\textbf{{{total:.1f}\\\\%}}$.\"\n",
    "    print(calculation)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "46f3b67128aa9e95477aaae345c3ea8b2d89ec91399f3d56304a3ee36b15e818"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}