VertexAICostOptimization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cca364d5-b0d2-4a38-9557-b13f045b2698",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Cost Optimization for Vertex Training/Pipelines\n",
    "#### ToDos (Strikethrough when complete)\n",
    " * ~~Be able to determine cost for a given training run~~\n",
    " * ~~Be able to determine cost for a given hyperparameter training run~~\n",
    " * Be able to determine pipeline cost (with arbitrary components)\n",
    " * Use Vizier to experiment with machine shapes for a given job\n",
    "   * Why is this useful?  Maybe there are some efficiencies that can be gained for a given job that is run on a schedule?\n",
    "   * How do you setup the parent, child relatioship with parameters\n",
    " * Get GPU Utilization usage for long running jobs - figure out how to return null or NAN if not\n",
    " * Populate results in BQ - report in datastudio?  Or could report directly in "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "65f95c6f-3652-4790-87e7-c70a6e6d373f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: google-cloud-billing in ./.local/lib/python3.7/site-packages (1.7.0)\n",
      "Collecting lxml\n",
      "  Downloading lxml-4.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (6.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.4/6.4 MB\u001b[0m \u001b[31m37.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: proto-plus<2.0.0dev,>=1.15.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-billing) (1.20.4)\n",
      "Requirement already satisfied: protobuf<4.0.0dev,>=3.19.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-billing) (3.19.4)\n",
      "Requirement already satisfied: grpc-google-iam-v1<1.0.0dev,>=0.12.4 in /opt/conda/lib/python3.7/site-packages (from google-cloud-billing) (0.12.4)\n",
      "Requirement already satisfied: google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0 in ./.local/lib/python3.7/site-packages (from google-cloud-billing) (2.8.2)\n",
      "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (2.27.1)\n",
      "Requirement already satisfied: google-auth<3.0dev,>=1.25.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (1.35.0)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in ./.local/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (1.56.4)\n",
      "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (1.47.0)\n",
      "Requirement already satisfied: grpcio-status<2.0dev,>=1.33.2 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (1.46.3)\n",
      "Requirement already satisfied: six>=1.9.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (1.16.0)\n",
      "Requirement already satisfied: setuptools>=40.3.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (59.8.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (4.8)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (0.2.7)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (4.2.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (2021.10.8)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (2.0.12)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (1.26.9)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-billing) (0.4.8)\n",
      "Installing collected packages: lxml\n",
      "Successfully installed lxml-4.9.1\n"
     ]
    }
   ],
   "source": [
    "!pip install -U --user google-cloud-billing lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a69c0489-7bed-4008-9925-cff6f8844ba4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Enable cloud billing API\n",
    "!gcloud services enable cloudbilling.googleapis.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "bd20636f-c765-415f-8f6b-046006ceaa1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Machine type</th>\n",
       "      <th>NVIDIA_TESLA_A100</th>\n",
       "      <th>NVIDIA_TESLA_K80</th>\n",
       "      <th>NVIDIA_TESLA_P4</th>\n",
       "      <th>NVIDIA_TESLA_P100</th>\n",
       "      <th>NVIDIA_TESLA_T4</th>\n",
       "      <th>NVIDIA_TESLA_V100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a2-highgpu-1g</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a2-highgpu-2g</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>a2-highgpu-4g</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>a2-highgpu-8g</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>a2-megagpu-16g</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>n1-standard-4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>n1-standard-8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>n1-standard-16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2, 4, 8</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>2, 4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>n1-standard-32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4, 8</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>n1-standard-64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>n1-standard-96</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>n1-highmem-2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>n1-highmem-4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>n1-highmem-8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>n1-highmem-16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2, 4, 8</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>2, 4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>n1-highmem-32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4, 8</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>n1-highmem-64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>n1-highmem-96</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>n1-highcpu-16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2, 4, 8</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>1, 2, 4</td>\n",
       "      <td>2, 4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>n1-highcpu-32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4, 8</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>2, 4</td>\n",
       "      <td>4, 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>n1-highcpu-64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>n1-highcpu-96</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Machine type  NVIDIA_TESLA_A100 NVIDIA_TESLA_K80 NVIDIA_TESLA_P4  \\\n",
       "4    a2-highgpu-1g                1.0                0               0   \n",
       "5    a2-highgpu-2g                2.0                0               0   \n",
       "6    a2-highgpu-4g                4.0                0               0   \n",
       "7    a2-highgpu-8g                8.0                0               0   \n",
       "8   a2-megagpu-16g               16.0                0               0   \n",
       "9    n1-standard-4                0.0       1, 2, 4, 8         1, 2, 4   \n",
       "10   n1-standard-8                0.0       1, 2, 4, 8         1, 2, 4   \n",
       "11  n1-standard-16                0.0          2, 4, 8         1, 2, 4   \n",
       "12  n1-standard-32                0.0             4, 8            2, 4   \n",
       "13  n1-standard-64                0.0                0               4   \n",
       "14  n1-standard-96                0.0                0               4   \n",
       "15    n1-highmem-2                0.0       1, 2, 4, 8         1, 2, 4   \n",
       "16    n1-highmem-4                0.0       1, 2, 4, 8         1, 2, 4   \n",
       "17    n1-highmem-8                0.0       1, 2, 4, 8         1, 2, 4   \n",
       "18   n1-highmem-16                0.0          2, 4, 8         1, 2, 4   \n",
       "19   n1-highmem-32                0.0             4, 8            2, 4   \n",
       "20   n1-highmem-64                0.0                0               4   \n",
       "21   n1-highmem-96                0.0                0               4   \n",
       "22   n1-highcpu-16                0.0          2, 4, 8         1, 2, 4   \n",
       "23   n1-highcpu-32                0.0             4, 8            2, 4   \n",
       "24   n1-highcpu-64                0.0                8               4   \n",
       "25   n1-highcpu-96                0.0                0               4   \n",
       "\n",
       "   NVIDIA_TESLA_P100 NVIDIA_TESLA_T4 NVIDIA_TESLA_V100  \n",
       "4                  0               0                 0  \n",
       "5                  0               0                 0  \n",
       "6                  0               0                 0  \n",
       "7                  0               0                 0  \n",
       "8                  0               0                 0  \n",
       "9            1, 2, 4         1, 2, 4        1, 2, 4, 8  \n",
       "10           1, 2, 4         1, 2, 4        1, 2, 4, 8  \n",
       "11           1, 2, 4         1, 2, 4           2, 4, 8  \n",
       "12              2, 4            2, 4              4, 8  \n",
       "13                 0               4                 8  \n",
       "14                 0               4                 8  \n",
       "15           1, 2, 4         1, 2, 4        1, 2, 4, 8  \n",
       "16           1, 2, 4         1, 2, 4        1, 2, 4, 8  \n",
       "17           1, 2, 4         1, 2, 4        1, 2, 4, 8  \n",
       "18           1, 2, 4         1, 2, 4           2, 4, 8  \n",
       "19              2, 4            2, 4              4, 8  \n",
       "20                 0               4                 8  \n",
       "21                 0               4                 8  \n",
       "22           1, 2, 4         1, 2, 4           2, 4, 8  \n",
       "23              2, 4            2, 4              4, 8  \n",
       "24                 4               4                 8  \n",
       "25                 0               4                 8  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Vertex AI Machine Shape, GPU Combination\n",
    "import pandas as pd\n",
    "\n",
    "table_vertex_compute = pd.read_html('https://cloud.google.com/vertex-ai/docs/training/configure-compute', attrs={\"id\": \"gpu-compatibility-table\"})[0]\n",
    "\n",
    "table_vertex_compute.columns = table_vertex_compute.columns.droplevel(0)\n",
    "\n",
    "# Get rid of a2-ultras & NVIDIA_A100_80GB for now until quota available\n",
    "table_vertex_compute = table_vertex_compute.loc[table_vertex_compute['Machine type'].str.contains(r'^(?!a2-ultra).*')]\n",
    "table_vertex_compute = table_vertex_compute.drop('NVIDIA_A100_80GB', axis=1)\n",
    "#table_vertex_compute = table_vertex_compute.fillna(0)\n",
    "\n",
    "table_vertex_compute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fed9779-3ec1-4416-b2d7-e52cdf06411b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Re-organize into map where a given machine shape has a child of ACCELERATOR_TYPE and each of those has a child for # of accelerators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "b102fb59-8e84-4a34-adaa-736a34727f11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NVIDIA_TESLA_A100',\n",
       " 'NVIDIA_TESLA_K80',\n",
       " 'NVIDIA_TESLA_P4',\n",
       " 'NVIDIA_TESLA_P100',\n",
       " 'NVIDIA_TESLA_T4',\n",
       " 'NVIDIA_TESLA_V100']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GPUS = table_vertex_compute.columns.tolist()[1:]\n",
    "GPUS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e902dd2-a4b9-4c91-bf28-cc041bc3dbd6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "198fdaf1-61cc-435c-a958-99ec8346b604",
   "metadata": {
    "id": "8NBduXsEaRKr",
    "tags": []
   },
   "source": [
    "### Create the study configuration\n",
    "\n",
    "The following is a sample study configuration, built as a hierarchical python dictionary. It is already filled out. Run the cell to configure the study."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8be24191-b9f2-438d-bf38-3895448b3bc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "#RJV - Hardcoding for testing - these values will be populated by the dataframe above\n",
    "param_machine_type = {\"parameter_id\": \"machine_type\",\"categorical_value_spec\": {\"values\":[\"n1-standard-4\",\"n1-standard-8\"]}}\n",
    "\n",
    "param_gpu_type = {\"parameter_id\": \"gpu_type\",\"categorical_value_spec\": {\"values\":[\"NVIDIA_TESLA_K80\",\"NVIDIA_TESLA_P4\",\"NVIDIA_TESLA_P100\",\"NVIDIA_TESLA_T4\",\"NVIDIA_TESLA_V100\"]}}\n",
    "\n",
    "param_gpu_count = {\"parameter_id\": \"gpu_count\",\"discrete_value_spec\": {\"values\":[1,2,4]}}\n",
    "# - maybe only explore - 2 & 8 (if it is, then maybe just 2)\n",
    "\n",
    "\n",
    "# Objective Metrics\n",
    "metric_training_time = {\"metric_id\": \"training_time\", \"goal\": \"MINIMIZE\"}\n",
    "\n",
    "metric_estimated_training_cost = {\"metric_id\": \"estimated_training_cost\", \"goal\": \"MINIMIZE\"}\n",
    "\n",
    "metric_gpu_utilization = {\"metric_id\": \"average_gpu_utilization\", \"goal\": \"MAXIMIZE\"}\n",
    "\n",
    "\n",
    "# Put it all together in a study configuration\n",
    "study = {\n",
    "    \"display_name\": 'Tabnet_GPU_Cost_Optimization_v2',\n",
    "    \"study_spec\": {\n",
    "        #\"algorithm\": \"RANDOM_SEARCH\",\n",
    "        \"parameters\": [\n",
    "            param_machine_type,\n",
    "            param_gpu_type,\n",
    "            param_gpu_count,\n",
    "        ],\n",
    "        \"metrics\": [metric_training_time, metric_estimated_training_cost, metric_gpu_utilization],\n",
    "    },\n",
    "}\n",
    "\n",
    "print(json.dumps(study, indent=2, sort_keys=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "ed907695-56b0-49b5-9228-069ee28dc030",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import billing_v1\n",
    "import re\n",
    "\n",
    "CACHED_PRICES=False\n",
    "GPU_PRICES= {}\n",
    "INSTANCE_CORE_PRICES = {}\n",
    "INSTANCE_RAM_PRICES = {}\n",
    "\n",
    "def normalize_price(response):\n",
    "    unit_price = response.pricing_info[0].pricing_expression.tiered_rates[0].unit_price\n",
    "    cents = unit_price.nanos/1000000000\n",
    "    if hasattr(unit_price, 'units'):\n",
    "        price= unit_price.units + cents\n",
    "    else:\n",
    "        price = cents\n",
    "    return price\n",
    "\n",
    "def get_machine_specs(machine_shape):\n",
    "    m_class,m_type,m_cores = machine_shape.split('-')\n",
    "    if m_class == 'a2':\n",
    "        #a2-highgpu: #12 vcpu, 72 Gb RAM per gpu\n",
    "        if m_type == 'highgpu':\n",
    "            gpu_mult = 12\n",
    "            ram_mult = 85\n",
    "        #a2-highgpu: #6 vcpu, 85 Gb RAM per gpu\n",
    "        if m_type == 'megagpu':\n",
    "            gpu_mult = 6\n",
    "            ram_mult = 85\n",
    "        #a2-highgpu: #6 vcpu, 85 Gb RAM per gpu\n",
    "        if m_type == 'ultragpu':\n",
    "            gpu_mult = 12\n",
    "            ram_mult = 170\n",
    "        gpus,_ = m_cores.split('g')\n",
    "        gpus = int(gpus)\n",
    "        return {'class':m_class,'cores':  gpus * gpu_mult, 'ram': gpus * ram_mult }\n",
    "    elif m_class == 'n1':\n",
    "        #n1-standard: 3.75 Gb RAM per core\n",
    "        if m_type == 'standard':\n",
    "            mult = 3.75        \n",
    "        #n1-highmem: 6.5 Gb RAM per core\n",
    "        if m_type == 'highmem':\n",
    "            mult = 6.5\n",
    "        #n1-highcpu: 1.8 Gb RAM per core\n",
    "        if m_type == 'highcpu':\n",
    "            mult = 1.8\n",
    "        return {'class':m_class,'cores':  int(m_cores), 'ram': int(m_cores) * mult }\n",
    "    elif m_class == 'e2' or m_class == 'c2':\n",
    "        #e2-standard: 4 Gb RAM per core\n",
    "        if m_type == 'standard':\n",
    "            mult = 4        \n",
    "        #e2-highmem: 8 Gb RAM per core\n",
    "        if m_type == 'highmem':\n",
    "            mult = 8\n",
    "         #e2-highcpu: 1 Gb RAM per core\n",
    "        if m_type == 'highcpu':\n",
    "            mult = 1\n",
    "        return {'class':m_class,'cores':  int(m_cores), 'ram': int(m_cores) * mult }\n",
    "\n",
    "\n",
    "def get_hourly_price(machine_shape, gpu_type='ACCELERATOR_TYPE_UNSPECIFIED', gpu_count=0):\n",
    "    global CACHED_PRICES\n",
    "    \n",
    "    # Create a client\n",
    "    \n",
    "    if not CACHED_PRICES:\n",
    "        client = billing_v1.CloudCatalogClient()\n",
    "\n",
    "        # Initialize request argument(s)\n",
    "        request = billing_v1.ListSkusRequest(\n",
    "            parent=\"services/C7E2-9256-1C43\",\n",
    "        )\n",
    "\n",
    "        # Make the request\n",
    "        page_result = client.list_skus(request=request)\n",
    "\n",
    "        # Look specifically for Vertex AI Training Americas\n",
    "        for response in page_result:\n",
    "            print(response)\n",
    "            #GPU\n",
    "            match = re.match(r'Vertex AI: Training/Pipelines.+Nvidia.+Americas',response.description)\n",
    "            if match is not None:\n",
    "                for gpu in GPUS:\n",
    "                    #_,gpu_id = gpu.split('NVIDIA_TESLA_')\n",
    "                    # Minor hack here to format change for a100_80g\n",
    "                    try:\n",
    "                        _,gpu_id = gpu.split('NVIDIA_TESLA_')\n",
    "                    except ValueError: # A100 80 Gb have a different format\n",
    "                        gpu_id = 'A100'\n",
    "                    gpu_match = re.search(gpu_id,response.description)\n",
    "                    if gpu_match is not None:\n",
    "                        price = normalize_price(response)\n",
    "                        GPU_PRICES[gpu]=price\n",
    "            match = re.match(r'Vertex AI: Training/Pipelines.+N1.+Americas',response.description)\n",
    "            if match is not None:\n",
    "                instance_match = re.search(\"Instance Core\",response.description)\n",
    "                if instance_match is not None:\n",
    "                    price = normalize_price(response)\n",
    "                    INSTANCE_CORE_PRICES['n1']=price\n",
    "                ram_match = re.search(\"Instance Ram\",response.description)\n",
    "                if ram_match is not None:\n",
    "                    price = normalize_price(response)\n",
    "                    INSTANCE_RAM_PRICES['n1']=price\n",
    "            match = re.match(r'Vertex AI: Training/Pipelines.+A2.+Americas',response.description)\n",
    "            if match is not None:\n",
    "                instance_match = re.search(\"Instance Core\",response.description)\n",
    "                if instance_match is not None:\n",
    "                    price = normalize_price(response)\n",
    "                    INSTANCE_CORE_PRICES['a2']=price\n",
    "                ram_match = re.search(\"Instance Ram\",response.description)\n",
    "                if ram_match is not None:\n",
    "                    price = normalize_price(response)\n",
    "                    INSTANCE_RAM_PRICES['a2']=price\n",
    "            match = re.match(r'Vertex AI: Training/Pipelines.+E2.+Americas',response.description)\n",
    "            if match is not None:\n",
    "                instance_match = re.search(\"Instance Core\",response.description)\n",
    "                if instance_match is not None:\n",
    "                    price = normalize_price(response)\n",
    "                    INSTANCE_CORE_PRICES['e2']=price\n",
    "                ram_match = re.search(\"Instance Ram\",response.description)\n",
    "                if ram_match is not None:\n",
    "                    price = normalize_price(response)\n",
    "                    INSTANCE_RAM_PRICES['e2']=price\n",
    "            match = re.match(r'Vertex AI: Training/Pipelines.+Compute optimized.+Americas',response.description)\n",
    "            if match is not None:\n",
    "                instance_match = re.search(\"Core\",response.description)\n",
    "                if instance_match is not None:\n",
    "                    price = normalize_price(response)\n",
    "                    INSTANCE_CORE_PRICES['c2']=price\n",
    "                ram_match = re.search(\"Ram\",response.description)\n",
    "                if ram_match is not None:\n",
    "                    price = normalize_price(response)\n",
    "                    INSTANCE_RAM_PRICES['c2']=price\n",
    "        CACHED_PRICES=True\n",
    "        \n",
    "        #Adding 'Unspecified' type for CPU-only jobs\n",
    "        GPU_PRICES['ACCELERATOR_TYPE_UNSPECIFIED']=0\n",
    "    \n",
    "    # Calculate prices\n",
    "    machine_specs = get_machine_specs(machine_shape)\n",
    "\n",
    "    if machine_specs['class']=='n1':\n",
    "        return (machine_specs['cores'] * INSTANCE_CORE_PRICES['n1']) + (machine_specs['ram'] * INSTANCE_RAM_PRICES['n1']) + (GPU_PRICES[gpu_type] * gpu_count)\n",
    "    \n",
    "    if machine_specs['class']=='a2':\n",
    "        return (machine_specs['cores'] * INSTANCE_CORE_PRICES['a2']) + (machine_specs['ram'] * INSTANCE_RAM_PRICES['a2']) + (GPU_PRICES[gpu_type] * gpu_count)\n",
    "    \n",
    "    if machine_specs['class']=='e2':\n",
    "        return (machine_specs['cores'] * INSTANCE_CORE_PRICES['e2']) + (machine_specs['ram'] * INSTANCE_RAM_PRICES['e2']) + (GPU_PRICES[gpu_type] * gpu_count)\n",
    "    \n",
    "    if machine_specs['class']=='c2':\n",
    "        return (machine_specs['cores'] * INSTANCE_CORE_PRICES['c2']) + (machine_specs['ram'] * INSTANCE_RAM_PRICES['c2']) + (GPU_PRICES[gpu_type] * gpu_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "ef06a4f1-57ef-4612-8b22-b55db05955e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.22439275"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_hourly_price('a2-highgpu-1g','NVIDIA_TESLA_A100',1)\n",
    "#get_hourly_price('e2-standard-4')\n",
    "#get_hourly_price('c2-standard-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "8ef8ea27-48e0-4dbf-af32-f22af65f446f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gpu_utilization(project_id, job_id, start_time, end_time,trial_id=False):\n",
    "    from google.cloud import monitoring_v3\n",
    "\n",
    "    client = monitoring_v3.MetricServiceClient()\n",
    "    project_name = f\"projects/{project_id}\"\n",
    "    \n",
    "    interval = monitoring_v3.TimeInterval(\n",
    "        {\n",
    "            \"end_time\": end_time,\n",
    "            #\"end_time\": {\"seconds\": end_time},\n",
    "            \"start_time\": start_time,\n",
    "            #\"start_time\": {\"seconds\": start_time},\n",
    "\n",
    "        }\n",
    "    )\n",
    "    \n",
    "    aggregation = monitoring_v3.Aggregation(\n",
    "        {\n",
    "            \"alignment_period\": {\"seconds\": 3600 * 24},  # 24 hours\n",
    "            \"per_series_aligner\": monitoring_v3.Aggregation.Aligner.ALIGN_MEAN,\n",
    "            \"cross_series_reducer\": monitoring_v3.Aggregation.Reducer.REDUCE_NONE,\n",
    "        }\n",
    "    )\n",
    "    filter = f'metric.type = \"ml.googleapis.com/training/accelerator/utilization\" AND resource.type=\"cloudml_job\" AND resource.labels.job_id =\"{job_id}\"'\n",
    "    \n",
    "    if trial_id:\n",
    "        filter += f' AND metric.labels.trial_id = \"{trial_id}\"'\n",
    "\n",
    "    results = client.list_time_series(\n",
    "        request={\n",
    "            \"name\": project_name,\n",
    "            \"filter\": filter,\n",
    "            \"interval\": interval,\n",
    "            \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
    "            \"aggregation\": aggregation,\n",
    "        }\n",
    "    )\n",
    "    \n",
    "    gpu_utilization = -1\n",
    "    for result in results:\n",
    "        #print(f\"utilization: {result.points[0].value.double_value}\")\n",
    "        gpu_utilization = result.points[0].value.double_value\n",
    "        break\n",
    "    return gpu_utilization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "de1de2b2-ecb1-4878-892d-70642e9942dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4981818181818182"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from google.protobuf.timestamp_pb2 import Timestamp\n",
    "\n",
    "start_time = Timestamp()\n",
    "end_time = Timestamp()\n",
    "\n",
    "start_time.FromJsonString('2022-08-12T20:14:00.0000Z')\n",
    "end_time.FromJsonString('2022-08-13T20:19:00.0000Z')\n",
    "\n",
    "get_gpu_utilization(project_id='gcp-ml-sandbox',job_id='7857859134383718400', start_time=start_time, end_time=end_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2400df6d-289c-412f-aa48-3d3b2ac97b1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import aiplatform\n",
    "\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "def get_job_details(\n",
    "    project: str,\n",
    "    custom_job: str,\n",
    "    location: str = \"us-central1\",\n",
    "    api_endpoint: str = \"us-central1-aiplatform.googleapis.com\",\n",
    "):\n",
    "    # The AI Platform services require regional API endpoints.\n",
    "    client_options = {\"api_endpoint\": api_endpoint}\n",
    "    # Initialize client that will be used to create and send requests.\n",
    "    # This client only needs to be created once, and can be reused for multiple requests.\n",
    "    client = aiplatform.gapic.JobServiceClient(client_options=client_options)\n",
    "    name = client.custom_job_path(\n",
    "        project=project, location=location, custom_job=custom_job\n",
    "    )\n",
    "    response = client.get_custom_job(name=name)\n",
    "    \n",
    "    gpu_utilization = get_gpu_utilization(project, custom_job, response.start_time, response.end_time)\n",
    "    job_duration = (response.end_time - response.start_time)/timedelta(hours=1)\n",
    "    job_cost = 0\n",
    "    \n",
    "    compute_resources = []\n",
    "    \n",
    "    for i, worker_pool_spec in enumerate(response.job_spec.worker_pool_specs):\n",
    "        machine_spec = worker_pool_spec.machine_spec\n",
    "        job_cost += get_hourly_price(machine_spec.machine_type, machine_spec.accelerator_type.name,machine_spec.accelerator_count) * worker_pool_spec.replica_count * job_duration\n",
    "        \n",
    "        compute_resources.append(\n",
    "            {\n",
    "            'worker_pool':i,\n",
    "            'machine_type': machine_spec.machine_type, \n",
    "            'accelerator_type':machine_spec.accelerator_type.name,\n",
    "            'accelerator_count':machine_spec.accelerator_count,\n",
    "            'replica_count':worker_pool_spec.replica_count\n",
    "            }\n",
    "        )\n",
    "    \n",
    "    return {'compute_resources': compute_resources,\n",
    "            'gpu_utilization':gpu_utilization,\n",
    "            'job_duration_hours': job_duration,\n",
    "            'job_cost': job_cost}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "3757e7c5-a882-428e-9a3a-c75765e8dd07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'compute_resources': [{'worker_pool': 0,\n",
       "   'machine_type': 'n1-standard-4',\n",
       "   'accelerator_type': 'NVIDIA_TESLA_T4',\n",
       "   'accelerator_count': 1,\n",
       "   'replica_count': 1},\n",
       "  {'worker_pool': 1,\n",
       "   'machine_type': 'n1-standard-4',\n",
       "   'accelerator_type': 'NVIDIA_TESLA_T4',\n",
       "   'accelerator_count': 1,\n",
       "   'replica_count': 3}],\n",
       " 'gpu_utilization': -1,\n",
       " 'job_duration_hours': 0.31805555555555554,\n",
       " 'job_cost': 0.7900485369444444}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_job_details('gcp-ml-sandbox','2891126792574205952')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "95f5f6b0-93db-456b-bf14-9f699a0564aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import aiplatform\n",
    "\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "def get_hpt_job_details(\n",
    "    project: str,\n",
    "    hyperparameter_tuning_job: str,\n",
    "    location: str = \"us-central1\",\n",
    "    api_endpoint: str = \"us-central1-aiplatform.googleapis.com\",\n",
    "):\n",
    "    # The AI Platform services require regional API endpoints.\n",
    "    client_options = {\"api_endpoint\": api_endpoint}\n",
    "\n",
    "    client = aiplatform.gapic.JobServiceClient(client_options=client_options)\n",
    "    name = client.hyperparameter_tuning_job_path(\n",
    "        project=project, location=location, hyperparameter_tuning_job=hyperparameter_tuning_job\n",
    "    )\n",
    "    response = client.get_hyperparameter_tuning_job(name=name)\n",
    "\n",
    "    machine_spec = response.trial_job_spec.worker_pool_specs[0].machine_spec\n",
    "    \n",
    "    \n",
    "    \n",
    "    total_trials = len(response.trials)\n",
    "    total_job_duration = 0\n",
    "    total_job_cost = 0\n",
    "    agg_gpu_utilization = 0\n",
    "    trial_dict = {\"trial_id\":[],\"job_duration\":[],\"job_cost\":[],\"gpu_utilization\":[]}\n",
    "    \n",
    "    hpt_trial_start_time = hpt_trial_end_time = 0\n",
    "    \n",
    "    for trial in response.trials:\n",
    "        if trial.id == '1':\n",
    "            hpt_trial_start_time = trial.start_time\n",
    "            \n",
    "        if trial.end_time:\n",
    "            job_duration = (trial.end_time - trial.start_time)/timedelta(hours=1)\n",
    "            job_cost = get_hourly_price(machine_spec.machine_type, machine_spec.accelerator_type.name,machine_spec.accelerator_count) * job_duration\n",
    "            \n",
    "            gpu_utilization = get_gpu_utilization(project, hyperparameter_tuning_job,trial.start_time, trial.end_time, trial.id)\n",
    "            \n",
    "            trial_dict['trial_id'].append(trial.id)\n",
    "            trial_dict['job_duration'].append(job_duration)\n",
    "            trial_dict['job_cost'].append(job_cost)\n",
    "            trial_dict['gpu_utilization'].append(gpu_utilization)\n",
    "            \n",
    "            total_job_duration+=job_duration\n",
    "            total_job_cost += job_cost\n",
    "            agg_gpu_utilization+=gpu_utilization\n",
    "            \n",
    "            #Lazily setting last trial's end time to hpt end time\n",
    "            hpt_trial_end_time = trial.end_time\n",
    "\n",
    "    \n",
    "    trial_df = pd.DataFrame.from_dict(trial_dict)\n",
    "    #trial_df.set_index('trial_id')\n",
    "    \n",
    "    # plt = trial_df.plot(y='job_cost',figsize=(20,6))\n",
    "    # plt.set_xlabel('trial #')\n",
    "    # plt.set_ylabel('trial cost ($)')\n",
    "    \n",
    "    return {'machine_type': machine_spec.machine_type, \n",
    "            'accelerator_type':machine_spec.accelerator_type.name,\n",
    "            'accelerator_count':machine_spec.accelerator_count,\n",
    "            'avg_gpu_utilization':agg_gpu_utilization/total_trials,\n",
    "            'hpt_job_cumulative_duration_hours': total_job_duration,\n",
    "            'hpt_total_trials': total_trials,\n",
    "            'hpt_trials_per_training_hour' : total_trials/total_job_duration,\n",
    "            'job_cost': total_job_cost,\n",
    "            'job_cost_per_trial_hour':total_job_cost/total_job_duration,\n",
    "            'job_cost_per_trial':total_job_cost/total_trials,\n",
    "            'trial_details':trial_df\n",
    "           }\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "2a03c8c0-ab96-4306-9af9-b01ca4c2176d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'machine_type': 'n1-standard-4',\n",
       " 'accelerator_type': 'NVIDIA_TESLA_T4',\n",
       " 'accelerator_count': 2,\n",
       " 'avg_gpu_utilization': -0.6666666666666666,\n",
       " 'hpt_job_cumulative_duration_hours': 0.49124607638888884,\n",
       " 'hpt_total_trials': 9,\n",
       " 'hpt_trials_per_training_hour': 18.320757014810766,\n",
       " 'job_cost': 0.50278979425104,\n",
       " 'job_cost_per_trial_hour': 1.0234988500000002,\n",
       " 'job_cost_per_trial': 0.055865532694559995,\n",
       " 'trial_details':   trial_id  job_duration  job_cost  gpu_utilization\n",
       " 0        1      0.082073  0.084001               -1\n",
       " 1        2      0.079851  0.081727               -1\n",
       " 2        3      0.084017  0.085991               -1\n",
       " 3        4      0.080472  0.082363               -1\n",
       " 4        5      0.081861  0.083785               -1\n",
       " 5        6      0.082972  0.084922               -1}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_hpt_job_details('gcp-ml-sandbox','9161651226507476992')"
   ]
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "common-cpu.m93",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m93"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}