GBDT-pytorch_flops_count.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "1\n",
      "1.13.0\n",
      "/media/hongyu/SSD/SSDUBUNTU/WUSL/Pixelhop-Pytorch\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "from skimage.util import view_as_windows\n",
    "\n",
    "from framework.pixelhop import *\n",
    "from framework.utils import *\n",
    "\n",
    "from skimage.measure import block_reduce\n",
    "import xgboost as xgb\n",
    "import warnings, gc\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(torch.cuda.is_available())\n",
    "print(torch.cuda.device_count())\n",
    "print(torch.__version__)\n",
    "\n",
    "import os\n",
    "print(os.getcwd())\n",
    "\n",
    "from framework.dftloss import *\n",
    "\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, plot_confusion_matrix\n",
    "import pandas as pd\n",
    "import seaborn as sn\n",
    "\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training with input: (50000, 32, 32, 3)\n",
      "(50000, 32, 32, 3)\n",
      "(100000, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "# ---------- Load MNIST data and split ----------\n",
    "(x_train, y_train), (x_test,y_test) = cifar10.load_data()\n",
    "# -----------Data Preprocessing-----------\n",
    "x_train = np.asarray(x_train,dtype='float32')\n",
    "x_test = np.asarray(x_test,dtype='float32')\n",
    "y_train = np.asarray(y_train,dtype='int')\n",
    "y_test = np.asarray(y_test,dtype='int')\n",
    "print(\"training with input:\", x_train.shape)\n",
    "\n",
    "# brutal force flip augmentation of trianing data:\n",
    "Xtrain = []\n",
    "for img in x_train:\n",
    "    \n",
    "    imgflip = cv2.flip(img,1)\n",
    "    Xtrain.append(imgflip)\n",
    "    \n",
    "Xtrain = np.array(Xtrain)\n",
    "print(Xtrain.shape)\n",
    "\n",
    "Xtrain = np.concatenate((Xtrain,x_train),axis=0)\n",
    "print(Xtrain.shape)\n",
    "\n",
    "Ytrain = np.concatenate((y_train,y_train),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model loaded at  ./model/model_d=535.pkl\n"
     ]
    }
   ],
   "source": [
    "module_filename = './model/model_d=535.pkl'\n",
    "mymodel = pickle.load(open(module_filename ,'rb'))\n",
    "print(\"model loaded at \", module_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t0 = time.time()\n",
    "\n",
    "feattrain = mymodel.transform(Xtrain)[-3] #hop -1\n",
    "# feattrain = mymodel.transform(Xtrain)[-2] # hop -2 5x5\n",
    "feattrain = feattrain.reshape(feattrain.shape[0],-1)\n",
    "print(feattrain.shape, Ytrain.shape)\n",
    "\n",
    "feattest = mymodel.transform(x_test)[-3]  #hop -1\n",
    "# feattest = mymodel.transform(x_test)[-2] # hop -2 5x5\n",
    "feattest = feattest.reshape(feattest.shape[0],-1)\n",
    "print(feattest.shape, y_test.shape)\n",
    "\n",
    "print(\"extraction time:\", time.time() - t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 535)\n",
      "(10000, 535)\n"
     ]
    }
   ],
   "source": [
    "feature_filename = './model/feat_train+test_d=535.pkl'\n",
    "feat_load = pickle.load(open(feature_filename ,'rb'))\n",
    "feattrain, feattest = feat_load\n",
    "\n",
    "print(feattrain.shape)\n",
    "print(feattest.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hongyu/anaconda3/envs/saab/lib/python3.8/site-packages/sklearn/preprocessing/_label.py:98: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n",
      "/home/hongyu/anaconda3/envs/saab/lib/python3.8/site-packages/sklearn/preprocessing/_label.py:133: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# xgb_clf = xgb.XGBClassifier(objective='multi:softmax', \n",
    "#                             num_class=3, \n",
    "#                             missing=1, \n",
    "#                             early_stopping_rounds=10, \n",
    "xgb_model = XGBClassifier(n_estimators = 2000, \\\n",
    "                            learning_rate=0.2,\\\n",
    "                            max_depth=5,\\\n",
    "                            num_class=10,\\\n",
    "                            objective='multi:softmax', \\\n",
    "                            tree_method='gpu_hist', \\\n",
    "                            gpu_id=0).fit(feattrain, Ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000,)\n",
      "Test Accuary: 71.56%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgcAAAGeCAYAAAAAIKItAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6HElEQVR4nO3de3hU1b3/8c/kNoSEJAZJJhGDoGiIIioojCj0SApi5MAhXrBRsXDg6AlUSEFNRQFRA1hFUYFqLVCR4uWILbSgMVYoNdxC8SACQuVHrDCJikkgmMll9u8PDyN7IJeBCXvivF8+63nM3nvWfAIk+Wattde2GYZhCAAA4P+EWR0AAAAEF4oDAABgQnEAAABMKA4AAIAJxQEAADChOAAAACYUBwAAwITiAAAAmFAcAAAAE4oDAABgEmF1gOPcnxRaHaFRnfr+l9URmmSPiLQ6QqOO1bmtjtCkuoZ6qyM0KZh3Nw8LC+7fLYL5z06SzovtaHWERlW4q62O0Kxvj+5r1f7rvv48YH1Fntutxdc2NDRoxowZWrZsmVwul1JTU3XPPfdo2rRpstlskr7/tz19+nS9/PLLqqioUP/+/bVw4UJ1797d28/hw4c1ceJErVq1SmFhYcrOztZzzz2n2NjYFuUI7q9uAABCyJw5c7Rw4UK98MIL2rVrl+bMmaO5c+fq+eef914zd+5czZ8/X4sWLdKmTZsUExOjIUOGqKamxntNTk6Odu7cqcLCQq1evVrr16/X+PHjW5zDFiwPXmLk4PQxcnD6GDk4fYwcnBlGDs5Mq48clO8NWF+RSd2bv+j/3HzzzUpOTtYrr7ziPZadna3o6GgtW7ZMhmEoNTVVv/zlLzVlyhRJUmVlpZKTk7VkyRKNGjVKu3btUkZGhrZs2aI+ffpIktauXaubbrpJ//rXv5SamtpsjuD+6gYAwAqGJ2DN7XarqqrK1NzuU//idO2116qoqEifffaZJOnjjz/Whg0bNHToUEnS/v375XK5lJmZ6X1NfHy8+vbtq+LiYklScXGxEhISvIWBJGVmZiosLEybNm1q0adPcQAAQCsqKChQfHy8qRUUFJzy2oceekijRo1Senq6IiMjdeWVV2rSpEnKycmRJLlcLklScnKy6XXJycnecy6XS0lJSabzERERSkxM9F7TnKBZkAgAQNDweALWVX5+vvLy8kzH7Hb7Ka9944039Nprr2n58uW69NJLtX37dk2aNEmpqakaPXp0wDI1h+IAAAAfhhG44sButzdaDPiaOnWqd/RAknr27KkDBw6ooKBAo0ePlsPhkCSVlZUpJSXF+7qysjJdccUVkiSHw6Hy8nJTv/X19Tp8+LD39c1hWgEAAF8eT+CaH44dO3bSYt/w8HB5/q+frl27yuFwqKioyHu+qqpKmzZtktPplCQ5nU5VVFSopKTEe80HH3wgj8ejvn37tigHIwcAAASJYcOG6YknnlBaWpouvfRS/eMf/9AzzzyjMWPGSJJsNpsmTZqkxx9/XN27d1fXrl31yCOPKDU1VSNGjJAk9ejRQzfeeKPGjRunRYsWqa6uThMmTNCoUaNadKeCRHEAAMDJAjit4I/nn39ejzzyiP77v/9b5eXlSk1N1X/913/p0Ucf9V7zwAMPqLq6WuPHj1dFRYWuu+46rV27Vu3atfNe89prr2nChAkaNGiQdxOk+fPntzgH+xy0APscnD72OTgzQfLleUrsc3Bm2OfgzLT2Pge1B7YFrK+oLlcFrK+zJbi/ugEAwFnHtAIAAL4smlYIFhQHAAD4CuA+B20R0woAAMDE75GDr7/+Wr/73e9UXFzs3YbR4XDo2muv1T333KNOnToFPCQAAGdTIDdBaov8GjnYsmWLLr74Ys2fP1/x8fEaMGCABgwYoPj4eM2fP1/p6enaunVra2UFAODssGgTpGDh18jBxIkTdeutt2rRokWy2Wymc4Zh6N5779XEiRO9T4ZqjNvtPvmJVLW1skdF+RMHAAC0Ar9GDj7++GNNnjz5pMJA+n7XpsmTJ2v79u3N9nOqJ1TN/e0Kf6IAANB6AvjI5rbIr5EDh8OhzZs3Kz09/ZTnN2/efNJjJE/lVE+o0r6/+RMFAIDW42mwOoGl/CoOpkyZovHjx6ukpESDBg3yFgJlZWUqKirSyy+/rF//+tfN9nOqJ1S5mVIAAASLNvobf6D4VRzk5ubq3HPP1bx587RgwQI1NHxfWYWHh6t3795asmSJbrvttlYJCgAAzg6/b2W8/fbbdfvtt6uurk5ff/21JOncc89VZGTw7u8PAIBf2uhdBoFy2jskRkZGKiUlJZBZAAAIDiE+rcAOiQAAwIRnKwAA4ItpBQAAcCLDCO1bGZlWAAAAJowcAADgK8QXJFIcAADgizUHAADAJMRHDlhzAAAATBg5AADAFw9eAgAAJkwrAAAA/ICRAwAAfHG3QnBIuPo/rY7QqKov/mp1hCa1T73e6giNCg8LtzpCkzyGYXUEtBIjyP9uK9zVVkdoVHL7c6yOYD2mFQAAAH4QNCMHAAAEDaYVAACASYgXB0wrAAAAE0YOAADwEeqPbKY4AADAV4hPK1AcAADgi1sZAQAAfsDIAQAAvkJ8WoGRAwAAfBmewDU/XHDBBbLZbCe13NxcSVJNTY1yc3PVsWNHxcbGKjs7W2VlZaY+SktLlZWVpfbt2yspKUlTp05VfX29XzkoDgAACBJbtmzRoUOHvK2wsFCSdOutt0qSJk+erFWrVunNN9/UunXrdPDgQY0cOdL7+oaGBmVlZam2tlYfffSRli5dqiVLlujRRx/1K4fNCJINyKOju1gdoVE8W+H0BfuzFepD/JntZyI8LLh/t/AE+bBwbFS01REa1RaerbC7fEur9v/dewsC1lf04P8+7ddOmjRJq1ev1t69e1VVVaVOnTpp+fLluuWWWyRJu3fvVo8ePVRcXKx+/fppzZo1uvnmm3Xw4EElJydLkhYtWqQHH3xQX331laKiolr0vsH91Q0AgBUCOK3gdrtVVVVlam63u9kItbW1WrZsmcaMGSObzaaSkhLV1dUpMzPTe016errS0tJUXFwsSSouLlbPnj29hYEkDRkyRFVVVdq5c2eLP32KAwAAWlFBQYHi4+NNraCgoNnXvfPOO6qoqNA999wjSXK5XIqKilJCQoLpuuTkZLlcLu81JxYGx88fP9dS3K0AAICvAE5L5efnKy8vz3TMbrc3+7pXXnlFQ4cOVWpqasCytBTFAQAAvgJYHNjt9hYVAyc6cOCA3n//fb399tveYw6HQ7W1taqoqDCNHpSVlcnhcHiv2bx5s6mv43czHL+mJQI+rfDFF19ozJgxge4WAICQsXjxYiUlJSkrK8t7rHfv3oqMjFRRUZH32J49e1RaWiqn0ylJcjqd2rFjh8rLy73XFBYWKi4uThkZGS1+/4CPHBw+fFhLly7V7373u0avcbvdJy3GMAxDNpst0HEAAPCfhdsnezweLV68WKNHj1ZExA8/puPj4zV27Fjl5eUpMTFRcXFxmjhxopxOp/r16ydJGjx4sDIyMnTXXXdp7ty5crlcmjZtmnJzc/0avfC7OPjTn/7U5PnPP/+82T4KCgo0c+ZM07Hw8DhFRib4GwcAgMCz8FbY999/X6WlpacchZ83b57CwsKUnZ0tt9utIUOGaMGCH267DA8P1+rVq3XffffJ6XQqJiZGo0eP1mOPPeZXBr/3OQgLC5PNZlNTL7PZbGpoaPz+8VONHCQlXRa0Iwfsc3D62Ofgx4t9Ds4M+xycmVbf5+CPcwPWV/TwBwLW19ni91d3SkqK3n77bXk8nlO2bdu2NduH3W5XXFycqQVrYQAAQKjxuzjo3bu3SkpKGj3f3KgCAABBz+MJXGuD/F5zMHXqVFVXVzd6/qKLLtJf/xrcw/AAADTJwgWJwcDv4uD665ue346JidHAgQNPOxAAALAWmyABAOCrjU4HBArFAQAAvkK8OAjue5EAAMBZx8gBAAC+QvyuO4oDAAB8Ma0AAADwA0YOAADwFeIjBxQHAAD4YhMkAABgEuIjB6w5AAAAJowcAADgi1sZAQCASYhPKwRNcdAxuoPVERoVndr0w6asVvnov1kdoVFXPrfb6ghNMhTcvx181+C2OkKjwmSzOkKTKtyNPz02GByrD96/288rD1kdARYLmuIAAICgwcgBAAAwCfFbGblbAQAAmDByAACAD8MT3OuRWhvFAQAAvkJ8zQHTCgAAwISRAwAAfIX4gkSKAwAAfLHmAAAAmLDmAAAA4AeMHAAA4CvERw4oDgAA8BXiT2VkWgEAAJgwcgAAgC+mFQAAgEmI38rItAIAADDxuzj47rvvtGHDBn366acnnaupqdHvf//7gAQDAMAyhidwrQ3yqzj47LPP1KNHDw0YMEA9e/bUwIEDdejQIe/5yspK/fznP2+2H7fbraqqKlMz2ugfIADgR8hjBK61QX4VBw8++KAuu+wylZeXa8+ePerQoYP69++v0tJSv960oKBA8fHxpnak5mu/+gAA4Mfoyy+/1J133qmOHTsqOjpaPXv21NatW73nDcPQo48+qpSUFEVHRyszM1N79+419XH48GHl5OQoLi5OCQkJGjt2rI4ePdriDH4VBx999JEKCgp07rnn6qKLLtKqVas0ZMgQXX/99fr8889b3E9+fr4qKytNrUO7c/2JAgBAqzE8noA1f3z77bfq37+/IiMjtWbNGn366ad6+umndc4553ivmTt3rubPn69FixZp06ZNiomJ0ZAhQ1RTU+O9JicnRzt37lRhYaFWr16t9evXa/z48S3O4dfdCt99950iIn54ic1m08KFCzVhwgQNHDhQy5cvb1E/drtddrvddMxmY20kACBIBHA6wO12y+12m46d6uegJM2ZM0fnn3++Fi9e7D3WtWtX7/8bhqFnn31W06ZN0/DhwyVJv//975WcnKx33nlHo0aN0q5du7R27Vpt2bJFffr0kSQ9//zzuummm/TrX/9aqampzWb26ydyenq6aWjjuBdeeEHDhw/Xv//7v/vTHQAAwSmACxJPNZVeUFBwyrf905/+pD59+ujWW29VUlKSrrzySr388sve8/v375fL5VJmZqb3WHx8vPr27avi4mJJUnFxsRISEryFgSRlZmYqLCxMmzZtatGn71dx8B//8R/6wx/+cMpzL7zwgu644w4ZIb7lJAAAJzrVVHp+fv4pr/3888+1cOFCde/eXe+++67uu+8+/eIXv9DSpUslSS6XS5KUnJxsel1ycrL3nMvlUlJSkul8RESEEhMTvdc0x6/iID8/X3/5y18aPb9gwQJ5QnxXKQDAj0AA71aw2+2Ki4sztVNNKUiSx+PRVVddpSeffFJXXnmlxo8fr3HjxmnRokVn9dNnoh8AAF8eT+CaH1JSUpSRkWE61qNHD+9dgQ6HQ5JUVlZmuqasrMx7zuFwqLy83HS+vr5ehw8f9l7THIoDAACCRP/+/bVnzx7Tsc8++0xdunSR9P3iRIfDoaKiIu/5qqoqbdq0SU6nU5LkdDpVUVGhkpIS7zUffPCBPB6P+vbt26IcPFsBAABfFm1eNHnyZF177bV68sknddttt2nz5s166aWX9NJLL0n6/i7BSZMm6fHHH1f37t3VtWtXPfLII0pNTdWIESMkfT/ScOONN3qnI+rq6jRhwgSNGjWqRXcqSBQHAACczKJde6+++mqtXLlS+fn5euyxx9S1a1c9++yzysnJ8V7zwAMPqLq6WuPHj1dFRYWuu+46rV27Vu3atfNe89prr2nChAkaNGiQwsLClJ2drfnz57c4h80IktsLOideZnWERrmOfmt1hCZVPvpvVkdo1JXP7bY6QpMMBcU//0Z91+Bu/iKLhMlmdYQmVbirrY7QJHdDndURGhUkPxaaVOv+V6v2X/3IbQHrK2bWGwHr62xh5AAAAF9t9JkIgUJxAACAD3+3Pf6x4W4FAABgwsgBAAC+mFYAAAAmFAcAAMDEolsZgwVrDgAAgEnQjBx8890RqyM0KswW3PdzXzZvh9URGvW/OedbHaFJFy3dZ3WEJl0b393qCI3aWPVPqyOglUSGB82PBuswrQAAAE5khHhxwLQCAAAwYeQAAABfIT5yQHEAAIAvdkgEAAD4ASMHAAD4YloBAACYhHhxwLQCAAAwYeQAAAAfhhHaIwcUBwAA+ArxaQWKAwAAfIV4ccCaAwAAYMLIAQAAPkL92QoUBwAA+Arx4oBpBQAAYMLIAQAAvkL70Qr+Fwe7du3Sxo0b5XQ6lZ6ert27d+u5556T2+3WnXfeqRtuuKE1cgIAcNaw5sAPa9eu1fDhwxUbG6tjx45p5cqVuvvuu9WrVy95PB4NHjxY7733XrMFgtvtltvtNh0zDEM2m83/zwAAAASUX2sOHnvsMU2dOlXffPONFi9erJ/97GcaN26cCgsLVVRUpKlTp2r27NnN9lNQUKD4+HhTq6+vPO1PAgCAgPIYgWttkF/Fwc6dO3XPPfdIkm677TYdOXJEt9xyi/d8Tk6O/vd//7fZfvLz81VZWWlqERHx/iUHAKC1eALY2iC/1xwcH/oPCwtTu3btFB//ww/1Dh06qLKy+REAu90uu91+yn4BAIC1/Bo5uOCCC7R3717vx8XFxUpLS/N+XFpaqpSUlMClAwDAAobHCFhri/waObjvvvvU0NDg/fiyyy4znV+zZg13KwAA2r42Oh0QKH4VB/fee2+T55988skzCgMAQDBoq7/xBwo7JAIAABN2SAQAwFeITyswcgAAgA/DE7jmjxkzZshms5laenq693xNTY1yc3PVsWNHxcbGKjs7W2VlZaY+SktLlZWVpfbt2yspKUlTp05VfX29XzkYOQAAIIhceumlev/9970fR0T88KN68uTJ+vOf/6w333xT8fHxmjBhgkaOHKm///3vkqSGhgZlZWXJ4XDoo48+0qFDh3T33XcrMjLSr3WBFAcAAPiycFohIiJCDofjpOOVlZV65ZVXtHz5cu+dgYsXL1aPHj20ceNG9evXT++9954+/fRTvf/++0pOTtYVV1yhWbNm6cEHH9SMGTMUFRXVogxMKwAA4COQ0wput1tVVVWm5vt8oRPt3btXqamp6tatm3JyclRaWipJKikpUV1dnTIzM73XpqenKy0tTcXFxZK+33+oZ8+eSk5O9l4zZMgQVVVVaefOnS3+/CkOAABoRad6nlBBQcEpr+3bt6+WLFmitWvXauHChdq/f7+uv/56HTlyRC6XS1FRUUpISDC9Jjk5WS6XS5LkcrlMhcHx88fPtRTTCgAA+ArgtEJ+fr7y8vJMx3wfIXDc0KFDvf9/+eWXq2/fvurSpYveeOMNRUdHBy5UMxg5AADARyCnFex2u+Li4kytseLAV0JCgi6++GLt27dPDodDtbW1qqioMF1TVlbmXaPgcDhOunvh+MenWsfQGIoDAACC1NGjR/XPf/5TKSkp6t27tyIjI1VUVOQ9v2fPHpWWlsrpdEqSnE6nduzYofLycu81hYWFiouLU0ZGRovfl2kFAAB8+Ls/QaBMmTJFw4YNU5cuXXTw4EFNnz5d4eHhuuOOOxQfH6+xY8cqLy9PiYmJiouL08SJE+V0OtWvXz9J0uDBg5WRkaG77rpLc+fOlcvl0rRp05Sbm9vi0QqJ4gAAgJNYVRz861//0h133KFvvvlGnTp10nXXXaeNGzeqU6dOkqR58+YpLCxM2dnZcrvdGjJkiBYsWOB9fXh4uFavXq377rtPTqdTMTExGj16tB577DG/ctgMwwiKp0vEtL/A6giNctfXWR2hSd3ig/cx2ccaaqyO0KS9T91odYQmnT/5T1ZHaFREWLjVEZrUPqKd1RGadPDoN1ZHaFSDp6H5iyxWV/tlq/Zf9pOfBKyv5A8/DFhfZwtrDgAAgAnTCgAA+LBqWiFYUBwAAODD8NisjmApphUAAIAJIwcAAPhgWgEAAJgYBtMKAAAAXowcAADgg2kFAABgwt0KAAAAJ2DkAAAAH8HxYAHrUBwAAOAj1KcVKA4AAPAR6sUBaw4AAIBJQEYODMOQzRbaVRYA4Mcj1NccBGTkwG63a9euXYHoCgAAyxkeW8BaW+TXyEFeXt4pjzc0NGj27Nnq2LGjJOmZZ54582QAAMASfhUHzz77rHr16qWEhATTccMwtGvXLsXExLRoesHtdsvtdp/UB1MTAIBgEOrPVvCrOHjyySf10ksv6emnn9YNN9zgPR4ZGaklS5YoIyOjRf0UFBRo5syZ5iAR8YqKTPAnDgAArSLUt0/2a83BQw89pNdff1333XefpkyZorq6utN60/z8fFVWVppaZET8afUFAAACy+8FiVdffbVKSkr01VdfqU+fPvrkk0/8ng6w2+2Ki4szNaYUAADBwmPYAtbaotO6lTE2NlZLly7VihUrlJmZqYaGhkDnAgDAMqw5OAOjRo3Sddddp5KSEnXp0iVQmQAAgIXOeBOkzp07q3PnzoHIAgBAUGir+xMECs9WAADAR6jvkEhxAACAj1AfOeDBSwAAwISRAwAAfLTVWxADheIAAAAfoX4rI9MKAADAhJEDAAB8cLcCAAAwCfU1B0wrAAAAE0YOAADwEeoLEikOAADwEeprDphWAAAgCM2ePVs2m02TJk3yHqupqVFubq46duyo2NhYZWdnq6yszPS60tJSZWVlqX379kpKStLUqVNVX1/v13sHzchBXYN/wc+mMFtwDy999V2F1REa5Uy82OoITer6yz9bHaFJXzw/0uoIjUq89w9WR2jTGjzB+6j7EP+lWZL1CxK3bNmi3/zmN7r88stNxydPnqw///nPevPNNxUfH68JEyZo5MiR+vvf/y5JamhoUFZWlhwOhz766CMdOnRId999tyIjI/Xkk0+2+P0ZOQAAwIdh2ALW/HX06FHl5OTo5Zdf1jnnnOM9XllZqVdeeUXPPPOMbrjhBvXu3VuLFy/WRx99pI0bN0qS3nvvPX366adatmyZrrjiCg0dOlSzZs3Siy++qNra2hZnoDgAAMCHx7AFrLndblVVVZma2+1u9L1zc3OVlZWlzMxM0/GSkhLV1dWZjqenpystLU3FxcWSpOLiYvXs2VPJycnea4YMGaKqqirt3LmzxZ8/xQEAAK2ooKBA8fHxplZQUHDKa1esWKFt27ad8rzL5VJUVJQSEhJMx5OTk+VyubzXnFgYHD9//FxLBc2aAwAAgkUg113k5+crLy/PdMxut5903RdffKH7779fhYWFateuXQAT+I+RAwAAfARyWsFutysuLs7UTlUclJSUqLy8XFdddZUiIiIUERGhdevWaf78+YqIiFBycrJqa2tVUVFhel1ZWZkcDockyeFwnHT3wvGPj1/TEhQHAAAEgUGDBmnHjh3avn27t/Xp00c5OTne/4+MjFRRUZH3NXv27FFpaamcTqckyel0aseOHSovL/deU1hYqLi4OGVkZLQ4C9MKAAD4sGKHxA4dOuiyyy4zHYuJiVHHjh29x8eOHau8vDwlJiYqLi5OEydOlNPpVL9+/SRJgwcPVkZGhu666y7NnTtXLpdL06ZNU25u7ilHKxpDcQAAgA+P1QEaMW/ePIWFhSk7O1tut1tDhgzRggULvOfDw8O1evVq3XfffXI6nYqJidHo0aP12GOP+fU+NsMIjk0io+ydrY7QZsVEWrtwpSnBvgnStqr9Vkdo0v7nhlsdoVHBvglSvL291RGa9M2xKqsjNCoofig0o772y1bt/2+OWwLW1/WutwLW19nCyAEAAD4MBffOuK2N4gAAAB+etjB80oq4WwEAAJgwcgAAgA8P0woAAOBErDkAAAAmwXor49nCmgMAAGByRiMH1dXVeuONN7Rv3z6lpKTojjvuUMeOHQOVDQAASzCt4IeMjAxt2LBBiYmJ+uKLLzRgwAB9++23uvjii/XPf/5Ts2bN0saNG9W1a9cm+3G73Sc9y9owDNlsof2XAQAIDkwr+GH37t2qr6+X9P0jKFNTU3XgwAFt3rxZBw4c0OWXX66HH3642X5O9WxrT8OR0/sMAABAQJ32moPi4mLNmDFD8fHxkqTY2FjNnDlTGzZsaPa1+fn5qqysNLWw8A6nGwUAgIDyBLC1RX6vOTg+9F9TU6OUlBTTufPOO09fffVVs33Y7faTng7FlAIAIFiw5sBPgwYNUkREhKqqqrRnzx7T4yUPHDjAgkQAANo4v4qD6dOnmz6OjY01fbxq1Spdf/31Z54KAAALeUJ74ODMigNfTz311BmFAQAgGIT69slsggQAAEzYPhkAAB8h/sRmigMAAHy11VsQA4XiAAAAH54Qv72eNQcAAMCEkQMAAHyw5gAAAJiE+poDphUAAIAJIwcAAPhgh0QAAGDCDokAAAAnYOQAAAAf3K0QJAwj1P8qTl94WPAOAG2r2m91hCbVNtRbHaFJjtw3rY7QqKrP11odoUmxXQdbHQFtWKivOQjenyoAAMASQTNyAABAsAj1fQ4oDgAA8BHqE90UBwAA+GDNAQAAwAkYOQAAwAdrDgAAgEmoFwdMKwAAECQWLlyoyy+/XHFxcYqLi5PT6dSaNWu852tqapSbm6uOHTsqNjZW2dnZKisrM/VRWlqqrKwstW/fXklJSZo6darq6/3b04XiAAAAH4YtcM0fnTt31uzZs1VSUqKtW7fqhhtu0PDhw7Vz505J0uTJk7Vq1Sq9+eabWrdunQ4ePKiRI0d6X9/Q0KCsrCzV1tbqo48+0tKlS7VkyRI9+uijfuWwGUGyNWFk1HlWR2iz4tvFWB2hURFh4VZHaFKw75DYYATv4ObXe1dbHaFJwb5DoscTvH+3QfFDoRn1tV+2av8Lzr8zYH399xfLzuj1iYmJeuqpp3TLLbeoU6dOWr58uW655RZJ0u7du9WjRw8VFxerX79+WrNmjW6++WYdPHhQycnJkqRFixbpwQcf1FdffaWoqKgWvScjBwAAtCK3262qqipTc7vdzb6uoaFBK1asUHV1tZxOp0pKSlRXV6fMzEzvNenp6UpLS1NxcbEkqbi4WD179vQWBpI0ZMgQVVVVeUcfWoLiAAAAH54AtoKCAsXHx5taQUFBo++9Y8cOxcbGym63695779XKlSuVkZEhl8ulqKgoJSQkmK5PTk6Wy+WSJLlcLlNhcPz88XMtxd0KAAD4COTUSn5+vvLy8kzH7HZ7o9dfcskl2r59uyorK/XWW29p9OjRWrduXQATNY/iAACAVmS325ssBnxFRUXpoosukiT17t1bW7Zs0XPPPafbb79dtbW1qqioMI0elJWVyeFwSJIcDoc2b95s6u/43QzHr2kJphUAAPDhsQWunXEWj0dut1u9e/dWZGSkioqKvOf27Nmj0tJSOZ1OSZLT6dSOHTtUXl7uvaawsFBxcXHKyMho8XsycgAAgA+r7iXJz8/X0KFDlZaWpiNHjmj58uX68MMP9e677yo+Pl5jx45VXl6eEhMTFRcXp4kTJ8rpdKpfv36SpMGDBysjI0N33XWX5s6dK5fLpWnTpik3N9ev0Qu/Rg62bdum/fv3ez9+9dVX1b9/f51//vm67rrrtGLFCn+6AwAgKAVyQaI/ysvLdffdd+uSSy7RoEGDtGXLFr377rv66U9/KkmaN2+ebr75ZmVnZ2vAgAFyOBx6++23va8PDw/X6tWrFR4eLqfTqTvvvFN33323HnvsMb9y+LXPQa9evfT0008rMzNTv/3tb/WLX/xC48aNU48ePbRnzx799re/1XPPPacxY8Y02Y/b7T7pNo7Ejumy2UL8MViniX0OTh/7HJw+9jk4M+xzcGZae5+Dp9MCt8/BL0vPbJ8DK/g1rbB37151795dkrRgwQI999xzGjdunPf81VdfrSeeeKLZ4qCgoEAzZ840HbOFxSo8PM6fOAAAtIq2UCC1Jr+mFdq3b6+vv/5akvTll1/qmmuuMZ3v27evadqhMfn5+aqsrDS1sLAO/kQBAKDVBNOCRCv4VRwMHTpUCxculCQNHDhQb731lun8G2+84b39oil2u937UInjjSkFAACCg1/TCnPmzFH//v01cOBA9enTR08//bQ+/PBD75qDjRs3auXKla2VFQCAsyJ4V4ScHX6NHKSmpuof//iHnE6n1q5dK8MwtHnzZr333nvq3Lmz/v73v+umm25qrawAAJwVRgBbW+T3PgcJCQmaPXu2Zs+e3Rp5AACAxdgECQAAH542+zt/YFAcAADggzUHAAAAJ2DkAAAAH6E9qUBxAADASUJ9WoHiAAAAH211Z8NAYc0BAAAwYeQAAAAf3MoIAABMQrs0YFoBAAD4YOQAAAAf3K0AAABMWHMQJMLCgneGwzCC+x9JdZ3b6giN6hjdweoITbKHR1odoUnB/G8vtutgqyM0qfLDX1sdoUmxA/KsjtCoiLBwqyPAYkFTHAAAECyCtyw/OygOAADwwZoDAABgEuprDoJ3oh8AAFiCkQMAAHyE9rgBxQEAACcJ9TUHTCsAAAATRg4AAPBhhPjEAsUBAAA+mFYAAAA4ASMHAAD4CPV9DigOAADwEdqlAdMKAADAByMHAAD4YFoBAACYhPrdChQHAAD4CPV9DvxaczBx4kT97W9/a60sAACEtIKCAl199dXq0KGDkpKSNGLECO3Zs8d0TU1NjXJzc9WxY0fFxsYqOztbZWVlpmtKS0uVlZWl9u3bKykpSVOnTlV9fX2Lc/hVHLz44ov6yU9+oosvvlhz5syRy+Xy5+VebrdbVVVVpmYYoV2lAQCChyeAzR/r1q1Tbm6uNm7cqMLCQtXV1Wnw4MGqrq72XjN58mStWrVKb775ptatW6eDBw9q5MiR3vMNDQ3KyspSbW2tPvroIy1dulRLlizRo48+2uIcNsOPn8phYWEqLCzUqlWr9Nprr6myslJDhw7VuHHjdNNNNyksrGW1xowZMzRz5kxz3+EdFBER3+LgZ1OwFy7hYeFWR2hUx+gOVkdoUrgtuG/YCeZ/e2XHKqyO0KTKD39tdYQmxQ7IszpCoyKC+HvKcTU1pa3a/88vyA5YX4v/3/+c9mu/+uorJSUlad26dRowYIAqKyvVqVMnLV++XLfccoskaffu3erRo4eKi4vVr18/rVmzRjfffLMOHjyo5ORkSdKiRYv04IMP6quvvlJUVFSz7+v3d8aePXvq2Wef1cGDB7Vs2TK53W6NGDFC559/vh5++GHt27ev2T7y8/NVWVlpauHhcf5GAQAg6J1qtNztdrfotZWVlZKkxMRESVJJSYnq6uqUmZnpvSY9PV1paWkqLi6WJBUXF6tnz57ewkCShgwZoqqqKu3cubNF73vavzZFRkbqtttu09q1a/X5559r3Lhxeu2113TJJZc0+1q73a64uDhTs9lspxsFAICACuS0QkFBgeLj402toKCg+QwejyZNmqT+/fvrsssukyS5XC5FRUUpISHBdG1ycrJ3qt/lcpkKg+Pnj59riYDcrZCWlqYZM2Zo+vTpev/99wPRJQAAlvEEcEovPz9feXnmaSS73d7s63Jzc/XJJ59ow4YNAcvSUn4VB126dFF4eONzUTabTT/96U/POBQAAD8Wdru9RcXAiSZMmKDVq1dr/fr16ty5s/e4w+FQbW2tKioqTKMHZWVlcjgc3ms2b95s6u/43QzHr2mOX9MK+/fvV8eOHf15CQAAbY4RwObX+xqGJkyYoJUrV+qDDz5Q165dTed79+6tyMhIFRUVeY/t2bNHpaWlcjqdkiSn06kdO3aovLzce01hYaHi4uKUkZHRohxsggQAgA+rtk/Ozc3V8uXL9cc//lEdOnTwrhGIj49XdHS04uPjNXbsWOXl5SkxMVFxcXGaOHGinE6n+vXrJ0kaPHiwMjIydNddd2nu3LlyuVyaNm2acnNzWzyCQXEAAECQWLhwoSTpJz/5ien44sWLdc8990iS5s2bp7CwMGVnZ8vtdmvIkCFasGCB99rw8HCtXr1a9913n5xOp2JiYjR69Gg99thjLc5BcQAAgA+rtk9uyd4m7dq104svvqgXX3yx0Wu6dOmiv/zlL6edg+IAAAAfPHgJAACYhPojm4N771gAAHDWMXIAAICPUH9kM8UBAAA+Qn3NAdMKAADAhJEDAAB8BPPj0s8GigMAAHxwtwIAAMAJgmbk4NzoOKsjNOpYndvqCE06Vh+8+b757ojVEZpU31BvdYQmhfbvLmcmbuAvrY7QpO8O/s3qCI1KvXCo1REsF+oLEoOmOAAAIFiE+q2MTCsAAAATRg4AAPAR6gsSKQ4AAPDBrYwAAMAk1BcksuYAAACYMHIAAICPUL9bgeIAAAAfob4gkWkFAABgwsgBAAA+uFsBAACYMK0AAABwAkYOAADwwd0KAADAxBPiaw6YVgAAACaMHAAA4CO0xw1OY+TghRde0N13360VK1ZIkl599VVlZGQoPT1dv/rVr1RfXx/wkAAAnE0eGQFrbZFfIwePP/645s6dq8GDB2vy5Mk6cOCAnnrqKU2ePFlhYWGaN2+eIiMjNXPmzCb7cbvdcrvdpmOG4ZHNxiwHAMB6bfWHeqD4VRwsWbJES5Ys0ciRI/Xxxx+rd+/eWrp0qXJyciRJ6enpeuCBB5otDgoKCk66JtZ+ruKik/yMDwAAAs2vX9UPHjyoPn36SJJ69eqlsLAwXXHFFd7zV111lQ4ePNhsP/n5+aqsrDS1Du3O9S85AACtxDCMgLW2yK/iwOFw6NNPP5Uk7d27Vw0NDd6PJWnnzp1KSmr+t3+73a64uDhTY0oBABAsWHPgh5ycHN19990aPny4ioqK9MADD2jKlCn65ptvZLPZ9MQTT+iWW25prawAAOAs8Ks4mDlzpqKjo1VcXKxx48bpoYceUq9evfTAAw/o2LFjGjZsmGbNmtVaWQEAOCtCfYdEmxEkEyLnnXOp1REadazO3fxFFjpWH7z5woJ8uqi+IbhvvQ2KL842KsxmszpCk6q/XG91hEalXjjU6gjN+qpyT6v23yfl+oD1tfXQ3wLW19kS3N+5AQDAWUdxAACAD6sWJK5fv17Dhg1TamqqbDab3nnnHdN5wzD06KOPKiUlRdHR0crMzNTevXtN1xw+fFg5OTmKi4tTQkKCxo4dq6NHj/qVg+IAAAAfVt3KWF1drV69eunFF1885fm5c+dq/vz5WrRokTZt2qSYmBgNGTJENTU13mtycnK0c+dOFRYWavXq1Vq/fr3Gjx/vVw7WHLQAaw5OH2sOzkxQfHG2Uaw5OH2sOZCudPQPWF8bD3xw0q7Adrtddru9ydfZbDatXLlSI0aMkPR9wZKamqpf/vKXmjJliiSpsrJSycnJWrJkiUaNGqVdu3YpIyNDW7Zs8e5LtHbtWt10003617/+pdTU1BZlDu7v3AAAWCCQ0woFBQWKj483tYKCAr8z7d+/Xy6XS5mZmd5j8fHx6tu3r4qLiyVJxcXFSkhI8BYGkpSZmamwsDBt2rSpxe/FUxkBAPARyFsZ8/PzlZeXZzrW3KjBqbhcLklScnKy6XhycrL3nMvlOmkzwoiICCUmJnqvaQmKAwAAfHgCOOPekimEYMO0AgAAbYDD4ZAklZWVmY6XlZV5zzkcDpWXl5vO19fX6/Dhw95rWoLiAAAAH0YA/wuUrl27yuFwqKioyHusqqpKmzZtktPplCQ5nU5VVFSopKTEe80HH3wgj8ejvn37tvi9mFYAAMBHIKcV/HH06FHt27fP+/H+/fu1fft2JSYmKi0tTZMmTdLjjz+u7t27q2vXrnrkkUeUmprqvaOhR48euvHGGzVu3DgtWrRIdXV1mjBhgkaNGtXiOxUkigMAAILG1q1b9W//9m/ej48vZBw9erSWLFmiBx54QNXV1Ro/frwqKip03XXXae3atWrXrp33Na+99pomTJigQYMGKSwsTNnZ2Zo/f75fOdjnoAXY5+D0sc/BmQmKL842in0OTh/7HEjpSVcHrK/d5VsC1tfZEjQjB0dqv7M6QqOS259jdYQm/b/Klt+ecraFhQd3cRDsgvnHW7AXLsFemKZ0u9HqCI06uPNNqyNYzqpphWAR3F89AADgrAuakQMAAIJFIO8yaIsoDgAA8MG0AgAAwAkYOQAAwAfTCgAAwMQwPFZHsBTFAQAAPjwhPnLAmgMAAGDCyAEAAD6CZPNgy1AcAADgg2kFAACAEzByAACAD6YVAACACTskAgAAnICRAwAAfLBDop8OHTqkhQsXasOGDTp06JDCwsLUrVs3jRgxQvfcc4/Cw8NbIycAAGdNqK858GtaYevWrerRo4f+8pe/qK6uTnv37lXv3r0VExOjKVOmaMCAATpy5EhrZQUAAGeBX8XBpEmTNHnyZG3dulV/+9vftGTJEn322WdasWKFPv/8cx07dkzTpk1rth+3262qqipTC/UqDQAQPDwyAtbaIr+Kg23btumuu+7yfvyzn/1M27ZtU1lZmc455xzNnTtXb731VrP9FBQUKD4+3tRq6yr8Dg8AQGswDCNgrS3yqzhISkrSoUOHvB+XlZWpvr5ecXFxkqTu3bvr8OHDzfaTn5+vyspKU4uKTPAvOQAArcRjGAFrbZFfCxJHjBihe++9V0899ZTsdrtmzZqlgQMHKjo6WpK0Z88enXfeec32Y7fbZbfbTcdsNps/UQAAQCvxqzh4/PHHdejQIQ0bNkwNDQ1yOp1atmyZ97zNZlNBQUHAQwIAcDa11emAQPGrOIiNjdXrr7+umpoa1dfXKzY21nR+8ODBAQ0HAIAV2upCwkA5rU2Q2rVrF+gcAAAgSLBDIgAAPphWAAAAJm31LoNA4cFLAADAhJEDAAB88OAlAABgwrQCAADACRg5AADAB3crAAAAE9YcAAAAk1AfOWDNAQAAMGHkAAAAH6E+ckBxAACAj9AuDSQZP0I1NTXG9OnTjZqaGqujnCSYsxkG+c5EMGczDPKdiWDOZhjkQ+DZDOPHN3ZSVVWl+Ph4VVZWKi4uzuo4JsGcTSLfmQjmbBL5zkQwZ5PIh8BjQSIAADChOAAAACYUBwAAwORHWRzY7XZNnz5ddrvd6ignCeZsEvnORDBnk8h3JoI5m0Q+BN6PckEiAAA4fT/KkQMAAHD6KA4AAIAJxQEAADChOAAAACYUBwAAwORHVxy8+OKLuuCCC9SuXTv17dtXmzdvtjqSJGn9+vUaNmyYUlNTZbPZ9M4771gdyaSgoEBXX321OnTooKSkJI0YMUJ79uyxOpYkaeHChbr88ssVFxenuLg4OZ1OrVmzxupYjZo9e7ZsNpsmTZpkdRRJ0owZM2Sz2UwtPT3d6lheX375pe6880517NhR0dHR6tmzp7Zu3Wp1LEnSBRdccNKfnc1mU25urtXRJEkNDQ165JFH1LVrV0VHR+vCCy/UrFmzguaJgkeOHNGkSZPUpUsXRUdH69prr9WWLVusjoUW+FEVB6+//rry8vI0ffp0bdu2Tb169dKQIUNUXl5udTRVV1erV69eevHFF62Ockrr1q1Tbm6uNm7cqMLCQtXV1Wnw4MGqrq62Opo6d+6s2bNnq6SkRFu3btUNN9yg4cOHa+fOnVZHO8mWLVv0m9/8RpdffrnVUUwuvfRSHTp0yNs2bNhgdSRJ0rfffqv+/fsrMjJSa9as0aeffqqnn35a55xzjtXRJH3/93nin1thYaEk6dZbb7U42ffmzJmjhQsX6oUXXtCuXbs0Z84czZ07V88//7zV0SRJ//mf/6nCwkK9+uqr2rFjhwYPHqzMzEx9+eWXVkdDcyx97FOAXXPNNUZubq7344aGBiM1NdUoKCiwMNXJJBkrV660OkaTysvLDUnGunXrrI5ySuecc47x29/+1uoYJkeOHDG6d+9uFBYWGgMHDjTuv/9+qyMZhmEY06dPN3r16mV1jFN68MEHjeuuu87qGC12//33GxdeeKHh8XisjmIYhmFkZWUZY8aMMR0bOXKkkZOTY1GiHxw7dswIDw83Vq9ebTp+1VVXGQ8//LBFqdBSP5qRg9raWpWUlCgzM9N7LCwsTJmZmSouLrYwWdtUWVkpSUpMTLQ4iVlDQ4NWrFih6upqOZ1Oq+OY5ObmKisry/RvMFjs3btXqamp6tatm3JyclRaWmp1JEnSn/70J/Xp00e33nqrkpKSdOWVV+rll1+2OtYp1dbWatmyZRozZoxsNpvVcSRJ1157rYqKivTZZ59Jkj7++GNt2LBBQ4cOtTiZVF9fr4aGBrVr1850PDo6OmhGrtC4CKsDBMrXX3+thoYGJScnm44nJydr9+7dFqVqmzwejyZNmqT+/fvrsssuszqOJGnHjh1yOp2qqalRbGysVq5cqYyMDKtjea1YsULbtm0LyvnUvn37asmSJbrkkkt06NAhzZw5U9dff70++eQTdejQwdJsn3/+uRYuXKi8vDz96le/0pYtW/SLX/xCUVFRGj16tKXZfL3zzjuqqKjQPffcY3UUr4ceekhVVVVKT09XeHi4Ghoa9MQTTygnJ8fqaOrQoYOcTqdmzZqlHj16KDk5WX/4wx9UXFysiy66yOp4aMaPpjhA4OTm5uqTTz4Jqur+kksu0fbt21VZWam33npLo0eP1rp164KiQPjiiy90//33q7Cw8KTfkoLBib9FXn755erbt6+6dOmiN954Q2PHjrUw2feFaJ8+ffTkk09Kkq688kp98sknWrRoUdAVB6+88oqGDh2q1NRUq6N4vfHGG3rttde0fPlyXXrppdq+fbsmTZqk1NTUoPjze/XVVzVmzBidd955Cg8P11VXXaU77rhDJSUlVkdDM340xcG5556r8PBwlZWVmY6XlZXJ4XBYlKrtmTBhglavXq3169erc+fOVsfxioqK8v620bt3b23ZskXPPfecfvOb31icTCopKVF5ebmuuuoq77GGhgatX79eL7zwgtxut8LDwy1MaJaQkKCLL75Y+/btszqKUlJSTirwevToof/5n/+xKNGpHThwQO+//77efvttq6OYTJ06VQ899JBGjRolSerZs6cOHDiggoKCoCgOLrzwQq1bt07V1dWqqqpSSkqKbr/9dnXr1s3qaGjGj2bNQVRUlHr37q2ioiLvMY/Ho6KioqCbmw5GhmFowoQJWrlypT744AN17drV6khN8ng8crvdVseQJA0aNEg7duzQ9u3bva1Pnz7KycnR9u3bg6owkKSjR4/qn//8p1JSUqyOov79+590y+xnn32mLl26WJTo1BYvXqykpCRlZWVZHcXk2LFjCgszfxsPDw+Xx+OxKNGpxcTEKCUlRd9++63effddDR8+3OpIaMaPZuRAkvLy8jR69Gj16dNH11xzjZ599llVV1fr5z//udXRdPToUdNvavv379f27duVmJiotLQ0C5N9Lzc3V8uXL9cf//hHdejQQS6XS5IUHx+v6OhoS7Pl5+dr6NChSktL05EjR7R8+XJ9+OGHevfddy3NdVyHDh1OWpsRExOjjh07BsWajSlTpmjYsGHq0qWLDh48qOnTpys8PFx33HGH1dE0efJkXXvttXryySd12223afPmzXrppZf00ksvWR3Ny+PxaPHixRo9erQiIoLrW+awYcP0xBNPKC0tTZdeeqn+8Y9/6JlnntGYMWOsjiZJevfdd2UYhi655BLt27dPU6dOVXp6elB8T0YzrL5dItCef/55Iy0tzYiKijKuueYaY+PGjVZHMgzDMP76178akk5qo0ePtjqaYRjGKbNJMhYvXmx1NGPMmDFGly5djKioKKNTp07GoEGDjPfee8/qWE0KplsZb7/9diMlJcWIiooyzjvvPOP222839u3bZ3Usr1WrVhmXXXaZYbfbjfT0dOOll16yOpLJu+++a0gy9uzZY3WUk1RVVRn333+/kZaWZrRr187o1q2b8fDDDxtut9vqaIZhGMbrr79udOvWzYiKijIcDoeRm5trVFRUWB0LLWAzjCDZSgsAAASFH82aAwAAEBgUBwAAwITiAAAAmFAcAAAAE4oDAABgQnEAAABMKA4AAIAJxQEAADChOAAAACYUBwAAwITiAAAAmPx/x39Uwi/6O90AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "y_test_preds = []\n",
    "y_test_preds = xgb_model.predict(feattest)\n",
    "print(y_test_preds.shape)\n",
    "\n",
    "\n",
    "pred_accuracy_score = accuracy_score(y_test, y_test_preds)\n",
    "# pred_recall_score = recall_score(y_test, y_test_pred, average='macro')\n",
    "print(\"Test Accuary: %.2f%%\" % (pred_accuracy_score* 100.0))\n",
    "# print('inference time:', time.time() - t0)\n",
    "\n",
    "cnf_matrix = confusion_matrix(y_test, y_test_preds)\n",
    "df_cm = pd.DataFrame(cnf_matrix)\n",
    "sn.heatmap(df_cm)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "xgb_dir = './model/xgb_hop3.pkl'\n",
    "\n",
    "# pickle.dump(xgb_model ,open(xgb_dir ,'wb'))\n",
    "xgb_model = pickle.load(open(xgb_dir ,'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hummingbird.ml import convert, load\n",
    "\n",
    "input = np.random.rand(10, 535)\n",
    "# pytorch_model = convert(xgb_model, 'pytorch', input)\n",
    "pytorch_model = convert(xgb_model, 'pytorch', input, device='cuda')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method Module.parameters of Executor(\n",
      "  (_operators): ModuleList(\n",
      "    (0): PerfectTreeTraversalGBDTImpl(\n",
      "      (nodes): ParameterList(\n",
      "          (0): Parameter containing: [torch.int64 of size 40000 (GPU 0)]\n",
      "          (1): Parameter containing: [torch.int64 of size 80000 (GPU 0)]\n",
      "          (2): Parameter containing: [torch.int64 of size 160000 (GPU 0)]\n",
      "          (3): Parameter containing: [torch.int64 of size 320000 (GPU 0)]\n",
      "      )\n",
      "      (biases): ParameterList(\n",
      "          (0): Parameter containing: [torch.float32 of size 40000 (GPU 0)]\n",
      "          (1): Parameter containing: [torch.float32 of size 80000 (GPU 0)]\n",
      "          (2): Parameter containing: [torch.float32 of size 160000 (GPU 0)]\n",
      "          (3): Parameter containing: [torch.float32 of size 320000 (GPU 0)]\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")>\n"
     ]
    }
   ],
   "source": [
    "xgb_pytorch = pytorch_model\n",
    "print(xgb_pytorch.model.parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'hummingbird.ml._executor.Executor'>\n",
      "Executor\n"
     ]
    }
   ],
   "source": [
    "from hummingbird.ml._executor import Executor\n",
    "print(xgb_pytorch.model.__class__)\n",
    "print(xgb_pytorch.model.__class__.__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "ename": "OutOfMemoryError",
     "evalue": "CUDA out of memory. Tried to allocate 1.49 GiB (GPU 0; 7.77 GiB total capacity; 5.45 GiB already allocated; 831.50 MiB free; 5.47 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOutOfMemoryError\u001b[0m                          Traceback (most recent call last)",
      "\u001b[1;32m/media/hongyu/SSD/SSDUBUNTU/WUSL/Pixelhop-Pytorch/GBDT-pytorch_flops_count.ipynb Cell 11\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/media/hongyu/SSD/SSDUBUNTU/WUSL/Pixelhop-Pytorch/GBDT-pytorch_flops_count.ipynb#X11sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m feattest_cuda \u001b[39m=\u001b[39m feattest_cuda\u001b[39m.\u001b[39mto(\u001b[39m'\u001b[39m\u001b[39mcuda\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m      <a href='vscode-notebook-cell:/media/hongyu/SSD/SSDUBUNTU/WUSL/Pixelhop-Pytorch/GBDT-pytorch_flops_count.ipynb#X11sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m \u001b[39m# verify the converted model:\u001b[39;00m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/media/hongyu/SSD/SSDUBUNTU/WUSL/Pixelhop-Pytorch/GBDT-pytorch_flops_count.ipynb#X11sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m y_test_preds \u001b[39m=\u001b[39m xgb_pytorch\u001b[39m.\u001b[39;49mpredict(feattest)\n\u001b[1;32m      <a href='vscode-notebook-cell:/media/hongyu/SSD/SSDUBUNTU/WUSL/Pixelhop-Pytorch/GBDT-pytorch_flops_count.ipynb#X11sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m \u001b[39mprint\u001b[39m(y_test_preds\u001b[39m.\u001b[39mshape)\n\u001b[1;32m      <a href='vscode-notebook-cell:/media/hongyu/SSD/SSDUBUNTU/WUSL/Pixelhop-Pytorch/GBDT-pytorch_flops_count.ipynb#X11sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m pred_accuracy_score \u001b[39m=\u001b[39m accuracy_score(y_test, y_test_preds)\n",
      "File \u001b[0;32m~/anaconda3/envs/saab/lib/python3.8/site-packages/hummingbird/ml/containers/_sklearn_api_containers.py:119\u001b[0m, in \u001b[0;36mSklearnContainerRegression.predict\u001b[0;34m(self, *inputs)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mpredict\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39minputs):\n\u001b[1;32m    113\u001b[0m     \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    114\u001b[0m \u001b[39m    Utility functions used to emulate the behavior of the Sklearn API.\u001b[39;00m\n\u001b[1;32m    115\u001b[0m \u001b[39m    On regression returns the predicted values.\u001b[39;00m\n\u001b[1;32m    116\u001b[0m \u001b[39m    On classification tasks returns the predicted class labels for the input data.\u001b[39;00m\n\u001b[1;32m    117\u001b[0m \u001b[39m    On anomaly detection (e.g. isolation forest) returns the predicted classes (-1 or 1).\u001b[39;00m\n\u001b[1;32m    118\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 119\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_predict, \u001b[39m*\u001b[39;49minputs)\n",
      "File \u001b[0;32m~/anaconda3/envs/saab/lib/python3.8/site-packages/hummingbird/ml/containers/_sklearn_api_containers.py:67\u001b[0m, in \u001b[0;36mSklearnContainer._run\u001b[0;34m(self, function, *inputs)\u001b[0m\n\u001b[1;32m     64\u001b[0m         splits \u001b[39m=\u001b[39m [inputs[input_names[idx]] \u001b[39mfor\u001b[39;00m idx \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mlen\u001b[39m(input_names))]\n\u001b[1;32m     65\u001b[0m         inputs \u001b[39m=\u001b[39m [df\u001b[39m.\u001b[39mto_numpy()\u001b[39m.\u001b[39mreshape(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, \u001b[39m1\u001b[39m) \u001b[39mfor\u001b[39;00m df \u001b[39min\u001b[39;00m splits]\n\u001b[0;32m---> 67\u001b[0m \u001b[39mreturn\u001b[39;00m function(\u001b[39m*\u001b[39;49minputs)\n",
      "File \u001b[0;32m~/anaconda3/envs/saab/lib/python3.8/site-packages/hummingbird/ml/containers/sklearn/pytorch_containers.py:196\u001b[0m, in \u001b[0;36mPyTorchSklearnContainerRegression._predict\u001b[0;34m(self, *inputs)\u001b[0m\n\u001b[1;32m    194\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39minputs)[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mcpu()\u001b[39m.\u001b[39mnumpy()\u001b[39m.\u001b[39mravel()\n\u001b[1;32m    195\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 196\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmodel\u001b[39m.\u001b[39;49mforward(\u001b[39m*\u001b[39;49minputs)[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mcpu()\u001b[39m.\u001b[39mnumpy()\u001b[39m.\u001b[39mravel()\n",
      "File \u001b[0;32m~/anaconda3/envs/saab/lib/python3.8/site-packages/hummingbird/ml/_executor.py:115\u001b[0m, in \u001b[0;36mExecutor.forward\u001b[0;34m(self, *inputs)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[39m# Evaluate all the operators in the topology by properly wiring inputs \\ outputs\u001b[39;00m\n\u001b[1;32m    114\u001b[0m \u001b[39mfor\u001b[39;00m operator \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_operators:\n\u001b[0;32m--> 115\u001b[0m     outputs \u001b[39m=\u001b[39m operator(\u001b[39m*\u001b[39;49m(variable_map[input_name] \u001b[39mfor\u001b[39;49;00m input_name \u001b[39min\u001b[39;49;00m operator\u001b[39m.\u001b[39;49minputs))\n\u001b[1;32m    117\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(operator\u001b[39m.\u001b[39moutputs) \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m    118\u001b[0m         variable_map[operator\u001b[39m.\u001b[39moutputs[\u001b[39m0\u001b[39m]] \u001b[39m=\u001b[39m outputs\n",
      "File \u001b[0;32m~/anaconda3/envs/saab/lib/python3.8/site-packages/torch/nn/modules/module.py:1190\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1186\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1187\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1188\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1189\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1190\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1191\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1192\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/anaconda3/envs/saab/lib/python3.8/site-packages/hummingbird/ml/operator_converters/_tree_implementations.py:394\u001b[0m, in \u001b[0;36mPerfectTreeTraversalTreeImpl.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    391\u001b[0m     gather_indices \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mindex_select(nodes, \u001b[39m0\u001b[39m, prev_indices)\u001b[39m.\u001b[39mview(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_trees)\n\u001b[1;32m    392\u001b[0m     features \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mgather(x, \u001b[39m1\u001b[39m, gather_indices)\u001b[39m.\u001b[39mview(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[1;32m    393\u001b[0m     prev_indices \u001b[39m=\u001b[39m (\n\u001b[0;32m--> 394\u001b[0m         factor \u001b[39m*\u001b[39m prev_indices \u001b[39m+\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdecision_cond(features, torch\u001b[39m.\u001b[39;49mindex_select(biases, \u001b[39m0\u001b[39;49m, prev_indices))\u001b[39m.\u001b[39;49mlong()\n\u001b[1;32m    395\u001b[0m     )\n\u001b[1;32m    397\u001b[0m output \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mindex_select(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mleaf_nodes, \u001b[39m0\u001b[39m, prev_indices)\u001b[39m.\u001b[39mview(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_trees, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mn_classes)\n\u001b[1;32m    399\u001b[0m output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39maggregation(output)\n",
      "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 1.49 GiB (GPU 0; 7.77 GiB total capacity; 5.45 GiB already allocated; 831.50 MiB free; 5.47 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"
     ]
    }
   ],
   "source": [
    "feattest_cuda = torch.FloatTensor(feattest)\n",
    "feattest_cuda = feattest_cuda.to('cuda')\n",
    "# verify the converted model:\n",
    "y_test_preds = xgb_pytorch.predict(feattest)\n",
    "print(y_test_preds.shape)\n",
    "pred_accuracy_score = accuracy_score(y_test, y_test_preds)\n",
    "# pred_recall_score = recall_score(y_test, y_test_pred, average='macro')\n",
    "print(\"Test Accuary: %.2f%%\" % (pred_accuracy_score* 100.0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "xgbpytorch_dir = './model/xgb_pytorch.pkl'\n",
    "\n",
    "pickle.dump(xgb_pytorch,open(xgbpytorch_dir ,'wb'))\n",
    "# xgb_pytorch.save(\"xgb_pytorch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-------------------------- DeepSpeed Flops Profiler --------------------------\n",
      "Profile Summary at step 1:\n",
      "Notations:\n",
      "data parallel size (dp_size), model parallel size(mp_size),\n",
      "number of parameters (params), number of multiply-accumulate operations(MACs),\n",
      "number of floating-point operations (flops), floating-point operations per second (FLOPS),\n",
      "fwd latency (forward propagation latency), bwd latency (backward propagation latency),\n",
      "step (weights update latency), iter latency (sum of fwd, bwd and step latency)\n",
      "\n",
      "params per gpu:                                               1.9 M   \n",
      "params of model = params per GPU * mp_size:                   1.9 M   \n",
      "fwd MACs per GPU:                                             0 MACs  \n",
      "fwd flops per GPU:                                            0       \n",
      "fwd flops of model = fwd flops per GPU * mp_size:             0       \n",
      "fwd latency:                                                  779.39 us\n",
      "fwd FLOPS per GPU = fwd flops per GPU / fwd latency:          0.0 FLOPS\n",
      "\n",
      "----------------------------- Aggregated Profile per GPU -----------------------------\n",
      "Top 1 modules in terms of params, MACs or fwd latency at different model depths:\n",
      "depth 0:\n",
      "    params      - {'Executor': '1.9 M'}\n",
      "    MACs        - {'Executor': '0 MACs'}\n",
      "    fwd latency - {'Executor': '779.39 us'}\n",
      "depth 1:\n",
      "    params      - {'ModuleList': '1.9 M'}\n",
      "    MACs        - {'ModuleList': '0 MACs'}\n",
      "    fwd latency - {'ModuleList': '643.73 us'}\n",
      "depth 2:\n",
      "    params      - {'PerfectTreeTraversalGBDTImpl': '1.9 M'}\n",
      "    MACs        - {'PerfectTreeTraversalGBDTImpl': '0 MACs'}\n",
      "    fwd latency - {'PerfectTreeTraversalGBDTImpl': '643.73 us'}\n",
      "\n",
      "------------------------------ Detailed Profile per GPU ------------------------------\n",
      "Each module profile is listed after its name in the following order: \n",
      "params, percentage of total params, MACs, percentage of total MACs, fwd latency, percentage of total fwd latency, fwd FLOPS\n",
      "\n",
      "Note: 1. A module can have torch.nn.module or torch.nn.functional to compute logits (e.g. CrossEntropyLoss). They are not counted as submodules, thus not to be printed out. However they make up the difference between a parent's MACs (or latency) and the sum of its submodules'.\n",
      "2. Number of floating-point operations is a theoretical estimation, thus FLOPS computed using that could be larger than the maximum system throughput.\n",
      "3. The fwd latency listed in the top module's profile is directly captured at the module forward function in PyTorch, thus it's less than the fwd latency shown above which is captured in DeepSpeed.\n",
      "\n",
      "Executor(\n",
      "  1.9 M, 100.00% Params, 0 MACs, 0.00% MACs, 779.39 us, 100.00% latency, 0.0 FLOPS, \n",
      "  (_operators): ModuleList(\n",
      "    1.9 M, 100.00% Params, 0 MACs, 0.00% MACs, 643.73 us, 82.59% latency, 0.0 FLOPS, \n",
      "    (0): PerfectTreeTraversalGBDTImpl(\n",
      "      1.9 M, 100.00% Params, 0 MACs, 0.00% MACs, 643.73 us, 82.59% latency, 0.0 FLOPS, \n",
      "      (nodes): ParameterList(\n",
      "        600.0 k, 31.58% Params, 0 MACs, 0.00% MACs, 0, 0.00% latency, 0.0 FLOPS,   (0): Parameter containing: [torch.int64 of size 40000 (GPU 0)]\n",
      "          (1): Parameter containing: [torch.int64 of size 80000 (GPU 0)]\n",
      "          (2): Parameter containing: [torch.int64 of size 160000 (GPU 0)]\n",
      "          (3): Parameter containing: [torch.int64 of size 320000 (GPU 0)]\n",
      "      )\n",
      "      (biases): ParameterList(\n",
      "        600.0 k, 31.58% Params, 0 MACs, 0.00% MACs, 0, 0.00% latency, 0.0 FLOPS,   (0): Parameter containing: [torch.float32 of size 40000 (GPU 0)]\n",
      "          (1): Parameter containing: [torch.float32 of size 80000 (GPU 0)]\n",
      "          (2): Parameter containing: [torch.float32 of size 160000 (GPU 0)]\n",
      "          (3): Parameter containing: [torch.float32 of size 320000 (GPU 0)]\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n",
      "------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# deepspeed.profiling.flops_profiler.profiler.get_model_profile(model, \\\n",
    "    # input_shape=None, args=[], kwargs={}, print_profile=True, detailed=True,\\\n",
    "        # module_depth=-1, top_modules=1, warm_up=1, as_string=True, output_file=None, ignore_modules=None)\n",
    "from deepspeed.profiling.flops_profiler.profiler import get_model_profile\n",
    "\n",
    "batch_size = 1\n",
    "flops, macs, params = get_model_profile(model=pytorch_model.model, input_shape=(batch_size, 535))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(tensor([6], device='cuda:0'), tensor([[7.3612e-02, 4.2954e-05, 1.8645e-03, 6.2914e-02, 6.5400e-03, 3.0134e-04,\n",
      "         8.1835e-01, 3.0043e-05, 3.6346e-02, 1.9313e-06]], device='cuda:0'))\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from pthflops import count_ops\n",
    "\n",
    "# Create a network and a corresponding input\n",
    "device = 'cuda:0'\n",
    "model = pytorch_model.model\n",
    "inp = torch.rand(1,535).to(device)\n",
    "\n",
    "test = model(inp)\n",
    "print(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-------------------------- DeepSpeed Flops Profiler --------------------------\n",
      "Profile Summary at step 1:\n",
      "Notations:\n",
      "data parallel size (dp_size), model parallel size(mp_size),\n",
      "number of parameters (params), number of multiply-accumulate operations(MACs),\n",
      "number of floating-point operations (flops), floating-point operations per second (FLOPS),\n",
      "fwd latency (forward propagation latency), bwd latency (backward propagation latency),\n",
      "step (weights update latency), iter latency (sum of fwd, bwd and step latency)\n",
      "\n",
      "params per gpu:                                               1.9 M   \n",
      "params of model = params per GPU * mp_size:                   1.9 M   \n",
      "fwd MACs per GPU:                                             0 MACs  \n",
      "fwd flops per GPU:                                            0       \n",
      "fwd flops of model = fwd flops per GPU * mp_size:             0       \n",
      "fwd latency:                                                  972.99 us\n",
      "fwd FLOPS per GPU = fwd flops per GPU / fwd latency:          0.0 FLOPS\n",
      "\n",
      "----------------------------- Aggregated Profile per GPU -----------------------------\n",
      "Top 1 modules in terms of params, MACs or fwd latency at different model depths:\n",
      "depth 0:\n",
      "    params      - {'Executor': '1.9 M'}\n",
      "    MACs        - {'Executor': '0 MACs'}\n",
      "    fwd latency - {'Executor': '972.99 us'}\n",
      "depth 1:\n",
      "    params      - {'ModuleList': '1.9 M'}\n",
      "    MACs        - {'ModuleList': '0 MACs'}\n",
      "    fwd latency - {'ModuleList': '865.22 us'}\n",
      "depth 2:\n",
      "    params      - {'PerfectTreeTraversalGBDTImpl': '1.9 M'}\n",
      "    MACs        - {'PerfectTreeTraversalGBDTImpl': '0 MACs'}\n",
      "    fwd latency - {'PerfectTreeTraversalGBDTImpl': '865.22 us'}\n",
      "\n",
      "------------------------------ Detailed Profile per GPU ------------------------------\n",
      "Each module profile is listed after its name in the following order: \n",
      "params, percentage of total params, MACs, percentage of total MACs, fwd latency, percentage of total fwd latency, fwd FLOPS\n",
      "\n",
      "Note: 1. A module can have torch.nn.module or torch.nn.functional to compute logits (e.g. CrossEntropyLoss). They are not counted as submodules, thus not to be printed out. However they make up the difference between a parent's MACs (or latency) and the sum of its submodules'.\n",
      "2. Number of floating-point operations is a theoretical estimation, thus FLOPS computed using that could be larger than the maximum system throughput.\n",
      "3. The fwd latency listed in the top module's profile is directly captured at the module forward function in PyTorch, thus it's less than the fwd latency shown above which is captured in DeepSpeed.\n",
      "\n",
      "Executor(\n",
      "  1.9 M, 100.00% Params, 0 MACs, 0.00% MACs, 972.99 us, 100.00% latency, 0.0 FLOPS, \n",
      "  (_operators): ModuleList(\n",
      "    1.9 M, 100.00% Params, 0 MACs, 0.00% MACs, 865.22 us, 88.92% latency, 0.0 FLOPS, \n",
      "    (0): PerfectTreeTraversalGBDTImpl(\n",
      "      1.9 M, 100.00% Params, 0 MACs, 0.00% MACs, 865.22 us, 88.92% latency, 0.0 FLOPS, \n",
      "      (nodes): ParameterList(\n",
      "        600.0 k, 31.58% Params, 0 MACs, 0.00% MACs, 0, 0.00% latency, 0.0 FLOPS,   (0): Parameter containing: [torch.int64 of size 40000 (GPU 0)]\n",
      "          (1): Parameter containing: [torch.int64 of size 80000 (GPU 0)]\n",
      "          (2): Parameter containing: [torch.int64 of size 160000 (GPU 0)]\n",
      "          (3): Parameter containing: [torch.int64 of size 320000 (GPU 0)]\n",
      "      )\n",
      "      (biases): ParameterList(\n",
      "        600.0 k, 31.58% Params, 0 MACs, 0.00% MACs, 0, 0.00% latency, 0.0 FLOPS,   (0): Parameter containing: [torch.float32 of size 40000 (GPU 0)]\n",
      "          (1): Parameter containing: [torch.float32 of size 80000 (GPU 0)]\n",
      "          (2): Parameter containing: [torch.float32 of size 160000 (GPU 0)]\n",
      "          (3): Parameter containing: [torch.float32 of size 320000 (GPU 0)]\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n",
      "------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# deepspeed.profiling.flops_profiler.profiler.FlopsProfiler\n",
    "from deepspeed.profiling.flops_profiler.profiler import FlopsProfiler as FP\n",
    "\n",
    "model = pytorch_model.model\n",
    "prof = FP(model)\n",
    "device = 'cuda:0'\n",
    "inp = torch.rand(1,535).to(device)\n",
    "prof.start_profile()\n",
    "\n",
    "model(inp)\n",
    "\n",
    "flops = prof.get_total_flops(as_string=True)\n",
    "params = prof.get_total_params(as_string=True)\n",
    "prof.print_model_profile()\n",
    "prof.end_profile()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80.000K\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hongyu/anaconda3/envs/saab/lib/python3.8/site-packages/torchprofile/profile.py:22: UserWarning: No handlers found: \"aten::index_select\". Skipped.\n",
      "  warnings.warn('No handlers found: \"{}\". Skipped.'.format(\n",
      "/home/hongyu/anaconda3/envs/saab/lib/python3.8/site-packages/torchprofile/profile.py:22: UserWarning: No handlers found: \"aten::gather\". Skipped.\n",
      "  warnings.warn('No handlers found: \"{}\". Skipped.'.format(\n",
      "/home/hongyu/anaconda3/envs/saab/lib/python3.8/site-packages/torchprofile/profile.py:22: UserWarning: No handlers found: \"aten::argmax\". Skipped.\n",
      "  warnings.warn('No handlers found: \"{}\". Skipped.'.format(\n"
     ]
    }
   ],
   "source": [
    "from torchprofile import profile_macs\n",
    "\n",
    "model = pytorch_model.model\n",
    "inputs = torch.rand(1,535)\n",
    "macs = profile_macs(model, inputs)\n",
    "\n",
    "from thop import clever_format\n",
    "macs = clever_format([macs], \"%.3f\")\n",
    "print(macs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "saab",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13 (default, Oct 21 2022, 23:50:54) \n[GCC 11.2.0]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "492f2b1cb6759606e5afbb8e792dc4d3b5e25cf4624a4af2a660ba1797daa55d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}