theo_predict_fine_tune_clip.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we are loading an existing model and features saved from unlabeled data from a single folder. Using these features we let the model make predictions for each image. Finally, we save the predictions to csv and json files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T16:08:17.800527Z",
     "start_time": "2024-07-18T16:08:17.787528Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 09:07:30.445555: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-08-07 09:07:30.445629: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-08-07 09:07:30.511155: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-08-07 09:07:30.642280: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-08-07 09:07:31.863616: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import DataFrame as df\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load validation set in tensorflow format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T16:09:36.114132Z",
     "start_time": "2024-07-18T16:09:35.762861Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 2932 instances from ../new_data_2024/validation.pk.\n",
      "X shape (2932, 512) and y shape (2932,) with labels:\n",
      "y\n",
      "Complex AI    2932\n",
      "Name: count, dtype: int64.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 09:07:56.824621: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.063740: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.063802: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.068419: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.068512: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.068545: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.303811: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.303912: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.303921: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2022] Could not identify NUMA node of platform GPU id 0, defaulting to 0.  Your kernel may not have been built with NUMA support.\n",
      "2024-08-07 09:07:57.303970: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:887] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2024-08-07 09:07:57.303995: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1929] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 1778 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3050 Laptop GPU, pci bus id: 0000:01:00.0, compute capability: 8.6\n"
     ]
    }
   ],
   "source": [
    "def load_dataset(path):\n",
    "    data = pd.read_pickle(path)\n",
    "    n_features = data['clip_features'][0].shape[-1]\n",
    "    class_names = data['y'].unique()\n",
    "    y = np.array([np.where(class_names == e)[0][0] for e in data['y']])\n",
    "    x = np.concatenate(data['clip_features'].to_numpy())\n",
    "    print(f'Loaded {len(x)} instances from {path}.')\n",
    "    print(f'X shape {x.shape} and y shape {y.shape} with labels:\\n{data[\"y\"].value_counts()}.')\n",
    "    train_ds = tf.data.Dataset.from_tensor_slices((x, y)).batch(16)\n",
    "    return train_ds, class_names, n_features, data['paths']\n",
    "\n",
    "path = '../new_data_2024/validation.pk'\n",
    "validation_ds, class_names, n_features, paths = load_dataset(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T16:09:41.228792Z",
     "start_time": "2024-07-18T16:09:41.102467Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paths</th>\n",
       "      <th>y</th>\n",
       "      <th>clip_features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.002533, 0.002254, 0.001475, 0.001476, 0.00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.001595, 0.003633, 0.002438, 0.001778, 0.00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.001514, 0.003183, 0.002155, 0.001897, 0.00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.001799, 0.002172, 0.00215, 0.001761, 0.001...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.001634, 0.001647, 0.002796, 0.001132, 0.00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2927</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.00221, 0.003422, 0.001049, 0.001612, 0.002...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2928</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.001391, 0.001732, 0.001799, 0.00176, 0.003...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2929</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.001863, 0.001716, 0.001486, 0.002432, 0.00...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2930</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.001982, 0.00182, 0.001515, 0.001769, 0.002...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2931</th>\n",
       "      <td>../new_data_2024/images_by_class/validation/Co...</td>\n",
       "      <td>Complex AI</td>\n",
       "      <td>[[0.001875, 0.00223, 0.001837, 0.002518, 0.002...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2932 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  paths           y  \\\n",
       "0     ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "1     ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "2     ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "3     ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "4     ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "...                                                 ...         ...   \n",
       "2927  ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "2928  ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "2929  ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "2930  ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "2931  ../new_data_2024/images_by_class/validation/Co...  Complex AI   \n",
       "\n",
       "                                          clip_features  \n",
       "0     [[0.002533, 0.002254, 0.001475, 0.001476, 0.00...  \n",
       "1     [[0.001595, 0.003633, 0.002438, 0.001778, 0.00...  \n",
       "2     [[0.001514, 0.003183, 0.002155, 0.001897, 0.00...  \n",
       "3     [[0.001799, 0.002172, 0.00215, 0.001761, 0.001...  \n",
       "4     [[0.001634, 0.001647, 0.002796, 0.001132, 0.00...  \n",
       "...                                                 ...  \n",
       "2927  [[0.00221, 0.003422, 0.001049, 0.001612, 0.002...  \n",
       "2928  [[0.001391, 0.001732, 0.001799, 0.00176, 0.003...  \n",
       "2929  [[0.001863, 0.001716, 0.001486, 0.002432, 0.00...  \n",
       "2930  [[0.001982, 0.00182, 0.001515, 0.001769, 0.002...  \n",
       "2931  [[0.001875, 0.00223, 0.001837, 0.002518, 0.002...  \n",
       "\n",
       "[2932 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_pickle(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T16:09:53.402238Z",
     "start_time": "2024-07-18T16:09:53.384245Z"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "p = Path(r\"../robot_or_brain_enriched_and_combined_data/images_by_class/validation\")\n",
    "class_list = [d.name for d in p.iterdir()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T16:09:54.762717Z",
     "start_time": "2024-07-18T16:09:54.720673Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Acting or Performing machine',\n",
       " 'Collaborative or Interactive AI',\n",
       " 'Complex AI',\n",
       " 'Learning or recognition machine',\n",
       " 'Mysterious AI',\n",
       " 'None of the above',\n",
       " 'Superior human',\n",
       " 'Thinking machine']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T16:09:58.347811Z",
     "start_time": "2024-07-18T16:09:57.754471Z"
    }
   },
   "outputs": [],
   "source": [
    "trues = [class_list[int(y)] for _x, y in validation_ds.unbatch()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Print class counts for each split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T16:10:00.939991Z",
     "start_time": "2024-07-18T16:10:00.869993Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Acting or Performing machine    2932\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df([y for y in trues]).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some of these counts are horrible. The number of examples in 'Learning or recognition machine', 'Superior human' and 'Mysterious AI' are really too small. Also, the 'None of the above' class is very large, although that's not really a problem."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load our trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-31T06:38:03.032480Z",
     "start_time": "2024-05-31T06:38:02.314398Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# model = tf.keras.models.load_model('./fine_tuned_model_3m6herki')\n",
    "# model = tf.keras.models.load_model('./fine_tuned_model_3f0vzk68')\n",
    "# model = tf.keras.models.load_model('./fine_tuned_model_qdesgan9')\n",
    "# model = tf.keras.models.load_model('./clip_features_model_kuw3ehqp')\n",
    "model = tf.keras.models.load_model('./clip_features_model_kf5cnvvi')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  1/184 [..............................] - ETA: 1:20"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 09:08:35.608377: W external/local_xla/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc:504] Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice. This may result in compilation or runtime failures, if the program we try to run uses routines from libdevice.\n",
      "Searched for CUDA in the following directories:\n",
      "  ./cuda_sdk_lib\n",
      "  /usr/local/cuda-12.2\n",
      "  /usr/local/cuda\n",
      "  /home/christiaan/anaconda3/envs/robot311/lib/python3.11/site-packages/tensorflow/python/platform/../../../nvidia/cuda_nvcc\n",
      "  /home/christiaan/anaconda3/envs/robot311/lib/python3.11/site-packages/tensorflow/python/platform/../../../../nvidia/cuda_nvcc\n",
      "  .\n",
      "You can choose the search directory by setting xla_gpu_cuda_data_dir in HloModule's DebugOptions.  For most apps, setting the environment variable XLA_FLAGS=--xla_gpu_cuda_data_dir=/path/to/cuda will work.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "184/184 [==============================] - 1s 2ms/step\n",
      "None of the above                  1853\n",
      "Collaborative or Interactive AI     432\n",
      "Acting or Performing machine        230\n",
      "Thinking machine                    221\n",
      "Complex AI                          182\n",
      "Superior human                       11\n",
      "Mysterious AI                         3\n",
      "Name: count, dtype: int64\n",
      "Acting or Performing machine    2932\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "logits = model.predict(validation_ds)\n",
    "predicted = [class_list[v] for v in np.argmax(logits, 1)]\n",
    "from pandas import DataFrame as df\n",
    "\n",
    "print(df(predicted).value_counts())\n",
    "print(df(trues).value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Complex AI',\n",
       " 'None of the above',\n",
       " 'None of the above',\n",
       " 'None of the above',\n",
       " 'None of the above',\n",
       " 'Collaborative or Interactive AI',\n",
       " 'Collaborative or Interactive AI',\n",
       " 'None of the above',\n",
       " 'Acting or Performing machine',\n",
       " 'None of the above']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Acting or Performing machine',\n",
       " 'Acting or Performing machine',\n",
       " 'Acting or Performing machine',\n",
       " 'Acting or Performing machine',\n",
       " 'Acting or Performing machine',\n",
       " 'Acting or Performing machine',\n",
       " 'Acting or Performing machine',\n",
       " 'Acting or Performing machine',\n",
       " 'Acting or Performing machine',\n",
       " 'Acting or Performing machine']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trues[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    ../new_data_2024/images_by_class/validation/Co...\n",
       "1    ../new_data_2024/images_by_class/validation/Co...\n",
       "2    ../new_data_2024/images_by_class/validation/Co...\n",
       "3    ../new_data_2024/images_by_class/validation/Co...\n",
       "4    ../new_data_2024/images_by_class/validation/Co...\n",
       "5    ../new_data_2024/images_by_class/validation/Co...\n",
       "6    ../new_data_2024/images_by_class/validation/Co...\n",
       "7    ../new_data_2024/images_by_class/validation/Co...\n",
       "8    ../new_data_2024/images_by_class/validation/Co...\n",
       "9    ../new_data_2024/images_by_class/validation/Co...\n",
       "Name: paths, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_predicted = df(predicted).reset_index()\n",
    "df_trues = df(trues).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_predicted = df_predicted.rename(columns={0:'predictions'})\n",
    "df_trues = df_trues.rename(columns={0:'folder_validation'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "for prediction, path in zip(predicted, paths):\n",
    "    r = {}\n",
    "    r['prediction'] = prediction\n",
    "    r['path'] = path\n",
    "    r['filename'] = path.name\n",
    "    results.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9634975668a4968a1c2451ff4321626",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(Label(value='Acting or Performing machine', layout=Layout(width='220px')), Image…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display\n",
    "from ipywidgets import widgets, HBox, VBox, Layout\n",
    "\n",
    "\n",
    "hboxes = []\n",
    "for class_name in class_list:    \n",
    "    label = widgets.Label(class_name, layout=Layout(width='220px'))\n",
    "    \n",
    "    image_paths = [str(r[1]['path']) for r in list(results.iterrows()) if r[1]['prediction'] == class_name]\n",
    "    images = [widgets.Image(value=open(p, 'rb').read(), layout=Layout(width='120px')) for p in image_paths[:5]]\n",
    "    # images=[widgets.Label(t,layout=Layout(width='220px')) for t in image_paths[:10]]\n",
    "    hbox = HBox([label] + images, layout=Layout(height='90px'))\n",
    "    hboxes += [hbox]\n",
    "vbox = VBox(hboxes)\n",
    "display(vbox)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.to_csv('predictions_2024_chris_wsl.csv', index=False)\n",
    "results.to_pickle('predictions_2024_chris_wsl.pkl')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}