notebooks/Automatic image labelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d0b45df9",
   "metadata": {},
   "source": [
    "# Fit and evaluate model for automatically labelling images in the collection\n",
    "\n",
    "This notebook is included to show how the classification model was trained. However, it requires the embeddings to run, which are not included in this GitHub repository. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caeccc5b",
   "metadata": {},
   "source": [
    "## Imports and type aliases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f6219bba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"OMP_NUM_THREADS\"] = \"32\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5402a8ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import json\n",
    "import warnings\n",
    "from collections.abc import Sequence\n",
    "from collections import defaultdict\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as mticker\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn.metrics as metrics\n",
    "\n",
    "from IPython.display import Markdown, display, HTML\n",
    "from pandas.io.formats.style import Styler\n",
    "from scipy.interpolate import interp1d\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score, KFold\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d59391cb",
   "metadata": {},
   "source": [
    "### Type aliases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "38ce97fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "EmbeddingType = str\n",
    "URL = str\n",
    "Label = int\n",
    "EvaluationDict = list[dict[str, float | int | EmbeddingType | list[Label] | list[URL]]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe706eb1",
   "metadata": {},
   "source": [
    "## Load image embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7629108a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading ViT\n",
      "Loading CLIP\n",
      "Loading SigLIP\n"
     ]
    }
   ],
   "source": [
    "data_dir = Path(\"../../image-embeddings/data\")\n",
    "\n",
    "embedding_dataframes = {}\n",
    "for embedding_type in [\"ViT\", \"CLIP\", \"SigLIP\"]:\n",
    "    print(f\"Loading {embedding_type}\", flush=True)\n",
    "    embedding_dir = data_dir / \"vectors/all_images\" / embedding_type\n",
    "    embedding_dataframes[embedding_type] = pd.concat(\n",
    "        [pd.read_parquet(p) for p in sorted(embedding_dir.glob(\"*.parquet\"))],\n",
    "        axis=\"index\"\n",
    "    ).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98ca1e04",
   "metadata": {},
   "source": [
    "## Load results from manual labelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6586080",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_df = pd.read_json(\"../data/labelled_data.json\")\n",
    "label_df.index.name = \"url\"\n",
    "label_df = label_df.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec3c0da5",
   "metadata": {},
   "source": [
    "## Merge training dataset with embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5c6aa7f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dfs = {embedding: pd.merge(label_df, df, on=\"url\", how=\"inner\") for embedding, df in embedding_dataframes.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4972ce03",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = {embedding:  np.stack((df[\"embedding\"].to_numpy().squeeze()), axis=0) for embedding, df in train_dfs.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd498b77",
   "metadata": {},
   "source": [
    "## Functions to fit and evaluate models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5b92fae9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(\n",
    "    classifier: LogisticRegression,\n",
    "    embedding_type: EmbeddingType,\n",
    "    Xs_train: dict[EmbeddingType, np.ndarray],\n",
    "    y_train: np.ndarray,\n",
    "    Xs_test: dict[EmbeddingType, np.ndarray],\n",
    "    y_test: np.ndarray,\n",
    "    urls: list[str]\n",
    ") -> EvaluationDict:\n",
    "    X_train, X_test = Xs_train[embedding_type], Xs_test[embedding_type]\n",
    "    \n",
    "    # Evaluate on train set\n",
    "    y_hat_train = classifier.predict(X_train)\n",
    "    f1_train = metrics.f1_score(y_train, y_hat_train, average=\"micro\")\n",
    "    \n",
    "    # Evaluate on test set\n",
    "    y_hat_test = classifier.predict(X_test)\n",
    "    \n",
    "    y_proba_test = classifier.predict_proba(X_test)[:, 1]\n",
    "    f1_test = metrics.f1_score(y_test, y_hat_test, average=\"macro\")\n",
    "    confusion_matrix_test = metrics.confusion_matrix(y_test, y_hat_test)\n",
    "    \n",
    "    num_negative = confusion_matrix_test[0].sum()\n",
    "    num_positive = confusion_matrix_test[1].sum()\n",
    "    \n",
    "    # Assemble results\n",
    "    return {\n",
    "        \"Embedding type\": embedding_type,\n",
    "        \"C\": classifier.C,\n",
    "        \"Number of iterations\": classifier.n_iter_.item(),\n",
    "        \n",
    "        \"F1 (train)\": f1_train,\n",
    "        \"F1 (test)\": f1_test,\n",
    "        \n",
    "        \"y_hat_test\": y_hat_test,\n",
    "        \"y_test\": y_test,\n",
    "        \"urls\": urls,\n",
    "    }\n",
    "\n",
    "\n",
    "def run_cross_validation_model_selection(\n",
    "    Xs: dict[EmbeddingType, np.ndarray],\n",
    "    y: np.ndarray,\n",
    "    folds: KFold\n",
    ") -> tuple[LogisticRegression, EmbeddingType]:\n",
    "    Cs = np.geomspace(1e-2, 1e4, 10)\n",
    "    performance: dict[tuple[float, EmbeddingType], float] = defaultdict(list)\n",
    "    \n",
    "    for train_idx, test_idx in tqdm(folds.split(y), total=folds.n_splits):\n",
    "        for C, embedding_type in itertools.product(Cs, Xs.keys()):\n",
    "            X_train = Xs[embedding_type][train_idx]\n",
    "            y_train = y[train_idx]\n",
    "            X_test = Xs[embedding_type][test_idx]\n",
    "            y_test = y[test_idx]\n",
    "            \n",
    "            model = LogisticRegression(C=C, random_state=2, max_iter=1000)  # Random state should not matter as logistic regression is unique\n",
    "            model.fit(X_train, y_train)\n",
    "            \n",
    "            y_test_hat = model.predict(X_test)\n",
    "            f1 = metrics.f1_score(y_test, y_test_hat, average=\"micro\")\n",
    "            performance[C, embedding_type].append(f1)\n",
    "\n",
    "    avg_performance = {k: np.mean(all_f1) for k, all_f1 in performance.items()}\n",
    "    C, embedding_type = pd.Series(avg_performance).idxmax()\n",
    "    X = Xs[embedding_type]\n",
    "    model = LogisticRegression(C=C, random_state=2, max_iter=200).fit(X, y)\n",
    "\n",
    "    return model, embedding_type\n",
    "\n",
    "\n",
    "def run_nested_crossvalidated_logistic_regression(\n",
    "    Xs: dict[EmbeddingType, np.ndarray],\n",
    "    y: np.ndarray,\n",
    "    urls: Sequence[URL]\n",
    ") -> list[EvaluationDict]:\n",
    "    n_outer_splits = 20\n",
    "    n_inner_splits = 10\n",
    "    outer_folds = KFold(n_splits=n_outer_splits, shuffle=True, random_state=0)\n",
    "    inner_folds = KFold(n_splits=n_inner_splits, shuffle=True, random_state=1)\n",
    "\n",
    "    results = []\n",
    "    false_positive_urls = []\n",
    "    false_negative_urls = []\n",
    "    for train_idx, test_idx in tqdm(outer_folds.split(y), total=n_outer_splits):\n",
    "        Xs_train = {embedding: X[train_idx] for embedding, X in Xs.items()}\n",
    "        y_train = y[train_idx]\n",
    "        Xs_test = {embedding: X[test_idx] for embedding, X in Xs.items()}\n",
    "        y_test = y[test_idx]\n",
    "        urls_test = urls[test_idx]\n",
    "\n",
    "\n",
    "        classifier, embedding_type = run_cross_validation_model_selection(Xs_train, y_train, folds=inner_folds)\n",
    "        res = evaluate(\n",
    "            classifier,\n",
    "            embedding_type,\n",
    "            Xs_train,\n",
    "            y_train,\n",
    "            Xs_test,\n",
    "            y_test,\n",
    "            urls_test,\n",
    "        )\n",
    "\n",
    "        results.append(res)\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dc5256f7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcfc2fd2537f4c9686d25ab948f67dd8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62b5949c124145a983f8b9d3f8625093",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "454c7a6b24bc46979d57cf2c7cfc5b49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d223f2bcfed46e88a8bf9188a395e46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "995ae1c8317a4c3e9c2d870db77237e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02164125086d4a43be389908995b2d66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b71a1dc792e946628fd3ece0322d7cde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11d9e283f68240f6af7c8f1156c36492",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd676c2e24be4f72bdef6e8df7f0c093",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee2271035c1447dd8502ef4bc517aee3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43bde63b1cbe410d9b25ab9f865631e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d538811f0d134f649bf4f61118f3e263",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d0d6bbb49b34e0e8dfc143e24953c7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e023c6f700f42f780b1b40867b8284b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ad9bb1820624c5081ee9c6cf7666f71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5821ccb0e8b24e23b7abe9759fddbe5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "177dda8d1b1a4c2b84a2e6ba22ac54d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6acfa065b8b48b4a1df3a4a9bccacfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e82150678f5548c4908f0c26025c8fdb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe4d615288d9492395a99f8f9d1176dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a30b6c6180d14a168c562d211cd9a468",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    results = run_nested_crossvalidated_logistic_regression(\n",
    "        embeddings,\n",
    "        np.argmax(label_df.drop(\"url\", axis=1).values, axis=1),\n",
    "        label_df[\"url\"].values\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d172df4",
   "metadata": {},
   "source": [
    "## Look at hyperparameters for the different models selected in the outer loops"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6264312",
   "metadata": {},
   "source": [
    "### Embedding types:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3975d6bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding type\n",
       "SigLIP    20\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(results)[\"Embedding type\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7469c992",
   "metadata": {},
   "source": [
    "### Complexity parameter:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a93b7c84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "C\n",
       "0.215443        11\n",
       "0.046416         5\n",
       "1.000000         2\n",
       "10000.000000     1\n",
       "464.158883       1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(results)[\"C\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fdfa064",
   "metadata": {},
   "source": [
    "## Store a CSV file with the true and estimated value for each URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ba708d43",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "res_df = pd.DataFrame(results).head()\n",
    "estimated = [label_df.drop(\"url\", axis=1).columns[i] for row in res_df.itertuples() for i in row.y_hat_test]\n",
    "true = [label_df.drop(\"url\", axis=1).columns[i] for row in res_df.itertuples() for i in row.y_test]\n",
    "urls = [url for row in res_df.itertuples() for url in row.urls]\n",
    "\n",
    "eval_df = pd.DataFrame({\"url\": urls, \"true_label\": true, \"estimated_label\": estimated})\n",
    "eval_df.to_csv(\"../data/test_fold_estimated_labels.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c510454f",
   "metadata": {},
   "source": [
    "## Look at cross-validated evaluation metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c248557",
   "metadata": {},
   "source": [
    "## F1 score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c0bd6c54",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9164543944213788, 0.05101429983736076)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(results)[\"F1 (test)\"].mean(), pd.DataFrame(results)[\"F1 (test)\"].std()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8c17513",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e23ac98a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>True class</th>\n",
       "      <th>Segmentation anomaly</th>\n",
       "      <th>Blank page</th>\n",
       "      <th>Graphical element</th>\n",
       "      <th>Illustration or photograph</th>\n",
       "      <th>Musical notation</th>\n",
       "      <th>Map</th>\n",
       "      <th>Mathematical chart</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predicted class</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Segmentation anomaly</th>\n",
       "      <td>496.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Blank page</th>\n",
       "      <td>11.0</td>\n",
       "      <td>339.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Graphical element</th>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>278.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Illustration or photograph</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>558.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Musical notation</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Map</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mathematical chart</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>39.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "True class                  Segmentation anomaly  Blank page  \\\n",
       "Predicted class                                                \n",
       "Segmentation anomaly                       496.0         5.0   \n",
       "Blank page                                  11.0       339.0   \n",
       "Graphical element                           14.0         2.0   \n",
       "Illustration or photograph                   1.0         3.0   \n",
       "Musical notation                             1.0         0.0   \n",
       "Map                                          1.0         0.0   \n",
       "Mathematical chart                           0.0         0.0   \n",
       "\n",
       "True class                  Graphical element  Illustration or photograph  \\\n",
       "Predicted class                                                             \n",
       "Segmentation anomaly                     28.0                         8.0   \n",
       "Blank page                                8.0                         1.0   \n",
       "Graphical element                       278.0                        15.0   \n",
       "Illustration or photograph               16.0                       558.0   \n",
       "Musical notation                          0.0                         0.0   \n",
       "Map                                       0.0                         2.0   \n",
       "Mathematical chart                        0.0                         8.0   \n",
       "\n",
       "True class                  Musical notation   Map  Mathematical chart  \n",
       "Predicted class                                                         \n",
       "Segmentation anomaly                     2.0   1.0                 2.0  \n",
       "Blank page                               0.0   0.0                 0.0  \n",
       "Graphical element                        1.0   0.0                 2.0  \n",
       "Illustration or photograph               1.0   2.0                 5.0  \n",
       "Musical notation                       109.0   0.0                 0.0  \n",
       "Map                                      0.0  41.0                 0.0  \n",
       "Mathematical chart                       0.0   0.0                39.0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm = np.zeros((7, 7))\n",
    "\n",
    "for res in results:\n",
    "    for yi, yhat_i in zip(res[\"y_test\"], res[\"y_hat_test\"]):\n",
    "        cm[yi, yhat_i] += 1\n",
    "\n",
    "        \n",
    "classes = [\n",
    "    \"Segmentation anomaly\",\n",
    "    \"Blank page\",\n",
    "    \"Graphical element\",\n",
    "    \"Illustration or photograph\",\n",
    "    \"Musical notation\",\n",
    "    \"Map\",\n",
    "    \"Mathematical chart\"\n",
    "]\n",
    "cols = label_df.drop(\"url\", axis=1).columns\n",
    "cm = pd.DataFrame(cm, index=cols.copy(), columns=cols.copy())\n",
    "cm.index.name = \"True class\"\n",
    "cm.columns.name = \"Predicted class\"\n",
    "cm = cm.loc[classes, classes].T\n",
    "cm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee85ebbc",
   "metadata": {},
   "source": [
    "## Format matrix as a LaTeX table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e8ac442c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{@{}lrrrrrrr}\n",
      "\\toprule\n",
      "\\rotatebox{90}{\\makecell{True\\\\class}} & \\rotatebox{90}{\\makecell{Segmentation\\\\anomaly}} & \\rotatebox{90}{\\makecell{Blank\\\\page}} & \\rotatebox{90}{\\makecell{Graphical\\\\element}} & \\rotatebox{90}{\\makecell{Illustration or\\\\photograph}} & \\rotatebox{90}{\\makecell{Musical\\\\notation}} & \\rotatebox{90}{\\makecell{Map}} & \\rotatebox{90}{\\makecell{Mathematical\\\\chart}} \\\\\n",
      "Predicted class &  &  &  &  &  &  &  \\\\\n",
      "\\midrule\n",
      "Segmentation anomaly & {\\cellcolor[HTML]{00682A}} \\color[HTML]{F1F1F1} 496 & {\\cellcolor[HTML]{F6FCF4}} \\color[HTML]{000000} 5 & {\\cellcolor[HTML]{F0F9ED}} \\color[HTML]{000000} 28 & {\\cellcolor[HTML]{F5FBF3}} \\color[HTML]{000000} 8 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 2 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 1 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 2 \\\\\n",
      "Blank page & {\\cellcolor[HTML]{F4FBF2}} \\color[HTML]{000000} 11 & {\\cellcolor[HTML]{48AE60}} \\color[HTML]{F1F1F1} 339 & {\\cellcolor[HTML]{F5FBF3}} \\color[HTML]{000000} 8 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 1 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 \\\\\n",
      "Graphical element & {\\cellcolor[HTML]{F4FBF1}} \\color[HTML]{000000} 14 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 2 & {\\cellcolor[HTML]{75C477}} \\color[HTML]{000000} 278 & {\\cellcolor[HTML]{F4FBF1}} \\color[HTML]{000000} 15 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 1 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 2 \\\\\n",
      "Illustration or photograph & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 1 & {\\cellcolor[HTML]{F6FCF4}} \\color[HTML]{000000} 3 & {\\cellcolor[HTML]{F3FAF0}} \\color[HTML]{000000} 16 & {\\cellcolor[HTML]{00441B}} \\color[HTML]{F1F1F1} 558 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 1 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 2 & {\\cellcolor[HTML]{F6FCF4}} \\color[HTML]{000000} 5 \\\\\n",
      "Musical notation & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 1 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{D4EECE}} \\color[HTML]{000000} 109 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 \\\\\n",
      "Map & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 1 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 2 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{EDF8E9}} \\color[HTML]{000000} 41 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 \\\\\n",
      "Mathematical chart & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F5FBF3}} \\color[HTML]{000000} 8 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{F7FCF5}} \\color[HTML]{000000} 0 & {\\cellcolor[HTML]{EDF8EA}} \\color[HTML]{000000} 39 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm_latex = cm.copy()\n",
    "\n",
    "cm_latex.columns = [\n",
    "    r'Segmentation\\\\anomaly', \n",
    "    r'Blank\\\\page',\n",
    "    r'Graphical\\\\element',\n",
    "    r'Illustration or\\\\photograph', \n",
    "    r'Musical\\\\notation', \n",
    "    r'Map',\n",
    "    r'Mathematical\\\\chart'\n",
    "]\n",
    "cm_latex.columns = [\n",
    "    f\"\\\\rotatebox{{90}}{{\\\\makecell{{{c}}}}}\" for c in cm_latex.columns\n",
    "]\n",
    "cm_latex.columns.name = cm.columns.name.replace(' ', r'\\\\')\n",
    "cm_latex.columns.name = f\"\\\\rotatebox{{90}}{{\\\\makecell{{{cm_latex.columns.name}}}}}\"\n",
    "styler = Styler(cm_latex, precision=0)\n",
    "(\n",
    "    styler.background_gradient(cmap=\"Greens\", vmin=0, vmax=cm.max().max())\n",
    ")\n",
    "\n",
    "print(styler.to_latex(convert_css=True, hrules=True, column_format=\"@{}lrrrrrrr\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f89aa0df",
   "metadata": {},
   "source": [
    "## Train final classifier and estimate label distribution on full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f2722d23",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5398f0b0b8ac4286832b473b3eee5823",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_labels = label_df.drop(\"url\", axis=1)\n",
    "folds = KFold(n_splits=10, shuffle=True, random_state=1)\n",
    "classifier, embedding_type = run_cross_validation_model_selection(\n",
    "    embeddings, np.argmax(train_labels.values, axis=1), folds=folds\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2708d199",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = label_df.drop(\"url\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d023b0c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_embeddings = np.stack(embedding_dataframes[\"SigLIP\"][\"embedding\"].values)\n",
    "urls = embedding_dataframes[\"SigLIP\"][\"url\"]\n",
    "classes = classifier.predict(selected_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4ebb7fc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df = pd.DataFrame({'url': urls.values, 'label': [train_labels.columns[c] for c in classes]})\n",
    "result_df.to_json(\"../data/estimated_labels.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b68924a2",
   "metadata": {},
   "source": [
    "## Inspect predicted class on subset of all images in the collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6af30d10",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>url</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009042803019_0104/442,259,1667,805/833,402/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2016122229002_0734/641,1483,575,617/287,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009020603007_0352/148,135,1472,1948/368,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2010021803056_0099/1180,887,336,168/336,168/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014102226004_0161/726,26,560,311/560,311/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2008121603011_0078/249,473,626,1119/313,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014070838045_0096/457,1096,1419,1174/354,293/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012020104107_0185/71,138,1779,2641/444,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014012124008_0676/0,87,2769,4436/345,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012012012002_0145/2798,1228,282,153/282,153/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Musical notation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012071613002_0320/0,0,3630,5664/453,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013073124006_0302/228,1148,1265,289/1265,289/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012070408119_0026/448,423,1405,2217/351,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2008082210006_0001/965,715,534,1436/267,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015120729001_0124/77,167,2194,2697/273,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2010070506051_0916/9,16,1339,181/1339,181/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2017082248017_0120/459,537,1262,2212/315,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Map</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013062438204_0119/125,125,4001,3335/500,416/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014071708086_0001/47,1023,1379,1154/344,288/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015060508070_0100/279,911,1045,1271/261,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013013124001_0516/361,929,510,338/510,338/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015022324004_0616/0,41,1799,3174/449,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009021703012_0069/746,723,200,798/200,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2011052320008_0074/395,383,1651,926/825,463/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2006112101053_0413/0,4,1859,965/929,482/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2020062626008_0262/421,1820,620,603/309,301/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015022327002_0310/647,2098,319,322/319,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2008040712002_0095/705,346,1926,3585/481,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2018020548232_0233/188,2516,1205,304/1205,304/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2020022048012_0058/276,295,1775,2128/443,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013121924010_0306/732,1678,1246,356/1246,356/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015012124002_0276/164,526,1447,880/723,439/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2016050948096_0303/61,69,2181,3235/272,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009103010001_0148/718,604,475,175/475,175/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2018092428003_0113/0,173,3066,2237/383,279/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Map</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014041524009_0737/0,2458,358,499/358,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2019110128001_0143/191,63,1736,2444/434,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2020032526003_0259/72,180,2114,3474/264,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012091108008_0037/8,53,3302,4241/412,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014031426011_0122/334,451,3678,2280/459,284/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2017092148003_0005/995,2423,562,273/562,273/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2010011103016_0053/347,1254,1108,815/553,407/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013081208201_0044/267,210,1373,630/686,315/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Mathematical chart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012090408089_0018/497,1842,375,362/375,362/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2010082723016_0092/810,1099,303,971/303,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2008072112002_0001/693,1063,228,298/228,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2011062123017_0282/1275,733,208,180/208,180/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2008042812014_0096/76,158,1656,2662/414,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014061848121_0480/320,36,2262,3058/282,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015120929004_0046/693,3227,1692,336/1692,336/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014111808052_0071/1655,1131,692,959/345,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009021703012_0186/380,402,620,612/310,306/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009013000007_0098/1585,599,366,1131/366,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2016051026006_0771/39,118,731,2063/365,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009012800059_0033/169,322,1391,1028/347,257/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Mathematical chart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009033103029_0353/283,934,1594,1127/398,281/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009103010001_0140/686,2621,1129,1337/282,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015110908170_0168/210,263,1281,1937/320,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009092303047_0007/1462,1961,456,207/456,207/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2019072348002_0197/67,146,1584,2856/395,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013100924003_0003/363,535,558,646/279,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014031428001_0533/158,444,903,1088/451,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009102300016_0182/346,54,3888,5222/486,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009030403037_0187/487,677,2688,3669/335,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015071429004_0153/458,459,276,169/276,169/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2007082010001_0115/196,417,1413,2122/353,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012052113001_0508/0,8,3504,5912/438,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2008050510001_0333/316,28,413,155/413,155/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014031808067_0010/200,2026,834,794/416,396/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015102907053_0210/156,1347,2047,682/1023,340/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2018082328005_0038/99,248,5439,9217/339,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009020203009_0320/439,142,558,1260/279,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013112928009_0620/293,584,1145,562/572,280/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2010021700062_0605/1046,866,753,468/753,468/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2019030628001_0224/1583,829,578,614/289,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2018092728004_0001/254,169,2115,1790/528,447/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009103010001_1317/1464,899,1373,1991/342,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2018051529001_0645/39,1068,1910,2114/477,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013040824075_0201/156,615,1714,2023/428,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014090324013_0184/329,377,2380,3871/297,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2017100548013_0160/79,295,2562,3419/319,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009050703010_0025/215,90,2387,3189/298,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015102348024_0101/210,290,2054,3155/256,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015062908059_0186/794,1069,1221,1603/305,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009030603005_0011/1119,1610,778,1243/389,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2018051529001_0151/11,353,1308,1143/326,285/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012052908115_0169/113,317,1298,283/1298,283/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2012012320004_0011/2786,224,1765,459/1765,459/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Musical notation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2016051148055_0001/39,303,1644,2358/410,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2016111648081_0302/199,815,1916,1505/478,376/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2013042608092_0099/825,295,736,368/736,368/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015032508049_0288/0,213,2109,3027/263,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2015022327003_0219/110,2442,1720,211/1720,211/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Graphical element</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014012124008_0412/4,0,2782,4523/347,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Blank page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2018040928002_0110/1543,943,7677,6095/479,380/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Map</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2009120812009_0335/74,3,1373,2208/342,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2014011608183_0074/331,1246,1064,961/531,480/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Segmentation anomaly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2017070507013_0137/306,711,1401,758/700,379/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Illustration or photograph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2018110728001_0115/18,793,2779,2670/347,333/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Map</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><img src=\"https://www.nb.no/services/image/resolver/URN:NBN:no-nb_digibok_2019072928002_0207/0,910,3754,5569/468,/0/default.jpg\" width=\"100px\" /></td>\n",
       "      <td>Map</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def format_image(url):\n",
    "    return f'<img src=\"{url}\" width=\"100px\" />'\n",
    "HTML(\n",
    "    result_df.sample(100, random_state=0).to_html(\n",
    "        formatters={\n",
    "            \"url\": format_image,\n",
    "          },\n",
    "        escape=False,\n",
    "        index=False\n",
    "      )\n",
    "  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "989d20c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}