notebooks/fairnesseval_Quick_Start.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# fairnesseval API\n",
    "Note: you can run **[this notebook live in Google Colab](https://colab.research.google.com/github/softlab-unimore/fairnesseval/blob/main/notebooks/fairnesseval_Quick_Start.ipynb)**."
   ],
   "metadata": {
    "id": "-D3BoNEC_YSD"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "!pip install git+https://github.com/softlab-unimore/fairnesseval@main"
   ],
   "metadata": {
    "id": "4nCp2TnDIWDN",
    "outputId": "757c8684-b0de-48d9-fff1-b197d71b89df",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "execution_count": 1,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Collecting git+https://github.com/softlab-unimore/fairnesseval@main\n",
      "  Cloning https://github.com/softlab-unimore/fairnesseval (to revision main) to /tmp/pip-req-build-ep0f0upk\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/softlab-unimore/fairnesseval /tmp/pip-req-build-ep0f0upk\n",
      "  Resolved https://github.com/softlab-unimore/fairnesseval to commit 328b4a965fe4b7fc019b9b807f9cc03c4262fea6\n",
      "  Preparing metadata (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "Collecting fairlearn@ git+https://github.com/MiroDudik/fairlearn.git@f11f99acd2ba724f9ef885973830d8f3390b0717#egg=fairlearn (from fairnesseval==0.1.0)\n",
      "  Cloning https://github.com/MiroDudik/fairlearn.git (to revision f11f99acd2ba724f9ef885973830d8f3390b0717) to /tmp/pip-install-xfld5u41/fairlearn_33c830718f824b9197327b9371915ee9\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/MiroDudik/fairlearn.git /tmp/pip-install-xfld5u41/fairlearn_33c830718f824b9197327b9371915ee9\n",
      "  Running command git rev-parse -q --verify 'sha^f11f99acd2ba724f9ef885973830d8f3390b0717'\n",
      "  Running command git fetch -q https://github.com/MiroDudik/fairlearn.git f11f99acd2ba724f9ef885973830d8f3390b0717\n",
      "  Running command git checkout -q f11f99acd2ba724f9ef885973830d8f3390b0717\n",
      "  Resolved https://github.com/MiroDudik/fairlearn.git to commit f11f99acd2ba724f9ef885973830d8f3390b0717\n",
      "  Installing build dependencies ... \u001B[?25l\u001B[?25hdone\n",
      "  Getting requirements to build wheel ... \u001B[?25l\u001B[?25hdone\n",
      "  Preparing metadata (pyproject.toml) ... \u001B[?25l\u001B[?25hdone\n",
      "Collecting fair-classification@ git+https://github.com/baraldian/fair-classification-python3@master (from fairnesseval==0.1.0)\n",
      "  Cloning https://github.com/baraldian/fair-classification-python3 (to revision master) to /tmp/pip-install-xfld5u41/fair-classification_642a0ddd50a94f739aedeb5b7ac2cc17\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/baraldian/fair-classification-python3 /tmp/pip-install-xfld5u41/fair-classification_642a0ddd50a94f739aedeb5b7ac2cc17\n",
      "  Resolved https://github.com/baraldian/fair-classification-python3 to commit a7c8a908d18dd86b8591db601691a4e42e75c5dc\n",
      "  Preparing metadata (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "Collecting aif360>=0.5.0 (from fairnesseval==0.1.0)\n",
      "  Downloading aif360-0.6.1-py3-none-any.whl.metadata (5.0 kB)\n",
      "Collecting folktables>=0.0.12 (from fairnesseval==0.1.0)\n",
      "  Downloading folktables-0.0.12-py3-none-any.whl.metadata (533 bytes)\n",
      "Requirement already satisfied: matplotlib>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (3.7.1)\n",
      "Requirement already satisfied: numpy<2,>=1.22.4 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (1.26.4)\n",
      "Requirement already satisfied: pandas>=1.4.2 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (2.1.4)\n",
      "Requirement already satisfied: Requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (2.32.3)\n",
      "Requirement already satisfied: scikit_learn>=1.0.2 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (1.3.2)\n",
      "Requirement already satisfied: scipy>=1.7.3 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (1.13.1)\n",
      "Requirement already satisfied: seaborn>=0.11.2 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (0.13.1)\n",
      "Requirement already satisfied: Send2Trash>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (1.8.3)\n",
      "Requirement already satisfied: setuptools>=67.6.1 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (71.0.4)\n",
      "Requirement already satisfied: tqdm>=4.64.0 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (4.66.5)\n",
      "Requirement already satisfied: cvxpy>=1.4.2 in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (1.5.3)\n",
      "Collecting dccp (from fairnesseval==0.1.0)\n",
      "  Downloading dccp-1.0.5-py3-none-any.whl.metadata (324 bytes)\n",
      "Requirement already satisfied: ipywidgets in /usr/local/lib/python3.10/dist-packages (from fairnesseval==0.1.0) (7.7.1)\n",
      "Requirement already satisfied: osqp>=0.6.2 in /usr/local/lib/python3.10/dist-packages (from cvxpy>=1.4.2->fairnesseval==0.1.0) (0.6.7.post0)\n",
      "Requirement already satisfied: ecos>=2 in /usr/local/lib/python3.10/dist-packages (from cvxpy>=1.4.2->fairnesseval==0.1.0) (2.0.14)\n",
      "Requirement already satisfied: clarabel>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from cvxpy>=1.4.2->fairnesseval==0.1.0) (0.9.0)\n",
      "Requirement already satisfied: scs>=3.2.4.post1 in /usr/local/lib/python3.10/dist-packages (from cvxpy>=1.4.2->fairnesseval==0.1.0) (3.2.7)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.7.1->fairnesseval==0.1.0) (1.3.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.7.1->fairnesseval==0.1.0) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.7.1->fairnesseval==0.1.0) (4.53.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.7.1->fairnesseval==0.1.0) (1.4.7)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.7.1->fairnesseval==0.1.0) (24.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.7.1->fairnesseval==0.1.0) (10.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.7.1->fairnesseval==0.1.0) (3.1.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.7.1->fairnesseval==0.1.0) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.4.2->fairnesseval==0.1.0) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.4.2->fairnesseval==0.1.0) (2024.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from Requests>=2.31.0->fairnesseval==0.1.0) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from Requests>=2.31.0->fairnesseval==0.1.0) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from Requests>=2.31.0->fairnesseval==0.1.0) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from Requests>=2.31.0->fairnesseval==0.1.0) (2024.8.30)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit_learn>=1.0.2->fairnesseval==0.1.0) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit_learn>=1.0.2->fairnesseval==0.1.0) (3.5.0)\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in /usr/local/lib/python3.10/dist-packages (from ipywidgets->fairnesseval==0.1.0) (5.5.6)\n",
      "Requirement already satisfied: ipython-genutils~=0.2.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets->fairnesseval==0.1.0) (0.2.0)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.10/dist-packages (from ipywidgets->fairnesseval==0.1.0) (5.7.1)\n",
      "Requirement already satisfied: widgetsnbextension~=3.6.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets->fairnesseval==0.1.0) (3.6.9)\n",
      "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets->fairnesseval==0.1.0) (7.34.0)\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets->fairnesseval==0.1.0) (3.0.13)\n",
      "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets->fairnesseval==0.1.0) (6.1.12)\n",
      "Requirement already satisfied: tornado>=4.2 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets->fairnesseval==0.1.0) (6.3.3)\n",
      "Collecting jedi>=0.16 (from ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0)\n",
      "  Using cached jedi-0.19.1-py2.py3-none-any.whl.metadata (22 kB)\n",
      "Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (4.4.2)\n",
      "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (0.7.5)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (3.0.47)\n",
      "Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (2.18.0)\n",
      "Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (0.2.0)\n",
      "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (0.1.7)\n",
      "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (4.9.0)\n",
      "Requirement already satisfied: qdldl in /usr/local/lib/python3.10/dist-packages (from osqp>=0.6.2->cvxpy>=1.4.2->fairnesseval==0.1.0) (0.1.7.post4)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib>=3.7.1->fairnesseval==0.1.0) (1.16.0)\n",
      "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.10/dist-packages (from widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (6.5.5)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (0.8.4)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (3.1.4)\n",
      "Requirement already satisfied: pyzmq<25,>=17 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (24.0.1)\n",
      "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (23.1.0)\n",
      "Requirement already satisfied: jupyter-core>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (5.7.2)\n",
      "Requirement already satisfied: nbformat in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (5.10.4)\n",
      "Requirement already satisfied: nbconvert>=5 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (6.5.4)\n",
      "Requirement already satisfied: nest-asyncio>=1.5 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.6.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.18.1)\n",
      "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.20.0)\n",
      "Requirement already satisfied: nbclassic>=0.4.7 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.1.0)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0->ipywidgets->fairnesseval==0.1.0) (0.2.13)\n",
      "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from jupyter-core>=4.6.1->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (4.3.4)\n",
      "Requirement already satisfied: notebook-shim>=0.2.3 in /usr/local/lib/python3.10/dist-packages (from nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.2.4)\n",
      "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (4.9.4)\n",
      "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (4.12.3)\n",
      "Requirement already satisfied: bleach in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (6.1.0)\n",
      "Requirement already satisfied: defusedxml in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.7.1)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.4)\n",
      "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.3.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (2.1.5)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.8.4)\n",
      "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.10.0)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.5.1)\n",
      "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.10/dist-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.3.0)\n",
      "Requirement already satisfied: fastjsonschema>=2.15 in /usr/local/lib/python3.10/dist-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (2.20.0)\n",
      "Requirement already satisfied: jsonschema>=2.6 in /usr/local/lib/python3.10/dist-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (4.23.0)\n",
      "Requirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.10/dist-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (21.2.0)\n",
      "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (24.2.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (2023.12.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.35.1)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.20.0)\n",
      "Requirement already satisfied: jupyter-server<3,>=1.8 in /usr/local/lib/python3.10/dist-packages (from notebook-shim>=0.2.3->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.24.0)\n",
      "Requirement already satisfied: cffi>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.17.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (2.6)\n",
      "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/dist-packages (from bleach->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (0.5.1)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (2.22)\n",
      "Requirement already satisfied: anyio<4,>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from jupyter-server<3,>=1.8->notebook-shim>=0.2.3->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (3.7.1)\n",
      "Requirement already satisfied: websocket-client in /usr/local/lib/python3.10/dist-packages (from jupyter-server<3,>=1.8->notebook-shim>=0.2.3->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.8.0)\n",
      "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4,>=3.1.0->jupyter-server<3,>=1.8->notebook-shim>=0.2.3->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.3.1)\n",
      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4,>=3.1.0->jupyter-server<3,>=1.8->notebook-shim>=0.2.3->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets->fairnesseval==0.1.0) (1.2.2)\n",
      "Downloading aif360-0.6.1-py3-none-any.whl (259 kB)\n",
      "\u001B[2K   \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m259.7/259.7 kB\u001B[0m \u001B[31m3.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hDownloading folktables-0.0.12-py3-none-any.whl (17 kB)\n",
      "Downloading dccp-1.0.5-py3-none-any.whl (7.1 kB)\n",
      "Using cached jedi-0.19.1-py2.py3-none-any.whl (1.6 MB)\n",
      "Building wheels for collected packages: fairnesseval, fair-classification, fairlearn\n",
      "  Building wheel for fairnesseval (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "  Created wheel for fairnesseval: filename=fairnesseval-0.1.0-py3-none-any.whl size=70145 sha256=ef386e9909327fd841eaa9f19a1f278d93b535e1f2af652633b78cb000787f41\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eaq_9fz1/wheels/f0/ae/a8/df048af1459ce9d3cd6f683b4db824dffeec6a5eae5f80b134\n",
      "  Building wheel for fair-classification (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "  Created wheel for fair-classification: filename=fair_classification-0.0.1-py3-none-any.whl size=40729 sha256=8e34c68e48b2edcbe3373fdf584803f716c6d8b5bbee6036b3e2b55def102153\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eaq_9fz1/wheels/2f/88/77/ce14494ec97ab23821c3661f45586fbb7cf5d91703dd6d6103\n",
      "  Building wheel for fairlearn (pyproject.toml) ... \u001B[?25l\u001B[?25hdone\n",
      "  Created wheel for fairlearn: filename=fairlearn-0.9.0.dev0-py3-none-any.whl size=235730 sha256=f256242f7b008c4e2eb6ac701bd88966536a66eb77be5824ca4d54a512841ffc\n",
      "  Stored in directory: /root/.cache/pip/wheels/34/2f/82/37825f0ebd1269db90994e0b3a366d0af2eded829c975bac38\n",
      "Successfully built fairnesseval fair-classification fairlearn\n",
      "Installing collected packages: jedi, fair-classification, folktables, fairlearn, aif360, dccp, fairnesseval\n",
      "Successfully installed aif360-0.6.1 dccp-1.0.5 fair-classification-0.0.1 fairlearn-0.9.0.dev0 fairnesseval-0.1.0 folktables-0.0.12 jedi-0.19.1\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Dataset download\n",
    "The following code will download the datasets following the instructions in aif360 errors.\n",
    "It should be changed according to your paths (python path especially)."
   ],
   "metadata": {
    "id": "L0Zlk_08JgHU"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# fairnesseval API\n",
    "This tool provides two interfaces for running fairness experiments on your data.\n",
    "\n",
    "**1. Python Interface**\n",
    "You can define the experiment settings in the form of a Python dictionary and use one of the following Python functions to run experiments:\n",
    "    \n",
    "1.1. `fairnesseval.run.launch_experiment_by_id` let you define and organize your experiments in a python module (default at `fairnesseval.experiment_definitions`). Then you will need to call this function by specifying only the id of the experiment you want to run. **This is the reccommended interface.**\n",
    "    \n",
    "1.2. `fairnesseval.run.launch_experiment_by_config` let you run an experiment by passing the dictionary of parameters of your experiment in input.\n",
    "\n",
    "**2. Command Line Interface**\n",
    "Alternatively, you can use the command line interface of `fairnesseval.run` to specify the experiment settings using traditional CLI parameters."
   ],
   "metadata": {
    "id": "ERXd-AUb2Met"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 1 Python Interface"
   ],
   "metadata": {
    "id": "WcWIJmEn2Qha"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "To launch an experiment you can run Python script that read experiment parameters from a module (default at `fairnesseval.experiment_definitions`).\n",
    "\n",
    "Loading experiment definitions is more powerful and flexible, it allows to:\n",
    "\n",
    "*   launch multiple experiments in a row.\n",
    "*   specify multiple datasets.\n",
    "*   specify multiple models.\n",
    "*   configurations are more organized and readable.\n",
    "*   have additional logging.\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "id": "gn5NRU1p2hob"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### TODO: Define your experiment in a file.\n",
    "(You can find example of experiment configuration in `fairnesseval.experiment_definitions`).\n",
    "\n",
    "Eg.: Create `exp_def.py` and define an experiment.\n",
    "```python\n",
    "RANDOM_SEEDs_RESTRICTED_V1 = [1]\n",
    "\n",
    "TRAIN_FRACTIONS_SMALLER_DATASETS_v1 = [0.063, 0.251, 1.]\n",
    "TRAIN_FRACTIONS_v1 = [0.001, 0.004, 0.016, 0.063, 0.251, 1]  # np.geomspace(0.001,1,7) np.linspace(0.001,1,7)\n",
    "\n",
    "experiment_definitions = [\n",
    "    {\n",
    "        'experiment_id': 'new_experiment',\n",
    "        'dataset_names': ('adult_sigmod',),\n",
    "        'model_names': ('LogisticRegression',),\n",
    "        'random_seeds': RANDOM_SEEDs_RESTRICTED_V1,\n",
    "        'results_path': './demo_results'\n",
    "    }\n",
    "]\n",
    "\n",
    "```\n",
    "\n"
   ],
   "metadata": {
    "id": "G_ylbpALJEml"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Run the experiment\n",
    "Copy the path to the experiment configuration file just defined.\n",
    "\n",
    "In my case: `/content/exp_def.py`\n",
    "\n",
    "Then run the experiment in Colab\n",
    "\n"
   ],
   "metadata": {
    "id": "ZllCbAb1M1IA"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ],
   "metadata": {
    "id": "f4pdt-EGVxh9"
   },
   "execution_count": 2,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "import fairnesseval as fe\n",
    "try:\n",
    "    fe.run.launch_experiment_by_id('new_experiment', '/content/exp_def.py')\n",
    "except ModuleNotFoundError as e:\n",
    "    print(e)\n",
    "    print('*'*50 + '\\nFollowing the previous instructions, you should \"Define your experiment in a file\" if you want torun this code.\\n'+ '*'*50)"
   ],
   "metadata": {
    "id": "X1BhG4l2OhRg",
    "outputId": "786b2808-c566-4787-b712-aa0c3f3d92f3",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "execution_count": 3,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "WARNING:root:No module named 'inFairness': SenSeI and SenSR will be unavailable. To install, run:\n",
      "pip install 'aif360[inFairness]'\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "No module named 'exp_def'\n",
      "**************************************************\n",
      "\n",
      "\n",
      "Following the previous instructions, you should \"Define your experiment in a file\" if you want torun this code.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "\n",
    "or save the following code in a .py file to run the experiments.\n",
    "\n",
    "\n",
    "```python\n",
    "# FILE runner.py\n",
    "import fairnesseval as fe\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    conf_todo = [\n",
    "        \"new_experiment\",\n",
    "        # ... (list of configurations to be executed)\n",
    "    ]\n",
    "    for x in conf_todo:\n",
    "        fe.run.launch_experiment_by_id(x, '/content/exp_def.py')\n",
    "\n",
    "```\n",
    "\n",
    "Then launch the python script"
   ],
   "metadata": {
    "id": "6TqlxFcPOhjx"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "!python -m runner"
   ],
   "metadata": {
    "id": "yjh9QpMb9kum",
    "outputId": "fcca32f6-5628-4196-ddf6-60be2e34f532",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "execution_count": 4,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "/usr/bin/python3: No module named runner\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Otherwise you can use `launch_experiment_by_config`.\n",
    "E.g.:"
   ],
   "metadata": {
    "id": "hRokI3ObADED"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "import fairnesseval as fe\n",
    "fe.run.launch_experiment_by_config(\n",
    "    {\n",
    "        'experiment_id': 'new_experiment',\n",
    "        'dataset_names': ['adult_sigmod_no_SA'],\n",
    "        'model_names': ['LogisticRegression'],\n",
    "        'random_seeds': [1],\n",
    "        'results_path': './demo_results'\n",
    "    }\n",
    "    )"
   ],
   "metadata": {
    "id": "VfYrY0e9DpEe",
    "outputId": "5cf60b18-a55b-4b20-aa81-cf8765365fe5",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "execution_count": 5,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "20/09/24 15:03:32 INFO:root: Parameters of experiment new_experiment\n",
      "{\"experiment_id\": \"new_experiment\",\n",
      "\t\"dataset_names\": [\"adult_sigmod_no_SA\"],\n",
      "\t\"model_names\": [\"LogisticRegression\"],\n",
      "\t\"random_seeds\": [1],\n",
      "\t\"results_path\": \"./demo_results\"}\n",
      "20/09/24 15:03:32 INFO:root: Started logging.\n",
      "20/09/24 15:03:32 INFO:root: Starting combination: base model: None, dataset_name: adult_sigmod_no_SA, model_name: LogisticRegression\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Configuration:\n",
      "experiment_id: new_experiment\n",
      "dataset_name: ['adult_sigmod_no_SA']\n",
      "model_name: ['LogisticRegression']\n",
      "results_path: ./demo_results\n",
      "train_fractions: [1]\n",
      "random_seeds: [1]\n",
      "metrics: default\n",
      "preprocessing: default\n",
      "split_strategy: StratifiedKFold\n",
      "train_test_fold: [0, 1, 2]\n",
      "model_params: {}\n",
      "dataset_params: {}\n",
      "debug: False\n",
      "states: None\n",
      "eps: None\n",
      "constraint_code: None\n",
      "expgrad_fractions: None\n",
      "grid_fractions: None\n",
      "exp_grid_ratio: None\n",
      "exp_subset: None\n",
      "run_linprog_step: None\n",
      "base_model_code: None\n",
      "train_test_seeds: [None]\n",
      "test_size: 0.3\n",
      "redo_tuning: False\n",
      "****************************************************************************************************\n",
      "Downloading adult.data in /usr/local/lib/python3.10/dist-packages/aif360/data/raw/adult/adult.data because it was not found.\n",
      "Downloading adult.test in /usr/local/lib/python3.10/dist-packages/aif360/data/raw/adult/adult.test because it was not found.\n",
      "Downloading adult.names in /usr/local/lib/python3.10/dist-packages/aif360/data/raw/adult/adult.names because it was not found.\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "\r0it [00:00, ?it/s]20/09/24 15:03:36 INFO:root: Starting step: random_seed: 1, train_test_seed: 1, train_test_fold: 0 \n",
      "{\"train_fractions\": 1}\n",
      "20/09/24 15:03:36 INFO:root: Starting fit...\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "20/09/24 15:03:37 INFO:root: Ended fit:  in: 0:00:00.824993 ||| Starting evaluation...\n",
      "20/09/24 15:03:38 INFO:root: Ended evaluation:  in: 0:00:01.600202\n",
      "20/09/24 15:03:38 INFO:root: Ended step in: 0:00:02.452659\n",
      "1it [00:03,  3.01s/it]20/09/24 15:03:38 INFO:root: Starting step: random_seed: 2, train_test_seed: 1, train_test_fold: 1 \n",
      "{\"train_fractions\": 1}\n",
      "20/09/24 15:03:38 INFO:root: Starting fit...\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Saving results in: ./demo_results/new_experiment/new_experiment_adult_sigmod_no_SA.csv\n",
      "\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "20/09/24 15:03:40 INFO:root: Ended fit:  in: 0:00:01.409505 ||| Starting evaluation...\n",
      "20/09/24 15:03:41 INFO:root: Ended evaluation:  in: 0:00:01.569132\n",
      "20/09/24 15:03:41 INFO:root: Ended step in: 0:00:03.007157\n",
      "2it [00:06,  3.02s/it]20/09/24 15:03:41 INFO:root: Starting step: random_seed: 3, train_test_seed: 1, train_test_fold: 2 \n",
      "{\"train_fractions\": 1}\n",
      "20/09/24 15:03:41 INFO:root: Starting fit...\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Saving results in: ./demo_results/new_experiment/new_experiment_adult_sigmod_no_SA.csv\n",
      "\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "20/09/24 15:03:42 INFO:root: Ended fit:  in: 0:00:00.715743 ||| Starting evaluation...\n",
      "20/09/24 15:03:43 INFO:root: Ended evaluation:  in: 0:00:00.801202\n",
      "20/09/24 15:03:43 INFO:root: Ended step in: 0:00:01.550385\n",
      "3it [00:07,  2.54s/it]\n",
      "20/09/24 15:03:43 INFO:root: Ended: None, dataset_name: adult_sigmod_no_SA, model_name: LogisticRegression in:\n",
      " 0:00:10.481135\n",
      "20/09/24 15:03:43 INFO:root: Ended experiment. It took: 0:00:10.828680\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Saving results in: ./demo_results/new_experiment/new_experiment_adult_sigmod_no_SA.csv\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## CLI interface\n",
    "The equivalent CLI call to run the experiment defined before is:"
   ],
   "metadata": {
    "id": "NlhpcpCTQQda"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "!python -m fairnesseval.run --dataset_name adult_sigmod_no_SA --model_name LogisticRegression --experiment_id new_experiment --random_seeds 1 --results_path /content/demo_results"
   ],
   "metadata": {
    "id": "kyf4gRHoQZnC",
    "outputId": "2328a7f9-4b29-427e-8ab5-cf33656fd392",
    "colab": {
     "base_uri": "https://localhost:8080/"
    }
   },
   "execution_count": 6,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "2024-09-20 15:03:46.551250: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-09-20 15:03:46.603595: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-09-20 15:03:46.619357: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-09-20 15:03:48.922411: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "WARNING:root:No module named 'inFairness': SenSeI and SenSR will be unavailable. To install, run:\n",
      "pip install 'aif360[inFairness]'\n",
      "/usr/lib/python3.10/runpy.py:126: RuntimeWarning: 'fairnesseval.run' found in sys.modules after import of package 'fairnesseval', but prior to execution of 'fairnesseval.run'; this may result in unpredictable behaviour\n",
      "  warn(RuntimeWarning(msg))\n",
      "Configuration:\n",
      "experiment_id: new_experiment\n",
      "dataset_name: ['adult_sigmod_no_SA']\n",
      "model_name: ['LogisticRegression']\n",
      "results_path: /content/demo_results\n",
      "train_fractions: [1]\n",
      "random_seeds: [1]\n",
      "metrics: default\n",
      "preprocessing: default\n",
      "split_strategy: StratifiedKFold\n",
      "train_test_fold: [0, 1, 2]\n",
      "model_params: {}\n",
      "dataset_params: {}\n",
      "debug: False\n",
      "states: None\n",
      "eps: None\n",
      "constraint_code: None\n",
      "expgrad_fractions: None\n",
      "grid_fractions: None\n",
      "exp_grid_ratio: None\n",
      "exp_subset: None\n",
      "run_linprog_step: None\n",
      "base_model_code: None\n",
      "train_test_seeds: [None]\n",
      "test_size: 0.3\n",
      "redo_tuning: False\n",
      "****************************************************************************************************\n",
      "0it [00:00, ?it/s]\n",
      "Saving results in: /content/demo_results/new_experiment/new_experiment_adult_sigmod_no_SA.csv\n",
      "1it [00:01,  1.47s/it]\n",
      "Saving results in: /content/demo_results/new_experiment/new_experiment_adult_sigmod_no_SA.csv\n",
      "2it [00:02,  1.42s/it]\n",
      "Saving results in: /content/demo_results/new_experiment/new_experiment_adult_sigmod_no_SA.csv\n",
      "3it [00:04,  1.38s/it]\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "id": "_JVjlhrfCuZJ"
   },
   "execution_count": 6,
   "outputs": []
  },
  {
   "metadata": {
    "id": "KpUKWAKwQ_HY"
   },
   "cell_type": "markdown",
   "source": [
    "# Saving additional information from models\n",
    "The `fairnesseval` library allows to save additional information from the models, such as the time of each phase of the prediction process or analytics about training process.\n",
    "To do this, you need to define a `get_stats_dict` method in your model class that returns a dictionary with the desired information."
   ]
  }
 ]
}