quickstart.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "VyyZAufDhLXr",
        "outputId": "08f8b912-0675-4962-d99f-96248b11ffe4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rHit:1 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu focal InRelease\n",
            "\r0% [Waiting for headers] [Waiting for headers] [Connected to cloud.r-project.or\r                                                                               \rHit:2 http://archive.ubuntu.com/ubuntu focal InRelease\n",
            "\r                                                                               \rGet:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB]\n",
            "\r                                                                               \rGet:4 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB]\n",
            "\r                                                                               \rHit:5 http://ppa.launchpad.net/cran/libgit2/ubuntu focal InRelease\n",
            "\r0% [3 InRelease 24.3 kB/114 kB 21%] [4 InRelease 110 kB/114 kB 96%] [Connected \r0% [3 InRelease 33.0 kB/114 kB 29%] [Connected to cloud.r-project.org (65.9.86.\r                                                                               \rHit:6 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal InRelease\n",
            "\r0% [3 InRelease 62.0 kB/114 kB 54%] [Waiting for headers] [Connecting to ppa.la\r                                                                               \rGet:7 https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/ InRelease [3,622 B]\n",
            "\r0% [3 InRelease 105 kB/114 kB 93%] [7 InRelease 3,622 B/3,622 B 100%] [Waiting \r                                                                               \r0% [3 InRelease 105 kB/114 kB 93%] [Waiting for headers] [Waiting for headers]\r                                                                              \r0% [Waiting for headers] [Waiting for headers] [Waiting for headers]\r                                                                    \rHit:8 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu focal InRelease\n",
            "\r0% [Waiting for headers] [Connecting to ppa.launchpad.net (185.125.190.52)] [Wa\r                                                                               \rGet:9 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB]\n",
            "\r0% [9 InRelease 15.6 kB/108 kB 14%] [Connecting to ppa.launchpad.net (185.125.1\r                                                                               \r0% [Connecting to ppa.launchpad.net (185.125.190.52)] [Waiting for headers]\r                                                                           \rHit:10 http://ppa.launchpad.net/ubuntugis/ppa/ubuntu focal InRelease\n",
            "Hit:11 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease\n",
            "Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,529 kB]\n",
            "Get:13 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,866 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [37.5 kB]\n",
            "Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [1,070 kB]\n",
            "Get:16 https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/ Packages [82.8 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [40.3 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,369 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [3,346 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,671 kB]\n",
            "Fetched 14.4 MB in 2s (6,938 kB/s)\n",
            "Reading package lists... Done\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "python3.8 is already the newest version (3.8.10-0ubuntu1~20.04.8).\n",
            "python3.8 set to manually installed.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 19 not upgraded.\n",
            "Python 3.10.12\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  python-pip-whl python3-setuptools python3-wheel\n",
            "Suggested packages:\n",
            "  python-setuptools-doc\n",
            "The following NEW packages will be installed:\n",
            "  python-pip-whl python3-pip python3-setuptools python3-wheel\n",
            "0 upgraded, 4 newly installed, 0 to remove and 19 not upgraded.\n",
            "Need to get 2,389 kB of archives.\n",
            "After this operation, 4,933 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python-pip-whl all 20.0.2-5ubuntu1.9 [1,805 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-setuptools all 45.2.0-1ubuntu0.1 [330 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-wheel all 0.34.2-1ubuntu0.1 [23.9 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-pip all 20.0.2-5ubuntu1.9 [231 kB]\n",
            "Fetched 2,389 kB in 0s (17.2 MB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 4.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package python-pip-whl.\n",
            "(Reading database ... 123105 files and directories currently installed.)\n",
            "Preparing to unpack .../python-pip-whl_20.0.2-5ubuntu1.9_all.deb ...\n",
            "Unpacking python-pip-whl (20.0.2-5ubuntu1.9) ...\n",
            "Selecting previously unselected package python3-setuptools.\n",
            "Preparing to unpack .../python3-setuptools_45.2.0-1ubuntu0.1_all.deb ...\n",
            "Unpacking python3-setuptools (45.2.0-1ubuntu0.1) ...\n",
            "Selecting previously unselected package python3-wheel.\n",
            "Preparing to unpack .../python3-wheel_0.34.2-1ubuntu0.1_all.deb ...\n",
            "Unpacking python3-wheel (0.34.2-1ubuntu0.1) ...\n",
            "Selecting previously unselected package python3-pip.\n",
            "Preparing to unpack .../python3-pip_20.0.2-5ubuntu1.9_all.deb ...\n",
            "Unpacking python3-pip (20.0.2-5ubuntu1.9) ...\n",
            "Setting up python3-setuptools (45.2.0-1ubuntu0.1) ...\n",
            "Setting up python3-wheel (0.34.2-1ubuntu0.1) ...\n",
            "Setting up python-pip-whl (20.0.2-5ubuntu1.9) ...\n",
            "Setting up python3-pip (20.0.2-5ubuntu1.9) ...\n",
            "Processing triggers for man-db (2.9.1-1) ...\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (23.1.2)\n",
            "Python 3.8.10\n"
          ]
        }
      ],
      "source": [
        "!sudo add-apt-repository ppa:deadsnakes/ppa\n",
        "!sudo apt-get update -y\n",
        "!sudo apt install python3.8-distutils\n",
        "!sudo apt-get install python3.8\n",
        "!python --version\n",
        "!sudo apt-get install python3-pip\n",
        "!python -m pip install --upgrade pip\n",
        "!ln -sf /usr/bin/python3.8 /usr/bin/python3\n",
        "!python --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0W9EBY1-hLXt",
        "outputId": "25758f23-37ca-4a4e-ed94-945cbb311cc1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "rm: cannot remove 'opentf/': No such file or directory\n",
            "Cloning into 'opentf'...\n",
            "remote: Enumerating objects: 6628, done.\u001b[K\n",
            "remote: Counting objects: 100% (1625/1625), done.\u001b[K\n",
            "remote: Compressing objects: 100% (1116/1116), done.\u001b[K\n",
            "remote: Total 6628 (delta 581), reused 1455 (delta 497), pack-reused 5003\u001b[K\n",
            "Receiving objects: 100% (6628/6628), 110.81 MiB | 14.61 MiB/s, done.\n",
            "Resolving deltas: 100% (2692/2692), done.\n",
            "Updating files: 100% (1745/1745), done.\n",
            "Submodule 'src/Adila' (https://github.com/fani-lab/Adila.git) registered for path 'src/Adila'\n",
            "Cloning into '/content/opentf/src/Adila'...\n",
            "remote: Enumerating objects: 778, done.        \n",
            "remote: Counting objects: 100% (108/108), done.        \n",
            "remote: Compressing objects: 100% (93/93), done.        \n",
            "remote: Total 778 (delta 62), reused 33 (delta 14), pack-reused 670        \n",
            "Receiving objects: 100% (778/778), 16.72 MiB | 16.79 MiB/s, done.\n",
            "Resolving deltas: 100% (348/348), done.\n",
            "Submodule path 'src/Adila': checked out 'ed7b732ab165177f652a49d6fbf1fa033cfb008e'\n",
            "/content/opentf\n",
            "Collecting virtualenv\n",
            "  Downloading virtualenv-20.23.1-py3-none-any.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 13.1 MB/s \n",
            "\u001b[?25hCollecting platformdirs<4,>=3.5.1\n",
            "  Downloading platformdirs-3.8.1-py3-none-any.whl (16 kB)\n",
            "Collecting filelock<4,>=3.12\n",
            "  Downloading filelock-3.12.2-py3-none-any.whl (10 kB)\n",
            "Collecting distlib<1,>=0.3.6\n",
            "  Downloading distlib-0.3.6-py2.py3-none-any.whl (468 kB)\n",
            "\u001b[K     |████████████████████████████████| 468 kB 66.9 MB/s \n",
            "\u001b[?25hInstalling collected packages: platformdirs, filelock, distlib, virtualenv\n",
            "Successfully installed distlib-0.3.6 filelock-3.12.2 platformdirs-3.8.1 virtualenv-20.23.1\n",
            "created virtual environment CPython3.8.10.final.0-64 in 864ms\n",
            "  creator CPython3Posix(dest=/content/opentf/opentf, clear=False, no_vcs_ignore=False, global=False)\n",
            "  seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/root/.local/share/virtualenv)\n",
            "    added seed packages: pip==23.1.2, setuptools==67.8.0, wheel==0.40.0\n",
            "  activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator\n",
            "Collecting torch>=1.9.0 (from -r requirements.txt (line 1))\n",
            "  Downloading torch-2.0.1-cp38-cp38-manylinux1_x86_64.whl (619.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m619.9/619.9 MB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting scipy==1.9.3 (from -r requirements.txt (line 2))\n",
            "  Downloading scipy-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (33.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m33.8/33.8 MB\u001b[0m \u001b[31m43.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting numpy==1.20.3 (from -r requirements.txt (line 3))\n",
            "  Downloading numpy-1.20.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.4/15.4 MB\u001b[0m \u001b[31m82.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tqdm==4.60.0 (from -r requirements.txt (line 4))\n",
            "  Downloading tqdm-4.60.0-py2.py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.8/75.8 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting PyYAML==5.4 (from -r requirements.txt (line 5))\n",
            "  Downloading PyYAML-5.4-cp38-cp38-manylinux1_x86_64.whl (662 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m662.2/662.2 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting scikit_learn==0.24.2 (from -r requirements.txt (line 6))\n",
            "  Downloading scikit_learn-0.24.2-cp38-cp38-manylinux2010_x86_64.whl (24.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.9/24.9 MB\u001b[0m \u001b[31m63.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pandas==1.3.3 (from -r requirements.txt (line 7))\n",
            "  Downloading pandas-1.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.5/11.5 MB\u001b[0m \u001b[31m79.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gensim==3.8.3 (from -r requirements.txt (line 8))\n",
            "  Downloading gensim-3.8.3-cp38-cp38-manylinux1_x86_64.whl (24.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.2/24.2 MB\u001b[0m \u001b[31m38.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pytrec-eval-terrier==0.5.2 (from -r requirements.txt (line 9))\n",
            "  Downloading pytrec_eval_terrier-0.5.2-cp38-cp38-manylinux2010_x86_64.whl (287 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m287.4/287.4 kB\u001b[0m \u001b[31m33.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting matplotlib==3.4.2 (from -r requirements.txt (line 10))\n",
            "  Downloading matplotlib-3.4.2-cp38-cp38-manylinux1_x86_64.whl (10.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.3/10.3 MB\u001b[0m \u001b[31m116.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting OpenNMT-py==2.2.0 (from -r requirements.txt (line 11))\n",
            "  Downloading OpenNMT_py-2.2.0-py3-none-any.whl (216 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m216.8/216.8 kB\u001b[0m \u001b[31m22.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting reranking==0.3.6 (from -r requirements.txt (line 12))\n",
            "  Downloading reranking-0.3.6-py3-none-any.whl (17 kB)\n",
            "Collecting python-dateutil==2.8.2 (from -r requirements.txt (line 14))\n",
            "  Downloading python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m247.7/247.7 kB\u001b[0m \u001b[31m30.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting joblib>=0.11 (from scikit_learn==0.24.2->-r requirements.txt (line 6))\n",
            "  Downloading joblib-1.3.1-py3-none-any.whl (301 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.0/302.0 kB\u001b[0m \u001b[31m30.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting threadpoolctl>=2.0.0 (from scikit_learn==0.24.2->-r requirements.txt (line 6))\n",
            "  Downloading threadpoolctl-3.1.0-py3-none-any.whl (14 kB)\n",
            "Collecting pytz>=2017.3 (from pandas==1.3.3->-r requirements.txt (line 7))\n",
            "  Downloading pytz-2023.3-py2.py3-none-any.whl (502 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m502.3/502.3 kB\u001b[0m \u001b[31m41.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting six>=1.5.0 (from gensim==3.8.3->-r requirements.txt (line 8))\n",
            "  Downloading six-1.16.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting smart-open>=1.8.1 (from gensim==3.8.3->-r requirements.txt (line 8))\n",
            "  Downloading smart_open-6.3.0-py3-none-any.whl (56 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cycler>=0.10 (from matplotlib==3.4.2->-r requirements.txt (line 10))\n",
            "  Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n",
            "Collecting kiwisolver>=1.0.1 (from matplotlib==3.4.2->-r requirements.txt (line 10))\n",
            "  Downloading kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m64.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pillow>=6.2.0 (from matplotlib==3.4.2->-r requirements.txt (line 10))\n",
            "  Downloading Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m75.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyparsing>=2.2.1 (from matplotlib==3.4.2->-r requirements.txt (line 10))\n",
            "  Downloading pyparsing-3.1.0-py3-none-any.whl (102 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.6/102.6 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchtext==0.5.0 (from OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading torchtext-0.5.0-py3-none-any.whl (73 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.2/73.2 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting configargparse (from OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading ConfigArgParse-1.5.5-py3-none-any.whl (25 kB)\n",
            "Collecting tensorboard>=2.3 (from OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading tensorboard-2.13.0-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m60.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting flask (from OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading Flask-2.3.2-py3-none-any.whl (96 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.9/96.9 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting waitress (from OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading waitress-2.1.2-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.7/57.7 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyonmttok<2,>=1.23 (from OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading pyonmttok-1.37.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.0/17.0 MB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting requests (from torchtext==0.5.0->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading requests-2.31.0-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.6/62.6 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece (from torchtext==0.5.0->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m81.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting filelock (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Using cached filelock-3.12.2-py3-none-any.whl (10 kB)\n",
            "Collecting typing-extensions (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading typing_extensions-4.7.1-py3-none-any.whl (33 kB)\n",
            "Collecting sympy (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading sympy-1.12-py3-none-any.whl (5.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m100.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting networkx (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading networkx-3.1-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m95.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jinja2 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading Jinja2-3.1.2-py3-none-any.whl (133 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m133.1/133.1 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-nvrtc-cu11==11.7.99 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m71.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu11==11.7.99 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m69.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu11==11.7.101 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl (11.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.8/11.8 MB\u001b[0m \u001b[31m107.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu11==8.5.0.96 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu11==11.10.3.66 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu11==10.9.0.58 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl (168.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m168.4/168.4 MB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu11==10.2.10.91 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl (54.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 MB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu11==11.4.0.1 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl (102.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.6/102.6 MB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu11==11.7.4.91 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl (173.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m173.2/173.2 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu11==2.14.3 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl (177.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.1/177.1 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu11==11.7.91 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl (98 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.6/98.6 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.0.0 (from torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (63.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.2/63.2 MB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: setuptools in ./opentf/lib/python3.8/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.9.0->-r requirements.txt (line 1)) (67.8.0)\n",
            "Requirement already satisfied: wheel in ./opentf/lib/python3.8/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.9.0->-r requirements.txt (line 1)) (0.40.0)\n",
            "Collecting cmake (from triton==2.0.0->torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading cmake-3.26.4-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (24.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.0/24.0 MB\u001b[0m \u001b[31m59.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting lit (from triton==2.0.0->torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading lit-16.0.6.tar.gz (153 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m153.7/153.7 kB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting absl-py>=0.4 (from tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading absl_py-1.4.0-py3-none-any.whl (126 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.5/126.5 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting grpcio>=1.48.2 (from tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading grpcio-1.56.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m67.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting google-auth<3,>=1.6.3 (from tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading google_auth-2.21.0-py2.py3-none-any.whl (182 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m182.1/182.1 kB\u001b[0m \u001b[31m21.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting google-auth-oauthlib<1.1,>=0.5 (from tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading google_auth_oauthlib-1.0.0-py2.py3-none-any.whl (18 kB)\n",
            "Collecting markdown>=2.6.8 (from tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading Markdown-3.4.3-py3-none-any.whl (93 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.9/93.9 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting protobuf>=3.19.6 (from tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl (304 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m304.5/304.5 kB\u001b[0m \u001b[31m26.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tensorboard-data-server<0.8.0,>=0.7.0 (from tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading tensorboard_data_server-0.7.1-py3-none-manylinux2014_x86_64.whl (6.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m64.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting werkzeug>=1.0.1 (from tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading Werkzeug-2.3.6-py3-none-any.whl (242 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m242.5/242.5 kB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting itsdangerous>=2.1.2 (from flask->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading itsdangerous-2.1.2-py3-none-any.whl (15 kB)\n",
            "Collecting click>=8.1.3 (from flask->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading click-8.1.4-py3-none-any.whl (98 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.2/98.2 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting blinker>=1.6.2 (from flask->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading blinker-1.6.2-py3-none-any.whl (13 kB)\n",
            "Collecting importlib-metadata>=3.6.0 (from flask->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading importlib_metadata-6.7.0-py3-none-any.whl (22 kB)\n",
            "Collecting MarkupSafe>=2.0 (from jinja2->torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\n",
            "Collecting mpmath>=0.19 (from sympy->torch>=1.9.0->-r requirements.txt (line 1))\n",
            "  Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m27.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cachetools<6.0,>=2.0.0 (from google-auth<3,>=1.6.3->tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading cachetools-5.3.1-py3-none-any.whl (9.3 kB)\n",
            "Collecting pyasn1-modules>=0.2.1 (from google-auth<3,>=1.6.3->tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading pyasn1_modules-0.3.0-py2.py3-none-any.whl (181 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m181.3/181.3 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rsa<5,>=3.1.4 (from google-auth<3,>=1.6.3->tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading rsa-4.9-py3-none-any.whl (34 kB)\n",
            "Collecting urllib3<2.0 (from google-auth<3,>=1.6.3->tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading urllib3-1.26.16-py2.py3-none-any.whl (143 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.1/143.1 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting requests-oauthlib>=0.7.0 (from google-auth-oauthlib<1.1,>=0.5->tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading requests_oauthlib-1.3.1-py2.py3-none-any.whl (23 kB)\n",
            "Collecting zipp>=0.5 (from importlib-metadata>=3.6.0->flask->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading zipp-3.15.0-py3-none-any.whl (6.8 kB)\n",
            "Collecting charset-normalizer<4,>=2 (from requests->torchtext==0.5.0->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (195 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m195.9/195.9 kB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting idna<4,>=2.5 (from requests->torchtext==0.5.0->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading idna-3.4-py3-none-any.whl (61 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.5/61.5 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting certifi>=2017.4.17 (from requests->torchtext==0.5.0->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading certifi-2023.5.7-py3-none-any.whl (156 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m157.0/157.0 kB\u001b[0m \u001b[31m20.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyasn1<0.6.0,>=0.4.6 (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading pyasn1-0.5.0-py2.py3-none-any.whl (83 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.9/83.9 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting oauthlib>=3.0.0 (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard>=2.3->OpenNMT-py==2.2.0->-r requirements.txt (line 11))\n",
            "  Downloading oauthlib-3.2.2-py3-none-any.whl (151 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m151.7/151.7 kB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: lit\n",
            "  Building wheel for lit (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for lit: filename=lit-16.0.6-py3-none-any.whl size=93582 sha256=aa4fb6f2f26770badb0e0705a36f1c3f387f6324bef45d4ddbbd05b63a607d77\n",
            "  Stored in directory: /root/.cache/pip/wheels/05/ab/f1/0102fea49a41c753f0e79a1a4012417d5d7ef0f93224694472\n",
            "Successfully built lit\n",
            "Installing collected packages: sentencepiece, pytz, mpmath, lit, cmake, zipp, waitress, urllib3, typing-extensions, tqdm, threadpoolctl, tensorboard-data-server, sympy, smart-open, six, reranking, PyYAML, pytrec-eval-terrier, pyparsing, pyonmttok, pyasn1, protobuf, pillow, oauthlib, nvidia-nvtx-cu11, nvidia-nccl-cu11, nvidia-cusparse-cu11, nvidia-curand-cu11, nvidia-cufft-cu11, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cuda-cupti-cu11, nvidia-cublas-cu11, numpy, networkx, MarkupSafe, kiwisolver, joblib, itsdangerous, idna, grpcio, filelock, cycler, configargparse, click, charset-normalizer, certifi, cachetools, blinker, absl-py, werkzeug, scipy, rsa, requests, python-dateutil, pyasn1-modules, nvidia-cusolver-cu11, nvidia-cudnn-cu11, jinja2, importlib-metadata, scikit_learn, requests-oauthlib, pandas, matplotlib, markdown, google-auth, gensim, flask, google-auth-oauthlib, tensorboard, triton, torch, torchtext, OpenNMT-py\n",
            "Successfully installed MarkupSafe-2.1.3 OpenNMT-py-2.2.0 PyYAML-5.4 absl-py-1.4.0 blinker-1.6.2 cachetools-5.3.1 certifi-2023.5.7 charset-normalizer-3.1.0 click-8.1.4 cmake-3.26.4 configargparse-1.5.5 cycler-0.11.0 filelock-3.12.2 flask-2.3.2 gensim-3.8.3 google-auth-2.21.0 google-auth-oauthlib-1.0.0 grpcio-1.56.0 idna-3.4 importlib-metadata-6.7.0 itsdangerous-2.1.2 jinja2-3.1.2 joblib-1.3.1 kiwisolver-1.4.4 lit-16.0.6 markdown-3.4.3 matplotlib-3.4.2 mpmath-1.3.0 networkx-3.1 numpy-1.20.3 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-cupti-cu11-11.7.101 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 nvidia-cufft-cu11-10.9.0.58 nvidia-curand-cu11-10.2.10.91 nvidia-cusolver-cu11-11.4.0.1 nvidia-cusparse-cu11-11.7.4.91 nvidia-nccl-cu11-2.14.3 nvidia-nvtx-cu11-11.7.91 oauthlib-3.2.2 pandas-1.3.3 pillow-10.0.0 protobuf-4.23.4 pyasn1-0.5.0 pyasn1-modules-0.3.0 pyonmttok-1.37.1 pyparsing-3.1.0 python-dateutil-2.8.2 pytrec-eval-terrier-0.5.2 pytz-2023.3 requests-2.31.0 requests-oauthlib-1.3.1 reranking-0.3.6 rsa-4.9 scikit_learn-0.24.2 scipy-1.9.3 sentencepiece-0.1.99 six-1.16.0 smart-open-6.3.0 sympy-1.12 tensorboard-2.13.0 tensorboard-data-server-0.7.1 threadpoolctl-3.1.0 torch-2.0.1 torchtext-0.5.0 tqdm-4.60.0 triton-2.0.0 typing-extensions-4.7.1 urllib3-1.26.16 waitress-2.1.2 werkzeug-2.3.6 zipp-3.15.0\n",
            "Collecting ipykernel\n",
            "  Downloading ipykernel-6.24.0-py3-none-any.whl (152 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.8/152.8 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting comm>=0.1.1 (from ipykernel)\n",
            "  Downloading comm-0.1.3-py3-none-any.whl (6.6 kB)\n",
            "Collecting debugpy>=1.6.5 (from ipykernel)\n",
            "  Downloading debugpy-1.6.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m55.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ipython>=7.23.1 (from ipykernel)\n",
            "  Downloading ipython-8.12.2-py3-none-any.whl (797 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m797.8/797.8 kB\u001b[0m \u001b[31m48.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jupyter-client>=6.1.12 (from ipykernel)\n",
            "  Downloading jupyter_client-8.3.0-py3-none-any.whl (103 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.2/103.2 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jupyter-core!=5.0.*,>=4.12 (from ipykernel)\n",
            "  Downloading jupyter_core-5.3.1-py3-none-any.whl (93 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.7/93.7 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting matplotlib-inline>=0.1 (from ipykernel)\n",
            "  Downloading matplotlib_inline-0.1.6-py3-none-any.whl (9.4 kB)\n",
            "Collecting nest-asyncio (from ipykernel)\n",
            "  Downloading nest_asyncio-1.5.6-py3-none-any.whl (5.2 kB)\n",
            "Collecting packaging (from ipykernel)\n",
            "  Downloading packaging-23.1-py3-none-any.whl (48 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.9/48.9 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting psutil (from ipykernel)\n",
            "  Downloading psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (282 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m282.1/282.1 kB\u001b[0m \u001b[31m23.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyzmq>=20 (from ipykernel)\n",
            "  Downloading pyzmq-25.1.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m65.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tornado>=6.1 (from ipykernel)\n",
            "  Downloading tornado-6.3.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (426 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m426.9/426.9 kB\u001b[0m \u001b[31m41.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting traitlets>=5.4.0 (from ipykernel)\n",
            "  Downloading traitlets-5.9.0-py3-none-any.whl (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.4/117.4 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting backcall (from ipython>=7.23.1->ipykernel)\n",
            "  Downloading backcall-0.2.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting decorator (from ipython>=7.23.1->ipykernel)\n",
            "  Downloading decorator-5.1.1-py3-none-any.whl (9.1 kB)\n",
            "Collecting jedi>=0.16 (from ipython>=7.23.1->ipykernel)\n",
            "  Downloading jedi-0.18.2-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m78.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pickleshare (from ipython>=7.23.1->ipykernel)\n",
            "  Downloading pickleshare-0.7.5-py2.py3-none-any.whl (6.9 kB)\n",
            "Collecting prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 (from ipython>=7.23.1->ipykernel)\n",
            "  Downloading prompt_toolkit-3.0.39-py3-none-any.whl (385 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m385.2/385.2 kB\u001b[0m \u001b[31m39.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pygments>=2.4.0 (from ipython>=7.23.1->ipykernel)\n",
            "  Downloading Pygments-2.15.1-py3-none-any.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m61.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting stack-data (from ipython>=7.23.1->ipykernel)\n",
            "  Downloading stack_data-0.6.2-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: typing-extensions in ./opentf/lib/python3.8/site-packages (from ipython>=7.23.1->ipykernel) (4.7.1)\n",
            "Collecting pexpect>4.3 (from ipython>=7.23.1->ipykernel)\n",
            "  Downloading pexpect-4.8.0-py2.py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.0/59.0 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata>=4.8.3 in ./opentf/lib/python3.8/site-packages (from jupyter-client>=6.1.12->ipykernel) (6.7.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in ./opentf/lib/python3.8/site-packages (from jupyter-client>=6.1.12->ipykernel) (2.8.2)\n",
            "Collecting platformdirs>=2.5 (from jupyter-core!=5.0.*,>=4.12->ipykernel)\n",
            "  Using cached platformdirs-3.8.1-py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: zipp>=0.5 in ./opentf/lib/python3.8/site-packages (from importlib-metadata>=4.8.3->jupyter-client>=6.1.12->ipykernel) (3.15.0)\n",
            "Collecting parso<0.9.0,>=0.8.0 (from jedi>=0.16->ipython>=7.23.1->ipykernel)\n",
            "  Downloading parso-0.8.3-py2.py3-none-any.whl (100 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100.8/100.8 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ptyprocess>=0.5 (from pexpect>4.3->ipython>=7.23.1->ipykernel)\n",
            "  Downloading ptyprocess-0.7.0-py2.py3-none-any.whl (13 kB)\n",
            "Collecting wcwidth (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=7.23.1->ipykernel)\n",
            "  Downloading wcwidth-0.2.6-py2.py3-none-any.whl (29 kB)\n",
            "Requirement already satisfied: six>=1.5 in ./opentf/lib/python3.8/site-packages (from python-dateutil>=2.8.2->jupyter-client>=6.1.12->ipykernel) (1.16.0)\n",
            "Collecting executing>=1.2.0 (from stack-data->ipython>=7.23.1->ipykernel)\n",
            "  Downloading executing-1.2.0-py2.py3-none-any.whl (24 kB)\n",
            "Collecting asttokens>=2.1.0 (from stack-data->ipython>=7.23.1->ipykernel)\n",
            "  Downloading asttokens-2.2.1-py2.py3-none-any.whl (26 kB)\n",
            "Collecting pure-eval (from stack-data->ipython>=7.23.1->ipykernel)\n",
            "  Downloading pure_eval-0.2.2-py3-none-any.whl (11 kB)\n",
            "Installing collected packages: wcwidth, pure-eval, ptyprocess, pickleshare, executing, backcall, traitlets, tornado, pyzmq, pygments, psutil, prompt-toolkit, platformdirs, pexpect, parso, packaging, nest-asyncio, decorator, debugpy, asttokens, stack-data, matplotlib-inline, jupyter-core, jedi, comm, jupyter-client, ipython, ipykernel\n",
            "Successfully installed asttokens-2.2.1 backcall-0.2.0 comm-0.1.3 debugpy-1.6.7 decorator-5.1.1 executing-1.2.0 ipykernel-6.24.0 ipython-8.12.2 jedi-0.18.2 jupyter-client-8.3.0 jupyter-core-5.3.1 matplotlib-inline-0.1.6 nest-asyncio-1.5.6 packaging-23.1 parso-0.8.3 pexpect-4.8.0 pickleshare-0.7.5 platformdirs-3.8.1 prompt-toolkit-3.0.39 psutil-5.9.5 ptyprocess-0.7.0 pure-eval-0.2.2 pygments-2.15.1 pyzmq-25.1.0 stack-data-0.6.2 tornado-6.3.2 traitlets-5.9.0 wcwidth-0.2.6\n",
            "/content/opentf/src\n"
          ]
        }
      ],
      "source": [
        "%rm -R opentf/\n",
        "!git clone --recursive https://github.com/Fani-Lab/opentf\n",
        "%cd opentf\n",
        "!pip install virtualenv\n",
        "!virtualenv opentf\n",
        "!opentf/bin/pip install -r requirements.txt\n",
        "!opentf/bin/pip install ipykernel\n",
        "import sys\n",
        "sys.path.append(\"opentf/lib/python3.8/site-packages\")\n",
        "%cd src/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FscMzVDAhLXt"
      },
      "outputs": [],
      "source": [
        "#!apt-get install --assume-yes p7zip-full\n",
        "#!wget -O ../data/raw/imdb/name.basics.tsv.gz https://datasets.imdbws.com/name.basics.tsv.gz\n",
        "#!wget -O ../data/raw/imdb/title.basics.tsv.gz https://datasets.imdbws.com/title.basics.tsv.gz\n",
        "#!wget -O ../data/raw/imdb/title.principals.tsv.gz https://datasets.imdbws.com/title.principals.tsv.gz\n",
        "#!7z e ../data/raw/imdb/name.basics.tsv.gz -o../data/raw/imdb/\n",
        "#!mv ../data/raw/imdb/data.tsv ../data/raw/imdb/name.basics.tsv\n",
        "#!7z e ../data/raw/imdb/title.basics.tsv.gz -o../data/raw/imdb/\n",
        "#!mv ../data/raw/imdb/data.tsv ../data/raw/imdb/title.basics.tsv\n",
        "#!7z e ../data/raw/imdb/title.principals.tsv.gz -o../data/raw/imdb/\n",
        "#!mv ../data/raw/imdb/data.tsv ../data/raw/imdb/title.principals.tsv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CrRnff_OhLXt"
      },
      "outputs": [],
      "source": [
        "#!apt-get install --assume-yes p7zip-full\n",
        "#!wget -O ../data/raw/dblp/dblp.v12.7z https://originalstatic.aminer.cn/misc/dblp.v12.7z\n",
        "#!7z e ../data/raw/dblp/dblp.v12.7z -o../data/raw/dblp/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mjGfQ7JkhLXt",
        "outputId": "a87975be-14e7-4aab-ff1f-bd9f57e2b150"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "import random\n",
            "import torch\n",
            "import numpy as np\n",
            "\n",
            "random.seed(0)\n",
            "torch.manual_seed(0)\n",
            "torch.cuda.manual_seed_all(0)\n",
            "\n",
            "np.random.seed(0)\n",
            "\n",
            "settings = {\n",
            "    'model':{\n",
            "        'baseline': {\n",
            "            'random': {\n",
            "                'b': 128\n",
            "            },\n",
            "            'fnn':{\n",
            "                'l': [128],  # list of number of nodes in each layer\n",
            "                'lr': 0.1,  # learning rate\n",
            "                'b': 128,  # batch size\n",
            "                'e': 20,  # epoch\n",
            "                'nns': 3,  # number of negative samples\n",
            "                'ns': 'unigram_b',  # 'uniform', 'unigram', 'unigram_b'\n",
            "            },\n",
            "            'bnn':{\n",
            "                'l': [128],  # list of number of nodes in each layer\n",
            "                'lr': 0.1,  # learning rate\n",
            "                'b': 128,  # batch size\n",
            "                'e': 20,  # epoch\n",
            "                'nns': 3,  # number of negative samples\n",
            "                'ns': 'unigram_b',  # 'uniform', 'unigram', 'unigram_b'\n",
            "                's': 1  # # sample_elbo for bnn\n",
            "            },\n",
            "            'nmt': {\n",
            "                'base_config': './mdl/nmt_config.yaml'\n",
            "            },\n",
            "            'caser': {},\n",
            "            'rrn': {\n",
            "                'with_zero': True\n",
            "            },\n",
            "            'emb':{\n",
            "                'd': 100,# embedding dimension\n",
            "                'e': 100,# epoch\n",
            "                'dm': 1,# training algorithm. 1: distributed memory (PV-DM), 0: distributed bag of words (PV-DBOW)\n",
            "                'w': 1 #cooccurrence window\n",
            "            }\n",
            "        },\n",
            "        'cmd': ['train', 'test', 'eval', 'fair'],  # 'train', 'test', 'eval', 'plot', 'agg', 'adila'\n",
            "        'nfolds': 5,\n",
            "        'train_test_split': 0.85,\n",
            "        'step_ahead': 2,#for now, it means that whatever are in the last [step_ahead] time interval will be the test set!\n",
            "    },\n",
            "    'data':{\n",
            "        'domain': {\n",
            "            'dblp':{},\n",
            "            'uspt':{},\n",
            "            'imdb':{},\n",
            "        },\n",
            "        'location_type': 'country', #should be one of 'city', 'state', 'country' and represents the location of members in teams (not the location of teams)\n",
            "        'filter': {\n",
            "            'min_nteam': 75,\n",
            "            'min_team_size': 3,\n",
            "        },\n",
            "        'parallel': 1,\n",
            "        'ncore': 0,# <= 0 for all\n",
            "        'bucket_size': 500\n",
            "    },\n",
            "    'fair': {'np_ratio': None,\n",
            "              'fairness': ['det_greedy',],\n",
            "              'k_max': None,\n",
            "              'fairness_metrics': {'ndkl'},\n",
            "              'utility_metrics': {'map_cut_2,5,10'},\n",
            "              'eq_op': False,\n",
            "              'mode': 0,\n",
            "              'core': -1,\n",
            "              'attribute': ['gender', 'popularity']},\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "!cat param.py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "orxVndmthLXu"
      },
      "outputs": [],
      "source": [
        "!rm -R ../data/preprocessed/dblp/toy.dblp.v12.json/\n",
        "!rm -R ../data/preprocessed/imdb/toy.title.basics.tsv/\n",
        "!rm -R ../data/preprocessed/uspt/toy.patent.tsv/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vr3rpQkXihw9",
        "outputId": "516ba3f6-06eb-4c84-d1b1-374f4d87617e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[0m\u001b[01;34mAdila\u001b[0m/  computecanada.sh  main.py  \u001b[01;34mmisc\u001b[0m/     readme.md\n",
            "\u001b[01;34mcmn\u001b[0m/    \u001b[01;34meval\u001b[0m/             \u001b[01;34mmdl\u001b[0m/     param.py\n"
          ]
        }
      ],
      "source": [
        "%ls"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "etihNlzkhLXu",
        "outputId": "dce3313f-4a0d-4999-84e6-3063dc7d6de2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loading sparse matrices from ./../data/preprocessed/dblp/toy.dblp.v12.json/teamsvecs.pkl ...\n",
            "File not found! Generating the sparse matrices ...\n",
            "Loading indexes pickle from ./../data/preprocessed/dblp/toy.dblp.v12.json/indexes.pkl ...\n",
            "Pickles not found! Reading raw data from ../data/raw/dblp/toy.dblp.v12.json (progress in bytes) ...\n",
            "\r  0% 0/46688 [00:00<?, ?it/s]JSONDecodeError: There has been error in loading json line `[\n",
            "`!\n",
            "Expecting value: line 2 column 1 (char 2)\n",
            "JSONDecodeError: There has been error in loading json line `]`!\n",
            "Expecting value: line 1 column 1 (char 0)\n",
            "\r100% 46664/46688 [00:00<00:00, 4585502.47it/s]\n",
            "Indexing members ...\n",
            "Indexing skills ...\n",
            "Indexing teams ...\n",
            "Indexing locations ...\n",
            "Indexing teams date (year) ...\n",
            "It took 0.000797271728515625 seconds to pickle the data into ./../data/preprocessed/dblp/toy.dblp.v12.json\n",
            "It took 0.07131552696228027 seconds to generate and store the sparse matrices of size (31, 53) at ./../data/preprocessed/dblp/toy.dblp.v12.json/teamsvecs.pkl\n",
            "Running for (dataset, model): (dblp, fnn) ... \n",
            "Fold 0/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 62.71426010131836, Time 0.023291826248168945, Overall 0.027387142181396484 \n",
            "Fold 0/4, Epoch 0/19, Running Loss train 3.135713005065918, Time 0.02346205711364746, Overall 0.027556180953979492 \n",
            "Fold 0/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 13.493390083312988, Time 0.02841019630432129, Overall 0.03250479698181152 \n",
            "Fold 0/4, Epoch 0/19, Running Loss valid 2.2488983472188315, Time 0.028549671173095703, Overall 0.032644033432006836 \n",
            "Fold 0/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 56.81795883178711, Time 0.044690847396850586, Overall 0.04878711700439453 \n",
            "Fold 0/4, Epoch 1/19, Running Loss train 2.8408979415893554, Time 0.04482150077819824, Overall 0.04891562461853027 \n",
            "Fold 0/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 14.17101001739502, Time 0.04969668388366699, Overall 0.05379152297973633 \n",
            "Fold 0/4, Epoch 1/19, Running Loss valid 2.36183500289917, Time 0.04983353614807129, Overall 0.053926944732666016 \n",
            "Fold 0/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 44.24309158325195, Time 0.0663306713104248, Overall 0.07042551040649414 \n",
            "Fold 0/4, Epoch 2/19, Running Loss train 2.212154579162598, Time 0.06648659706115723, Overall 0.07058072090148926 \n",
            "Fold 0/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 19.75441551208496, Time 0.0720522403717041, Overall 0.07614684104919434 \n",
            "Fold 0/4, Epoch 2/19, Running Loss valid 3.2924025853474936, Time 0.07219958305358887, Overall 0.0762934684753418 \n",
            "Fold 0/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 29.45379066467285, Time 0.0910179615020752, Overall 0.09511327743530273 \n",
            "Fold 0/4, Epoch 3/19, Running Loss train 1.4726895332336425, Time 0.09116387367248535, Overall 0.09525799751281738 \n",
            "Fold 0/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 23.601545333862305, Time 0.09623193740844727, Overall 0.1003270149230957 \n",
            "Fold 0/4, Epoch 3/19, Running Loss valid 3.933590888977051, Time 0.09640216827392578, Overall 0.10049605369567871 \n",
            "Fold 0/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 23.174551010131836, Time 0.11269807815551758, Overall 0.11679315567016602 \n",
            "Fold 0/4, Epoch 4/19, Running Loss train 1.1587275505065917, Time 0.1128530502319336, Overall 0.11694717407226562 \n",
            "Fold 0/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 25.024845123291016, Time 0.11773109436035156, Overall 0.1218254566192627 \n",
            "Fold 0/4, Epoch 4/19, Running Loss valid 4.170807520548503, Time 0.11786270141601562, Overall 0.12195682525634766 \n",
            "Fold 0/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 27.327308654785156, Time 0.13366174697875977, Overall 0.1377565860748291 \n",
            "Fold 0/4, Epoch 5/19, Running Loss train 1.3663654327392578, Time 0.13380742073059082, Overall 0.13790154457092285 \n",
            "Fold 0/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 26.28115463256836, Time 0.13866376876831055, Overall 0.14275813102722168 \n",
            "Fold 0/4, Epoch 5/19, Running Loss valid 4.3801924387613935, Time 0.13880085945129395, Overall 0.14289450645446777 \n",
            "Fold 0/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 16.219404220581055, Time 0.15802836418151855, Overall 0.162123441696167 \n",
            "Fold 0/4, Epoch 6/19, Running Loss train 0.8109702110290528, Time 0.15840554237365723, Overall 0.16249966621398926 \n",
            "Fold 0/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 30.575420379638672, Time 0.16352248191833496, Overall 0.1676170825958252 \n",
            "Fold 0/4, Epoch 6/19, Running Loss valid 5.095903396606445, Time 0.16365504264831543, Overall 0.16774868965148926 \n",
            "Fold 0/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 18.457120895385742, Time 0.18178224563598633, Overall 0.18587756156921387 \n",
            "Fold 0/4, Epoch 7/19, Running Loss train 0.9228560447692871, Time 0.1819312572479248, Overall 0.18602514266967773 \n",
            "Fold 0/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 31.59212303161621, Time 0.1870584487915039, Overall 0.19115328788757324 \n",
            "Fold 0/4, Epoch 7/19, Running Loss valid 5.265353838602702, Time 0.18720316886901855, Overall 0.1912975311279297 \n",
            "Fold 0/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 20.644256591796875, Time 0.20476245880126953, Overall 0.20885777473449707 \n",
            "Fold 0/4, Epoch 8/19, Running Loss train 1.0322128295898438, Time 0.2049083709716797, Overall 0.20900273323059082 \n",
            "Fold 0/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 30.48838996887207, Time 0.2098691463470459, Overall 0.21396374702453613 \n",
            "Fold 0/4, Epoch 8/19, Running Loss valid 5.081398328145345, Time 0.2100074291229248, Overall 0.21410131454467773 \n",
            "Fold 0/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 19.780677795410156, Time 0.22650909423828125, Overall 0.23060393333435059 \n",
            "Fold 0/4, Epoch 9/19, Running Loss train 0.9890338897705078, Time 0.22666144371032715, Overall 0.23075532913208008 \n",
            "Fold 0/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 30.420711517333984, Time 0.23180723190307617, Overall 0.2359018325805664 \n",
            "Fold 0/4, Epoch 9/19, Running Loss valid 5.070118586222331, Time 0.23195672035217285, Overall 0.23605060577392578 \n",
            "Fold 0/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 9.82799243927002, Time 0.24846291542053223, Overall 0.25255846977233887 \n",
            "Fold 0/4, Epoch 10/19, Running Loss train 0.49139962196350095, Time 0.24860620498657227, Overall 0.2527003288269043 \n",
            "Fold 0/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 40.203880310058594, Time 0.25382018089294434, Overall 0.25791501998901367 \n",
            "Fold 0/4, Epoch 10/19, Running Loss valid 6.700646718343099, Time 0.25398874282836914, Overall 0.25808262825012207 \n",
            "Fold 0/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 18.801475524902344, Time 0.27049922943115234, Overall 0.2745943069458008 \n",
            "Fold 0/4, Epoch 11/19, Running Loss train 0.9400737762451172, Time 0.27065277099609375, Overall 0.2747471332550049 \n",
            "Fold 0/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 30.595874786376953, Time 0.2756679058074951, Overall 0.27976250648498535 \n",
            "Fold 0/4, Epoch 11/19, Running Loss valid 5.099312464396159, Time 0.27583932876586914, Overall 0.27993297576904297 \n",
            "Epoch 00012: reducing learning rate of group 0 to 5.0000e-02.\n",
            "Fold 0/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 14.113418579101562, Time 0.29285597801208496, Overall 0.2969508171081543 \n",
            "Fold 0/4, Epoch 12/19, Running Loss train 0.7056709289550781, Time 0.2929971218109131, Overall 0.297091007232666 \n",
            "Fold 0/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 30.890506744384766, Time 0.29799747467041016, Overall 0.3020923137664795 \n",
            "Fold 0/4, Epoch 12/19, Running Loss valid 5.148417790730794, Time 0.29813551902770996, Overall 0.3022294044494629 \n",
            "Fold 0/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 18.77577018737793, Time 0.31416916847229004, Overall 0.3182647228240967 \n",
            "Fold 0/4, Epoch 13/19, Running Loss train 0.9387885093688965, Time 0.31431126594543457, Overall 0.3184051513671875 \n",
            "Fold 0/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 36.75393295288086, Time 0.31932592391967773, Overall 0.32342028617858887 \n",
            "Fold 0/4, Epoch 13/19, Running Loss valid 6.12565549214681, Time 0.31946563720703125, Overall 0.3235597610473633 \n",
            "Fold 0/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 11.035259246826172, Time 0.33812761306762695, Overall 0.3422226905822754 \n",
            "Fold 0/4, Epoch 14/19, Running Loss train 0.5517629623413086, Time 0.33826732635498047, Overall 0.3423612117767334 \n",
            "Fold 0/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 33.634498596191406, Time 0.34337902069091797, Overall 0.3474738597869873 \n",
            "Fold 0/4, Epoch 14/19, Running Loss valid 5.605749766031901, Time 0.34352660179138184, Overall 0.34762048721313477 \n",
            "Fold 0/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 21.083236694335938, Time 0.36053943634033203, Overall 0.36463475227355957 \n",
            "Fold 0/4, Epoch 15/19, Running Loss train 1.054161834716797, Time 0.36068272590637207, Overall 0.3647770881652832 \n",
            "Fold 0/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 38.38726806640625, Time 0.36572790145874023, Overall 0.36982297897338867 \n",
            "Fold 0/4, Epoch 15/19, Running Loss valid 6.397878011067708, Time 0.36585569381713867, Overall 0.3699495792388916 \n",
            "Fold 0/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 15.614261627197266, Time 0.3882107734680176, Overall 0.3923056125640869 \n",
            "Fold 0/4, Epoch 16/19, Running Loss train 0.7807130813598633, Time 0.3883535861968994, Overall 0.39244723320007324 \n",
            "Fold 0/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 40.47874450683594, Time 0.3925015926361084, Overall 0.39659595489501953 \n",
            "Fold 0/4, Epoch 16/19, Running Loss valid 6.746457417805989, Time 0.3926270008087158, Overall 0.39672064781188965 \n",
            "Fold 0/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 11.675141334533691, Time 0.40753650665283203, Overall 0.41163134574890137 \n",
            "Fold 0/4, Epoch 17/19, Running Loss train 0.5837570667266846, Time 0.4076530933380127, Overall 0.4117467403411865 \n",
            "Fold 0/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 34.45796203613281, Time 0.41234683990478516, Overall 0.4164414405822754 \n",
            "Fold 0/4, Epoch 17/19, Running Loss valid 5.742993672688802, Time 0.41248464584350586, Overall 0.4165785312652588 \n",
            "Fold 0/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 8.832579612731934, Time 0.42753005027770996, Overall 0.4316246509552002 \n",
            "Fold 0/4, Epoch 18/19, Running Loss train 0.44162898063659667, Time 0.4276423454284668, Overall 0.4317359924316406 \n",
            "Fold 0/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 34.035743713378906, Time 0.4321165084838867, Overall 0.43621110916137695 \n",
            "Fold 0/4, Epoch 18/19, Running Loss valid 5.672623952229817, Time 0.43224191665649414, Overall 0.43633604049682617 \n",
            "Fold 0/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 25.159082412719727, Time 0.44965028762817383, Overall 0.45374512672424316 \n",
            "Fold 0/4, Epoch 19/19, Running Loss train 1.2579541206359863, Time 0.44979000091552734, Overall 0.45388364791870117 \n",
            "Fold 0/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 33.77218246459961, Time 0.45450901985168457, Overall 0.4586038589477539 \n",
            "Fold 0/4, Epoch 19/19, Running Loss valid 5.6286970774332685, Time 0.45464515686035156, Overall 0.4587392807006836 \n",
            "Fold 1/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 62.89884948730469, Time 0.016277790069580078, Overall 0.47821640968322754 \n",
            "Fold 1/4, Epoch 0/19, Running Loss train 2.995183308919271, Time 0.016415834426879883, Overall 0.47835254669189453 \n",
            "Fold 1/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 11.704445838928223, Time 0.020780086517333984, Overall 0.48271846771240234 \n",
            "Fold 1/4, Epoch 0/19, Running Loss valid 2.3408891677856447, Time 0.020926237106323242, Overall 0.4828624725341797 \n",
            "Fold 1/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 51.396934509277344, Time 0.038411617279052734, Overall 0.5003492832183838 \n",
            "Fold 1/4, Epoch 1/19, Running Loss train 2.4474730718703497, Time 0.03855156898498535, Overall 0.50048828125 \n",
            "Fold 1/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 11.400910377502441, Time 0.04282975196838379, Overall 0.5047667026519775 \n",
            "Fold 1/4, Epoch 1/19, Running Loss valid 2.2801820755004885, Time 0.04297614097595215, Overall 0.5049128532409668 \n",
            "Fold 1/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 32.789268493652344, Time 0.05945301055908203, Overall 0.5213909149169922 \n",
            "Fold 1/4, Epoch 2/19, Running Loss train 1.5613937377929688, Time 0.05960512161254883, Overall 0.5215415954589844 \n",
            "Fold 1/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 9.632389068603516, Time 0.06375694274902344, Overall 0.5256936550140381 \n",
            "Fold 1/4, Epoch 2/19, Running Loss valid 1.926477813720703, Time 0.06389880180358887, Overall 0.5258347988128662 \n",
            "Fold 1/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 17.599075317382812, Time 0.08010554313659668, Overall 0.5420427322387695 \n",
            "Fold 1/4, Epoch 3/19, Running Loss train 0.8380512055896577, Time 0.08023929595947266, Overall 0.54217529296875 \n",
            "Fold 1/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 12.045702934265137, Time 0.08449029922485352, Overall 0.5464272499084473 \n",
            "Fold 1/4, Epoch 3/19, Running Loss valid 2.4091405868530273, Time 0.08463287353515625, Overall 0.5465700626373291 \n",
            "Fold 1/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 22.20928192138672, Time 0.10094690322875977, Overall 0.5628840923309326 \n",
            "Fold 1/4, Epoch 4/19, Running Loss train 1.0575848533993675, Time 0.10112333297729492, Overall 0.5630598068237305 \n",
            "Fold 1/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 6.162274360656738, Time 0.10517621040344238, Overall 0.5671133995056152 \n",
            "Fold 1/4, Epoch 4/19, Running Loss valid 1.2324548721313477, Time 0.10530638694763184, Overall 0.5672428607940674 \n",
            "Fold 1/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 31.390235900878906, Time 0.12347555160522461, Overall 0.5854134559631348 \n",
            "Fold 1/4, Epoch 5/19, Running Loss train 1.4947731381370908, Time 0.12363529205322266, Overall 0.5855722427368164 \n",
            "Fold 1/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 7.986185073852539, Time 0.12810611724853516, Overall 0.5900435447692871 \n",
            "Fold 1/4, Epoch 5/19, Running Loss valid 1.5972370147705077, Time 0.1282353401184082, Overall 0.5901718139648438 \n",
            "Fold 1/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 24.28365707397461, Time 0.14486980438232422, Overall 0.6068069934844971 \n",
            "Fold 1/4, Epoch 6/19, Running Loss train 1.1563646225702195, Time 0.14500021934509277, Overall 0.6069364547729492 \n",
            "Fold 1/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 14.989298820495605, Time 0.1491556167602539, Overall 0.6110928058624268 \n",
            "Fold 1/4, Epoch 6/19, Running Loss valid 2.997859764099121, Time 0.1492927074432373, Overall 0.6112287044525146 \n",
            "Fold 1/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 25.836502075195312, Time 0.16577529907226562, Overall 0.6277129650115967 \n",
            "Fold 1/4, Epoch 7/19, Running Loss train 1.2303096226283483, Time 0.1659250259399414, Overall 0.627861499786377 \n",
            "Fold 1/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 12.855713844299316, Time 0.17022371292114258, Overall 0.6321604251861572 \n",
            "Fold 1/4, Epoch 7/19, Running Loss valid 2.5711427688598634, Time 0.17035889625549316, Overall 0.6322956085205078 \n",
            "Fold 1/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 19.26841926574707, Time 0.1869966983795166, Overall 0.6489334106445312 \n",
            "Fold 1/4, Epoch 8/19, Running Loss train 0.9175437745593843, Time 0.18712329864501953, Overall 0.6490600109100342 \n",
            "Fold 1/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 21.173856735229492, Time 0.1912069320678711, Overall 0.6531438827514648 \n",
            "Fold 1/4, Epoch 8/19, Running Loss valid 4.234771347045898, Time 0.19133710861206055, Overall 0.6532738208770752 \n",
            "Fold 1/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 15.552143096923828, Time 0.2072892189025879, Overall 0.6692264080047607 \n",
            "Fold 1/4, Epoch 9/19, Running Loss train 0.7405782427106585, Time 0.2074108123779297, Overall 0.6693470478057861 \n",
            "Fold 1/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 20.14832305908203, Time 0.21149563789367676, Overall 0.6734328269958496 \n",
            "Fold 1/4, Epoch 9/19, Running Loss valid 4.029664611816406, Time 0.21165871620178223, Overall 0.6735951900482178 \n",
            "Fold 1/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 25.021390914916992, Time 0.23045849800109863, Overall 0.6923961639404297 \n",
            "Fold 1/4, Epoch 10/19, Running Loss train 1.1914948054722376, Time 0.23059463500976562, Overall 0.6925323009490967 \n",
            "Fold 1/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 23.885204315185547, Time 0.23475885391235352, Overall 0.6966953277587891 \n",
            "Fold 1/4, Epoch 10/19, Running Loss valid 4.77704086303711, Time 0.23490357398986816, Overall 0.6968402862548828 \n",
            "Fold 1/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 25.38947105407715, Time 0.25092196464538574, Overall 0.7128586769104004 \n",
            "Fold 1/4, Epoch 11/19, Running Loss train 1.209022431146531, Time 0.25103163719177246, Overall 0.712968111038208 \n",
            "Fold 1/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 26.60730743408203, Time 0.2550077438354492, Overall 0.7169449329376221 \n",
            "Fold 1/4, Epoch 11/19, Running Loss valid 5.321461486816406, Time 0.2551305294036865, Overall 0.7170665264129639 \n",
            "Fold 1/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 16.845849990844727, Time 0.2710392475128174, Overall 0.7329761981964111 \n",
            "Fold 1/4, Epoch 12/19, Running Loss train 0.802183332897368, Time 0.27115821838378906, Overall 0.7330946922302246 \n",
            "Fold 1/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 29.94708824157715, Time 0.2751736640930176, Overall 0.7371103763580322 \n",
            "Fold 1/4, Epoch 12/19, Running Loss valid 5.989417648315429, Time 0.27530360221862793, Overall 0.7372403144836426 \n",
            "Fold 1/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 20.71160316467285, Time 0.2918431758880615, Overall 0.7537803649902344 \n",
            "Fold 1/4, Epoch 13/19, Running Loss train 0.9862668173653739, Time 0.2919652462005615, Overall 0.7539012432098389 \n",
            "Fold 1/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 31.966880798339844, Time 0.29608631134033203, Overall 0.7580235004425049 \n",
            "Fold 1/4, Epoch 13/19, Running Loss valid 6.393376159667969, Time 0.2962174415588379, Overall 0.7581536769866943 \n",
            "Fold 1/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 16.83731460571289, Time 0.3134610652923584, Overall 0.7753984928131104 \n",
            "Fold 1/4, Epoch 14/19, Running Loss train 0.8017768859863281, Time 0.31357860565185547, Overall 0.7755155563354492 \n",
            "Fold 1/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 29.739187240600586, Time 0.31763482093811035, Overall 0.7795712947845459 \n",
            "Fold 1/4, Epoch 14/19, Running Loss valid 5.947837448120117, Time 0.31775355339050293, Overall 0.7796897888183594 \n",
            "Fold 1/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 17.817535400390625, Time 0.33371496200561523, Overall 0.7956521511077881 \n",
            "Fold 1/4, Epoch 15/19, Running Loss train 0.8484540666852679, Time 0.33385300636291504, Overall 0.7957897186279297 \n",
            "Fold 1/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 30.439796447753906, Time 0.3379175662994385, Overall 0.7998545169830322 \n",
            "Fold 1/4, Epoch 15/19, Running Loss valid 6.087959289550781, Time 0.33803796768188477, Overall 0.7999744415283203 \n",
            "Epoch 00016: reducing learning rate of group 0 to 5.0000e-02.\n",
            "Fold 1/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 19.479488372802734, Time 0.35690951347351074, Overall 0.8188467025756836 \n",
            "Fold 1/4, Epoch 16/19, Running Loss train 0.9275946844191778, Time 0.3570406436920166, Overall 0.818976640701294 \n",
            "Fold 1/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 30.08045768737793, Time 0.36094117164611816, Overall 0.822878360748291 \n",
            "Fold 1/4, Epoch 16/19, Running Loss valid 6.016091537475586, Time 0.36112022399902344, Overall 0.823056697845459 \n",
            "Fold 1/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 23.848888397216797, Time 0.3771970272064209, Overall 0.8391339778900146 \n",
            "Fold 1/4, Epoch 17/19, Running Loss train 1.135661352248419, Time 0.3773157596588135, Overall 0.8392517566680908 \n",
            "Fold 1/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 28.601160049438477, Time 0.38142943382263184, Overall 0.8433666229248047 \n",
            "Fold 1/4, Epoch 17/19, Running Loss valid 5.720232009887695, Time 0.3815591335296631, Overall 0.8434958457946777 \n",
            "Fold 1/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 10.932768821716309, Time 0.39749693870544434, Overall 0.8594341278076172 \n",
            "Fold 1/4, Epoch 18/19, Running Loss train 0.520608039129348, Time 0.3976128101348877, Overall 0.859548807144165 \n",
            "Fold 1/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 32.29781723022461, Time 0.4017298221588135, Overall 0.8636670112609863 \n",
            "Fold 1/4, Epoch 18/19, Running Loss valid 6.459563446044922, Time 0.4018821716308594, Overall 0.863818883895874 \n",
            "Fold 1/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 6.717739582061768, Time 0.41903114318847656, Overall 0.8809688091278076 \n",
            "Fold 1/4, Epoch 19/19, Running Loss train 0.31989236105056035, Time 0.4191617965698242, Overall 0.8810980319976807 \n",
            "Fold 1/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 26.951889038085938, Time 0.423537015914917, Overall 0.885474443435669 \n",
            "Fold 1/4, Epoch 19/19, Running Loss valid 5.390377807617187, Time 0.42368197441101074, Overall 0.8856182098388672 \n",
            "Fold 2/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 67.0121841430664, Time 0.014915227890014648, Overall 0.9036264419555664 \n",
            "Fold 2/4, Epoch 0/19, Running Loss train 3.191056387765067, Time 0.015042543411254883, Overall 0.9037532806396484 \n",
            "Fold 2/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 12.900025367736816, Time 0.03729081153869629, Overall 0.9260015487670898 \n",
            "Fold 2/4, Epoch 0/19, Running Loss valid 2.580005073547363, Time 0.03744387626647949, Overall 0.9261534214019775 \n",
            "Fold 2/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 64.3685302734375, Time 0.05318570137023926, Overall 0.9418954849243164 \n",
            "Fold 2/4, Epoch 1/19, Running Loss train 3.0651681082589284, Time 0.053298234939575195, Overall 0.9420080184936523 \n",
            "Fold 2/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 6.960123538970947, Time 0.05734539031982422, Overall 0.9460551738739014 \n",
            "Fold 2/4, Epoch 1/19, Running Loss valid 1.3920247077941894, Time 0.057463884353637695, Overall 0.9461736679077148 \n",
            "Fold 2/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 42.58934020996094, Time 0.07383513450622559, Overall 0.9625458717346191 \n",
            "Fold 2/4, Epoch 2/19, Running Loss train 2.0280638195219494, Time 0.07398104667663574, Overall 0.9626905918121338 \n",
            "Fold 2/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 6.630851745605469, Time 0.07833456993103027, Overall 0.9670450687408447 \n",
            "Fold 2/4, Epoch 2/19, Running Loss valid 1.3261703491210937, Time 0.07846522331237793, Overall 0.967174768447876 \n",
            "Fold 2/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 32.77325439453125, Time 0.1012105941772461, Overall 0.9899213314056396 \n",
            "Fold 2/4, Epoch 3/19, Running Loss train 1.5606311616443453, Time 0.10134100914001465, Overall 0.9900505542755127 \n",
            "Fold 2/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 3.7333309650421143, Time 0.1056528091430664, Overall 0.99436354637146 \n",
            "Fold 2/4, Epoch 3/19, Running Loss valid 0.7466661930084229, Time 0.10579347610473633, Overall 0.9945027828216553 \n",
            "Fold 2/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 31.483177185058594, Time 0.12230229377746582, Overall 1.0110125541687012 \n",
            "Fold 2/4, Epoch 4/19, Running Loss train 1.4991989135742188, Time 0.1224508285522461, Overall 1.0111749172210693 \n",
            "Fold 2/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 3.6342058181762695, Time 0.1266183853149414, Overall 1.015329122543335 \n",
            "Fold 2/4, Epoch 4/19, Running Loss valid 0.7268411636352539, Time 0.12674999237060547, Overall 1.0154602527618408 \n",
            "Fold 2/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 18.197547912597656, Time 0.14301085472106934, Overall 1.031721591949463 \n",
            "Fold 2/4, Epoch 5/19, Running Loss train 0.8665499005998883, Time 0.1431596279144287, Overall 1.031869649887085 \n",
            "Fold 2/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 3.428286075592041, Time 0.14743566513061523, Overall 1.036146879196167 \n",
            "Fold 2/4, Epoch 5/19, Running Loss valid 0.6856572151184082, Time 0.14757657051086426, Overall 1.0362863540649414 \n",
            "Fold 2/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 14.279206275939941, Time 0.1639862060546875, Overall 1.052696704864502 \n",
            "Fold 2/4, Epoch 6/19, Running Loss train 0.6799622036161876, Time 0.1641836166381836, Overall 1.0528936386108398 \n",
            "Fold 2/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 7.982638835906982, Time 0.168548583984375, Overall 1.0572590827941895 \n",
            "Fold 2/4, Epoch 6/19, Running Loss valid 1.5965277671813964, Time 0.1687147617340088, Overall 1.0574250221252441 \n",
            "Fold 2/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 16.575489044189453, Time 0.18333220481872559, Overall 1.0720431804656982 \n",
            "Fold 2/4, Epoch 7/19, Running Loss train 0.7893090021042597, Time 0.1835014820098877, Overall 1.072211503982544 \n",
            "Fold 2/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 15.821167945861816, Time 0.18897032737731934, Overall 1.0776808261871338 \n",
            "Fold 2/4, Epoch 7/19, Running Loss valid 3.164233589172363, Time 0.18915390968322754, Overall 1.077864170074463 \n",
            "Fold 2/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 35.862083435058594, Time 0.20563340187072754, Overall 1.094344139099121 \n",
            "Fold 2/4, Epoch 8/19, Running Loss train 1.707718258812314, Time 0.20576238632202148, Overall 1.0944724082946777 \n",
            "Fold 2/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 18.033451080322266, Time 0.21144318580627441, Overall 1.1001536846160889 \n",
            "Fold 2/4, Epoch 8/19, Running Loss valid 3.606690216064453, Time 0.21158099174499512, Overall 1.1002907752990723 \n",
            "Fold 2/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 36.48210525512695, Time 0.22792482376098633, Overall 1.116635799407959 \n",
            "Fold 2/4, Epoch 9/19, Running Loss train 1.7372431073869978, Time 0.22808098793029785, Overall 1.1167912483215332 \n",
            "Fold 2/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 20.3493709564209, Time 0.2322688102722168, Overall 1.1209790706634521 \n",
            "Fold 2/4, Epoch 9/19, Running Loss valid 4.06987419128418, Time 0.23239874839782715, Overall 1.1211082935333252 \n",
            "Fold 2/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 34.15285110473633, Time 0.24904274940490723, Overall 1.1377534866333008 \n",
            "Fold 2/4, Epoch 10/19, Running Loss train 1.6263262430826824, Time 0.2491910457611084, Overall 1.1379013061523438 \n",
            "Fold 2/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 2.8085319995880127, Time 0.2535843849182129, Overall 1.1422948837280273 \n",
            "Fold 2/4, Epoch 10/19, Running Loss valid 0.5617063999176025, Time 0.253711462020874, Overall 1.142420768737793 \n",
            "Fold 2/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 19.24724769592285, Time 0.27016305923461914, Overall 1.1588735580444336 \n",
            "Fold 2/4, Epoch 11/19, Running Loss train 0.9165356045677548, Time 0.27028536796569824, Overall 1.1589953899383545 \n",
            "Fold 2/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 5.2179460525512695, Time 0.27488279342651367, Overall 1.1635932922363281 \n",
            "Fold 2/4, Epoch 11/19, Running Loss valid 1.043589210510254, Time 0.2750129699707031, Overall 1.1637229919433594 \n",
            "Fold 2/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 33.7504768371582, Time 0.2919468879699707, Overall 1.1806576251983643 \n",
            "Fold 2/4, Epoch 12/19, Running Loss train 1.6071655636742002, Time 0.2920875549316406, Overall 1.1807973384857178 \n",
            "Fold 2/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 6.168686389923096, Time 0.29639101028442383, Overall 1.1851017475128174 \n",
            "Fold 2/4, Epoch 12/19, Running Loss valid 1.2337372779846192, Time 0.29653310775756836, Overall 1.1852433681488037 \n",
            "Fold 2/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 23.207921981811523, Time 0.3125908374786377, Overall 1.2013013362884521 \n",
            "Fold 2/4, Epoch 13/19, Running Loss train 1.105139141991025, Time 0.31270408630371094, Overall 1.201413869857788 \n",
            "Fold 2/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 8.781145095825195, Time 0.31696128845214844, Overall 1.205672025680542 \n",
            "Fold 2/4, Epoch 13/19, Running Loss valid 1.7562290191650392, Time 0.3171060085296631, Overall 1.2058155536651611 \n",
            "Fold 2/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 18.25078773498535, Time 0.3329293727874756, Overall 1.221639633178711 \n",
            "Fold 2/4, Epoch 14/19, Running Loss train 0.8690851302373976, Time 0.33304667472839355, Overall 1.2217564582824707 \n",
            "Fold 2/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 2.2615103721618652, Time 0.33719611167907715, Overall 1.2259066104888916 \n",
            "Fold 2/4, Epoch 14/19, Running Loss valid 0.452302074432373, Time 0.33732008934020996, Overall 1.226029634475708 \n",
            "Fold 2/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 28.77548599243164, Time 0.3533594608306885, Overall 1.242070198059082 \n",
            "Fold 2/4, Epoch 15/19, Running Loss train 1.37026123773484, Time 0.3535149097442627, Overall 1.2422246932983398 \n",
            "Fold 2/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 8.8562593460083, Time 0.35761475563049316, Overall 1.2463252544403076 \n",
            "Fold 2/4, Epoch 15/19, Running Loss valid 1.7712518692016601, Time 0.3577454090118408, Overall 1.246455192565918 \n",
            "Fold 2/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 27.104217529296875, Time 0.373584508895874, Overall 1.262294054031372 \n",
            "Fold 2/4, Epoch 16/19, Running Loss train 1.2906770252046131, Time 0.3737001419067383, Overall 1.2624101638793945 \n",
            "Fold 2/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 5.100404739379883, Time 0.3778219223022461, Overall 1.2665324211120605 \n",
            "Fold 2/4, Epoch 16/19, Running Loss valid 1.0200809478759765, Time 0.3779566287994385, Overall 1.2666661739349365 \n",
            "Fold 2/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 32.517494201660156, Time 0.3985867500305176, Overall 1.2872974872589111 \n",
            "Fold 2/4, Epoch 17/19, Running Loss train 1.5484521048409599, Time 0.39879941940307617, Overall 1.2875096797943115 \n",
            "Fold 2/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 6.071877479553223, Time 0.40229082107543945, Overall 1.291001319885254 \n",
            "Fold 2/4, Epoch 17/19, Running Loss valid 1.2143754959106445, Time 0.40243005752563477, Overall 1.291139841079712 \n",
            "Fold 2/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 22.86707305908203, Time 0.4178905487060547, Overall 1.30660080909729 \n",
            "Fold 2/4, Epoch 18/19, Running Loss train 1.0889082409086681, Time 0.4180030822753906, Overall 1.3067123889923096 \n",
            "Fold 2/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 6.941075325012207, Time 0.42203664779663086, Overall 1.310746431350708 \n",
            "Fold 2/4, Epoch 18/19, Running Loss valid 1.3882150650024414, Time 0.422177791595459, Overall 1.3108878135681152 \n",
            "Fold 2/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 19.33847427368164, Time 0.43777894973754883, Overall 1.3264896869659424 \n",
            "Fold 2/4, Epoch 19/19, Running Loss train 0.9208797273181734, Time 0.43790459632873535, Overall 1.3266141414642334 \n",
            "Fold 2/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 1.7035934925079346, Time 0.44190025329589844, Overall 1.330610752105713 \n",
            "Fold 2/4, Epoch 19/19, Running Loss valid 0.3407186985015869, Time 0.442018985748291, Overall 1.3307280540466309 \n",
            "Fold 3/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 67.87638854980469, Time 0.01731729507446289, Overall 1.350951910018921 \n",
            "Fold 3/4, Epoch 0/19, Running Loss train 3.232208978562128, Time 0.01743793487548828, Overall 1.351072072982788 \n",
            "Fold 3/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 20.320114135742188, Time 0.02141857147216797, Overall 1.3550527095794678 \n",
            "Fold 3/4, Epoch 0/19, Running Loss valid 4.064022827148437, Time 0.021543025970458984, Overall 1.3551771640777588 \n",
            "Fold 3/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 50.53109359741211, Time 0.039156436920166016, Overall 1.3727912902832031 \n",
            "Fold 3/4, Epoch 1/19, Running Loss train 2.4062425522577193, Time 0.040236473083496094, Overall 1.373871088027954 \n",
            "Fold 3/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 15.164619445800781, Time 0.045419931411743164, Overall 1.3790538311004639 \n",
            "Fold 3/4, Epoch 1/19, Running Loss valid 3.0329238891601564, Time 0.04554128646850586, Overall 1.3791749477386475 \n",
            "Fold 3/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 34.52818298339844, Time 0.06157183647155762, Overall 1.3952064514160156 \n",
            "Fold 3/4, Epoch 2/19, Running Loss train 1.6441991896856398, Time 0.06169772148132324, Overall 1.3953313827514648 \n",
            "Fold 3/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 16.593807220458984, Time 0.06580018997192383, Overall 1.3994345664978027 \n",
            "Fold 3/4, Epoch 2/19, Running Loss valid 3.318761444091797, Time 0.06593036651611328, Overall 1.399564266204834 \n",
            "Fold 3/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 28.82463836669922, Time 0.0812523365020752, Overall 1.414886236190796 \n",
            "Fold 3/4, Epoch 3/19, Running Loss train 1.372601826985677, Time 0.0813591480255127, Overall 1.4149928092956543 \n",
            "Fold 3/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 21.980443954467773, Time 0.08523917198181152, Overall 1.4188728332519531 \n",
            "Fold 3/4, Epoch 3/19, Running Loss valid 4.396088790893555, Time 0.08534812927246094, Overall 1.4189817905426025 \n",
            "Fold 3/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 26.640588760375977, Time 0.10136723518371582, Overall 1.4350011348724365 \n",
            "Fold 3/4, Epoch 4/19, Running Loss train 1.2685994647798085, Time 0.10148167610168457, Overall 1.435114860534668 \n",
            "Fold 3/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 23.140783309936523, Time 0.10553979873657227, Overall 1.4391741752624512 \n",
            "Fold 3/4, Epoch 4/19, Running Loss valid 4.628156661987305, Time 0.10566306114196777, Overall 1.4392971992492676 \n",
            "Fold 3/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 12.317014694213867, Time 0.1213693618774414, Overall 1.4550034999847412 \n",
            "Fold 3/4, Epoch 5/19, Running Loss train 0.5865245092482794, Time 0.1214745044708252, Overall 1.4551076889038086 \n",
            "Fold 3/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 41.924041748046875, Time 0.12526321411132812, Overall 1.458897352218628 \n",
            "Fold 3/4, Epoch 5/19, Running Loss valid 8.384808349609376, Time 0.12537074089050293, Overall 1.4590041637420654 \n",
            "Fold 3/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 20.624347686767578, Time 0.14078545570373535, Overall 1.474419355392456 \n",
            "Fold 3/4, Epoch 6/19, Running Loss train 0.9821117946079799, Time 0.14090347290039062, Overall 1.4745373725891113 \n",
            "Fold 3/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 33.35353088378906, Time 0.1466841697692871, Overall 1.4803187847137451 \n",
            "Fold 3/4, Epoch 6/19, Running Loss valid 6.670706176757813, Time 0.14926505088806152, Overall 1.4828994274139404 \n",
            "Fold 3/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 19.836772918701172, Time 0.16666913032531738, Overall 1.5003035068511963 \n",
            "Fold 3/4, Epoch 7/19, Running Loss train 0.9446082342238653, Time 0.1667804718017578, Overall 1.5004143714904785 \n",
            "Fold 3/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 51.578956604003906, Time 0.1706552505493164, Overall 1.5042893886566162 \n",
            "Fold 3/4, Epoch 7/19, Running Loss valid 10.31579132080078, Time 0.17077970504760742, Overall 1.5044136047363281 \n",
            "Fold 3/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 14.64824390411377, Time 0.1875319480895996, Overall 1.5211665630340576 \n",
            "Fold 3/4, Epoch 8/19, Running Loss train 0.6975354240054176, Time 0.18764591217041016, Overall 1.5212795734405518 \n",
            "Fold 3/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 38.29242706298828, Time 0.19189000129699707, Overall 1.5255248546600342 \n",
            "Fold 3/4, Epoch 8/19, Running Loss valid 7.658485412597656, Time 0.19202136993408203, Overall 1.525655746459961 \n",
            "Fold 3/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 16.29279899597168, Time 0.20899415016174316, Overall 1.5426294803619385 \n",
            "Fold 3/4, Epoch 9/19, Running Loss train 0.7758475712367466, Time 0.2091214656829834, Overall 1.542754888534546 \n",
            "Fold 3/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 47.07456588745117, Time 0.2136077880859375, Overall 1.5472428798675537 \n",
            "Fold 3/4, Epoch 9/19, Running Loss valid 9.414913177490234, Time 0.21374297142028809, Overall 1.547377586364746 \n",
            "Fold 3/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 14.519548416137695, Time 0.2299790382385254, Overall 1.5636138916015625 \n",
            "Fold 3/4, Epoch 10/19, Running Loss train 0.6914070674351284, Time 0.23010945320129395, Overall 1.5637433528900146 \n",
            "Fold 3/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 64.30027770996094, Time 0.23445463180541992, Overall 1.568089246749878 \n",
            "Fold 3/4, Epoch 10/19, Running Loss valid 12.860055541992187, Time 0.23458552360534668, Overall 1.5682199001312256 \n",
            "Fold 3/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 11.734477996826172, Time 0.2557106018066406, Overall 1.589346170425415 \n",
            "Fold 3/4, Epoch 11/19, Running Loss train 0.558784666515532, Time 0.2558400630950928, Overall 1.5894737243652344 \n",
            "Fold 3/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 72.63427734375, Time 0.25998735427856445, Overall 1.5936214923858643 \n",
            "Fold 3/4, Epoch 11/19, Running Loss valid 14.52685546875, Time 0.26013708114624023, Overall 1.593770980834961 \n",
            "Fold 3/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 16.287330627441406, Time 0.2758619785308838, Overall 1.6094963550567627 \n",
            "Fold 3/4, Epoch 12/19, Running Loss train 0.7755871727353051, Time 0.27597737312316895, Overall 1.6096115112304688 \n",
            "Fold 3/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 76.4717788696289, Time 0.2799875736236572, Overall 1.6136219501495361 \n",
            "Fold 3/4, Epoch 12/19, Running Loss valid 15.294355773925782, Time 0.28011202812194824, Overall 1.613745927810669 \n",
            "Epoch 00013: reducing learning rate of group 0 to 5.0000e-02.\n",
            "Fold 3/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 8.214170455932617, Time 0.2959256172180176, Overall 1.6295602321624756 \n",
            "Fold 3/4, Epoch 13/19, Running Loss train 0.39115097409202937, Time 0.2960360050201416, Overall 1.629669189453125 \n",
            "Fold 3/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 76.00479888916016, Time 0.2998223304748535, Overall 1.6334567070007324 \n",
            "Fold 3/4, Epoch 13/19, Running Loss valid 15.200959777832031, Time 0.29995179176330566, Overall 1.6335852146148682 \n",
            "Fold 3/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 19.75815200805664, Time 0.31542158126831055, Overall 1.6490559577941895 \n",
            "Fold 3/4, Epoch 14/19, Running Loss train 0.9408643813360305, Time 0.31552743911743164, Overall 1.6491611003875732 \n",
            "Fold 3/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 79.45247650146484, Time 0.319227933883667, Overall 1.6528618335723877 \n",
            "Fold 3/4, Epoch 14/19, Running Loss valid 15.890495300292969, Time 0.3193626403808594, Overall 1.652996301651001 \n",
            "Fold 3/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 4.506253242492676, Time 0.33455967903137207, Overall 1.6681947708129883 \n",
            "Fold 3/4, Epoch 15/19, Running Loss train 0.21458348773774646, Time 0.33467769622802734, Overall 1.668311595916748 \n",
            "Fold 3/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 99.70198059082031, Time 0.33830809593200684, Overall 1.6719419956207275 \n",
            "Fold 3/4, Epoch 15/19, Running Loss valid 19.940396118164063, Time 0.3384280204772949, Overall 1.6721596717834473 \n",
            "Fold 3/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 8.886896133422852, Time 0.3544917106628418, Overall 1.6881263256072998 \n",
            "Fold 3/4, Epoch 16/19, Running Loss train 0.42318553016299293, Time 0.35460948944091797, Overall 1.6882431507110596 \n",
            "Fold 3/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 114.34217834472656, Time 0.35840725898742676, Overall 1.6920416355133057 \n",
            "Fold 3/4, Epoch 16/19, Running Loss valid 22.868435668945313, Time 0.35851573944091797, Overall 1.6921494007110596 \n",
            "Fold 3/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 8.537256240844727, Time 0.373307466506958, Overall 1.706941843032837 \n",
            "Fold 3/4, Epoch 17/19, Running Loss train 0.4065360114687965, Time 0.37341856956481934, Overall 1.707052230834961 \n",
            "Fold 3/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 101.61919403076172, Time 0.3772909641265869, Overall 1.7109248638153076 \n",
            "Fold 3/4, Epoch 17/19, Running Loss valid 20.323838806152345, Time 0.37741589546203613, Overall 1.7110493183135986 \n",
            "Fold 3/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 5.183207988739014, Time 0.39163899421691895, Overall 1.7252733707427979 \n",
            "Fold 3/4, Epoch 18/19, Running Loss train 0.24681942803519114, Time 0.39174437522888184, Overall 1.7253780364990234 \n",
            "Fold 3/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 81.25391387939453, Time 0.3955256938934326, Overall 1.7291595935821533 \n",
            "Fold 3/4, Epoch 18/19, Running Loss valid 16.250782775878907, Time 0.39563894271850586, Overall 1.7292726039886475 \n",
            "Fold 3/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 16.77395248413086, Time 0.4103424549102783, Overall 1.7439773082733154 \n",
            "Fold 3/4, Epoch 19/19, Running Loss train 0.7987596421014695, Time 0.4104642868041992, Overall 1.74409818649292 \n",
            "Fold 3/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 100.36331939697266, Time 0.4150044918060303, Overall 1.7486388683319092 \n",
            "Fold 3/4, Epoch 19/19, Running Loss valid 20.072663879394533, Time 0.4151465892791748, Overall 1.7487976551055908 \n",
            "Fold 4/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 65.69478607177734, Time 0.014950037002563477, Overall 1.7666981220245361 \n",
            "Fold 4/4, Epoch 0/19, Running Loss train 3.1283231462751115, Time 0.015103340148925781, Overall 1.7668497562408447 \n",
            "Fold 4/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 17.102703094482422, Time 0.01941537857055664, Overall 1.7711622714996338 \n",
            "Fold 4/4, Epoch 0/19, Running Loss valid 3.420540618896484, Time 0.019554615020751953, Overall 1.77130126953125 \n",
            "Fold 4/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 50.872047424316406, Time 0.0369720458984375, Overall 1.7887194156646729 \n",
            "Fold 4/4, Epoch 1/19, Running Loss train 2.422478448776972, Time 0.037138938903808594, Overall 1.7888853549957275 \n",
            "Fold 4/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 13.65554141998291, Time 0.041292428970336914, Overall 1.793039321899414 \n",
            "Fold 4/4, Epoch 1/19, Running Loss valid 2.731108283996582, Time 0.04150247573852539, Overall 1.7932491302490234 \n",
            "Fold 4/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 38.99824523925781, Time 0.05672740936279297, Overall 1.8084747791290283 \n",
            "Fold 4/4, Epoch 2/19, Running Loss train 1.8570592971075148, Time 0.0568547248840332, Overall 1.808600902557373 \n",
            "Fold 4/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 13.229655265808105, Time 0.060800790786743164, Overall 1.8125474452972412 \n",
            "Fold 4/4, Epoch 2/19, Running Loss valid 2.645931053161621, Time 0.06093764305114746, Overall 1.812683343887329 \n",
            "Fold 4/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 34.05793762207031, Time 0.07741904258728027, Overall 1.8291661739349365 \n",
            "Fold 4/4, Epoch 3/19, Running Loss train 1.6218065534319197, Time 0.07755303382873535, Overall 1.829298973083496 \n",
            "Fold 4/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 16.238861083984375, Time 0.08171629905700684, Overall 1.8334624767303467 \n",
            "Fold 4/4, Epoch 3/19, Running Loss valid 3.247772216796875, Time 0.0818338394165039, Overall 1.8335800170898438 \n",
            "Fold 4/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 25.692148208618164, Time 0.09806418418884277, Overall 1.849811315536499 \n",
            "Fold 4/4, Epoch 4/19, Running Loss train 1.2234356289818173, Time 0.09819698333740234, Overall 1.8499433994293213 \n",
            "Fold 4/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 20.763479232788086, Time 0.10240030288696289, Overall 1.854146957397461 \n",
            "Fold 4/4, Epoch 4/19, Running Loss valid 4.152695846557617, Time 0.1025245189666748, Overall 1.8542704582214355 \n",
            "Fold 4/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 24.96883201599121, Time 0.11858916282653809, Overall 1.8703360557556152 \n",
            "Fold 4/4, Epoch 5/19, Running Loss train 1.1889920007614863, Time 0.11872339248657227, Overall 1.8704700469970703 \n",
            "Fold 4/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 27.65744972229004, Time 0.12295389175415039, Overall 1.8747007846832275 \n",
            "Fold 4/4, Epoch 5/19, Running Loss valid 5.5314899444580075, Time 0.12310099601745605, Overall 1.8748481273651123 \n",
            "Fold 4/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 39.857261657714844, Time 0.1443166732788086, Overall 1.8960635662078857 \n",
            "Fold 4/4, Epoch 6/19, Running Loss train 1.897964840843564, Time 0.1444706916809082, Overall 1.896216869354248 \n",
            "Fold 4/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 31.511104583740234, Time 0.1486344337463379, Overall 1.900381088256836 \n",
            "Fold 4/4, Epoch 6/19, Running Loss valid 6.3022209167480465, Time 0.14896559715270996, Overall 1.900712013244629 \n",
            "Fold 4/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 25.28790283203125, Time 0.16809439659118652, Overall 1.9198410511016846 \n",
            "Fold 4/4, Epoch 7/19, Running Loss train 1.2041858491443453, Time 0.1682736873626709, Overall 1.9200201034545898 \n",
            "Fold 4/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 29.65654945373535, Time 0.17253661155700684, Overall 1.924283742904663 \n",
            "Fold 4/4, Epoch 7/19, Running Loss valid 5.9313098907470705, Time 0.17286014556884766, Overall 1.924607276916504 \n",
            "Fold 4/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 28.05084228515625, Time 0.18799805641174316, Overall 1.9397451877593994 \n",
            "Fold 4/4, Epoch 8/19, Running Loss train 1.33575439453125, Time 0.18814659118652344, Overall 1.939892292022705 \n",
            "Fold 4/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 35.61595153808594, Time 0.19242215156555176, Overall 1.9441814422607422 \n",
            "Fold 4/4, Epoch 8/19, Running Loss valid 7.123190307617188, Time 0.19256377220153809, Overall 1.944309949874878 \n",
            "Fold 4/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 32.6871452331543, Time 0.20849204063415527, Overall 1.9602386951446533 \n",
            "Fold 4/4, Epoch 9/19, Running Loss train 1.5565307253882998, Time 0.20861506462097168, Overall 1.9603610038757324 \n",
            "Fold 4/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 38.44313049316406, Time 0.21268033981323242, Overall 1.9644269943237305 \n",
            "Fold 4/4, Epoch 9/19, Running Loss valid 7.688626098632812, Time 0.21280694007873535, Overall 1.9645533561706543 \n",
            "Fold 4/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 20.750194549560547, Time 0.22958874702453613, Overall 1.9813358783721924 \n",
            "Fold 4/4, Epoch 10/19, Running Loss train 0.988104502360026, Time 0.22970890998840332, Overall 1.9814550876617432 \n",
            "Fold 4/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 40.615089416503906, Time 0.23374104499816895, Overall 1.985487937927246 \n",
            "Fold 4/4, Epoch 10/19, Running Loss valid 8.123017883300781, Time 0.23386669158935547, Overall 1.9856131076812744 \n",
            "Fold 4/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 21.579883575439453, Time 0.24947762489318848, Overall 2.0012245178222656 \n",
            "Fold 4/4, Epoch 11/19, Running Loss train 1.0276135035923548, Time 0.24959301948547363, Overall 2.0013391971588135 \n",
            "Fold 4/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 42.39402770996094, Time 0.25369691848754883, Overall 2.005443811416626 \n",
            "Fold 4/4, Epoch 11/19, Running Loss valid 8.478805541992188, Time 0.2538273334503174, Overall 2.0055735111236572 \n",
            "Fold 4/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 31.401512145996094, Time 0.26978445053100586, Overall 2.021531343460083 \n",
            "Fold 4/4, Epoch 12/19, Running Loss train 1.4953101021902901, Time 0.2699098587036133, Overall 2.0216565132141113 \n",
            "Fold 4/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 41.34917068481445, Time 0.2741851806640625, Overall 2.0259320735931396 \n",
            "Fold 4/4, Epoch 12/19, Running Loss valid 8.26983413696289, Time 0.27434372901916504, Overall 2.026090383529663 \n",
            "Fold 4/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 25.72704315185547, Time 0.2897815704345703, Overall 2.0415284633636475 \n",
            "Fold 4/4, Epoch 13/19, Running Loss train 1.2250972929454984, Time 0.2899007797241211, Overall 2.041646957397461 \n",
            "Fold 4/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 42.34473419189453, Time 0.29387688636779785, Overall 2.045623540878296 \n",
            "Fold 4/4, Epoch 13/19, Running Loss valid 8.468946838378907, Time 0.2940044403076172, Overall 2.045750617980957 \n",
            "Epoch 00014: reducing learning rate of group 0 to 5.0000e-02.\n",
            "Fold 4/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 16.366348266601562, Time 0.3099853992462158, Overall 2.061732530593872 \n",
            "Fold 4/4, Epoch 14/19, Running Loss train 0.7793499174572173, Time 0.3101320266723633, Overall 2.0618791580200195 \n",
            "Fold 4/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 41.790992736816406, Time 0.31426072120666504, Overall 2.066007137298584 \n",
            "Fold 4/4, Epoch 14/19, Running Loss valid 8.358198547363282, Time 0.31438398361206055, Overall 2.0661306381225586 \n",
            "Fold 4/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 22.504878997802734, Time 0.331207275390625, Overall 2.082980155944824 \n",
            "Fold 4/4, Epoch 15/19, Running Loss train 1.0716609046572732, Time 0.33136963844299316, Overall 2.083115816116333 \n",
            "Fold 4/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 41.174530029296875, Time 0.3354976177215576, Overall 2.0872440338134766 \n",
            "Fold 4/4, Epoch 15/19, Running Loss valid 8.234906005859376, Time 0.3356170654296875, Overall 2.0873632431030273 \n",
            "Fold 4/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 24.712417602539062, Time 0.3517632484436035, Overall 2.1035103797912598 \n",
            "Fold 4/4, Epoch 16/19, Running Loss train 1.1767817905970983, Time 0.35189008712768555, Overall 2.1036362648010254 \n",
            "Fold 4/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 41.049705505371094, Time 0.3559854030609131, Overall 2.107732057571411 \n",
            "Fold 4/4, Epoch 16/19, Running Loss valid 8.209941101074218, Time 0.3561391830444336, Overall 2.1078853607177734 \n",
            "Fold 4/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 21.40627670288086, Time 0.3721199035644531, Overall 2.1238670349121094 \n",
            "Fold 4/4, Epoch 17/19, Running Loss train 1.0193465096609933, Time 0.37224245071411133, Overall 2.123988389968872 \n",
            "Fold 4/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 40.72153091430664, Time 0.3758993148803711, Overall 2.1276462078094482 \n",
            "Fold 4/4, Epoch 17/19, Running Loss valid 8.144306182861328, Time 0.3760511875152588, Overall 2.127797842025757 \n",
            "Fold 4/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 31.412397384643555, Time 0.3924243450164795, Overall 2.1441714763641357 \n",
            "Fold 4/4, Epoch 18/19, Running Loss train 1.4958284468877883, Time 0.3925628662109375, Overall 2.1443090438842773 \n",
            "Fold 4/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 50.5351676940918, Time 0.3966073989868164, Overall 2.1483542919158936 \n",
            "Fold 4/4, Epoch 18/19, Running Loss valid 10.10703353881836, Time 0.39675235748291016, Overall 2.148498773574829 \n",
            "Fold 4/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 16.676877975463867, Time 0.4131460189819336, Overall 2.16489315032959 \n",
            "Fold 4/4, Epoch 19/19, Running Loss train 0.7941370464506603, Time 0.413285493850708, Overall 2.165031671524048 \n",
            "Fold 4/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 42.07284164428711, Time 0.41756415367126465, Overall 2.1693108081817627 \n",
            "Fold 4/4, Epoch 19/19, Running Loss valid 8.414568328857422, Time 0.41770434379577637, Overall 2.169450283050537 \n",
            "It took 2.171104907989502 to train the model.\n",
            "Figure(640x480)\n",
            "Figure(640x480)\n",
            "Figure(640x480)\n",
            "Figure(640x480)\n",
            "Figure(640x480)\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f1.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 87.62it/s]\n",
            "100% 5/5 [00:00<00:00, 66156.21it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 3874.29it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f1.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.09128832817077637\n",
            "####################################################################################################\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f3.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 94.49it/s]\n",
            "100% 5/5 [00:00<00:00, 66576.25it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 4628.45it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f3.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.08298778533935547\n",
            "####################################################################################################\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f0.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 91.39it/s]\n",
            "100% 5/5 [00:00<00:00, 64133.09it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 4545.19it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f0.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.08613228797912598\n",
            "####################################################################################################\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f4.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 82.42it/s]\n",
            "100% 5/5 [00:00<00:00, 59578.18it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 4635.61it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f4.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.09069395065307617\n",
            "####################################################################################################\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f2.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 90.73it/s]\n",
            "100% 5/5 [00:00<00:00, 56679.78it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 4337.44it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/fnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b/f2.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.08569788932800293\n",
            "####################################################################################################\n",
            "Running for (dataset, model): (dblp, bnn) ... \n",
            "Fold 0/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 292.9632263183594, Time 0.023603439331054688, Overall 0.025895357131958008 \n",
            "Fold 0/4, Epoch 0/19, Running Loss train 14.648161315917969, Time 0.023758649826049805, Overall 0.026049137115478516 \n",
            "Fold 0/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 75.3116683959961, Time 0.033686161041259766, Overall 0.03597712516784668 \n",
            "Fold 0/4, Epoch 0/19, Running Loss valid 12.551944732666016, Time 0.034581661224365234, Overall 0.036872148513793945 \n",
            "Fold 0/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 230.18264770507812, Time 0.05815839767456055, Overall 0.06044960021972656 \n",
            "Fold 0/4, Epoch 1/19, Running Loss train 11.509132385253906, Time 0.05830264091491699, Overall 0.0605928897857666 \n",
            "Fold 0/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 82.22288513183594, Time 0.067108154296875, Overall 0.06939864158630371 \n",
            "Fold 0/4, Epoch 1/19, Running Loss valid 13.703814188639322, Time 0.06723666191101074, Overall 0.06952667236328125 \n",
            "Fold 0/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 300.65887451171875, Time 0.08925557136535645, Overall 0.09154748916625977 \n",
            "Fold 0/4, Epoch 2/19, Running Loss train 15.032943725585938, Time 0.08939361572265625, Overall 0.09168410301208496 \n",
            "Fold 0/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 67.91845703125, Time 0.09810495376586914, Overall 0.10039567947387695 \n",
            "Fold 0/4, Epoch 2/19, Running Loss valid 11.319742838541666, Time 0.09823393821716309, Overall 0.1005241870880127 \n",
            "Fold 0/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 220.3152313232422, Time 0.12032413482666016, Overall 0.12261605262756348 \n",
            "Fold 0/4, Epoch 3/19, Running Loss train 11.015761566162109, Time 0.1204831600189209, Overall 0.12277340888977051 \n",
            "Fold 0/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 107.8074951171875, Time 0.12974929809570312, Overall 0.13204050064086914 \n",
            "Fold 0/4, Epoch 3/19, Running Loss valid 17.967915852864582, Time 0.12991929054260254, Overall 0.13220977783203125 \n",
            "Fold 0/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 313.1582336425781, Time 0.15855884552001953, Overall 0.16085052490234375 \n",
            "Fold 0/4, Epoch 4/19, Running Loss train 15.657911682128907, Time 0.1587061882019043, Overall 0.1609961986541748 \n",
            "Fold 0/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 70.32586669921875, Time 0.17449402809143066, Overall 0.17678523063659668 \n",
            "Fold 0/4, Epoch 4/19, Running Loss valid 11.720977783203125, Time 0.1746675968170166, Overall 0.176957368850708 \n",
            "Fold 0/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 337.654541015625, Time 0.200423002243042, Overall 0.2027139663696289 \n",
            "Fold 0/4, Epoch 5/19, Running Loss train 16.88272705078125, Time 0.20057344436645508, Overall 0.20286345481872559 \n",
            "Fold 0/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 70.96583557128906, Time 0.21004939079284668, Overall 0.2123403549194336 \n",
            "Fold 0/4, Epoch 5/19, Running Loss valid 11.82763926188151, Time 0.21018362045288086, Overall 0.21247386932373047 \n",
            "Fold 0/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 281.9571838378906, Time 0.2335515022277832, Overall 0.23584246635437012 \n",
            "Fold 0/4, Epoch 6/19, Running Loss train 14.097859191894532, Time 0.23369383811950684, Overall 0.23598408699035645 \n",
            "Fold 0/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 63.27661895751953, Time 0.24394869804382324, Overall 0.24624037742614746 \n",
            "Fold 0/4, Epoch 6/19, Running Loss valid 10.546103159586588, Time 0.24413037300109863, Overall 0.24642086029052734 \n",
            "Fold 0/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 233.1710968017578, Time 0.27228379249572754, Overall 0.27457547187805176 \n",
            "Fold 0/4, Epoch 7/19, Running Loss train 11.658554840087891, Time 0.27242445945739746, Overall 0.27471399307250977 \n",
            "Fold 0/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 89.43740844726562, Time 0.28403282165527344, Overall 0.28632473945617676 \n",
            "Fold 0/4, Epoch 7/19, Running Loss valid 14.906234741210938, Time 0.284207820892334, Overall 0.2864975929260254 \n",
            "Fold 0/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 190.81585693359375, Time 0.3152804374694824, Overall 0.31757211685180664 \n",
            "Fold 0/4, Epoch 8/19, Running Loss train 9.540792846679688, Time 0.31543564796447754, Overall 0.31772565841674805 \n",
            "Fold 0/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 82.36827087402344, Time 0.3256111145019531, Overall 0.32790231704711914 \n",
            "Fold 0/4, Epoch 8/19, Running Loss valid 13.728045145670572, Time 0.3259925842285156, Overall 0.32828330993652344 \n",
            "Fold 0/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 232.82611083984375, Time 0.35097455978393555, Overall 0.35326576232910156 \n",
            "Fold 0/4, Epoch 9/19, Running Loss train 11.641305541992187, Time 0.35111546516418457, Overall 0.353405237197876 \n",
            "Fold 0/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 84.60344696044922, Time 0.3614461421966553, Overall 0.363736629486084 \n",
            "Fold 0/4, Epoch 9/19, Running Loss valid 14.100574493408203, Time 0.36159729957580566, Overall 0.3638875484466553 \n",
            "Fold 0/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 237.59576416015625, Time 0.3847067356109619, Overall 0.38699841499328613 \n",
            "Fold 0/4, Epoch 10/19, Running Loss train 11.879788208007813, Time 0.38491129875183105, Overall 0.38720202445983887 \n",
            "Fold 0/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 77.48917388916016, Time 0.39318275451660156, Overall 0.3954732418060303 \n",
            "Fold 0/4, Epoch 10/19, Running Loss valid 12.914862314860025, Time 0.3933229446411133, Overall 0.3956127166748047 \n",
            "Fold 0/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 235.72750854492188, Time 0.4166591167449951, Overall 0.4189603328704834 \n",
            "Fold 0/4, Epoch 11/19, Running Loss train 11.786375427246094, Time 0.41681623458862305, Overall 0.41910672187805176 \n",
            "Fold 0/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 96.12178802490234, Time 0.4260399341583252, Overall 0.428330659866333 \n",
            "Fold 0/4, Epoch 11/19, Running Loss valid 16.02029800415039, Time 0.4261918067932129, Overall 0.4284818172454834 \n",
            "Fold 0/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 200.4329833984375, Time 0.44898080825805664, Overall 0.45127201080322266 \n",
            "Fold 0/4, Epoch 12/19, Running Loss train 10.021649169921876, Time 0.44911623001098633, Overall 0.45140671730041504 \n",
            "Fold 0/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 62.82769775390625, Time 0.46274232864379883, Overall 0.46503305435180664 \n",
            "Fold 0/4, Epoch 12/19, Running Loss valid 10.471282958984375, Time 0.4629018306732178, Overall 0.4651918411254883 \n",
            "Fold 0/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 288.7845458984375, Time 0.4854586124420166, Overall 0.4877493381500244 \n",
            "Fold 0/4, Epoch 13/19, Running Loss train 14.439227294921874, Time 0.48558783531188965, Overall 0.48787808418273926 \n",
            "Fold 0/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 70.79632568359375, Time 0.4949030876159668, Overall 0.4971933364868164 \n",
            "Fold 0/4, Epoch 13/19, Running Loss valid 11.799387613932291, Time 0.49504876136779785, Overall 0.49733853340148926 \n",
            "Fold 0/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 235.23240661621094, Time 0.517784833908081, Overall 0.5200765132904053 \n",
            "Fold 0/4, Epoch 14/19, Running Loss train 11.761620330810548, Time 0.5179455280303955, Overall 0.5202357769012451 \n",
            "Fold 0/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 84.63516235351562, Time 0.5274538993835449, Overall 0.5297446250915527 \n",
            "Fold 0/4, Epoch 14/19, Running Loss valid 14.105860392252604, Time 0.527597188949585, Overall 0.5298871994018555 \n",
            "Fold 0/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 141.32164001464844, Time 0.5510985851287842, Overall 0.5533897876739502 \n",
            "Fold 0/4, Epoch 15/19, Running Loss train 7.066082000732422, Time 0.5512404441833496, Overall 0.553530216217041 \n",
            "Fold 0/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 79.43212127685547, Time 0.5619268417358398, Overall 0.5642178058624268 \n",
            "Fold 0/4, Epoch 15/19, Running Loss valid 13.238686879475912, Time 0.5621058940887451, Overall 0.5643956661224365 \n",
            "Fold 0/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 189.02139282226562, Time 0.585547685623169, Overall 0.5878386497497559 \n",
            "Fold 0/4, Epoch 16/19, Running Loss train 9.451069641113282, Time 0.5857295989990234, Overall 0.5880203247070312 \n",
            "Fold 0/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 70.0833511352539, Time 0.595327615737915, Overall 0.597618818283081 \n",
            "Fold 0/4, Epoch 16/19, Running Loss valid 11.680558522542318, Time 0.5955421924591064, Overall 0.5978326797485352 \n",
            "Fold 0/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 207.38453674316406, Time 0.6186099052429199, Overall 0.6209011077880859 \n",
            "Fold 0/4, Epoch 17/19, Running Loss train 10.369226837158203, Time 0.6187920570373535, Overall 0.6210823059082031 \n",
            "Fold 0/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 88.61091613769531, Time 0.627941370010376, Overall 0.6302323341369629 \n",
            "Fold 0/4, Epoch 17/19, Running Loss valid 14.768486022949219, Time 0.6280865669250488, Overall 0.6303765773773193 \n",
            "Fold 0/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 186.1289825439453, Time 0.6508595943450928, Overall 0.6531503200531006 \n",
            "Fold 0/4, Epoch 18/19, Running Loss train 9.306449127197265, Time 0.6509819030761719, Overall 0.6532719135284424 \n",
            "Fold 0/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 87.49095153808594, Time 0.6611874103546143, Overall 0.663477897644043 \n",
            "Fold 0/4, Epoch 18/19, Running Loss valid 14.581825256347656, Time 0.6613810062408447, Overall 0.6636717319488525 \n",
            "Fold 0/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 132.07272338867188, Time 0.6830992698669434, Overall 0.6853909492492676 \n",
            "Fold 0/4, Epoch 19/19, Running Loss train 6.603636169433594, Time 0.6832437515258789, Overall 0.6855337619781494 \n",
            "Fold 0/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 55.914634704589844, Time 0.6924123764038086, Overall 0.6947031021118164 \n",
            "Fold 0/4, Epoch 19/19, Running Loss valid 9.319105784098307, Time 0.6925458908081055, Overall 0.694835901260376 \n",
            "Fold 1/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 233.13058471679688, Time 0.02234816551208496, Overall 0.7210748195648193 \n",
            "Fold 1/4, Epoch 0/19, Running Loss train 11.101456415085565, Time 0.022488117218017578, Overall 0.7212140560150146 \n",
            "Fold 1/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 76.23406982421875, Time 0.031072139739990234, Overall 0.7297985553741455 \n",
            "Fold 1/4, Epoch 0/19, Running Loss valid 15.24681396484375, Time 0.03122234344482422, Overall 0.729947566986084 \n",
            "Fold 1/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 223.3173370361328, Time 0.05572795867919922, Overall 0.7544541358947754 \n",
            "Fold 1/4, Epoch 1/19, Running Loss train 10.634158906482515, Time 0.05586600303649902, Overall 0.7545912265777588 \n",
            "Fold 1/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 48.88222885131836, Time 0.06906938552856445, Overall 0.7677958011627197 \n",
            "Fold 1/4, Epoch 1/19, Running Loss valid 9.776445770263672, Time 0.06921219825744629, Overall 0.767937421798706 \n",
            "Fold 1/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 316.66741943359375, Time 0.09419107437133789, Overall 0.7929174900054932 \n",
            "Fold 1/4, Epoch 2/19, Running Loss train 15.079400925409226, Time 0.09436178207397461, Overall 0.7930874824523926 \n",
            "Fold 1/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 83.17504119873047, Time 0.10317611694335938, Overall 0.8019022941589355 \n",
            "Fold 1/4, Epoch 2/19, Running Loss valid 16.635008239746092, Time 0.10331940650939941, Overall 0.8020448684692383 \n",
            "Fold 1/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 288.5155334472656, Time 0.128068208694458, Overall 0.8267946243286133 \n",
            "Fold 1/4, Epoch 3/19, Running Loss train 13.738834926060267, Time 0.1282198429107666, Overall 0.8269455432891846 \n",
            "Fold 1/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 51.33122253417969, Time 0.13754868507385254, Overall 0.8362758159637451 \n",
            "Fold 1/4, Epoch 3/19, Running Loss valid 10.266244506835937, Time 0.1377089023590088, Overall 0.8364346027374268 \n",
            "Fold 1/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 381.9440612792969, Time 0.1619892120361328, Overall 0.8607158660888672 \n",
            "Fold 1/4, Epoch 4/19, Running Loss train 18.18781244187128, Time 0.16213130950927734, Overall 0.8608574867248535 \n",
            "Fold 1/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 43.8880615234375, Time 0.17108726501464844, Overall 0.8698139190673828 \n",
            "Fold 1/4, Epoch 4/19, Running Loss valid 8.7776123046875, Time 0.1712498664855957, Overall 0.8699750900268555 \n",
            "Fold 1/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 265.59283447265625, Time 0.19585013389587402, Overall 0.8945772647857666 \n",
            "Fold 1/4, Epoch 5/19, Running Loss train 12.64727783203125, Time 0.19598007202148438, Overall 0.8947060108184814 \n",
            "Fold 1/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 63.467872619628906, Time 0.20458769798278809, Overall 0.9033136367797852 \n",
            "Fold 1/4, Epoch 5/19, Running Loss valid 12.693574523925781, Time 0.20473718643188477, Overall 0.9034631252288818 \n",
            "Fold 1/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 214.4427947998047, Time 0.22861194610595703, Overall 0.9273388385772705 \n",
            "Fold 1/4, Epoch 6/19, Running Loss train 10.211561657133556, Time 0.22876715660095215, Overall 0.927492618560791 \n",
            "Fold 1/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 22.49260139465332, Time 0.23719263076782227, Overall 0.9359192848205566 \n",
            "Fold 1/4, Epoch 6/19, Running Loss valid 4.498520278930664, Time 0.23734593391418457, Overall 0.9360716342926025 \n",
            "Fold 1/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 230.1715087890625, Time 0.2612900733947754, Overall 0.9600167274475098 \n",
            "Fold 1/4, Epoch 7/19, Running Loss train 10.960548037574405, Time 0.2614579200744629, Overall 0.9601831436157227 \n",
            "Fold 1/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 57.55039978027344, Time 0.2702951431274414, Overall 0.9690213203430176 \n",
            "Fold 1/4, Epoch 7/19, Running Loss valid 11.510079956054687, Time 0.2704508304595947, Overall 0.9691762924194336 \n",
            "Fold 1/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 172.38070678710938, Time 0.2937638759613037, Overall 0.9924900531768799 \n",
            "Fold 1/4, Epoch 8/19, Running Loss train 8.208605085100446, Time 0.2939143180847168, Overall 0.9926400184631348 \n",
            "Fold 1/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 70.96222686767578, Time 0.30263805389404297, Overall 1.0013642311096191 \n",
            "Fold 1/4, Epoch 8/19, Running Loss valid 14.192445373535156, Time 0.30278730392456055, Overall 1.0015122890472412 \n",
            "Fold 1/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 225.63726806640625, Time 0.32673144340515137, Overall 1.025458574295044 \n",
            "Fold 1/4, Epoch 9/19, Running Loss train 10.744631812686013, Time 0.3268749713897705, Overall 1.0256006717681885 \n",
            "Fold 1/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 48.345672607421875, Time 0.3415849208831787, Overall 1.0403118133544922 \n",
            "Fold 1/4, Epoch 9/19, Running Loss valid 9.669134521484375, Time 0.34175896644592285, Overall 1.0404841899871826 \n",
            "Fold 1/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 148.19667053222656, Time 0.373309850692749, Overall 1.0720360279083252 \n",
            "Fold 1/4, Epoch 10/19, Running Loss train 7.056984311058407, Time 0.3734726905822754, Overall 1.0721981525421143 \n",
            "Fold 1/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 51.626197814941406, Time 0.38273000717163086, Overall 1.081456184387207 \n",
            "Fold 1/4, Epoch 10/19, Running Loss valid 10.325239562988282, Time 0.3828763961791992, Overall 1.0816011428833008 \n",
            "Fold 1/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 150.4284210205078, Time 0.4063699245452881, Overall 1.1050963401794434 \n",
            "Fold 1/4, Epoch 11/19, Running Loss train 7.163258143833706, Time 0.40653324127197266, Overall 1.1052591800689697 \n",
            "Fold 1/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 35.01679611206055, Time 0.4150702953338623, Overall 1.1137957572937012 \n",
            "Fold 1/4, Epoch 11/19, Running Loss valid 7.00335922241211, Time 0.4152567386627197, Overall 1.1139826774597168 \n",
            "Fold 1/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 204.74961853027344, Time 0.4388456344604492, Overall 1.1375718116760254 \n",
            "Fold 1/4, Epoch 12/19, Running Loss train 9.749981834774925, Time 0.4389796257019043, Overall 1.137704849243164 \n",
            "Fold 1/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 92.60681915283203, Time 0.4487903118133545, Overall 1.1475162506103516 \n",
            "Fold 1/4, Epoch 12/19, Running Loss valid 18.521363830566408, Time 0.44894862174987793, Overall 1.1476740837097168 \n",
            "Fold 1/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 189.57662963867188, Time 0.47182130813598633, Overall 1.1705477237701416 \n",
            "Fold 1/4, Epoch 13/19, Running Loss train 9.02745855422247, Time 0.47196269035339355, Overall 1.1706886291503906 \n",
            "Fold 1/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 87.39735412597656, Time 0.480541467666626, Overall 1.1792676448822021 \n",
            "Fold 1/4, Epoch 13/19, Running Loss valid 17.479470825195314, Time 0.4806995391845703, Overall 1.1794254779815674 \n",
            "Fold 1/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 140.98756408691406, Time 0.5039770603179932, Overall 1.2027037143707275 \n",
            "Fold 1/4, Epoch 14/19, Running Loss train 6.713693527948289, Time 0.5041124820709229, Overall 1.2028381824493408 \n",
            "Fold 1/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 49.87782669067383, Time 0.5135045051574707, Overall 1.212230920791626 \n",
            "Fold 1/4, Epoch 14/19, Running Loss valid 9.975565338134766, Time 0.5136656761169434, Overall 1.2123913764953613 \n",
            "Fold 1/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 221.27532958984375, Time 0.5377042293548584, Overall 1.2364306449890137 \n",
            "Fold 1/4, Epoch 15/19, Running Loss train 10.536920456659226, Time 0.5378322601318359, Overall 1.2365577220916748 \n",
            "Fold 1/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 34.009559631347656, Time 0.546614408493042, Overall 1.2453405857086182 \n",
            "Fold 1/4, Epoch 15/19, Running Loss valid 6.801911926269531, Time 0.5467581748962402, Overall 1.2454841136932373 \n",
            "Fold 1/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 270.1790771484375, Time 0.5698347091674805, Overall 1.2685608863830566 \n",
            "Fold 1/4, Epoch 16/19, Running Loss train 12.865670340401786, Time 0.5699687004089355, Overall 1.2686944007873535 \n",
            "Fold 1/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 47.926109313964844, Time 0.5788161754608154, Overall 1.2775423526763916 \n",
            "Fold 1/4, Epoch 16/19, Running Loss valid 9.585221862792968, Time 0.5789604187011719, Overall 1.277686357498169 \n",
            "Fold 1/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 154.51690673828125, Time 0.6023643016815186, Overall 1.301090955734253 \n",
            "Fold 1/4, Epoch 17/19, Running Loss train 7.357947939918155, Time 0.6024928092956543, Overall 1.3012187480926514 \n",
            "Fold 1/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 45.9198112487793, Time 0.6111633777618408, Overall 1.3098900318145752 \n",
            "Fold 1/4, Epoch 17/19, Running Loss valid 9.18396224975586, Time 0.6113100051879883, Overall 1.3100354671478271 \n",
            "Epoch 00018: reducing learning rate of group 0 to 5.0000e-02.\n",
            "Fold 1/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 155.67198181152344, Time 0.6349625587463379, Overall 1.333688497543335 \n",
            "Fold 1/4, Epoch 18/19, Running Loss train 7.412951514834449, Time 0.6350917816162109, Overall 1.333817481994629 \n",
            "Fold 1/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 51.898658752441406, Time 0.6459753513336182, Overall 1.3447015285491943 \n",
            "Fold 1/4, Epoch 18/19, Running Loss valid 10.379731750488281, Time 0.6461136341094971, Overall 1.344839334487915 \n",
            "Fold 1/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 166.86676025390625, Time 0.6735155582427979, Overall 1.372241735458374 \n",
            "Fold 1/4, Epoch 19/19, Running Loss train 7.946036202566964, Time 0.6736681461334229, Overall 1.3723938465118408 \n",
            "Fold 1/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 90.33082580566406, Time 0.6816666126251221, Overall 1.3803930282592773 \n",
            "Fold 1/4, Epoch 19/19, Running Loss valid 18.066165161132812, Time 0.6817917823791504, Overall 1.380516767501831 \n",
            "Fold 2/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 251.04244995117188, Time 0.02286052703857422, Overall 1.4068162441253662 \n",
            "Fold 2/4, Epoch 0/19, Running Loss train 11.954402378627233, Time 0.022992610931396484, Overall 1.406947374343872 \n",
            "Fold 2/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 55.264686584472656, Time 0.03163290023803711, Overall 1.4155888557434082 \n",
            "Fold 2/4, Epoch 0/19, Running Loss valid 11.052937316894532, Time 0.031771183013916016, Overall 1.4157264232635498 \n",
            "Fold 2/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 274.7178039550781, Time 0.0558171272277832, Overall 1.4397737979888916 \n",
            "Fold 2/4, Epoch 1/19, Running Loss train 13.081800188337054, Time 0.05596351623535156, Overall 1.4399189949035645 \n",
            "Fold 2/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 70.75019073486328, Time 0.06469178199768066, Overall 1.4486477375030518 \n",
            "Fold 2/4, Epoch 1/19, Running Loss valid 14.150038146972657, Time 0.06484603881835938, Overall 1.4488022327423096 \n",
            "Fold 2/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 286.3109436035156, Time 0.08965349197387695, Overall 1.4736101627349854 \n",
            "Fold 2/4, Epoch 2/19, Running Loss train 13.633854457310267, Time 0.08980035781860352, Overall 1.4737565517425537 \n",
            "Fold 2/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 79.45699310302734, Time 0.09858584403991699, Overall 1.4825425148010254 \n",
            "Fold 2/4, Epoch 2/19, Running Loss valid 15.891398620605468, Time 0.09875679016113281, Overall 1.482712984085083 \n",
            "Fold 2/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 325.7330017089844, Time 0.12291169166564941, Overall 1.5068683624267578 \n",
            "Fold 2/4, Epoch 3/19, Running Loss train 15.511095319475446, Time 0.12305307388305664, Overall 1.5070087909698486 \n",
            "Fold 2/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 55.68498229980469, Time 0.13262176513671875, Overall 1.5165784358978271 \n",
            "Fold 2/4, Epoch 3/19, Running Loss valid 11.136996459960937, Time 0.13281822204589844, Overall 1.5167744159698486 \n",
            "Fold 2/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 310.8350524902344, Time 0.15825557708740234, Overall 1.542212963104248 \n",
            "Fold 2/4, Epoch 4/19, Running Loss train 14.801669166201638, Time 0.15843605995178223, Overall 1.5423924922943115 \n",
            "Fold 2/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 40.19318771362305, Time 0.1677405834197998, Overall 1.551697015762329 \n",
            "Fold 2/4, Epoch 4/19, Running Loss valid 8.03863754272461, Time 0.16791939735412598, Overall 1.5518755912780762 \n",
            "Fold 2/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 330.1227111816406, Time 0.19356155395507812, Overall 1.5775189399719238 \n",
            "Fold 2/4, Epoch 5/19, Running Loss train 15.720129103887649, Time 0.19371414184570312, Overall 1.5776700973510742 \n",
            "Fold 2/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 54.17022705078125, Time 0.202866792678833, Overall 1.5868241786956787 \n",
            "Fold 2/4, Epoch 5/19, Running Loss valid 10.83404541015625, Time 0.20308470726013184, Overall 1.587040662765503 \n",
            "Fold 2/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 259.5484313964844, Time 0.22676897048950195, Overall 1.6107258796691895 \n",
            "Fold 2/4, Epoch 6/19, Running Loss train 12.359449114118304, Time 0.22691965103149414, Overall 1.6108758449554443 \n",
            "Fold 2/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 48.34206008911133, Time 0.2360858917236328, Overall 1.6200425624847412 \n",
            "Fold 2/4, Epoch 6/19, Running Loss valid 9.668412017822266, Time 0.23627901077270508, Overall 1.6202354431152344 \n",
            "Fold 2/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 313.8331298828125, Time 0.26046156883239746, Overall 1.644418716430664 \n",
            "Fold 2/4, Epoch 7/19, Running Loss train 14.944434756324405, Time 0.260601282119751, Overall 1.6445567607879639 \n",
            "Fold 2/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 96.63296508789062, Time 0.26938748359680176, Overall 1.653343915939331 \n",
            "Fold 2/4, Epoch 7/19, Running Loss valid 19.326593017578126, Time 0.269533634185791, Overall 1.653489589691162 \n",
            "Fold 2/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 259.4864196777344, Time 0.30159473419189453, Overall 1.6855518817901611 \n",
            "Fold 2/4, Epoch 8/19, Running Loss train 12.356496175130209, Time 0.3017454147338867, Overall 1.6857011318206787 \n",
            "Fold 2/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 72.74503326416016, Time 0.3107302188873291, Overall 1.6946868896484375 \n",
            "Fold 2/4, Epoch 8/19, Running Loss valid 14.54900665283203, Time 0.31088900566101074, Overall 1.694845199584961 \n",
            "Fold 2/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 237.438232421875, Time 0.33376407623291016, Overall 1.7177200317382812 \n",
            "Fold 2/4, Epoch 9/19, Running Loss train 11.306582496279763, Time 0.33390212059020996, Overall 1.7178587913513184 \n",
            "Fold 2/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 59.57831573486328, Time 0.3424057960510254, Overall 1.7263622283935547 \n",
            "Fold 2/4, Epoch 9/19, Running Loss valid 11.915663146972657, Time 0.3425619602203369, Overall 1.726517677307129 \n",
            "Fold 2/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 232.652099609375, Time 0.36618638038635254, Overall 1.7501428127288818 \n",
            "Fold 2/4, Epoch 10/19, Running Loss train 11.078671409970237, Time 0.36634230613708496, Overall 1.7502985000610352 \n",
            "Fold 2/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 47.20024108886719, Time 0.3751704692840576, Overall 1.759127140045166 \n",
            "Fold 2/4, Epoch 10/19, Running Loss valid 9.440048217773438, Time 0.37534332275390625, Overall 1.7592997550964355 \n",
            "Fold 2/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 241.0546875, Time 0.39928674697875977, Overall 1.7832434177398682 \n",
            "Fold 2/4, Epoch 11/19, Running Loss train 11.478794642857142, Time 0.3994331359863281, Overall 1.7833890914916992 \n",
            "Fold 2/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 51.70051574707031, Time 0.40849804878234863, Overall 1.792454481124878 \n",
            "Fold 2/4, Epoch 11/19, Running Loss valid 10.340103149414062, Time 0.4086465835571289, Overall 1.7926025390625 \n",
            "Fold 2/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 280.18670654296875, Time 0.43265295028686523, Overall 1.8166096210479736 \n",
            "Fold 2/4, Epoch 12/19, Running Loss train 13.34222412109375, Time 0.43280625343322754, Overall 1.8167624473571777 \n",
            "Fold 2/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 95.89515686035156, Time 0.4418618679046631, Overall 1.8258185386657715 \n",
            "Fold 2/4, Epoch 12/19, Running Loss valid 19.179031372070312, Time 0.44200682640075684, Overall 1.8259625434875488 \n",
            "Fold 2/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 294.2493896484375, Time 0.46816349029541016, Overall 1.8521199226379395 \n",
            "Fold 2/4, Epoch 13/19, Running Loss train 14.011875697544642, Time 0.46828532218933105, Overall 1.852241039276123 \n",
            "Fold 2/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 39.639915466308594, Time 0.47660112380981445, Overall 1.8605575561523438 \n",
            "Fold 2/4, Epoch 13/19, Running Loss valid 7.927983093261719, Time 0.47674560546875, Overall 1.860701322555542 \n",
            "Fold 2/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 231.20469665527344, Time 0.5026915073394775, Overall 1.8866474628448486 \n",
            "Fold 2/4, Epoch 14/19, Running Loss train 11.009747459774925, Time 0.5028202533721924, Overall 1.8867759704589844 \n",
            "Fold 2/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 32.70665740966797, Time 0.5107784271240234, Overall 1.8947341442108154 \n",
            "Fold 2/4, Epoch 14/19, Running Loss valid 6.541331481933594, Time 0.5109188556671143, Overall 1.894874095916748 \n",
            "Fold 2/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 201.15550231933594, Time 0.5334072113037109, Overall 1.9173634052276611 \n",
            "Fold 2/4, Epoch 15/19, Running Loss train 9.578833443777901, Time 0.5335190296173096, Overall 1.9174742698669434 \n",
            "Fold 2/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 46.54168701171875, Time 0.5414245128631592, Overall 1.9253811836242676 \n",
            "Fold 2/4, Epoch 15/19, Running Loss valid 9.30833740234375, Time 0.5415630340576172, Overall 1.9255189895629883 \n",
            "Fold 2/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 247.6414031982422, Time 0.5641818046569824, Overall 1.9481379985809326 \n",
            "Fold 2/4, Epoch 16/19, Running Loss train 11.792447771344866, Time 0.5643012523651123, Overall 1.9482567310333252 \n",
            "Fold 2/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 54.708072662353516, Time 0.5724732875823975, Overall 1.9564294815063477 \n",
            "Fold 2/4, Epoch 16/19, Running Loss valid 10.941614532470703, Time 0.5726025104522705, Overall 1.9565582275390625 \n",
            "Fold 2/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 181.28436279296875, Time 0.5950155258178711, Overall 1.9789726734161377 \n",
            "Fold 2/4, Epoch 17/19, Running Loss train 8.632588704427084, Time 0.5951724052429199, Overall 1.9791285991668701 \n",
            "Fold 2/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 47.28829574584961, Time 0.6037423610687256, Overall 1.987699031829834 \n",
            "Fold 2/4, Epoch 17/19, Running Loss valid 9.457659149169922, Time 0.6039314270019531, Overall 1.9878873825073242 \n",
            "Fold 2/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 241.25619506835938, Time 0.6270074844360352, Overall 2.0109641551971436 \n",
            "Fold 2/4, Epoch 18/19, Running Loss train 11.488390241350446, Time 0.627152681350708, Overall 2.011108160018921 \n",
            "Fold 2/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 41.25358963012695, Time 0.6354751586914062, Overall 2.0194318294525146 \n",
            "Fold 2/4, Epoch 18/19, Running Loss valid 8.250717926025391, Time 0.6356048583984375, Overall 2.0195603370666504 \n",
            "Fold 2/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 171.02684020996094, Time 0.6624550819396973, Overall 2.0464117527008057 \n",
            "Fold 2/4, Epoch 19/19, Running Loss train 8.144135248093377, Time 0.6626043319702148, Overall 2.046560049057007 \n",
            "Fold 2/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 64.29452514648438, Time 0.6720881462097168, Overall 2.056044340133667 \n",
            "Fold 2/4, Epoch 19/19, Running Loss valid 12.858905029296874, Time 0.6722400188446045, Overall 2.0561954975128174 \n",
            "Fold 3/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 284.57928466796875, Time 0.021750688552856445, Overall 2.0825796127319336 \n",
            "Fold 3/4, Epoch 0/19, Running Loss train 13.551394507998513, Time 0.021926403045654297, Overall 2.0827550888061523 \n",
            "Fold 3/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 68.97999572753906, Time 0.03060603141784668, Overall 2.091435194015503 \n",
            "Fold 3/4, Epoch 0/19, Running Loss valid 13.795999145507812, Time 0.030747652053833008, Overall 2.091576337814331 \n",
            "Fold 3/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 218.075439453125, Time 0.053748369216918945, Overall 2.1145780086517334 \n",
            "Fold 3/4, Epoch 1/19, Running Loss train 10.384544735863095, Time 0.053879499435424805, Overall 2.114708423614502 \n",
            "Fold 3/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 54.116539001464844, Time 0.06216907501220703, Overall 2.122997999191284 \n",
            "Fold 3/4, Epoch 1/19, Running Loss valid 10.82330780029297, Time 0.06231331825256348, Overall 2.1231417655944824 \n",
            "Fold 3/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 237.24542236328125, Time 0.08487367630004883, Overall 2.1457033157348633 \n",
            "Fold 3/4, Epoch 2/19, Running Loss train 11.297401064918155, Time 0.08501076698303223, Overall 2.145839214324951 \n",
            "Fold 3/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 64.46322631835938, Time 0.09344983100891113, Overall 2.1542789936065674 \n",
            "Fold 3/4, Epoch 2/19, Running Loss valid 12.892645263671875, Time 0.09358906745910645, Overall 2.1544177532196045 \n",
            "Fold 3/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 287.84808349609375, Time 0.12066531181335449, Overall 2.181494951248169 \n",
            "Fold 3/4, Epoch 3/19, Running Loss train 13.707051595052084, Time 0.12082910537719727, Overall 2.181657552719116 \n",
            "Fold 3/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 92.86085510253906, Time 0.1288471221923828, Overall 2.18967604637146 \n",
            "Fold 3/4, Epoch 3/19, Running Loss valid 18.572171020507813, Time 0.12899017333984375, Overall 2.1898186206817627 \n",
            "Fold 3/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 278.00042724609375, Time 0.15009427070617676, Overall 2.210923910140991 \n",
            "Fold 3/4, Epoch 4/19, Running Loss train 13.238115583147321, Time 0.1502385139465332, Overall 2.2110674381256104 \n",
            "Fold 3/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 71.73455810546875, Time 0.1581132411956787, Overall 2.218942165374756 \n",
            "Fold 3/4, Epoch 4/19, Running Loss valid 14.34691162109375, Time 0.15825843811035156, Overall 2.2190866470336914 \n",
            "Fold 3/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 299.4591369628906, Time 0.18108749389648438, Overall 2.241917610168457 \n",
            "Fold 3/4, Epoch 5/19, Running Loss train 14.259958902994791, Time 0.18121886253356934, Overall 2.2420475482940674 \n",
            "Fold 3/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 45.2863655090332, Time 0.18959736824035645, Overall 2.2504262924194336 \n",
            "Fold 3/4, Epoch 5/19, Running Loss valid 9.05727310180664, Time 0.18972253799438477, Overall 2.2505509853363037 \n",
            "Fold 3/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 272.1653747558594, Time 0.22245430946350098, Overall 2.2832839488983154 \n",
            "Fold 3/4, Epoch 6/19, Running Loss train 12.960255940755209, Time 0.22437286376953125, Overall 2.2852017879486084 \n",
            "Fold 3/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 54.699012756347656, Time 0.236006498336792, Overall 2.296835422515869 \n",
            "Fold 3/4, Epoch 6/19, Running Loss valid 10.939802551269532, Time 0.23616862297058105, Overall 2.296997547149658 \n",
            "Fold 3/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 289.58477783203125, Time 0.2601184844970703, Overall 2.320948362350464 \n",
            "Fold 3/4, Epoch 7/19, Running Loss train 13.789751325334821, Time 0.26029276847839355, Overall 2.3211214542388916 \n",
            "Fold 3/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 37.71748733520508, Time 0.26960206031799316, Overall 2.3304312229156494 \n",
            "Fold 3/4, Epoch 7/19, Running Loss valid 7.543497467041016, Time 0.26975131034851074, Overall 2.3305795192718506 \n",
            "Fold 3/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 233.6993408203125, Time 0.2933046817779541, Overall 2.3541338443756104 \n",
            "Fold 3/4, Epoch 8/19, Running Loss train 11.1285400390625, Time 0.2934255599975586, Overall 2.3542537689208984 \n",
            "Fold 3/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 23.62826156616211, Time 0.30186986923217773, Overall 2.362698554992676 \n",
            "Fold 3/4, Epoch 8/19, Running Loss valid 4.7256523132324215, Time 0.3020021915435791, Overall 2.36283016204834 \n",
            "Fold 3/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 364.88385009765625, Time 0.33016180992126465, Overall 2.390991687774658 \n",
            "Fold 3/4, Epoch 9/19, Running Loss train 17.375421433221724, Time 0.33237481117248535, Overall 2.3932037353515625 \n",
            "Fold 3/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 73.34457397460938, Time 0.34125709533691406, Overall 2.402104377746582 \n",
            "Fold 3/4, Epoch 9/19, Running Loss valid 14.668914794921875, Time 0.34182214736938477, Overall 2.4026505947113037 \n",
            "Fold 3/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 264.67889404296875, Time 0.36809492111206055, Overall 2.428924322128296 \n",
            "Fold 3/4, Epoch 10/19, Running Loss train 12.603756859188987, Time 0.3682568073272705, Overall 2.4290852546691895 \n",
            "Fold 3/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 39.553985595703125, Time 0.3775503635406494, Overall 2.4383792877197266 \n",
            "Fold 3/4, Epoch 10/19, Running Loss valid 7.910797119140625, Time 0.3777015209197998, Overall 2.438530206680298 \n",
            "Fold 3/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 319.6342468261719, Time 0.40037989616394043, Overall 2.4612205028533936 \n",
            "Fold 3/4, Epoch 11/19, Running Loss train 15.220678420293899, Time 0.4005265235900879, Overall 2.4613544940948486 \n",
            "Fold 3/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 62.903282165527344, Time 0.40912961959838867, Overall 2.469958543777466 \n",
            "Fold 3/4, Epoch 11/19, Running Loss valid 12.580656433105469, Time 0.4092895984649658, Overall 2.4701180458068848 \n",
            "Fold 3/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 200.37261962890625, Time 0.4313089847564697, Overall 2.492138624191284 \n",
            "Fold 3/4, Epoch 12/19, Running Loss train 9.541553315662203, Time 0.4314405918121338, Overall 2.4922690391540527 \n",
            "Fold 3/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 51.46501541137695, Time 0.44047069549560547, Overall 2.501300096511841 \n",
            "Fold 3/4, Epoch 12/19, Running Loss valid 10.29300308227539, Time 0.44063234329223633, Overall 2.501460552215576 \n",
            "Fold 3/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 161.8535614013672, Time 0.4589214324951172, Overall 2.5197503566741943 \n",
            "Fold 3/4, Epoch 13/19, Running Loss train 7.707312447684152, Time 0.4590480327606201, Overall 2.519876480102539 \n",
            "Fold 3/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 66.5828857421875, Time 0.4654364585876465, Overall 2.5262644290924072 \n",
            "Fold 3/4, Epoch 13/19, Running Loss valid 13.3165771484375, Time 0.46556639671325684, Overall 2.5263943672180176 \n",
            "Fold 3/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 208.2784881591797, Time 0.4793281555175781, Overall 2.540156364440918 \n",
            "Fold 3/4, Epoch 14/19, Running Loss train 9.918023245675224, Time 0.47948646545410156, Overall 2.540314197540283 \n",
            "Fold 3/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 71.21064758300781, Time 0.48479413986206055, Overall 2.5456225872039795 \n",
            "Fold 3/4, Epoch 14/19, Running Loss valid 14.242129516601562, Time 0.48492956161499023, Overall 2.545757532119751 \n",
            "Fold 3/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 215.9420623779297, Time 0.49925804138183594, Overall 2.560086727142334 \n",
            "Fold 3/4, Epoch 15/19, Running Loss train 10.282955351329985, Time 0.49938011169433594, Overall 2.5602080821990967 \n",
            "Fold 3/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 99.7041015625, Time 0.5046396255493164, Overall 2.5654683113098145 \n",
            "Fold 3/4, Epoch 15/19, Running Loss valid 19.9408203125, Time 0.5047681331634521, Overall 2.565596103668213 \n",
            "Fold 3/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 128.1316680908203, Time 0.525076150894165, Overall 2.585904598236084 \n",
            "Fold 3/4, Epoch 16/19, Running Loss train 6.101508004324777, Time 0.525183916091919, Overall 2.5860114097595215 \n",
            "Fold 3/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 67.00112915039062, Time 0.5304155349731445, Overall 2.5912435054779053 \n",
            "Fold 3/4, Epoch 16/19, Running Loss valid 13.400225830078124, Time 0.5305318832397461, Overall 2.5913591384887695 \n",
            "Fold 3/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 169.28045654296875, Time 0.5445880889892578, Overall 2.6054165363311768 \n",
            "Fold 3/4, Epoch 17/19, Running Loss train 8.06097412109375, Time 0.5447242259979248, Overall 2.6055517196655273 \n",
            "Fold 3/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 93.85348510742188, Time 0.5498747825622559, Overall 2.6107029914855957 \n",
            "Fold 3/4, Epoch 17/19, Running Loss valid 18.770697021484374, Time 0.55000901222229, Overall 2.6108367443084717 \n",
            "Fold 3/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 166.16456604003906, Time 0.5639400482177734, Overall 2.6247682571411133 \n",
            "Fold 3/4, Epoch 18/19, Running Loss train 7.912598382859003, Time 0.5640518665313721, Overall 2.6248795986175537 \n",
            "Fold 3/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 54.570396423339844, Time 0.569284200668335, Overall 2.630112409591675 \n",
            "Fold 3/4, Epoch 18/19, Running Loss valid 10.914079284667968, Time 0.5694394111633301, Overall 2.6302671432495117 \n",
            "Fold 3/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 245.57669067382812, Time 0.5836789608001709, Overall 2.6445069313049316 \n",
            "Fold 3/4, Epoch 19/19, Running Loss train 11.694128127325149, Time 0.583784818649292, Overall 2.6446123123168945 \n",
            "Fold 3/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 62.96432876586914, Time 0.5890383720397949, Overall 2.6498661041259766 \n",
            "Fold 3/4, Epoch 19/19, Running Loss valid 12.592865753173829, Time 0.5891475677490234, Overall 2.649975061416626 \n",
            "Epoch 00020: reducing learning rate of group 0 to 5.0000e-02.\n",
            "Fold 4/4, Epoch 0/19, Minibatch 0/0, Phase train, Running Loss train 292.6813659667969, Time 0.013022661209106445, Overall 2.66566801071167 \n",
            "Fold 4/4, Epoch 0/19, Running Loss train 13.937207903180804, Time 0.013118982315063477, Overall 2.6657638549804688 \n",
            "Fold 4/4, Epoch 0/19, Minibatch 0/0, Phase valid, Running Loss valid 66.30386352539062, Time 0.018940210342407227, Overall 2.6715855598449707 \n",
            "Fold 4/4, Epoch 0/19, Running Loss valid 13.260772705078125, Time 0.01905035972595215, Overall 2.671694755554199 \n",
            "Fold 4/4, Epoch 1/19, Minibatch 0/0, Phase train, Running Loss train 232.3562774658203, Time 0.03268313407897949, Overall 2.685328245162964 \n",
            "Fold 4/4, Epoch 1/19, Running Loss train 11.064584641229539, Time 0.03277897834777832, Overall 2.6854236125946045 \n",
            "Fold 4/4, Epoch 1/19, Minibatch 0/0, Phase valid, Running Loss valid 80.86601257324219, Time 0.03829383850097656, Overall 2.690938711166382 \n",
            "Fold 4/4, Epoch 1/19, Running Loss valid 16.173202514648438, Time 0.03840231895446777, Overall 2.691047191619873 \n",
            "Fold 4/4, Epoch 2/19, Minibatch 0/0, Phase train, Running Loss train 258.8118896484375, Time 0.051902055740356445, Overall 2.7045888900756836 \n",
            "Fold 4/4, Epoch 2/19, Running Loss train 12.324375697544642, Time 0.05202889442443848, Overall 2.7046737670898438 \n",
            "Fold 4/4, Epoch 2/19, Minibatch 0/0, Phase valid, Running Loss valid 57.9990119934082, Time 0.05710649490356445, Overall 2.709751605987549 \n",
            "Fold 4/4, Epoch 2/19, Running Loss valid 11.599802398681641, Time 0.057225704193115234, Overall 2.7098705768585205 \n",
            "Fold 4/4, Epoch 3/19, Minibatch 0/0, Phase train, Running Loss train 314.4598693847656, Time 0.07106375694274902, Overall 2.7237091064453125 \n",
            "Fold 4/4, Epoch 3/19, Running Loss train 14.974279494512649, Time 0.07117438316345215, Overall 2.7238190174102783 \n",
            "Fold 4/4, Epoch 3/19, Minibatch 0/0, Phase valid, Running Loss valid 85.5087661743164, Time 0.07639718055725098, Overall 2.7290422916412354 \n",
            "Fold 4/4, Epoch 3/19, Running Loss valid 17.10175323486328, Time 0.07654166221618652, Overall 2.729186534881592 \n",
            "Fold 4/4, Epoch 4/19, Minibatch 0/0, Phase train, Running Loss train 230.591796875, Time 0.09156322479248047, Overall 2.744208574295044 \n",
            "Fold 4/4, Epoch 4/19, Running Loss train 10.980561755952381, Time 0.09166550636291504, Overall 2.744310140609741 \n",
            "Fold 4/4, Epoch 4/19, Minibatch 0/0, Phase valid, Running Loss valid 123.52296447753906, Time 0.09680366516113281, Overall 2.749448537826538 \n",
            "Fold 4/4, Epoch 4/19, Running Loss valid 24.704592895507812, Time 0.09691643714904785, Overall 2.749561071395874 \n",
            "Fold 4/4, Epoch 5/19, Minibatch 0/0, Phase train, Running Loss train 286.60186767578125, Time 0.11042571067810059, Overall 2.763071060180664 \n",
            "Fold 4/4, Epoch 5/19, Running Loss train 13.647707984561013, Time 0.11052703857421875, Overall 2.763171672821045 \n",
            "Fold 4/4, Epoch 5/19, Minibatch 0/0, Phase valid, Running Loss valid 78.61717987060547, Time 0.11610960960388184, Overall 2.7687559127807617 \n",
            "Fold 4/4, Epoch 5/19, Running Loss valid 15.723435974121093, Time 0.11623191833496094, Overall 2.7688775062561035 \n",
            "Fold 4/4, Epoch 6/19, Minibatch 0/0, Phase train, Running Loss train 281.26922607421875, Time 0.13875460624694824, Overall 2.79140043258667 \n",
            "Fold 4/4, Epoch 6/19, Running Loss train 13.393772670200892, Time 0.13887500762939453, Overall 2.7915196418762207 \n",
            "Fold 4/4, Epoch 6/19, Minibatch 0/0, Phase valid, Running Loss valid 102.33460235595703, Time 0.1440291404724121, Overall 2.7966744899749756 \n",
            "Fold 4/4, Epoch 6/19, Running Loss valid 20.466920471191408, Time 0.14415335655212402, Overall 2.796797752380371 \n",
            "Fold 4/4, Epoch 7/19, Minibatch 0/0, Phase train, Running Loss train 304.8406066894531, Time 0.15809106826782227, Overall 2.8107361793518066 \n",
            "Fold 4/4, Epoch 7/19, Running Loss train 14.516219366164435, Time 0.15820574760437012, Overall 2.8108503818511963 \n",
            "Fold 4/4, Epoch 7/19, Minibatch 0/0, Phase valid, Running Loss valid 116.80184936523438, Time 0.16351008415222168, Overall 2.8161559104919434 \n",
            "Fold 4/4, Epoch 7/19, Running Loss valid 23.360369873046874, Time 0.16362309455871582, Overall 2.816267967224121 \n",
            "Fold 4/4, Epoch 8/19, Minibatch 0/0, Phase train, Running Loss train 218.84918212890625, Time 0.1777958869934082, Overall 2.8304412364959717 \n",
            "Fold 4/4, Epoch 8/19, Running Loss train 10.421389625186013, Time 0.17789363861083984, Overall 2.830538511276245 \n",
            "Fold 4/4, Epoch 8/19, Minibatch 0/0, Phase valid, Running Loss valid 84.73246765136719, Time 0.18301868438720703, Overall 2.8356642723083496 \n",
            "Fold 4/4, Epoch 8/19, Running Loss valid 16.946493530273436, Time 0.18316340446472168, Overall 2.835808515548706 \n",
            "Fold 4/4, Epoch 9/19, Minibatch 0/0, Phase train, Running Loss train 311.3430480957031, Time 0.19709348678588867, Overall 2.849738597869873 \n",
            "Fold 4/4, Epoch 9/19, Running Loss train 14.82585943312872, Time 0.19719767570495605, Overall 2.8498423099517822 \n",
            "Fold 4/4, Epoch 9/19, Minibatch 0/0, Phase valid, Running Loss valid 63.94920349121094, Time 0.20232486724853516, Overall 2.8549699783325195 \n",
            "Fold 4/4, Epoch 9/19, Running Loss valid 12.789840698242188, Time 0.20244550704956055, Overall 2.855090379714966 \n",
            "Fold 4/4, Epoch 10/19, Minibatch 0/0, Phase train, Running Loss train 182.5721435546875, Time 0.2171034812927246, Overall 2.869748592376709 \n",
            "Fold 4/4, Epoch 10/19, Running Loss train 8.693911597842263, Time 0.21720647811889648, Overall 2.8698513507843018 \n",
            "Fold 4/4, Epoch 10/19, Minibatch 0/0, Phase valid, Running Loss valid 84.10595703125, Time 0.22233295440673828, Overall 2.8749780654907227 \n",
            "Fold 4/4, Epoch 10/19, Running Loss valid 16.82119140625, Time 0.22245526313781738, Overall 2.8751003742218018 \n",
            "Fold 4/4, Epoch 11/19, Minibatch 0/0, Phase train, Running Loss train 269.44464111328125, Time 0.23626923561096191, Overall 2.8889145851135254 \n",
            "Fold 4/4, Epoch 11/19, Running Loss train 12.830697195870536, Time 0.23636507987976074, Overall 2.889009714126587 \n",
            "Fold 4/4, Epoch 11/19, Minibatch 0/0, Phase valid, Running Loss valid 94.4412841796875, Time 0.2415001392364502, Overall 2.8941454887390137 \n",
            "Fold 4/4, Epoch 11/19, Running Loss valid 18.8882568359375, Time 0.24161052703857422, Overall 2.8942551612854004 \n",
            "Fold 4/4, Epoch 12/19, Minibatch 0/0, Phase train, Running Loss train 250.40087890625, Time 0.2554666996002197, Overall 2.9081122875213623 \n",
            "Fold 4/4, Epoch 12/19, Running Loss train 11.923851376488095, Time 0.2555665969848633, Overall 2.9082114696502686 \n",
            "Fold 4/4, Epoch 12/19, Minibatch 0/0, Phase valid, Running Loss valid 109.69734954833984, Time 0.2607452869415283, Overall 2.9133903980255127 \n",
            "Fold 4/4, Epoch 12/19, Running Loss valid 21.93946990966797, Time 0.2608613967895508, Overall 2.913506031036377 \n",
            "Fold 4/4, Epoch 13/19, Minibatch 0/0, Phase train, Running Loss train 136.77487182617188, Time 0.2755320072174072, Overall 2.92817759513855 \n",
            "Fold 4/4, Epoch 13/19, Running Loss train 6.513089134579613, Time 0.2756350040435791, Overall 2.9282796382904053 \n",
            "Fold 4/4, Epoch 13/19, Minibatch 0/0, Phase valid, Running Loss valid 84.2767105102539, Time 0.28108906745910645, Overall 2.933734655380249 \n",
            "Fold 4/4, Epoch 13/19, Running Loss valid 16.85534210205078, Time 0.2812080383300781, Overall 2.9338529109954834 \n",
            "Epoch 00014: reducing learning rate of group 0 to 5.0000e-02.\n",
            "Fold 4/4, Epoch 14/19, Minibatch 0/0, Phase train, Running Loss train 286.5164794921875, Time 0.2956721782684326, Overall 2.948317289352417 \n",
            "Fold 4/4, Epoch 14/19, Running Loss train 13.643641880580358, Time 0.2957925796508789, Overall 2.948437452316284 \n",
            "Fold 4/4, Epoch 14/19, Minibatch 0/0, Phase valid, Running Loss valid 64.07288360595703, Time 0.3011133670806885, Overall 2.953758716583252 \n",
            "Fold 4/4, Epoch 14/19, Running Loss valid 12.814576721191406, Time 0.3012392520904541, Overall 2.9538838863372803 \n",
            "Fold 4/4, Epoch 15/19, Minibatch 0/0, Phase train, Running Loss train 200.6886749267578, Time 0.31529879570007324, Overall 2.9679441452026367 \n",
            "Fold 4/4, Epoch 15/19, Running Loss train 9.556603567940849, Time 0.3154001235961914, Overall 2.9680447578430176 \n",
            "Fold 4/4, Epoch 15/19, Minibatch 0/0, Phase valid, Running Loss valid 68.1082534790039, Time 0.32121729850769043, Overall 2.973862648010254 \n",
            "Fold 4/4, Epoch 15/19, Running Loss valid 13.621650695800781, Time 0.32134461402893066, Overall 2.973989248275757 \n",
            "Fold 4/4, Epoch 16/19, Minibatch 0/0, Phase train, Running Loss train 211.82745361328125, Time 0.3355753421783447, Overall 2.98822021484375 \n",
            "Fold 4/4, Epoch 16/19, Running Loss train 10.08702160063244, Time 0.3357052803039551, Overall 2.9883501529693604 \n",
            "Fold 4/4, Epoch 16/19, Minibatch 0/0, Phase valid, Running Loss valid 64.29167175292969, Time 0.34104061126708984, Overall 2.9936859607696533 \n",
            "Fold 4/4, Epoch 16/19, Running Loss valid 12.858334350585938, Time 0.341170072555542, Overall 2.993814706802368 \n",
            "Fold 4/4, Epoch 17/19, Minibatch 0/0, Phase train, Running Loss train 211.84683227539062, Time 0.3553049564361572, Overall 3.0079503059387207 \n",
            "Fold 4/4, Epoch 17/19, Running Loss train 10.08794439406622, Time 0.3554372787475586, Overall 3.008082389831543 \n",
            "Fold 4/4, Epoch 17/19, Minibatch 0/0, Phase valid, Running Loss valid 110.48436737060547, Time 0.361663818359375, Overall 3.0143091678619385 \n",
            "Fold 4/4, Epoch 17/19, Running Loss valid 22.096873474121093, Time 0.3618357181549072, Overall 3.014482021331787 \n",
            "Fold 4/4, Epoch 18/19, Minibatch 0/0, Phase train, Running Loss train 216.23460388183594, Time 0.378741979598999, Overall 3.0313875675201416 \n",
            "Fold 4/4, Epoch 18/19, Running Loss train 10.296885899135045, Time 0.37887120246887207, Overall 3.0315158367156982 \n",
            "Fold 4/4, Epoch 18/19, Minibatch 0/0, Phase valid, Running Loss valid 83.25033569335938, Time 0.38559389114379883, Overall 3.0382392406463623 \n",
            "Fold 4/4, Epoch 18/19, Running Loss valid 16.650067138671876, Time 0.38571619987487793, Overall 3.038360834121704 \n",
            "Fold 4/4, Epoch 19/19, Minibatch 0/0, Phase train, Running Loss train 189.91323852539062, Time 0.3992147445678711, Overall 3.0518598556518555 \n",
            "Fold 4/4, Epoch 19/19, Running Loss train 9.043487548828125, Time 0.39931273460388184, Overall 3.051957130432129 \n",
            "Fold 4/4, Epoch 19/19, Minibatch 0/0, Phase valid, Running Loss valid 81.71934509277344, Time 0.4044933319091797, Overall 3.057138442993164 \n",
            "Fold 4/4, Epoch 19/19, Running Loss valid 16.343869018554688, Time 0.4045987129211426, Overall 3.057243585586548 \n",
            "It took 3.0587947368621826 to train the model.\n",
            "Figure(640x480)\n",
            "Figure(640x480)\n",
            "Figure(640x480)\n",
            "Figure(640x480)\n",
            "Figure(640x480)\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'P_2,5,10', 'ndcg_cut_2,5,10', 'map_cut_2,5,10', 'recall_2,5,10'} ...\n",
            "Averaging ...\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f1.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 129.71it/s]\n",
            "100% 5/5 [00:00<00:00, 81284.96it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 7592.87it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f1.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.05859541893005371\n",
            "####################################################################################################\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f3.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 148.42it/s]\n",
            "100% 5/5 [00:00<00:00, 77385.68it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 7989.15it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f3.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.05463123321533203\n",
            "####################################################################################################\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f0.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 137.24it/s]\n",
            "100% 5/5 [00:00<00:00, 83220.32it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 7481.81it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f0.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.057726383209228516\n",
            "####################################################################################################\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f4.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 121.14it/s]\n",
            "100% 5/5 [00:00<00:00, 59747.92it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 4657.23it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f4.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.06519198417663574\n",
            "####################################################################################################\n",
            "####################################################################################################\n",
            "Reranking for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f2.test.pred ...\n",
            "Loading attribute labels ...\n",
            "Loading attribute labels failed! Generating popularity labels at ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/rerank/stats.pkl ...\n",
            "Loading reranking results ...\n",
            "Loading re-ranking results failed! Reranking the predictions based on det_greedy for top-None ...\n",
            "100% 5/5 [00:00<00:00, 142.20it/s]\n",
            "100% 5/5 [00:00<00:00, 97541.95it/s]\n",
            "Loading fairness evaluation results before and after reranking ...\n",
            "Loading fairness results failed! Evaluating fairness metric {'ndkl'} ...\n",
            "100% 5/5 [00:00<00:00, 7839.82it/s]\n",
            "Loading utility metric evaluation results before and after reranking ...\n",
            " Loading utility metric results failed! Evaluating utility metric {'map_cut_2,5,10'} ...\n",
            "Building pytrec_eval input for 5 instances ...\n",
            "Evaluating {'map_cut_2,5,10'} ...\n",
            "Averaging ...\n",
            "Pipeline for the baseline ../output/toy.dblp.v12.json/bnn/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/t31.s10.m13.l[128].lr0.1.b128.e20.nns3.nsunigram_b.s1/f2.test.pred completed by <_MainProcess name='MainProcess' parent=None started>! 0.05569291114807129\n",
            "####################################################################################################\n"
          ]
        }
      ],
      "source": [
        "!../opentf/bin/python3 main.py -data ../data/raw/dblp/toy.dblp.v12.json -domain dblp -model fnn bnn -fairness det_greedy -attribute popularity"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I1ktJb2ghLXu",
        "outputId": "a74e0874-049e-4d77-ed8f-a59e8729e743"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "bnn  fnn\n"
          ]
        }
      ],
      "source": [
        "!ls ../output/toy.dblp.v12.json/"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 2
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython2",
      "version": "2.7.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}