kiva-cf-polara.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kiva collaborative filtering\n",
    "## Polara framework"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# essentials\n",
    "import os\n",
    "import sys\n",
    "import csv\n",
    "import itertools\n",
    "import copy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score, precision_recall_curve, auc\n",
    "from scipy.sparse import csr_matrix, lil_matrix\n",
    "\n",
    "# polara framework\n",
    "from polara.recommender.data import RecommenderData\n",
    "from polara.recommender.models import SVDModel, PopularityModel, RandomModel, CooccurrenceModel\n",
    "from polara.recommender.external.implicit.ialswrapper import ImplicitALS\n",
    "from polara.recommender.external.implicit.ibprwrapper import ImplicitBPR\n",
    "from polara.datasets.movielens import get_movielens_data\n",
    "from polara.evaluation import evaluation_engine as ee\n",
    "from polara.evaluation.plotting import show_hit_rates, show_precision_recall, show_ranking, show_relevance\n",
    "\n",
    "# utilities\n",
    "import codecs\n",
    "import logging\n",
    "import time\n",
    "import tqdm\n",
    "\n",
    "# visualization libraries\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# serialization\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', 40)\n",
    "pd.set_option('display.max_rows', 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: MKL_NUM_THREADS=1\n"
     ]
    }
   ],
   "source": [
    "%env MKL_NUM_THREADS=1\n",
    "logging.basicConfig(level=logging.DEBUG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Pickle loads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loans_table = pickle.load(open(\"pickle/loans_table.p\", \"rb\"))\n",
    "funded_loans_table = pickle.load(open(\"pickle/funded_loans_table.p\", \"rb\"))\n",
    "funded_loan_ids_set = pickle.load(open(\"pickle/funded_loan_ids_set.p\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utility_matrix = pickle.load(open(\"pickle/utility_matrix.p\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utility_matrix_df = pickle.load(open(\"pickle/utility_matrix_df.p\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utility_matrix_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Dataset reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loans_table = pd.read_csv('additional-kiva-snapshot/loans.csv')\n",
    "loans_table = loans_table.sort_values(by='raised_time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "funded_loans_table = loans_table[loans_table.status == 'funded']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "START_DATE = '2013-10-01'\n",
    "END_DATE = '2015-05-01'\n",
    "\n",
    "mask = (funded_loans_table['raised_time'] > START_DATE) & (funded_loans_table['raised_time'] <= END_DATE)\n",
    "funded_loans_table = funded_loans_table.loc[mask]\n",
    "del mask\n",
    "\n",
    "funded_loan_ids_set = set(funded_loans_table['loan_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(loans_table, open(\"pickle/loans_table.p\", \"wb\"))\n",
    "# pickle.dump(funded_loans_table, open(\"pickle/funded_loans_table.p\", \"wb\"))\n",
    "# pickle.dump(funded_loan_ids_set, open(\"pickle/funded_loan_ids_set.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# free ram\n",
    "del loans_table\n",
    "del funded_loans_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loans = set()\n",
    "lenders = set()\n",
    "loans_lenders_dict = {}\n",
    "\n",
    "with open('additional-kiva-snapshot/loans_lenders.csv', newline='', encoding=\"utf8\") as csvfile:\n",
    "    csv_reader = csv.reader(csvfile)\n",
    "    line_num = 0\n",
    "    for row in csv_reader:\n",
    "        if line_num == 0:\n",
    "            line_num += 1\n",
    "            continue\n",
    "        loan_id, lender_ids = row\n",
    "        loan_id = int(loan_id)\n",
    "        if loan_id not in funded_loan_ids_set:\n",
    "            continue\n",
    "        \n",
    "        loans.add(loan_id)\n",
    "        new_lenders = set(lender_ids.split(\", \"))\n",
    "        loans_lenders_dict[loan_id] = new_lenders\n",
    "        lenders.update(new_lenders)\n",
    "        line_num += 1\n",
    "\n",
    "loans = list(loans)\n",
    "lenders = list(lenders)\n",
    "\n",
    "print('Loans-lenders dict filled')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Utility matrix creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lenders_reverse_index = {k: v for v, k in enumerate(lenders)}\n",
    "utility_matrix = lil_matrix((len(loans), len(lenders)), dtype=np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for loan_index, loan in enumerate(loans):\n",
    "    for lender in loans_lenders_dict[loan]:\n",
    "        lender_index = lenders_reverse_index[lender]\n",
    "        utility_matrix[loan_index, lender_index] = 1.0\n",
    "\n",
    "print('Filled utility matrix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utility_matrix = utility_matrix.tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(utility_matrix, open(\"pickle/utility_matrix.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Conversion to SparseDataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utility_matrix_df = pd.SparseDataFrame(utility_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(utility_matrix_df, open(\"pickle/utility_matrix_df.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_matrix_to_csv(matrix, filename):\n",
    "    if not isinstance(matrix, scipy.sparse.coo_matrix):\n",
    "        matrix = matrix.tocoo()\n",
    "    \n",
    "    with open(filename, 'w', encoding='utf-8') as f:\n",
    "        f.write(\"%s,%s,%s\\n\" % ('itemid', 'userid', 'feedback'))\n",
    "        for i,j,v in zip(matrix.row, matrix.col, matrix.data):\n",
    "            f.write(\"%s,%s,%s\\n\" % (i, j, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Kiva dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "kiva_dataframe = pd.read_csv('kiva_dataframe.csv', engine='c')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5339455, 3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kiva_dataframe.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>itemid</th>\n",
       "      <th>userid</th>\n",
       "      <th>feedback</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7703</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>8354</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>9000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>10247</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>21284</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   itemid  userid  feedback\n",
       "0       0    7703       1.0\n",
       "1       0    8354       1.0\n",
       "2       0    9000       1.0\n",
       "3       0   10247       1.0\n",
       "4       0   21284       1.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kiva_dataframe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Polara example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define models\n",
    "ml_data = get_movielens_data(get_genres=False)\n",
    "data_model = RecommenderData(ml_data, 'userid', 'movieid', 'rating')\n",
    "\n",
    "# data_model = RecommenderData(kiva_dataframe, 'userid', 'itemid', 'feedback')\n",
    "\n",
    "svd = SVDModel(data_model)\n",
    "popular = PopularityModel(data_model)\n",
    "random = RandomModel(data_model)\n",
    "models = [svd, popular, random]\n",
    "\n",
    "metrics = ['ranking', 'relevance'] # metrics for evaluation: NDGC, Precision, Recall, etc.\n",
    "folds = [1, 2, 3, 4, 5] # use all 5 folds for cross-validation (default)\n",
    "topk_values = [1, 5, 10, 20, 50] # values of k to experiment with\n",
    "\n",
    "# run 5-fold CV experiment\n",
    "result = ee.run_cv_experiment(models, folds, metrics,\n",
    "                              fold_experiment=ee.topk_test,\n",
    "                              topk_list=topk_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate average values across all folds for e.g. relevance metrics\n",
    "scores = result.mean(axis=0, level=['top-n', 'model']) # use .std instead of .mean for standard deviation\n",
    "# scores.xs('nDCG', level='metric', axis=1).unstack('model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing data...\n",
      "207 unique itemid's within 228 holdout interactions were filtered. Reason: not in the training data.\n",
      "85452 unique userid's within 136096 holdout interactions were filtered. Reason: not in the training data.\n",
      "114 of 44225 userid's were filtered out from holdout. Reason: incompatible number of items.\n",
      "Done.\n",
      "There are 5070574 events in the training and 132333 events in the holdout.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:root:Prepared data in 123.20s\n"
     ]
    }
   ],
   "source": [
    "data_model = RecommenderData(kiva_dataframe, 'userid', 'itemid', 'feedback', seed=0)\n",
    "data_model.warm_start = False\n",
    "\n",
    "start = time.time()\n",
    "data_model.prepare()\n",
    "logging.debug(\"Prepared data in %0.2fs\", time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "random = RandomModel(data_model)\n",
    "popular = PopularityModel(data_model)\n",
    "svd = SVDModel(data_model)\n",
    "\n",
    "# bpr = ImplicitBPR(data_model)\n",
    "# bpr.rank = 200\n",
    "# bpr.num_epochs = 100\n",
    "# bpr.use_gpu = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate a list of ALS models by parameter grid:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_base_model(model_name):\n",
    "    if (model_name == 'bpr'):\n",
    "        model = ImplicitBPR(data_model)\n",
    "    else:\n",
    "        model = ImplicitALS(data_model)\n",
    "        model.epsilon = 1e-8\n",
    "        model.weight_func = np.log2\n",
    "    \n",
    "    model.use_gpu = False\n",
    "    return model\n",
    "\n",
    "def get_grid_models(cv_param_grid, model_name):\n",
    "    models = []\n",
    "    keys, values = zip(*cv_param_grid.items())\n",
    "    for v in itertools.product(*values):\n",
    "        params = dict(zip(keys, v))\n",
    "        next_model = get_base_model(model_name)\n",
    "        name = next_model.method\n",
    "        for k, v in params.items():\n",
    "            setattr(next_model, k, v)\n",
    "            name += '_%s-%s' % (k, v)\n",
    "        \n",
    "        next_model.method = name\n",
    "        models.append(next_model)\n",
    "    return models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cross-validation setup:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_param_grid = {\n",
    "    'learning_rate': [0.05, 0.1, 0.5, 1]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "basic_models = [random, popular, svd]\n",
    "\n",
    "# als_models = get_grid_models(cv_param_grid)\n",
    "# models = basic_models + als_models\n",
    "\n",
    "bpr_models = get_grid_models(cv_param_grid, 'bpr')\n",
    "models = basic_models + bpr_models\n",
    "\n",
    "metrics = ['ranking', 'relevance'] # metrics for evaluation: NDGC, Precision, Recall, etc.\n",
    "folds = [1, 2, 3]\n",
    "topk_values = [5, 10, 20] # values of k to experiment with"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing data...\n",
      "211 unique itemid's within 234 holdout interactions were filtered. Reason: not in the training data.\n",
      "85443 unique userid's within 136172 holdout interactions were filtered. Reason: not in the training data.\n",
      "100 of 44237 userid's were filtered out from holdout. Reason: incompatible number of items.\n",
      "Done.\n",
      "There are 5070446 events in the training and 132411 events in the holdout.\n",
      "PureSVD training time: 3.850s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|█████████████████████████████████████████████████| 100/100 [04:20<00:00,  2.59s/it, correct=99.57%, skipped=0.74%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.05 training time: 04m:22s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|█████████████████████████████████████████████████| 100/100 [04:16<00:00,  2.39s/it, correct=99.62%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.1 training time: 04m:18s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|██████████████████████████████████████████████████| 100/100 [03:32<00:00,  2.19s/it, correct=0.00%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.5 training time: 03m:34s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|██████████████████████████████████████████████████| 100/100 [03:29<00:00,  2.07s/it, correct=0.00%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-1 training time: 03m:31s\n",
      "Evaluated model RND in 268.44s\n",
      "Evaluated model MP in 116.86s\n",
      "Evaluated model PureSVD in 182.33s\n",
      "Evaluated model BPR_learning_rate-0.05 in 174.98s\n",
      "Evaluated model BPR_learning_rate-0.1 in 173.77s\n",
      "Evaluated model BPR_learning_rate-0.5 in 117.55s\n",
      "Evaluated model BPR_learning_rate-1 in 117.44s\n",
      "Evaluated model RND in 0.47s\n",
      "Evaluated model MP in 0.47s\n",
      "Evaluated model PureSVD in 0.47s\n",
      "Evaluated model BPR_learning_rate-0.05 in 0.48s\n",
      "Evaluated model BPR_learning_rate-0.1 in 0.47s\n",
      "Evaluated model BPR_learning_rate-0.5 in 0.47s\n",
      "Evaluated model BPR_learning_rate-1 in 0.47s\n",
      "Evaluated model RND in 0.45s\n",
      "Evaluated model MP in 0.45s\n",
      "Evaluated model PureSVD in 0.44s\n",
      "Evaluated model BPR_learning_rate-0.05 in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.1 in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.5 in 0.47s\n",
      "Evaluated model BPR_learning_rate-1 in 0.45s\n",
      "Preparing data...\n",
      "182 unique itemid's within 197 holdout interactions were filtered. Reason: not in the training data.\n",
      "85704 unique userid's within 136225 holdout interactions were filtered. Reason: not in the training data.\n",
      "89 of 43984 userid's were filtered out from holdout. Reason: incompatible number of items.\n",
      "Done.\n",
      "There are 5071181 events in the training and 131685 events in the holdout.\n",
      "PureSVD training time: 3.261s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|█████████████████████████████████████████████████| 100/100 [03:24<00:00,  2.05s/it, correct=99.59%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.05 training time: 03m:26s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|█████████████████████████████████████████████████| 100/100 [03:25<00:00,  2.04s/it, correct=99.61%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.1 training time: 03m:27s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|██████████████████████████████████████████████████| 100/100 [03:19<00:00,  1.93s/it, correct=0.00%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.5 training time: 03m:21s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|██████████████████████████████████████████████████| 100/100 [03:14<00:00,  1.94s/it, correct=0.00%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-1 training time: 03m:16s\n",
      "Evaluated model RND in 258.03s\n",
      "Evaluated model MP in 115.57s\n",
      "Evaluated model PureSVD in 190.55s\n",
      "Evaluated model BPR_learning_rate-0.05 in 164.58s\n",
      "Evaluated model BPR_learning_rate-0.1 in 161.48s\n",
      "Evaluated model BPR_learning_rate-0.5 in 113.01s\n",
      "Evaluated model BPR_learning_rate-1 in 113.21s\n",
      "Evaluated model RND in 0.47s\n",
      "Evaluated model MP in 0.47s\n",
      "Evaluated model PureSVD in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.05 in 0.47s\n",
      "Evaluated model BPR_learning_rate-0.1 in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.5 in 0.47s\n",
      "Evaluated model BPR_learning_rate-1 in 0.47s\n",
      "Evaluated model RND in 0.44s\n",
      "Evaluated model MP in 0.45s\n",
      "Evaluated model PureSVD in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.05 in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.1 in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.5 in 0.45s\n",
      "Evaluated model BPR_learning_rate-1 in 0.45s\n",
      "Preparing data...\n",
      "215 unique itemid's within 227 holdout interactions were filtered. Reason: not in the training data.\n",
      "85431 unique userid's within 135552 holdout interactions were filtered. Reason: not in the training data.\n",
      "97 of 44250 userid's were filtered out from holdout. Reason: incompatible number of items.\n",
      "Done.\n",
      "There are 5071031 events in the training and 132459 events in the holdout.\n",
      "PureSVD training time: 3.407s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|█████████████████████████████████████████████████| 100/100 [03:13<00:00,  1.94s/it, correct=99.58%, skipped=0.74%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.05 training time: 03m:15s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|█████████████████████████████████████████████████| 100/100 [03:13<00:00,  1.94s/it, correct=99.61%, skipped=0.74%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.1 training time: 03m:15s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|██████████████████████████████████████████████████| 100/100 [03:13<00:00,  1.93s/it, correct=0.00%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-0.5 training time: 03m:14s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:implicit:Running 100 BPR training epochs\n",
      "100%|██████████████████████████████████████████████████| 100/100 [03:12<00:00,  1.93s/it, correct=0.00%, skipped=0.73%]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BPR_learning_rate-1 training time: 03m:14s\n",
      "Evaluated model RND in 259.43s\n",
      "Evaluated model MP in 111.17s\n",
      "Evaluated model PureSVD in 169.83s\n",
      "Evaluated model BPR_learning_rate-0.05 in 168.55s\n",
      "Evaluated model BPR_learning_rate-0.1 in 163.83s\n",
      "Evaluated model BPR_learning_rate-0.5 in 113.70s\n",
      "Evaluated model BPR_learning_rate-1 in 114.03s\n",
      "Evaluated model RND in 0.47s\n",
      "Evaluated model MP in 0.45s\n",
      "Evaluated model PureSVD in 0.47s\n",
      "Evaluated model BPR_learning_rate-0.05 in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.1 in 0.47s\n",
      "Evaluated model BPR_learning_rate-0.5 in 0.47s\n",
      "Evaluated model BPR_learning_rate-1 in 0.45s\n",
      "Evaluated model RND in 0.47s\n",
      "Evaluated model MP in 0.44s\n",
      "Evaluated model PureSVD in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.05 in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.1 in 0.45s\n",
      "Evaluated model BPR_learning_rate-0.5 in 0.45s\n",
      "Evaluated model BPR_learning_rate-1 in 0.45s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:root:Cross-validation experiment finished in 6335.30s\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# run 3-fold CV experiment\n",
    "result = ee.run_cv_experiment(models, folds, metrics,\n",
    "                              fold_experiment=ee.topk_test,\n",
    "                              topk_list=topk_values)\n",
    "\n",
    "\n",
    "logging.debug(\"Cross-validation experiment finished in %0.2fs\", time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(result, open(\"eval_results/result_7.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th colspan=\"5\" halign=\"left\">relevance</th>\n",
       "      <th colspan=\"2\" halign=\"left\">ranking</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fallout</th>\n",
       "      <th>specifity</th>\n",
       "      <th>miss_rate</th>\n",
       "      <th>nDCG</th>\n",
       "      <th>nDCL</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fold</th>\n",
       "      <th>top-n</th>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">5</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999977</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.001115</td>\n",
       "      <td>0.001858</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998142</td>\n",
       "      <td>0.001654</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.001355</td>\n",
       "      <td>0.002258</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.997742</td>\n",
       "      <td>0.002124</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.002388</td>\n",
       "      <td>0.003980</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.996020</td>\n",
       "      <td>0.003662</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.002660</td>\n",
       "      <td>0.004433</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995567</td>\n",
       "      <td>0.004208</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">10</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999970</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.000834</td>\n",
       "      <td>0.002779</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.997221</td>\n",
       "      <td>0.002047</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.000931</td>\n",
       "      <td>0.003104</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.996896</td>\n",
       "      <td>0.002503</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.001881</td>\n",
       "      <td>0.006268</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.993732</td>\n",
       "      <td>0.004693</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.002023</td>\n",
       "      <td>0.006744</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.993256</td>\n",
       "      <td>0.005254</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">20</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999940</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.000896</td>\n",
       "      <td>0.005974</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.994026</td>\n",
       "      <td>0.003209</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.000676</td>\n",
       "      <td>0.004509</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995491</td>\n",
       "      <td>0.003006</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.001380</td>\n",
       "      <td>0.009199</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.990801</td>\n",
       "      <td>0.005725</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.001562</td>\n",
       "      <td>0.010415</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.989585</td>\n",
       "      <td>0.006556</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.000106</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999894</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.000106</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999894</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">2</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">5</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999977</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.001180</td>\n",
       "      <td>0.001967</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998033</td>\n",
       "      <td>0.001668</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.001176</td>\n",
       "      <td>0.001959</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998041</td>\n",
       "      <td>0.001879</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.002201</td>\n",
       "      <td>0.003668</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.996332</td>\n",
       "      <td>0.003425</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.002433</td>\n",
       "      <td>0.004055</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995945</td>\n",
       "      <td>0.003716</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999992</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999992</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">10</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999954</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.000875</td>\n",
       "      <td>0.002916</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.997084</td>\n",
       "      <td>0.002077</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.000932</td>\n",
       "      <td>0.003106</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.996894</td>\n",
       "      <td>0.002402</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.001702</td>\n",
       "      <td>0.005673</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.994327</td>\n",
       "      <td>0.004327</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.001961</td>\n",
       "      <td>0.006538</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.993462</td>\n",
       "      <td>0.004835</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999985</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999985</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">20</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000076</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999924</td>\n",
       "      <td>0.000037</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.000945</td>\n",
       "      <td>0.006303</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.993697</td>\n",
       "      <td>0.003311</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.000687</td>\n",
       "      <td>0.004579</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995421</td>\n",
       "      <td>0.002935</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.001305</td>\n",
       "      <td>0.008703</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.991297</td>\n",
       "      <td>0.005401</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.001472</td>\n",
       "      <td>0.009811</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.990189</td>\n",
       "      <td>0.005991</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000031</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999795</td>\n",
       "      <td>0.000076</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000031</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999795</td>\n",
       "      <td>0.000076</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">3</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">5</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999977</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.001173</td>\n",
       "      <td>0.001955</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998045</td>\n",
       "      <td>0.001704</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.000942</td>\n",
       "      <td>0.001570</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998430</td>\n",
       "      <td>0.001443</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.002156</td>\n",
       "      <td>0.003594</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.996406</td>\n",
       "      <td>0.003390</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.002605</td>\n",
       "      <td>0.004341</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995659</td>\n",
       "      <td>0.004005</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999992</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999992</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">10</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999932</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.000815</td>\n",
       "      <td>0.002718</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.997282</td>\n",
       "      <td>0.002032</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.000786</td>\n",
       "      <td>0.002620</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.997380</td>\n",
       "      <td>0.001926</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.001712</td>\n",
       "      <td>0.005707</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.994293</td>\n",
       "      <td>0.004347</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.002038</td>\n",
       "      <td>0.006795</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.993205</td>\n",
       "      <td>0.005109</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999977</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999977</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">20</th>\n",
       "      <th>RND</th>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000136</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999864</td>\n",
       "      <td>0.000059</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MP</th>\n",
       "      <td>0.000887</td>\n",
       "      <td>0.005911</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.994089</td>\n",
       "      <td>0.003188</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PureSVD</th>\n",
       "      <td>0.000582</td>\n",
       "      <td>0.003880</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.996120</td>\n",
       "      <td>0.002375</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.05</th>\n",
       "      <td>0.001302</td>\n",
       "      <td>0.008682</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.991318</td>\n",
       "      <td>0.005403</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.1</th>\n",
       "      <td>0.001553</td>\n",
       "      <td>0.010350</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.989650</td>\n",
       "      <td>0.006364</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-0.5</th>\n",
       "      <td>0.000029</td>\n",
       "      <td>0.000196</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999804</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BPR_learning_rate-1</th>\n",
       "      <td>0.000029</td>\n",
       "      <td>0.000196</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999804</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "type                              relevance                              \\\n",
       "metric                            precision    recall fallout specifity   \n",
       "fold top-n model                                                          \n",
       "1    5     RND                     0.000014  0.000023    None      None   \n",
       "           MP                      0.001115  0.001858    None      None   \n",
       "           PureSVD                 0.001355  0.002258    None      None   \n",
       "           BPR_learning_rate-0.05  0.002388  0.003980    None      None   \n",
       "           BPR_learning_rate-0.1   0.002660  0.004433    None      None   \n",
       "           BPR_learning_rate-0.5   0.000000  0.000000    None      None   \n",
       "           BPR_learning_rate-1     0.000000  0.000000    None      None   \n",
       "     10    RND                     0.000009  0.000030    None      None   \n",
       "           MP                      0.000834  0.002779    None      None   \n",
       "           PureSVD                 0.000931  0.003104    None      None   \n",
       "           BPR_learning_rate-0.05  0.001881  0.006268    None      None   \n",
       "           BPR_learning_rate-0.1   0.002023  0.006744    None      None   \n",
       "           BPR_learning_rate-0.5   0.000000  0.000000    None      None   \n",
       "           BPR_learning_rate-1     0.000000  0.000000    None      None   \n",
       "     20    RND                     0.000009  0.000060    None      None   \n",
       "           MP                      0.000896  0.005974    None      None   \n",
       "           PureSVD                 0.000676  0.004509    None      None   \n",
       "           BPR_learning_rate-0.05  0.001380  0.009199    None      None   \n",
       "           BPR_learning_rate-0.1   0.001562  0.010415    None      None   \n",
       "           BPR_learning_rate-0.5   0.000016  0.000106    None      None   \n",
       "           BPR_learning_rate-1     0.000016  0.000106    None      None   \n",
       "2    5     RND                     0.000014  0.000023    None      None   \n",
       "           MP                      0.001180  0.001967    None      None   \n",
       "           PureSVD                 0.001176  0.001959    None      None   \n",
       "           BPR_learning_rate-0.05  0.002201  0.003668    None      None   \n",
       "           BPR_learning_rate-0.1   0.002433  0.004055    None      None   \n",
       "           BPR_learning_rate-0.5   0.000005  0.000008    None      None   \n",
       "           BPR_learning_rate-1     0.000005  0.000008    None      None   \n",
       "     10    RND                     0.000014  0.000046    None      None   \n",
       "           MP                      0.000875  0.002916    None      None   \n",
       "           PureSVD                 0.000932  0.003106    None      None   \n",
       "           BPR_learning_rate-0.05  0.001702  0.005673    None      None   \n",
       "           BPR_learning_rate-0.1   0.001961  0.006538    None      None   \n",
       "           BPR_learning_rate-0.5   0.000005  0.000015    None      None   \n",
       "           BPR_learning_rate-1     0.000005  0.000015    None      None   \n",
       "     20    RND                     0.000011  0.000076    None      None   \n",
       "           MP                      0.000945  0.006303    None      None   \n",
       "           PureSVD                 0.000687  0.004579    None      None   \n",
       "           BPR_learning_rate-0.05  0.001305  0.008703    None      None   \n",
       "           BPR_learning_rate-0.1   0.001472  0.009811    None      None   \n",
       "           BPR_learning_rate-0.5   0.000031  0.000205    None      None   \n",
       "           BPR_learning_rate-1     0.000031  0.000205    None      None   \n",
       "3    5     RND                     0.000014  0.000023    None      None   \n",
       "           MP                      0.001173  0.001955    None      None   \n",
       "           PureSVD                 0.000942  0.001570    None      None   \n",
       "           BPR_learning_rate-0.05  0.002156  0.003594    None      None   \n",
       "           BPR_learning_rate-0.1   0.002605  0.004341    None      None   \n",
       "           BPR_learning_rate-0.5   0.000005  0.000008    None      None   \n",
       "           BPR_learning_rate-1     0.000005  0.000008    None      None   \n",
       "     10    RND                     0.000020  0.000068    None      None   \n",
       "           MP                      0.000815  0.002718    None      None   \n",
       "           PureSVD                 0.000786  0.002620    None      None   \n",
       "           BPR_learning_rate-0.05  0.001712  0.005707    None      None   \n",
       "           BPR_learning_rate-0.1   0.002038  0.006795    None      None   \n",
       "           BPR_learning_rate-0.5   0.000007  0.000023    None      None   \n",
       "           BPR_learning_rate-1     0.000007  0.000023    None      None   \n",
       "     20    RND                     0.000020  0.000136    None      None   \n",
       "           MP                      0.000887  0.005911    None      None   \n",
       "           PureSVD                 0.000582  0.003880    None      None   \n",
       "           BPR_learning_rate-0.05  0.001302  0.008682    None      None   \n",
       "           BPR_learning_rate-0.1   0.001553  0.010350    None      None   \n",
       "           BPR_learning_rate-0.5   0.000029  0.000196    None      None   \n",
       "           BPR_learning_rate-1     0.000029  0.000196    None      None   \n",
       "\n",
       "type                                          ranking        \n",
       "metric                            miss_rate      nDCG  nDCL  \n",
       "fold top-n model                                             \n",
       "1    5     RND                     0.999977  0.000027  None  \n",
       "           MP                      0.998142  0.001654  None  \n",
       "           PureSVD                 0.997742  0.002124  None  \n",
       "           BPR_learning_rate-0.05  0.996020  0.003662  None  \n",
       "           BPR_learning_rate-0.1   0.995567  0.004208  None  \n",
       "           BPR_learning_rate-0.5   1.000000  0.000000  None  \n",
       "           BPR_learning_rate-1     1.000000  0.000000  None  \n",
       "     10    RND                     0.999970  0.000030  None  \n",
       "           MP                      0.997221  0.002047  None  \n",
       "           PureSVD                 0.996896  0.002503  None  \n",
       "           BPR_learning_rate-0.05  0.993732  0.004693  None  \n",
       "           BPR_learning_rate-0.1   0.993256  0.005254  None  \n",
       "           BPR_learning_rate-0.5   1.000000  0.000000  None  \n",
       "           BPR_learning_rate-1     1.000000  0.000000  None  \n",
       "     20    RND                     0.999940  0.000041  None  \n",
       "           MP                      0.994026  0.003209  None  \n",
       "           PureSVD                 0.995491  0.003006  None  \n",
       "           BPR_learning_rate-0.05  0.990801  0.005725  None  \n",
       "           BPR_learning_rate-0.1   0.989585  0.006556  None  \n",
       "           BPR_learning_rate-0.5   0.999894  0.000035  None  \n",
       "           BPR_learning_rate-1     0.999894  0.000035  None  \n",
       "2    5     RND                     0.999977  0.000017  None  \n",
       "           MP                      0.998033  0.001668  None  \n",
       "           PureSVD                 0.998041  0.001879  None  \n",
       "           BPR_learning_rate-0.05  0.996332  0.003425  None  \n",
       "           BPR_learning_rate-0.1   0.995945  0.003716  None  \n",
       "           BPR_learning_rate-0.5   0.999992  0.000011  None  \n",
       "           BPR_learning_rate-1     0.999992  0.000011  None  \n",
       "     10    RND                     0.999954  0.000027  None  \n",
       "           MP                      0.997084  0.002077  None  \n",
       "           PureSVD                 0.996894  0.002402  None  \n",
       "           BPR_learning_rate-0.05  0.994327  0.004327  None  \n",
       "           BPR_learning_rate-0.1   0.993462  0.004835  None  \n",
       "           BPR_learning_rate-0.5   0.999985  0.000014  None  \n",
       "           BPR_learning_rate-1     0.999985  0.000014  None  \n",
       "     20    RND                     0.999924  0.000037  None  \n",
       "           MP                      0.993697  0.003311  None  \n",
       "           PureSVD                 0.995421  0.002935  None  \n",
       "           BPR_learning_rate-0.05  0.991297  0.005401  None  \n",
       "           BPR_learning_rate-0.1   0.990189  0.005991  None  \n",
       "           BPR_learning_rate-0.5   0.999795  0.000076  None  \n",
       "           BPR_learning_rate-1     0.999795  0.000076  None  \n",
       "3    5     RND                     0.999977  0.000014  None  \n",
       "           MP                      0.998045  0.001704  None  \n",
       "           PureSVD                 0.998430  0.001443  None  \n",
       "           BPR_learning_rate-0.05  0.996406  0.003390  None  \n",
       "           BPR_learning_rate-0.1   0.995659  0.004005  None  \n",
       "           BPR_learning_rate-0.5   0.999992  0.000005  None  \n",
       "           BPR_learning_rate-1     0.999992  0.000005  None  \n",
       "     10    RND                     0.999932  0.000035  None  \n",
       "           MP                      0.997282  0.002032  None  \n",
       "           PureSVD                 0.997380  0.001926  None  \n",
       "           BPR_learning_rate-0.05  0.994293  0.004347  None  \n",
       "           BPR_learning_rate-0.1   0.993205  0.005109  None  \n",
       "           BPR_learning_rate-0.5   0.999977  0.000011  None  \n",
       "           BPR_learning_rate-1     0.999977  0.000011  None  \n",
       "     20    RND                     0.999864  0.000059  None  \n",
       "           MP                      0.994089  0.003188  None  \n",
       "           PureSVD                 0.996120  0.002375  None  \n",
       "           BPR_learning_rate-0.05  0.991318  0.005403  None  \n",
       "           BPR_learning_rate-0.1   0.989650  0.006364  None  \n",
       "           BPR_learning_rate-0.5   0.999804  0.000067  None  \n",
       "           BPR_learning_rate-1     0.999804  0.000067  None  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate average values across all folds for e.g. relevance metrics\n",
    "scores = result.mean(axis=0, level=['top-n', 'model']) # use .std instead of .mean for standard deviation\n",
    "# scores.xs('recall', level='metric', axis=1).unstack('model')\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "### Variable sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               RecommenderData:   2.0KiB\n",
      "                   ImplicitALS:   2.0KiB\n",
      "                      SVDModel:   1.4KiB\n",
      "               PopularityModel:   1.4KiB\n",
      "                   RandomModel:   1.4KiB\n",
      "                    csr_matrix:   1.0KiB\n",
      "                    lil_matrix:   1.0KiB\n",
      "             CooccurrenceModel:   1.0KiB\n",
      "                           _i1:   917.0B\n",
      "                    als_models:   768.0B\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "def sizeof_fmt(num, suffix='B'):\n",
    "    ''' By Fred Cirera, after https://stackoverflow.com/a/1094933/1870254'''\n",
    "    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n",
    "        if abs(num) < 1024.0:\n",
    "            return \"%3.1f%s%s\" % (num, unit, suffix)\n",
    "        num /= 1024.0\n",
    "    return \"%.1f%s%s\" % (num, 'Yi', suffix)\n",
    "\n",
    "for name, size in sorted(((name, sys.getsizeof(value)) for name,value in locals().items()),\n",
    "                         key= lambda x: -x[1])[:10]:\n",
    "    print(\"{:>30}: {:>8}\".format(name,sizeof_fmt(size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "del kiva_dataframe"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}