mckinsey-recommendation-challenge-solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Approach\n",
    "\n",
    "One way to approach this problem is to treat it as a sequence prediction problem for NLP, where you need to predict the next word given the previous words.\n",
    "\n",
    "Now there may be many ways to predict using this hypothesis . for example calculating conditional probabilities.\n",
    "\n",
    "I have used here a simple method of co-occurence matrix which essentially will be used to predict the next challenge.\n",
    "\n",
    "I will be using both train and test data to calculate the co-occurence matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import the required libraries\n",
    "\n",
    "from collections import Counter\n",
    "from scipy import sparse\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading train and test file\n",
    "\n",
    "train = pd.read_csv(\"./data/train.csv\")\n",
    "test = pd.read_csv(\"./data/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_sequence</th>\n",
       "      <th>user_id</th>\n",
       "      <th>challenge_sequence</th>\n",
       "      <th>challenge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4576_1</td>\n",
       "      <td>4576</td>\n",
       "      <td>1</td>\n",
       "      <td>CI23714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4576_2</td>\n",
       "      <td>4576</td>\n",
       "      <td>2</td>\n",
       "      <td>CI23855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4576_3</td>\n",
       "      <td>4576</td>\n",
       "      <td>3</td>\n",
       "      <td>CI24917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4576_4</td>\n",
       "      <td>4576</td>\n",
       "      <td>4</td>\n",
       "      <td>CI23663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4576_5</td>\n",
       "      <td>4576</td>\n",
       "      <td>5</td>\n",
       "      <td>CI23933</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_sequence  user_id  challenge_sequence challenge\n",
       "0        4576_1     4576                   1   CI23714\n",
       "1        4576_2     4576                   2   CI23855\n",
       "2        4576_3     4576                   3   CI24917\n",
       "3        4576_4     4576                   4   CI23663\n",
       "4        4576_5     4576                   5   CI23933"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating dataset in the required form for Co-occurence matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the train in the long format to wide format\n",
    "\n",
    "wide_train = train.pivot_table(index = \"user_id\", columns=\"challenge_sequence\", values=\"challenge\", aggfunc= lambda x : x).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dropping the user_id, since we won't be needing those for our co-occurrence matrix\n",
    "\n",
    "wide_train.drop([\"user_id\"], axis =1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert each row for a user into a string\n",
    "\n",
    "rows = []\n",
    "for index, row in wide_train.iterrows():\n",
    "    r = \" \".join(row.map(str))\n",
    "    rows.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# converting test to wide format\n",
    "\n",
    "wide_test = test.pivot_table(index = \"user_id\", columns=\"challenge_sequence\", values=\"challenge\", aggfunc= lambda x : x).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving test user_id for future use\n",
    "\n",
    "test_ids = wide_test['user_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dropping user_id from wide test\n",
    "\n",
    "wide_test.drop([\"user_id\"], axis =1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# appending the test strings into the train strings (vertically)\n",
    "\n",
    "for index, row in wide_test.iterrows():\n",
    "    r = \" \".join(row.map(str))\n",
    "    rows.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a corpus\n",
    "\n",
    "thefile = open(\"corpus.txt\",\"w\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for element in rows:\n",
    "    thefile.write(\"%s\\n\"%element)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "thefile.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating co-occurence matrix from the corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading the corpus\n",
    "\n",
    "corpus = open(\"corpus.txt\",\"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a dictionary with key = challenge_name and value = frequency\n",
    "\n",
    "vocab = Counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# updating the vocab dictionary with each line in the corpus\n",
    "\n",
    "for line in corpus:\n",
    "    tokens = line.strip().split()\n",
    "    vocab.update(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# modifying the vocab dictionary to begin creating a mapping of challenge_id to integers\n",
    "\n",
    "vocab = {word: (i, freq) for i, (word, freq) in enumerate(vocab.items())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a reverse mapping from integer to challenge_id to decode the predictions made.\n",
    "\n",
    "id2word = dict((i, word) for word, (i, _) in enumerate(vocab.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a square co-occurence matrix\n",
    "\n",
    "cooccurrences = sparse.lil_matrix((vocab_size, vocab_size),dtype=np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# context window size\n",
    "\n",
    "window_size = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = open(\"corpus.txt\",\"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is where the actual magic is.\n",
    "\n",
    "# Tuneable parameters : window_size, distance\n",
    "\n",
    "\n",
    "for i, line in enumerate(corpus):\n",
    "    tokens = line.strip().split()\n",
    "    token_ids = [vocab[word][0] for word in tokens]\n",
    "    \n",
    "    for center_i, center_id in enumerate(token_ids):\n",
    "        # Collect all word IDs in left window of center word\n",
    "        context_ids = token_ids[max(0, center_i - window_size) : center_i]\n",
    "        contexts_len = len(context_ids)\n",
    "\n",
    "#         print(\"context_ids = %s\" %context_ids)\n",
    "#         print(\"contexts_len = %s\" %contexts_len)\n",
    "\n",
    "        for left_i, left_id in enumerate(context_ids):\n",
    "            # Distance from center word\n",
    "\n",
    "#             print(\"left_i = %s\" %left_i)\n",
    "#             print(\"left_id = %s\" %left_id)\n",
    "\n",
    "            distance = contexts_len - left_i\n",
    "\n",
    "#             print(\"distance = %s\"  %distance)\n",
    "#             print(\"-------------\")\n",
    "\n",
    "            # Weight by inverse of distance between words\n",
    "            increment = 1.0 / float(distance)\n",
    "\n",
    "            # Build co-occurrence matrix symmetrically (pretend we\n",
    "            # are calculating right contexts as well)\n",
    "            cooccurrences[center_id, left_id] += increment\n",
    "            cooccurrences[left_id, center_id] += increment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If anything other than None will exclude challenges whose frequencies are below this value.\n",
    "\n",
    "min_count = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filling the values in a matrix form\n",
    "\n",
    "co_matrix = np.zeros([len(id2word),len(id2word)])\n",
    "\n",
    "for i, (row, data) in enumerate(zip(cooccurrences.rows,cooccurrences.data)):\n",
    "    if min_count is not None and vocab[id2word[i]][0] < min_count:\n",
    "        continue\n",
    "        \n",
    "    for data_idx, j in enumerate(row):\n",
    "        if min_count is not None and vocab[id2word[j]][0] < min_count:\n",
    "            continue\n",
    "            \n",
    "        #print(i, j, data[data_idx])\n",
    "        co_matrix[i,j] = data[data_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0.        , 902.5       , 754.16666667, ...,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [902.5       ,   0.        , 962.66666667, ...,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [754.16666667, 962.66666667,   0.        , ...,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       ...,\n",
       "       [  0.        ,   0.        ,   0.        , ...,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        , ...,   0.        ,\n",
       "          0.        ,   0.        ],\n",
       "       [  0.        ,   0.        ,   0.        , ...,   0.        ,\n",
       "          0.        ,   0.        ]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# have a peek at the co-occurence matrix\n",
    "\n",
    "co_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving the mapping to a dictionary\n",
    "\n",
    "import pickle\n",
    "\n",
    "pickle_path = \"./vocab_mapping.pkl\"\n",
    "pickle_mapping = open(pickle_path,\"wb\")\n",
    "pickle.dump(id2word, pickle_mapping)\n",
    "pickle_mapping.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the co-occurence matrix as a dataframe\n",
    "\n",
    "co_occurence_dataframe = pd.DataFrame(co_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = {v:k for k,v in id2word.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "co_occurence_dataframe =co_occurence_dataframe.rename(columns=res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "co_occurence_dataframe = co_occurence_dataframe.rename(index=res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "co_occurence_dataframe.to_csv(\"co_matrix_with_window_size_3.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>challenge_sequence</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CI23855</td>\n",
       "      <td>CI23933</td>\n",
       "      <td>CI24917</td>\n",
       "      <td>CI24915</td>\n",
       "      <td>CI23714</td>\n",
       "      <td>CI23663</td>\n",
       "      <td>CI24958</td>\n",
       "      <td>CI25135</td>\n",
       "      <td>CI25727</td>\n",
       "      <td>CI24530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CI23663</td>\n",
       "      <td>CI23855</td>\n",
       "      <td>CI24917</td>\n",
       "      <td>CI23933</td>\n",
       "      <td>CI23975</td>\n",
       "      <td>CI23714</td>\n",
       "      <td>CI25135</td>\n",
       "      <td>CI24915</td>\n",
       "      <td>CI24958</td>\n",
       "      <td>CI23781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CI26939</td>\n",
       "      <td>CI26940</td>\n",
       "      <td>CI26941</td>\n",
       "      <td>CI26942</td>\n",
       "      <td>CI26943</td>\n",
       "      <td>CI26944</td>\n",
       "      <td>CI26945</td>\n",
       "      <td>CI26947</td>\n",
       "      <td>CI26948</td>\n",
       "      <td>CI26954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CI23663</td>\n",
       "      <td>CI23855</td>\n",
       "      <td>CI23975</td>\n",
       "      <td>CI23714</td>\n",
       "      <td>CI23848</td>\n",
       "      <td>CI23933</td>\n",
       "      <td>CI25135</td>\n",
       "      <td>CI23781</td>\n",
       "      <td>CI24530</td>\n",
       "      <td>CI23667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CI23855</td>\n",
       "      <td>CI23975</td>\n",
       "      <td>CI25135</td>\n",
       "      <td>CI23848</td>\n",
       "      <td>CI23714</td>\n",
       "      <td>CI24917</td>\n",
       "      <td>CI23929</td>\n",
       "      <td>CI25733</td>\n",
       "      <td>CI25126</td>\n",
       "      <td>CI23913</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "challenge_sequence       1        2        3        4        5        6   \\\n",
       "0                   CI23855  CI23933  CI24917  CI24915  CI23714  CI23663   \n",
       "1                   CI23663  CI23855  CI24917  CI23933  CI23975  CI23714   \n",
       "2                   CI26939  CI26940  CI26941  CI26942  CI26943  CI26944   \n",
       "3                   CI23663  CI23855  CI23975  CI23714  CI23848  CI23933   \n",
       "4                   CI23855  CI23975  CI25135  CI23848  CI23714  CI24917   \n",
       "\n",
       "challenge_sequence       7        8        9        10  \n",
       "0                   CI24958  CI25135  CI25727  CI24530  \n",
       "1                   CI25135  CI24915  CI24958  CI23781  \n",
       "2                   CI26945  CI26947  CI26948  CI26954  \n",
       "3                   CI25135  CI23781  CI24530  CI23667  \n",
       "4                   CI23929  CI25733  CI25126  CI23913  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wide_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39732, 10)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wide_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# making predictions with the co-occurence_matrix based on last attemped/predicted\n",
    "final_predictions = []\n",
    "\n",
    "for i in range(0,39732):\n",
    "    predictions = [wide_test.loc[i,10]]\n",
    "    counter = 0\n",
    "    for stimulus in predictions:\n",
    "        #print(co_occurence_dataframe.loc[co_occurence_dataframe[stimulus]== co_occurence_dataframe[stimulus].max(),stimulus])\n",
    "        predictions.append(co_occurence_dataframe[stimulus].idxmax())\n",
    "        counter+=1\n",
    "        if counter == 3:\n",
    "            break\n",
    "            \n",
    "    final_predictions.append(predictions[1:])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# making predictions with the co-occurence_matrix based on 10th challenge only\n",
    "final_predictions_new = []\n",
    "\n",
    "for i in range(0,39732):\n",
    "    stimulus = wide_test.loc[i,10]\n",
    "    #print(co_occurence_dataframe.loc[co_occurence_dataframe[stimulus]== co_occurence_dataframe[stimulus].max(),stimulus])\n",
    "    final_predictions_new.append(list(co_occurence_dataframe[stimulus].nlargest(3).index))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "largest_3 = pd.DataFrame(final_predictions_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "largest_3['user_id'] = test_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "largest_3_long = pd.melt(largest_3,id_vars=\"user_id\",var_name=\"sequence\", value_name=\"challenge\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "largest_3_long['sequence'] = largest_3_long['sequence'].map({0:'11',1:'12',2:\"13\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "largest_3_long['user_sequence'] = largest_3_long['user_id'].map(str)+\"_\"+largest_3_long['sequence'].map(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "largest_3_long[['user_sequence','challenge']].to_csv(\"rnn-co-occurence.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try Experimenting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}