capstone_anshika.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "da978749",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import mannwhitneyu\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
    "from sklearn.metrics import mean_squared_error, r2_score, silhouette_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eccd75d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "spotify_data = pd.read_csv(\"spotify52kData.csv\")\n",
    "rating_data = pd.read_csv(\"starRatings.csv\", header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6c09c572",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>songNumber</th>\n",
       "      <th>artists</th>\n",
       "      <th>album_name</th>\n",
       "      <th>track_name</th>\n",
       "      <th>popularity</th>\n",
       "      <th>duration</th>\n",
       "      <th>explicit</th>\n",
       "      <th>danceability</th>\n",
       "      <th>energy</th>\n",
       "      <th>key</th>\n",
       "      <th>loudness</th>\n",
       "      <th>mode</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>acousticness</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>liveness</th>\n",
       "      <th>valence</th>\n",
       "      <th>tempo</th>\n",
       "      <th>time_signature</th>\n",
       "      <th>track_genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Gen Hoshino</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>73</td>\n",
       "      <td>230666</td>\n",
       "      <td>False</td>\n",
       "      <td>0.676</td>\n",
       "      <td>0.4610</td>\n",
       "      <td>1</td>\n",
       "      <td>-6.746</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1430</td>\n",
       "      <td>0.03220</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.3580</td>\n",
       "      <td>0.715</td>\n",
       "      <td>87.917</td>\n",
       "      <td>4</td>\n",
       "      <td>acoustic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Ben Woodward</td>\n",
       "      <td>Ghost (Acoustic)</td>\n",
       "      <td>Ghost - Acoustic</td>\n",
       "      <td>55</td>\n",
       "      <td>149610</td>\n",
       "      <td>False</td>\n",
       "      <td>0.420</td>\n",
       "      <td>0.1660</td>\n",
       "      <td>1</td>\n",
       "      <td>-17.235</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.92400</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.1010</td>\n",
       "      <td>0.267</td>\n",
       "      <td>77.489</td>\n",
       "      <td>4</td>\n",
       "      <td>acoustic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Ingrid Michaelson;ZAYN</td>\n",
       "      <td>To Begin Again</td>\n",
       "      <td>To Begin Again</td>\n",
       "      <td>57</td>\n",
       "      <td>210826</td>\n",
       "      <td>False</td>\n",
       "      <td>0.438</td>\n",
       "      <td>0.3590</td>\n",
       "      <td>0</td>\n",
       "      <td>-9.734</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0557</td>\n",
       "      <td>0.21000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.1170</td>\n",
       "      <td>0.120</td>\n",
       "      <td>76.332</td>\n",
       "      <td>4</td>\n",
       "      <td>acoustic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Kina Grannis</td>\n",
       "      <td>Crazy Rich Asians (Original Motion Picture Sou...</td>\n",
       "      <td>Can't Help Falling In Love</td>\n",
       "      <td>71</td>\n",
       "      <td>201933</td>\n",
       "      <td>False</td>\n",
       "      <td>0.266</td>\n",
       "      <td>0.0596</td>\n",
       "      <td>0</td>\n",
       "      <td>-18.515</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0363</td>\n",
       "      <td>0.90500</td>\n",
       "      <td>0.000071</td>\n",
       "      <td>0.1320</td>\n",
       "      <td>0.143</td>\n",
       "      <td>181.740</td>\n",
       "      <td>3</td>\n",
       "      <td>acoustic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Chord Overstreet</td>\n",
       "      <td>Hold On</td>\n",
       "      <td>Hold On</td>\n",
       "      <td>82</td>\n",
       "      <td>198853</td>\n",
       "      <td>False</td>\n",
       "      <td>0.618</td>\n",
       "      <td>0.4430</td>\n",
       "      <td>2</td>\n",
       "      <td>-9.681</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0526</td>\n",
       "      <td>0.46900</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0829</td>\n",
       "      <td>0.167</td>\n",
       "      <td>119.949</td>\n",
       "      <td>4</td>\n",
       "      <td>acoustic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51995</th>\n",
       "      <td>51995</td>\n",
       "      <td>Don Omar;Trebol Clan</td>\n",
       "      <td>Perreo Tenebroso Vol. 3</td>\n",
       "      <td>Tu Cuerpo Me Arrebata</td>\n",
       "      <td>0</td>\n",
       "      <td>203653</td>\n",
       "      <td>False</td>\n",
       "      <td>0.819</td>\n",
       "      <td>0.6450</td>\n",
       "      <td>10</td>\n",
       "      <td>-6.707</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0481</td>\n",
       "      <td>0.23200</td>\n",
       "      <td>0.000863</td>\n",
       "      <td>0.1780</td>\n",
       "      <td>0.724</td>\n",
       "      <td>94.998</td>\n",
       "      <td>4</td>\n",
       "      <td>hip-hop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51996</th>\n",
       "      <td>51996</td>\n",
       "      <td>Don Omar</td>\n",
       "      <td>Feliz Cumpleaños con Perreo</td>\n",
       "      <td>Dale Don Dale</td>\n",
       "      <td>1</td>\n",
       "      <td>212413</td>\n",
       "      <td>False</td>\n",
       "      <td>0.831</td>\n",
       "      <td>0.8180</td>\n",
       "      <td>2</td>\n",
       "      <td>-7.827</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0824</td>\n",
       "      <td>0.02450</td>\n",
       "      <td>0.000319</td>\n",
       "      <td>0.0892</td>\n",
       "      <td>0.819</td>\n",
       "      <td>95.022</td>\n",
       "      <td>4</td>\n",
       "      <td>hip-hop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51997</th>\n",
       "      <td>51997</td>\n",
       "      <td>Don Omar</td>\n",
       "      <td>Última Misión Reggaeton</td>\n",
       "      <td>Dale Don Dale</td>\n",
       "      <td>0</td>\n",
       "      <td>212413</td>\n",
       "      <td>False</td>\n",
       "      <td>0.831</td>\n",
       "      <td>0.8180</td>\n",
       "      <td>2</td>\n",
       "      <td>-7.827</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0824</td>\n",
       "      <td>0.02450</td>\n",
       "      <td>0.000319</td>\n",
       "      <td>0.0892</td>\n",
       "      <td>0.819</td>\n",
       "      <td>95.022</td>\n",
       "      <td>4</td>\n",
       "      <td>hip-hop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51998</th>\n",
       "      <td>51998</td>\n",
       "      <td>Moneybagg Yo;Pooh Shiesty;Lil Durk;EST Gee</td>\n",
       "      <td>Energy Hip Hop</td>\n",
       "      <td>Switches &amp; Dracs</td>\n",
       "      <td>0</td>\n",
       "      <td>168620</td>\n",
       "      <td>True</td>\n",
       "      <td>0.727</td>\n",
       "      <td>0.6470</td>\n",
       "      <td>11</td>\n",
       "      <td>-7.383</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2800</td>\n",
       "      <td>0.03290</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.2460</td>\n",
       "      <td>0.529</td>\n",
       "      <td>159.959</td>\n",
       "      <td>4</td>\n",
       "      <td>hip-hop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51999</th>\n",
       "      <td>51999</td>\n",
       "      <td>Don Omar;Juan Magán</td>\n",
       "      <td>Perreo Tenebroso Vol. 4</td>\n",
       "      <td>No Sigue Modas Aka Ella No Sigue Modas</td>\n",
       "      <td>0</td>\n",
       "      <td>232000</td>\n",
       "      <td>False</td>\n",
       "      <td>0.685</td>\n",
       "      <td>0.8620</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.611</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0627</td>\n",
       "      <td>0.00757</td>\n",
       "      <td>0.001400</td>\n",
       "      <td>0.0226</td>\n",
       "      <td>0.884</td>\n",
       "      <td>128.032</td>\n",
       "      <td>4</td>\n",
       "      <td>hip-hop</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>52000 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       songNumber                                     artists  \\\n",
       "0               0                                 Gen Hoshino   \n",
       "1               1                                Ben Woodward   \n",
       "2               2                      Ingrid Michaelson;ZAYN   \n",
       "3               3                                Kina Grannis   \n",
       "4               4                            Chord Overstreet   \n",
       "...           ...                                         ...   \n",
       "51995       51995                        Don Omar;Trebol Clan   \n",
       "51996       51996                                    Don Omar   \n",
       "51997       51997                                    Don Omar   \n",
       "51998       51998  Moneybagg Yo;Pooh Shiesty;Lil Durk;EST Gee   \n",
       "51999       51999                         Don Omar;Juan Magán   \n",
       "\n",
       "                                              album_name  \\\n",
       "0                                                 Comedy   \n",
       "1                                       Ghost (Acoustic)   \n",
       "2                                         To Begin Again   \n",
       "3      Crazy Rich Asians (Original Motion Picture Sou...   \n",
       "4                                                Hold On   \n",
       "...                                                  ...   \n",
       "51995                            Perreo Tenebroso Vol. 3   \n",
       "51996                        Feliz Cumpleaños con Perreo   \n",
       "51997                            Última Misión Reggaeton   \n",
       "51998                                     Energy Hip Hop   \n",
       "51999                            Perreo Tenebroso Vol. 4   \n",
       "\n",
       "                                   track_name  popularity  duration  explicit  \\\n",
       "0                                      Comedy          73    230666     False   \n",
       "1                            Ghost - Acoustic          55    149610     False   \n",
       "2                              To Begin Again          57    210826     False   \n",
       "3                  Can't Help Falling In Love          71    201933     False   \n",
       "4                                     Hold On          82    198853     False   \n",
       "...                                       ...         ...       ...       ...   \n",
       "51995                   Tu Cuerpo Me Arrebata           0    203653     False   \n",
       "51996                           Dale Don Dale           1    212413     False   \n",
       "51997                           Dale Don Dale           0    212413     False   \n",
       "51998                        Switches & Dracs           0    168620      True   \n",
       "51999  No Sigue Modas Aka Ella No Sigue Modas           0    232000     False   \n",
       "\n",
       "       danceability  energy  key  loudness  mode  speechiness  acousticness  \\\n",
       "0             0.676  0.4610    1    -6.746     0       0.1430       0.03220   \n",
       "1             0.420  0.1660    1   -17.235     1       0.0763       0.92400   \n",
       "2             0.438  0.3590    0    -9.734     1       0.0557       0.21000   \n",
       "3             0.266  0.0596    0   -18.515     1       0.0363       0.90500   \n",
       "4             0.618  0.4430    2    -9.681     1       0.0526       0.46900   \n",
       "...             ...     ...  ...       ...   ...          ...           ...   \n",
       "51995         0.819  0.6450   10    -6.707     0       0.0481       0.23200   \n",
       "51996         0.831  0.8180    2    -7.827     1       0.0824       0.02450   \n",
       "51997         0.831  0.8180    2    -7.827     1       0.0824       0.02450   \n",
       "51998         0.727  0.6470   11    -7.383     0       0.2800       0.03290   \n",
       "51999         0.685  0.8620    0    -4.611     1       0.0627       0.00757   \n",
       "\n",
       "       instrumentalness  liveness  valence    tempo  time_signature  \\\n",
       "0              0.000001    0.3580    0.715   87.917               4   \n",
       "1              0.000006    0.1010    0.267   77.489               4   \n",
       "2              0.000000    0.1170    0.120   76.332               4   \n",
       "3              0.000071    0.1320    0.143  181.740               3   \n",
       "4              0.000000    0.0829    0.167  119.949               4   \n",
       "...                 ...       ...      ...      ...             ...   \n",
       "51995          0.000863    0.1780    0.724   94.998               4   \n",
       "51996          0.000319    0.0892    0.819   95.022               4   \n",
       "51997          0.000319    0.0892    0.819   95.022               4   \n",
       "51998          0.000000    0.2460    0.529  159.959               4   \n",
       "51999          0.001400    0.0226    0.884  128.032               4   \n",
       "\n",
       "      track_genre  \n",
       "0        acoustic  \n",
       "1        acoustic  \n",
       "2        acoustic  \n",
       "3        acoustic  \n",
       "4        acoustic  \n",
       "...           ...  \n",
       "51995     hip-hop  \n",
       "51996     hip-hop  \n",
       "51997     hip-hop  \n",
       "51998     hip-hop  \n",
       "51999     hip-hop  \n",
       "\n",
       "[52000 rows x 20 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spotify_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "81bfc181",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>songNumber</th>\n",
       "      <th>artists</th>\n",
       "      <th>album_name</th>\n",
       "      <th>track_name</th>\n",
       "      <th>popularity</th>\n",
       "      <th>duration</th>\n",
       "      <th>explicit</th>\n",
       "      <th>danceability</th>\n",
       "      <th>energy</th>\n",
       "      <th>key</th>\n",
       "      <th>loudness</th>\n",
       "      <th>mode</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>acousticness</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>liveness</th>\n",
       "      <th>valence</th>\n",
       "      <th>tempo</th>\n",
       "      <th>time_signature</th>\n",
       "      <th>track_genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3000</th>\n",
       "      <td>3000</td>\n",
       "      <td>The Neighbourhood</td>\n",
       "      <td>Wiped Out!</td>\n",
       "      <td>Daddy Issues</td>\n",
       "      <td>87</td>\n",
       "      <td>260173</td>\n",
       "      <td>False</td>\n",
       "      <td>0.588</td>\n",
       "      <td>0.521</td>\n",
       "      <td>10</td>\n",
       "      <td>-9.461</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0329</td>\n",
       "      <td>0.0678</td>\n",
       "      <td>0.149</td>\n",
       "      <td>0.123</td>\n",
       "      <td>0.337</td>\n",
       "      <td>85.012</td>\n",
       "      <td>4</td>\n",
       "      <td>alternative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      songNumber            artists  album_name    track_name  popularity  \\\n",
       "3000        3000  The Neighbourhood  Wiped Out!  Daddy Issues          87   \n",
       "\n",
       "      duration  explicit  danceability  energy  key  loudness  mode  \\\n",
       "3000    260173     False         0.588   0.521   10    -9.461     1   \n",
       "\n",
       "      speechiness  acousticness  instrumentalness  liveness  valence   tempo  \\\n",
       "3000       0.0329        0.0678             0.149     0.123    0.337  85.012   \n",
       "\n",
       "      time_signature  track_genre  \n",
       "3000               4  alternative  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spotify_data[spotify_data['songNumber']==3000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6c09c5a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>songNumber</th>\n",
       "      <th>artists</th>\n",
       "      <th>album_name</th>\n",
       "      <th>track_name</th>\n",
       "      <th>popularity</th>\n",
       "      <th>duration</th>\n",
       "      <th>explicit</th>\n",
       "      <th>danceability</th>\n",
       "      <th>energy</th>\n",
       "      <th>key</th>\n",
       "      <th>loudness</th>\n",
       "      <th>mode</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>acousticness</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>liveness</th>\n",
       "      <th>valence</th>\n",
       "      <th>tempo</th>\n",
       "      <th>time_signature</th>\n",
       "      <th>track_genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2000</th>\n",
       "      <td>2000</td>\n",
       "      <td>The Neighbourhood</td>\n",
       "      <td>Wiped Out!</td>\n",
       "      <td>Daddy Issues</td>\n",
       "      <td>87</td>\n",
       "      <td>260173</td>\n",
       "      <td>False</td>\n",
       "      <td>0.588</td>\n",
       "      <td>0.521</td>\n",
       "      <td>10</td>\n",
       "      <td>-9.461</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0329</td>\n",
       "      <td>0.0678</td>\n",
       "      <td>0.149</td>\n",
       "      <td>0.123</td>\n",
       "      <td>0.337</td>\n",
       "      <td>85.012</td>\n",
       "      <td>4</td>\n",
       "      <td>alt-rock</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      songNumber            artists  album_name    track_name  popularity  \\\n",
       "2000        2000  The Neighbourhood  Wiped Out!  Daddy Issues          87   \n",
       "\n",
       "      duration  explicit  danceability  energy  key  loudness  mode  \\\n",
       "2000    260173     False         0.588   0.521   10    -9.461     1   \n",
       "\n",
       "      speechiness  acousticness  instrumentalness  liveness  valence   tempo  \\\n",
       "2000       0.0329        0.0678             0.149     0.123    0.337  85.012   \n",
       "\n",
       "      time_signature track_genre  \n",
       "2000               4    alt-rock  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spotify_data[spotify_data['songNumber']==2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "790b9283",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(17951972)\n",
    "random.seed(17951972)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e28e85e0",
   "metadata": {},
   "source": [
    "## Q7\n",
    "\n",
    "Can you predict whether a song is in major or minor key from valence using logistic regression or a support vector machine? If so, how good is this prediction? If not, is there a better one?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6b99cf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Load and preprocess data\n",
    "spotify_data = pd.read_csv(\"spotify52kData.csv\")\n",
    "star_ratings = pd.read_csv(\"starRatings.csv\", header=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ac4acb9",
   "metadata": {},
   "source": [
    "## Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "14f9530d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.6435\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00       713\n",
      "           1       0.64      1.00      0.78      1287\n",
      "\n",
      "    accuracy                           0.64      2000\n",
      "   macro avg       0.32      0.50      0.39      2000\n",
      "weighted avg       0.41      0.64      0.50      2000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/anshikagupta/Library/Python/3.9/lib/python/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/anshikagupta/Library/Python/3.9/lib/python/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/anshikagupta/Library/Python/3.9/lib/python/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# Merge datasets\n",
    "merged_data = pd.merge(spotify_data, star_ratings, left_on=\"songNumber\", right_index=True)\n",
    "\n",
    "# Extract relevant features and target variable\n",
    "features = merged_data[['valence']]\n",
    "target = merged_data['mode']  # Assuming 'mode' is the binary categorical variable indicating major/minor\n",
    "\n",
    "# Handle missing values\n",
    "features = features.dropna()\n",
    "target = target.dropna()\n",
    "\n",
    "# Train-test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, random_state=42)\n",
    "\n",
    "# Standardize features (optional but can be beneficial for some algorithms)\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "\n",
    "# Logistic Regression model\n",
    "model = LogisticRegression()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Predictions\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# Model evaluation\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "report = classification_report(y_test, y_pred)\n",
    "\n",
    "print(f\"Accuracy: {accuracy}\")\n",
    "print(\"Classification Report:\")\n",
    "print(report)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "722aa80c",
   "metadata": {},
   "source": [
    "## Using SVM, we get:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "917ed65b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.6435\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00       713\n",
      "           1       0.64      1.00      0.78      1287\n",
      "\n",
      "    accuracy                           0.64      2000\n",
      "   macro avg       0.32      0.50      0.39      2000\n",
      "weighted avg       0.41      0.64      0.50      2000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/anshikagupta/Library/Python/3.9/lib/python/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/anshikagupta/Library/Python/3.9/lib/python/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/anshikagupta/Library/Python/3.9/lib/python/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "model = SVC(kernel='linear', C=1.0)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Predictions\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# Model evaluation\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "report = classification_report(y_test, y_pred)\n",
    "\n",
    "print(f\"Accuracy: {accuracy}\")\n",
    "print(\"Classification Report:\")\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86927fdf",
   "metadata": {},
   "source": [
    "Both Logistic regression and SVM produce similar results and the accuracy is pretty low.\n",
    "\n",
    "Let's try,\n",
    "\n",
    "## 1. GBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "52605c7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.646\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.02      0.04       713\n",
      "           1       0.65      0.99      0.78      1287\n",
      "\n",
      "    accuracy                           0.65      2000\n",
      "   macro avg       0.63      0.51      0.41      2000\n",
      "weighted avg       0.63      0.65      0.52      2000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "# Replace SVM model with Gradient Boosting\n",
    "model = GradientBoostingClassifier(n_estimators=100, random_state=42)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Predictions\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# Model evaluation\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "report = classification_report(y_test, y_pred)\n",
    "\n",
    "print(f\"Accuracy: {accuracy}\")\n",
    "print(\"Classification Report:\")\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04923e74",
   "metadata": {},
   "source": [
    "## 2. Random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6c1b1f5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.6245\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.45      0.25      0.33       713\n",
      "           1       0.67      0.83      0.74      1287\n",
      "\n",
      "    accuracy                           0.62      2000\n",
      "   macro avg       0.56      0.54      0.53      2000\n",
      "weighted avg       0.59      0.62      0.59      2000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Replace SVM model with Random Forest\n",
    "model = RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "# Predictions\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# Model evaluation\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "report = classification_report(y_test, y_pred)\n",
    "\n",
    "print(f\"Accuracy: {accuracy}\")\n",
    "print(\"Classification Report:\")\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4dcc373",
   "metadata": {},
   "source": [
    "## Q8\n",
    "Can you predict genre by using the 10 song features from question 4 directly or the principal components you extracted in question 6 with a neural network? How well does this work?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "df620940",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.24980769230769231\n",
      "Classification Report:\n",
      "                precision    recall  f1-score   support\n",
      "\n",
      "      acoustic       0.16      0.22      0.19       205\n",
      "      afrobeat       0.19      0.16      0.17       197\n",
      "      alt-rock       0.10      0.04      0.05       194\n",
      "   alternative       0.20      0.03      0.06       210\n",
      "       ambient       0.34      0.63      0.45       190\n",
      "         anime       0.17      0.08      0.11       208\n",
      "   black-metal       0.26      0.23      0.25       217\n",
      "     bluegrass       0.29      0.40      0.34       207\n",
      "         blues       0.20      0.14      0.16       190\n",
      "        brazil       0.14      0.04      0.07       206\n",
      "     breakbeat       0.18      0.15      0.16       192\n",
      "       british       0.10      0.01      0.03       200\n",
      "      cantopop       0.20      0.44      0.28       199\n",
      " chicago-house       0.30      0.40      0.34       205\n",
      "      children       0.34      0.36      0.35       195\n",
      "         chill       0.22      0.26      0.23       211\n",
      "     classical       0.49      0.45      0.47       196\n",
      "          club       0.22      0.23      0.22       199\n",
      "        comedy       0.91      0.78      0.84       204\n",
      "       country       0.19      0.29      0.23       191\n",
      "         dance       0.19      0.30      0.23       208\n",
      "     dancehall       0.25      0.29      0.27       191\n",
      "   death-metal       0.28      0.24      0.26       206\n",
      "    deep-house       0.16      0.32      0.22       182\n",
      "detroit-techno       0.45      0.49      0.47       206\n",
      "         disco       0.18      0.27      0.21       188\n",
      "        disney       0.24      0.17      0.20       206\n",
      " drum-and-bass       0.38      0.50      0.43       175\n",
      "           dub       0.21      0.05      0.08       220\n",
      "       dubstep       0.20      0.38      0.27       203\n",
      "           edm       0.15      0.18      0.16       200\n",
      "       electro       0.16      0.10      0.12       236\n",
      "    electronic       0.05      0.01      0.02       218\n",
      "           emo       0.07      0.02      0.03       191\n",
      "          folk       0.12      0.09      0.11       196\n",
      "         forro       0.41      0.65      0.50       201\n",
      "        french       0.13      0.10      0.12       200\n",
      "          funk       0.27      0.28      0.27       203\n",
      "        garage       0.10      0.06      0.07       212\n",
      "        german       0.32      0.06      0.10       205\n",
      "        gospel       0.22      0.33      0.26       197\n",
      "          goth       0.08      0.03      0.04       189\n",
      "     grindcore       0.37      0.53      0.44       182\n",
      "        groove       0.06      0.03      0.04       197\n",
      "        grunge       0.12      0.10      0.11       184\n",
      "        guitar       0.30      0.35      0.32       173\n",
      "         happy       0.31      0.39      0.34       210\n",
      "     hard-rock       0.15      0.07      0.10       209\n",
      "      hardcore       0.20      0.18      0.19       203\n",
      "     hardstyle       0.34      0.57      0.43       185\n",
      "   heavy-metal       0.22      0.38      0.28       215\n",
      "       hip-hop       0.16      0.30      0.21       193\n",
      "\n",
      "      accuracy                           0.25     10400\n",
      "     macro avg       0.23      0.25      0.23     10400\n",
      "  weighted avg       0.23      0.25      0.23     10400\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "\n",
    "\n",
    "# Extract relevant features and target variable\n",
    "features = spotify_data.iloc[:, 8:18]  # Assuming columns 8 to 17 are the 10 song features\n",
    "target = spotify_data['track_genre']\n",
    "\n",
    "# Handle missing values\n",
    "features = features.dropna()\n",
    "target = target.dropna()\n",
    "\n",
    "# Train-test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(features, target, test_size=0.2, random_state=42)\n",
    "\n",
    "# Standardize features\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "# Neural Network model\n",
    "model = MLPClassifier(hidden_layer_sizes=(100,), max_iter=1000, random_state=42)\n",
    "model.fit(X_train_scaled, y_train)\n",
    "\n",
    "# Predictions\n",
    "y_pred = model.predict(X_test_scaled)\n",
    "\n",
    "# Model evaluation\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "report = classification_report(y_test, y_pred)\n",
    "\n",
    "print(f\"Accuracy: {accuracy}\")\n",
    "print(\"Classification Report:\")\n",
    "print(report)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d207738",
   "metadata": {},
   "source": [
    "## Q9\n",
    "In recommender systems, the popularity based model is an important baseline. We have a two part question in this regard: a) Is there a relationship between popularity and average star rating for the 5k songs we have explicit feedback for? b) Which 10 songs are in the “greatest hits” (out of the 5k songs), on the basis of the popularity based model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "aa48861d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Star Ratings for Each Song:\n"
     ]
    }
   ],
   "source": [
    "# average start ratings\n",
    "import pandas as pd\n",
    "\n",
    "# Load star ratings data\n",
    "star_ratings = pd.read_csv(\"starRatings.csv\", header=None)\n",
    "\n",
    "# Assuming the first 5k songs correspond to the first 5k columns in star_ratings\n",
    "# If the mapping is different, adjust the column selection accordingly\n",
    "\n",
    "# Calculate average star ratings for each song\n",
    "average_ratings = star_ratings.iloc[:, :5000].mean(axis=0)\n",
    "\n",
    "# Display the average star ratings\n",
    "print(\"Average Star Ratings for Each Song:\")\n",
    "# print(average_ratings)\n",
    "\n",
    "merged_data['average_ratings'] = average_ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "6d27cb8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>popularity</th>\n",
       "      <th>average_ratings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>73</td>\n",
       "      <td>3.196939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>55</td>\n",
       "      <td>1.361029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>57</td>\n",
       "      <td>2.186377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>71</td>\n",
       "      <td>1.576998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>82</td>\n",
       "      <td>3.038835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>42</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>43</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>42</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>42</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>44</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      popularity  average_ratings\n",
       "0             73         3.196939\n",
       "1             55         1.361029\n",
       "2             57         2.186377\n",
       "3             71         1.576998\n",
       "4             82         3.038835\n",
       "...          ...              ...\n",
       "9995          42              NaN\n",
       "9996          43              NaN\n",
       "9997          42              NaN\n",
       "9998          42              NaN\n",
       "9999          44              NaN\n",
       "\n",
       "[10000 rows x 2 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_data[['popularity', 'average_ratings']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "014d537f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correlation between Popularity and Average Star Rating: 0.5693914205195068\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Assuming you have loaded the data into the 'merged_data' DataFrame\n",
    "# Make sure that the 'popularity' and 'star_rating' columns are available\n",
    "\n",
    "# Extract relevant columns\n",
    "popularity_and_ratings = merged_data[['popularity', 'average_ratings']]\n",
    "\n",
    "# Drop rows with missing values\n",
    "popularity_and_ratings = popularity_and_ratings.dropna()\n",
    "\n",
    "# Calculate correlation coefficient\n",
    "correlation = popularity_and_ratings['popularity'].corr(popularity_and_ratings['average_ratings'])\n",
    "\n",
    "print(f\"Correlation between Popularity and Average Star Rating: {correlation}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "8804d304",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 10 Songs Based on Popularity:\n",
      "              track_name                   artists  popularity\n",
      "2003     Sweater Weather         The Neighbourhood          93\n",
      "3003     Sweater Weather         The Neighbourhood          93\n",
      "2000        Daddy Issues         The Neighbourhood          87\n",
      "3000        Daddy Issues         The Neighbourhood          87\n",
      "3300            Miss You  Oliver Tree;Robin Schulz          87\n",
      "3257      Mr. Brightside               The Killers          86\n",
      "3004             abcdefu                     GAYLE          86\n",
      "2002            Softcore         The Neighbourhood          86\n",
      "3002            Softcore         The Neighbourhood          86\n",
      "2106      Mr. Brightside               The Killers          86\n",
      "3006          In the End               Linkin Park          85\n",
      "3154               Creep                 Radiohead          85\n",
      "2053               Creep                 Radiohead          85\n",
      "3217   Seven Nation Army         The White Stripes          84\n",
      "2057   Seven Nation Army         The White Stripes          84\n",
      "2203  Losing My Religion                    R.E.M.          84\n",
      "8003   Seven Nation Army         The White Stripes          84\n",
      "4002          Apocalypse      Cigarettes After Sex          84\n",
      "3255      Feel Good Inc.                  Gorillaz          84\n",
      "3569            Everlong              Foo Fighters          84\n"
     ]
    }
   ],
   "source": [
    "# Rank songs based on popularity\n",
    "top_songs = merged_data.sort_values(by='popularity', ascending=False).head(20)\n",
    "\n",
    "# Display the top 10 songs\n",
    "print(\"Top 10 Songs Based on Popularity:\")\n",
    "print(top_songs[['track_name', 'artists', 'popularity']])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56de6040",
   "metadata": {},
   "source": [
    "The results obtained above have duplicate song names as observed from the original dataset, wherein one song has multiple song numbers. If we take the approach of dropping duplicates from the entire spotify dataset, we will also have to make necessary adjustments in the star_ratings dataset. I'm unsure of the repurcurssions this would lead to, not to mention it would be time consuming.\n",
    "\n",
    "Given this situation and our problem statement, we only need the top 10 songs with most popular ratings. This concerns columns: Popularity and Avg star ratings mainly. \n",
    "\n",
    "We can also observe in the output above that even though the duplicates exist, the popularity ratings for all of them is the exact same. \n",
    "\n",
    "Hence, it seems safe to filter out the duplicates at this stage and select the top 10 songs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "1e3e541c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_name</th>\n",
       "      <th>artists</th>\n",
       "      <th>popularity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2003</th>\n",
       "      <td>Sweater Weather</td>\n",
       "      <td>The Neighbourhood</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000</th>\n",
       "      <td>Daddy Issues</td>\n",
       "      <td>The Neighbourhood</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3300</th>\n",
       "      <td>Miss You</td>\n",
       "      <td>Oliver Tree;Robin Schulz</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3257</th>\n",
       "      <td>Mr. Brightside</td>\n",
       "      <td>The Killers</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3004</th>\n",
       "      <td>abcdefu</td>\n",
       "      <td>GAYLE</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2002</th>\n",
       "      <td>Softcore</td>\n",
       "      <td>The Neighbourhood</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3006</th>\n",
       "      <td>In the End</td>\n",
       "      <td>Linkin Park</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3154</th>\n",
       "      <td>Creep</td>\n",
       "      <td>Radiohead</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3217</th>\n",
       "      <td>Seven Nation Army</td>\n",
       "      <td>The White Stripes</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2203</th>\n",
       "      <td>Losing My Religion</td>\n",
       "      <td>R.E.M.</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              track_name                   artists  popularity\n",
       "2003     Sweater Weather         The Neighbourhood          93\n",
       "2000        Daddy Issues         The Neighbourhood          87\n",
       "3300            Miss You  Oliver Tree;Robin Schulz          87\n",
       "3257      Mr. Brightside               The Killers          86\n",
       "3004             abcdefu                     GAYLE          86\n",
       "2002            Softcore         The Neighbourhood          86\n",
       "3006          In the End               Linkin Park          85\n",
       "3154               Creep                 Radiohead          85\n",
       "3217   Seven Nation Army         The White Stripes          84\n",
       "2203  Losing My Religion                    R.E.M.          84"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_songs[['track_name', 'artists', 'popularity']].drop_duplicates()[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c08d6a1",
   "metadata": {},
   "source": [
    "## Q10\n",
    "You want to create a “personal mixtape” for all 10k users we have explicit feedback for. This mixtape contains individualized recommendations as to which 10 songs (out of the 5k) a given user will enjoy most. How do these recommendations compare to the “greatest hits” from the previous question and how good is your recommender system in making recommendations?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "2399581e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>songNumber</th>\n",
       "      <th>artists</th>\n",
       "      <th>album_name</th>\n",
       "      <th>track_name</th>\n",
       "      <th>popularity</th>\n",
       "      <th>duration</th>\n",
       "      <th>explicit</th>\n",
       "      <th>danceability</th>\n",
       "      <th>energy</th>\n",
       "      <th>key</th>\n",
       "      <th>...</th>\n",
       "      <th>4991</th>\n",
       "      <th>4992</th>\n",
       "      <th>4993</th>\n",
       "      <th>4994</th>\n",
       "      <th>4995</th>\n",
       "      <th>4996</th>\n",
       "      <th>4997</th>\n",
       "      <th>4998</th>\n",
       "      <th>4999</th>\n",
       "      <th>average_ratings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Gen Hoshino</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>73</td>\n",
       "      <td>230666</td>\n",
       "      <td>False</td>\n",
       "      <td>0.676</td>\n",
       "      <td>0.4610</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.196939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Ben Woodward</td>\n",
       "      <td>Ghost (Acoustic)</td>\n",
       "      <td>Ghost - Acoustic</td>\n",
       "      <td>55</td>\n",
       "      <td>149610</td>\n",
       "      <td>False</td>\n",
       "      <td>0.420</td>\n",
       "      <td>0.1660</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.361029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Ingrid Michaelson;ZAYN</td>\n",
       "      <td>To Begin Again</td>\n",
       "      <td>To Begin Again</td>\n",
       "      <td>57</td>\n",
       "      <td>210826</td>\n",
       "      <td>False</td>\n",
       "      <td>0.438</td>\n",
       "      <td>0.3590</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.186377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Kina Grannis</td>\n",
       "      <td>Crazy Rich Asians (Original Motion Picture Sou...</td>\n",
       "      <td>Can't Help Falling In Love</td>\n",
       "      <td>71</td>\n",
       "      <td>201933</td>\n",
       "      <td>False</td>\n",
       "      <td>0.266</td>\n",
       "      <td>0.0596</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.576998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Chord Overstreet</td>\n",
       "      <td>Hold On</td>\n",
       "      <td>Hold On</td>\n",
       "      <td>82</td>\n",
       "      <td>198853</td>\n",
       "      <td>False</td>\n",
       "      <td>0.618</td>\n",
       "      <td>0.4430</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.038835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>9995</td>\n",
       "      <td>Armandinho</td>\n",
       "      <td>Acústico (Ao Vivo)</td>\n",
       "      <td>O Que Meu Pai Falou Pra Mim - Ao Vivo</td>\n",
       "      <td>42</td>\n",
       "      <td>252390</td>\n",
       "      <td>False</td>\n",
       "      <td>0.629</td>\n",
       "      <td>0.5810</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>9996</td>\n",
       "      <td>Canção &amp; Louvor</td>\n",
       "      <td>Profetizando Vida</td>\n",
       "      <td>Dependente</td>\n",
       "      <td>43</td>\n",
       "      <td>270791</td>\n",
       "      <td>False</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.4590</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>9997</td>\n",
       "      <td>Planet Hemp</td>\n",
       "      <td>Usuário</td>\n",
       "      <td>Não Compre, Plante!</td>\n",
       "      <td>42</td>\n",
       "      <td>251493</td>\n",
       "      <td>False</td>\n",
       "      <td>0.652</td>\n",
       "      <td>0.7800</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>9998</td>\n",
       "      <td>Cidade Negra;Lulu Santos</td>\n",
       "      <td>Quanto Mais Curtido Melhor</td>\n",
       "      <td>Sábado a Noite (feat. Lulu Santos)</td>\n",
       "      <td>42</td>\n",
       "      <td>267306</td>\n",
       "      <td>False</td>\n",
       "      <td>0.773</td>\n",
       "      <td>0.9400</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>9999</td>\n",
       "      <td>Legião Urbana</td>\n",
       "      <td>O Descobrimento Do Brasil</td>\n",
       "      <td>Vamos Fazer Um Filme</td>\n",
       "      <td>44</td>\n",
       "      <td>261866</td>\n",
       "      <td>False</td>\n",
       "      <td>0.674</td>\n",
       "      <td>0.4370</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 5021 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      songNumber                   artists  \\\n",
       "0              0               Gen Hoshino   \n",
       "1              1              Ben Woodward   \n",
       "2              2    Ingrid Michaelson;ZAYN   \n",
       "3              3              Kina Grannis   \n",
       "4              4          Chord Overstreet   \n",
       "...          ...                       ...   \n",
       "9995        9995                Armandinho   \n",
       "9996        9996           Canção & Louvor   \n",
       "9997        9997               Planet Hemp   \n",
       "9998        9998  Cidade Negra;Lulu Santos   \n",
       "9999        9999             Legião Urbana   \n",
       "\n",
       "                                             album_name  \\\n",
       "0                                                Comedy   \n",
       "1                                      Ghost (Acoustic)   \n",
       "2                                        To Begin Again   \n",
       "3     Crazy Rich Asians (Original Motion Picture Sou...   \n",
       "4                                               Hold On   \n",
       "...                                                 ...   \n",
       "9995                                 Acústico (Ao Vivo)   \n",
       "9996                                  Profetizando Vida   \n",
       "9997                                            Usuário   \n",
       "9998                         Quanto Mais Curtido Melhor   \n",
       "9999                          O Descobrimento Do Brasil   \n",
       "\n",
       "                                 track_name  popularity  duration  explicit  \\\n",
       "0                                    Comedy          73    230666     False   \n",
       "1                          Ghost - Acoustic          55    149610     False   \n",
       "2                            To Begin Again          57    210826     False   \n",
       "3                Can't Help Falling In Love          71    201933     False   \n",
       "4                                   Hold On          82    198853     False   \n",
       "...                                     ...         ...       ...       ...   \n",
       "9995  O Que Meu Pai Falou Pra Mim - Ao Vivo          42    252390     False   \n",
       "9996                             Dependente          43    270791     False   \n",
       "9997                    Não Compre, Plante!          42    251493     False   \n",
       "9998     Sábado a Noite (feat. Lulu Santos)          42    267306     False   \n",
       "9999                   Vamos Fazer Um Filme          44    261866     False   \n",
       "\n",
       "      danceability  energy  key  ...  4991  4992  4993  4994  4995  4996  \\\n",
       "0            0.676  0.4610    1  ...   0.0   NaN   NaN   NaN   NaN   NaN   \n",
       "1            0.420  0.1660    1  ...   NaN   NaN   NaN   NaN   NaN   NaN   \n",
       "2            0.438  0.3590    0  ...   2.0   1.0   NaN   NaN   3.0   NaN   \n",
       "3            0.266  0.0596    0  ...   NaN   NaN   NaN   NaN   NaN   NaN   \n",
       "4            0.618  0.4430    2  ...   NaN   NaN   NaN   NaN   NaN   NaN   \n",
       "...            ...     ...  ...  ...   ...   ...   ...   ...   ...   ...   \n",
       "9995         0.629  0.5810    7  ...   NaN   NaN   NaN   NaN   NaN   NaN   \n",
       "9996         0.647  0.4590    2  ...   NaN   NaN   NaN   NaN   NaN   NaN   \n",
       "9997         0.652  0.7800    0  ...   NaN   NaN   NaN   NaN   NaN   NaN   \n",
       "9998         0.773  0.9400    1  ...   NaN   NaN   NaN   NaN   NaN   NaN   \n",
       "9999         0.674  0.4370    0  ...   1.0   NaN   NaN   NaN   0.0   NaN   \n",
       "\n",
       "      4997  4998  4999 average_ratings  \n",
       "0      NaN   NaN   NaN        3.196939  \n",
       "1      NaN   NaN   NaN        1.361029  \n",
       "2      NaN   NaN   0.0        2.186377  \n",
       "3      NaN   NaN   NaN        1.576998  \n",
       "4      NaN   NaN   NaN        3.038835  \n",
       "...    ...   ...   ...             ...  \n",
       "9995   NaN   NaN   0.0             NaN  \n",
       "9996   NaN   NaN   NaN             NaN  \n",
       "9997   NaN   NaN   NaN             NaN  \n",
       "9998   NaN   NaN   NaN             NaN  \n",
       "9999   NaN   NaN   NaN             NaN  \n",
       "\n",
       "[10000 rows x 5021 columns]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "6b843a2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>4990</th>\n",
       "      <th>4991</th>\n",
       "      <th>4992</th>\n",
       "      <th>4993</th>\n",
       "      <th>4994</th>\n",
       "      <th>4995</th>\n",
       "      <th>4996</th>\n",
       "      <th>4997</th>\n",
       "      <th>4998</th>\n",
       "      <th>4999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 5000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      0     1     2     3     4     5     6     7     8     9     ...  4990  \\\n",
       "0      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "1      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "2      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "3      0.0   3.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "4      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   4.0   \n",
       "...    ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  ...   ...   \n",
       "9995   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "9996   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "9997   3.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "9998   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "9999   0.0   1.0   0.0   0.0   0.0   0.0   0.0   4.0   0.0   0.0  ...   0.0   \n",
       "\n",
       "      4991  4992  4993  4994  4995  4996  4997  4998  4999  \n",
       "0      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "1      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2      2.0   1.0   0.0   0.0   3.0   0.0   0.0   0.0   0.0  \n",
       "3      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "...    ...   ...   ...   ...   ...   ...   ...   ...   ...  \n",
       "9995   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "9996   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "9997   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "9998   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "9999   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[10000 rows x 5000 columns]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_ratings = merged_data.iloc[:, 20:]\n",
    "user_ratings.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "71ee5637",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Recommendations for User 1:\n",
      "Chama Arteira by Samuca e a Selva (Popularity: 19)\n",
      "Living Dead Girl by Rob Zombie (Popularity: 0)\n",
      "Isn't It Romantic? by Ella Fitzgerald (Popularity: 37)\n",
      "Heart-Shaped Box by Nirvana (Popularity: 79)\n",
      "Suzume (From \"Suzume no Tojimari\") - Orchestral Lullaby Version by Pharozen (Popularity: 45)\n",
      "Poetas no Topo 1 by Makalister;BK;Menestrel;Djonga;Sant;JXNV$;Pineapple StormTv;Slim Beat (Popularity: 49)\n",
      "To the Wind by Chris Cresswell (Popularity: 25)\n",
      "One Headlight by The Wallflowers (Popularity: 0)\n",
      "De to spellemenn - 2013 edition by Myrkgrav (Popularity: 20)\n",
      "Tired of Being Alone by Al Green (Popularity: 23)\n",
      "\n",
      "Recommendations for User 2:\n",
      "District Six - Live by Hugh Masekela (Popularity: 0)\n",
      "The Departure by Max Richter (Popularity: 61)\n",
      "Shampoo by Plastilina Mosh (Popularity: 28)\n",
      "I'm Not Asking by Odina (Popularity: 39)\n",
      "HandClap by Fitz and The Tantrums (Popularity: 65)\n",
      "Courting Baphomet by Cradle Of Filth (Popularity: 0)\n",
      "Solitude by Billie Holiday (Popularity: 67)\n",
      "トリコになれ by Macaroni Empitsu (Popularity: 41)\n",
      "Adán vs Zeus Rap by MegaR;Kballero Rap (Popularity: 39)\n",
      "Must Reach by J Prince (Popularity: 17)\n",
      "\n",
      "Recommendations for User 3:\n",
      "What You Want by Angeloh (Popularity: 17)\n",
      "Batom by ATR;Luedji Luna (Popularity: 21)\n",
      "Stars and Butterflies - From \"Pride & Prejudice\" Soundtrack by Dario Marianelli;Jean-Yves Thibaudet (Popularity: 50)\n",
      "The Departure by Max Richter (Popularity: 61)\n",
      "Contact by Violet Cold (Popularity: 36)\n",
      "Walk by Saint Chaos;Sam Tinnesz (Popularity: 60)\n",
      "A Million Dreams - Acoustic by John Adams (Popularity: 48)\n",
      "You Are My Sunshine by Kina Grannis (Popularity: 60)\n",
      "Muerte En Hawaii by Calle 13 (Popularity: 71)\n",
      "Berceuse by Felix Rösch (Popularity: 51)\n",
      "\n",
      "Recommendations for User 4:\n",
      "Mediterráneo by Jorge Drexler (Popularity: 33)\n",
      "You Got to Roll It Slow (Remix) by The Love Doctor (Popularity: 31)\n",
      "Say It Ain't So by Weezer (Popularity: 2)\n",
      "The Devil's Bleeding Crown by Volbeat (Popularity: 0)\n",
      "嘘 by SID (Popularity: 60)\n",
      "Kamoukié by Salif Keita (Popularity: 0)\n",
      "Sleigh Ride by Ella Fitzgerald (Popularity: 0)\n",
      "Naticongo - Ao Vivo by Natiruts (Popularity: 43)\n",
      "Estrechez De Corazón by Los Prisioneros (Popularity: 0)\n",
      "Bouncing Bona by Dele Ojo (Popularity: 16)\n",
      "\n",
      "Recommendations for User 5:\n",
      "Hold On - Remix by Chord Overstreet;Deepend (Popularity: 56)\n",
      "Say Something by A Great Big World;Christina Aguilera (Popularity: 74)\n",
      "2U - Acoustic Version by Clementine Duo (Popularity: 56)\n",
      "Porta Do Teu Prédio by Maneva (Popularity: 57)\n",
      "Bedroom Talk - LIVE by Official HIGE DANdism (Popularity: 24)\n",
      "Spring Fra Livet by Kvelertak (Popularity: 20)\n",
      "レイン by SID (Popularity: 49)\n",
      "Woodland Cathedral by Wolves In The Throne Room (Popularity: 18)\n",
      "O Seu Amor por Mim - Ao Vivo by Weslei Santos (Popularity: 47)\n",
      "Marieva by Projota (Popularity: 42)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "\n",
    "# Load data\n",
    "# spotify_data = pd.read_csv(\"spotify52kData.csv\")\n",
    "# star_ratings = pd.read_csv(\"starRatings.csv\", header=None)\n",
    "\n",
    "# Merge datasets\n",
    "merged_data = pd.merge(spotify_data, star_ratings, left_on=\"songNumber\", right_index=True)\n",
    "\n",
    "# Extract relevant columns\n",
    "user_ratings = merged_data.iloc[:, 20:].fillna(0)  # Assuming the ratings are in columns 20 and onwards\n",
    "\n",
    "# Initialize Nearest Neighbors model\n",
    "knn_model = NearestNeighbors(n_neighbors=10, metric='cosine', algorithm='brute')\n",
    "knn_model.fit(user_ratings)\n",
    "\n",
    "# Create personalized mixtape recommendations for each user\n",
    "user_mixtapes = []\n",
    "for user_index in range(user_ratings.shape[0]):\n",
    "    user_rating = user_ratings.iloc[user_index, :].values.reshape(1, -1)\n",
    "    _, indices = knn_model.kneighbors(user_rating, n_neighbors=10 + 1)  # Exclude the user itself\n",
    "\n",
    "    recommended_songs = []\n",
    "    for index in indices.flatten()[1:]:  # Exclude the user itself\n",
    "        recommended_songs.append(merged_data.iloc[index][['track_name', 'artists', 'popularity']].values)\n",
    "\n",
    "    user_mixtapes.append(recommended_songs)\n",
    "\n",
    "# Display the recommendations for a few users (adjust as needed)\n",
    "for user_index in range(5):\n",
    "    print(f\"\\nRecommendations for User {user_index + 1}:\")\n",
    "    for song_info in user_mixtapes[user_index]:\n",
    "        print(f\"{song_info[0]} by {song_info[1]} (Popularity: {song_info[2]})\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "064b9a97",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}