Model Predictions - Tuning and Blending.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from scipy.stats import norm\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "from scipy import stats\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_validate, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.svm import LinearSVR\n",
    "from sklearn.linear_model import LinearRegression, SGDRegressor \n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from yellowbrick.text import TSNEVisualizer\n",
    "from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comments</th>\n",
       "      <th>description</th>\n",
       "      <th>duration</th>\n",
       "      <th>languages</th>\n",
       "      <th>main_speaker</th>\n",
       "      <th>name</th>\n",
       "      <th>num_speaker</th>\n",
       "      <th>related_talks</th>\n",
       "      <th>speaker_occupation</th>\n",
       "      <th>tags</th>\n",
       "      <th>...</th>\n",
       "      <th>Fascinating</th>\n",
       "      <th>Unconvincing</th>\n",
       "      <th>Persuasive</th>\n",
       "      <th>Jaw-dropping</th>\n",
       "      <th>OK</th>\n",
       "      <th>Obnoxious</th>\n",
       "      <th>Inspiring</th>\n",
       "      <th>max_rating</th>\n",
       "      <th>yr</th>\n",
       "      <th>month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4553</td>\n",
       "      <td>Sir Ken Robinson makes an entertaining and pro...</td>\n",
       "      <td>1164</td>\n",
       "      <td>60</td>\n",
       "      <td>Ken Robinson</td>\n",
       "      <td>Ken Robinson: Do schools kill creativity?</td>\n",
       "      <td>1</td>\n",
       "      <td>[{'id': 865, 'hero': 'https://pe.tedcdn.com/im...</td>\n",
       "      <td>Author/educator</td>\n",
       "      <td>['children', 'creativity', 'culture', 'dance',...</td>\n",
       "      <td>...</td>\n",
       "      <td>10581</td>\n",
       "      <td>300</td>\n",
       "      <td>10704</td>\n",
       "      <td>4439</td>\n",
       "      <td>1174</td>\n",
       "      <td>209</td>\n",
       "      <td>24924</td>\n",
       "      <td>Inspiring</td>\n",
       "      <td>2006</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>265</td>\n",
       "      <td>With the same humor and humanity he exuded in ...</td>\n",
       "      <td>977</td>\n",
       "      <td>43</td>\n",
       "      <td>Al Gore</td>\n",
       "      <td>Al Gore: Averting the climate crisis</td>\n",
       "      <td>1</td>\n",
       "      <td>[{'id': 243, 'hero': 'https://pe.tedcdn.com/im...</td>\n",
       "      <td>Climate advocate</td>\n",
       "      <td>['alternative energy', 'cars', 'climate change...</td>\n",
       "      <td>...</td>\n",
       "      <td>132</td>\n",
       "      <td>258</td>\n",
       "      <td>268</td>\n",
       "      <td>116</td>\n",
       "      <td>203</td>\n",
       "      <td>131</td>\n",
       "      <td>413</td>\n",
       "      <td>Funny</td>\n",
       "      <td>2006</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>124</td>\n",
       "      <td>New York Times columnist David Pogue takes aim...</td>\n",
       "      <td>1286</td>\n",
       "      <td>26</td>\n",
       "      <td>David Pogue</td>\n",
       "      <td>David Pogue: Simplicity sells</td>\n",
       "      <td>1</td>\n",
       "      <td>[{'id': 1725, 'hero': 'https://pe.tedcdn.com/i...</td>\n",
       "      <td>Technology columnist</td>\n",
       "      <td>['computers', 'entertainment', 'interface desi...</td>\n",
       "      <td>...</td>\n",
       "      <td>166</td>\n",
       "      <td>104</td>\n",
       "      <td>230</td>\n",
       "      <td>54</td>\n",
       "      <td>146</td>\n",
       "      <td>142</td>\n",
       "      <td>230</td>\n",
       "      <td>Funny</td>\n",
       "      <td>2006</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>200</td>\n",
       "      <td>In an emotionally charged talk, MacArthur-winn...</td>\n",
       "      <td>1116</td>\n",
       "      <td>35</td>\n",
       "      <td>Majora Carter</td>\n",
       "      <td>Majora Carter: Greening the ghetto</td>\n",
       "      <td>1</td>\n",
       "      <td>[{'id': 1041, 'hero': 'https://pe.tedcdn.com/i...</td>\n",
       "      <td>Activist for environmental justice</td>\n",
       "      <td>['MacArthur grant', 'activism', 'business', 'c...</td>\n",
       "      <td>...</td>\n",
       "      <td>132</td>\n",
       "      <td>36</td>\n",
       "      <td>460</td>\n",
       "      <td>230</td>\n",
       "      <td>85</td>\n",
       "      <td>35</td>\n",
       "      <td>1070</td>\n",
       "      <td>Inspiring</td>\n",
       "      <td>2006</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>593</td>\n",
       "      <td>You've never seen data presented like this. Wi...</td>\n",
       "      <td>1190</td>\n",
       "      <td>48</td>\n",
       "      <td>Hans Rosling</td>\n",
       "      <td>Hans Rosling: The best stats you've ever seen</td>\n",
       "      <td>1</td>\n",
       "      <td>[{'id': 2056, 'hero': 'https://pe.tedcdn.com/i...</td>\n",
       "      <td>Global health expert; data visionary</td>\n",
       "      <td>['Africa', 'Asia', 'Google', 'demo', 'economic...</td>\n",
       "      <td>...</td>\n",
       "      <td>4606</td>\n",
       "      <td>67</td>\n",
       "      <td>2542</td>\n",
       "      <td>3736</td>\n",
       "      <td>248</td>\n",
       "      <td>61</td>\n",
       "      <td>2893</td>\n",
       "      <td>Informative</td>\n",
       "      <td>2006</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   comments                                        description  duration  \\\n",
       "0      4553  Sir Ken Robinson makes an entertaining and pro...      1164   \n",
       "1       265  With the same humor and humanity he exuded in ...       977   \n",
       "2       124  New York Times columnist David Pogue takes aim...      1286   \n",
       "3       200  In an emotionally charged talk, MacArthur-winn...      1116   \n",
       "4       593  You've never seen data presented like this. Wi...      1190   \n",
       "\n",
       "   languages   main_speaker                                           name  \\\n",
       "0         60   Ken Robinson      Ken Robinson: Do schools kill creativity?   \n",
       "1         43        Al Gore           Al Gore: Averting the climate crisis   \n",
       "2         26    David Pogue                  David Pogue: Simplicity sells   \n",
       "3         35  Majora Carter             Majora Carter: Greening the ghetto   \n",
       "4         48   Hans Rosling  Hans Rosling: The best stats you've ever seen   \n",
       "\n",
       "   num_speaker                                      related_talks  \\\n",
       "0            1  [{'id': 865, 'hero': 'https://pe.tedcdn.com/im...   \n",
       "1            1  [{'id': 243, 'hero': 'https://pe.tedcdn.com/im...   \n",
       "2            1  [{'id': 1725, 'hero': 'https://pe.tedcdn.com/i...   \n",
       "3            1  [{'id': 1041, 'hero': 'https://pe.tedcdn.com/i...   \n",
       "4            1  [{'id': 2056, 'hero': 'https://pe.tedcdn.com/i...   \n",
       "\n",
       "                     speaker_occupation  \\\n",
       "0                       Author/educator   \n",
       "1                      Climate advocate   \n",
       "2                  Technology columnist   \n",
       "3    Activist for environmental justice   \n",
       "4  Global health expert; data visionary   \n",
       "\n",
       "                                                tags  ...   Fascinating  \\\n",
       "0  ['children', 'creativity', 'culture', 'dance',...  ...         10581   \n",
       "1  ['alternative energy', 'cars', 'climate change...  ...           132   \n",
       "2  ['computers', 'entertainment', 'interface desi...  ...           166   \n",
       "3  ['MacArthur grant', 'activism', 'business', 'c...  ...           132   \n",
       "4  ['Africa', 'Asia', 'Google', 'demo', 'economic...  ...          4606   \n",
       "\n",
       "  Unconvincing  Persuasive Jaw-dropping    OK  Obnoxious  Inspiring  \\\n",
       "0          300       10704         4439  1174        209      24924   \n",
       "1          258         268          116   203        131        413   \n",
       "2          104         230           54   146        142        230   \n",
       "3           36         460          230    85         35       1070   \n",
       "4           67        2542         3736   248         61       2893   \n",
       "\n",
       "    max_rating    yr  month  \n",
       "0    Inspiring  2006      6  \n",
       "1        Funny  2006      6  \n",
       "2        Funny  2006      6  \n",
       "3    Inspiring  2006      6  \n",
       "4  Informative  2006      6  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main = pd.read_csv(\"datasets/cleaned.csv\").drop(columns=\"Unnamed: 0\")\n",
    "main.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestRegressor(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "svr = LinearSVR(random_state=42, C=1.49)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "transcript = main[\"transcript\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = main[\"views\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['comments', 'description', 'duration', 'languages', 'main_speaker',\n",
       "       'name', 'num_speaker', 'related_talks', 'speaker_occupation', 'tags',\n",
       "       'title', 'url', 'views', 'transcript', 'Funny', 'Beautiful',\n",
       "       'Ingenious', 'Courageous', 'Longwinded', 'Confusing', 'Informative',\n",
       "       'Fascinating', 'Unconvincing', 'Persuasive', 'Jaw-dropping', 'OK',\n",
       "       'Obnoxious', 'Inspiring', 'max_rating', 'yr', 'month'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['comments', 'duration', 'languages', 'num_speaker', 'Funny',\n",
       "       'Beautiful', 'Ingenious', 'Courageous', 'Longwinded', 'Confusing',\n",
       "       'Informative', 'Fascinating', 'Unconvincing', 'Persuasive',\n",
       "       'Jaw-dropping', 'OK', 'Obnoxious', 'Inspiring', 'max_rating', 'yr',\n",
       "       'month'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_data = main.drop(columns=[\"description\", \"main_speaker\", \"name\", \"related_talks\", \"speaker_occupation\", \n",
    "                            \"tags\", \"title\", \"url\", \"transcript\", \"views\"])\n",
    "rf_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "rf_data[\"max_rating\"] = le.fit_transform(rf_data[\"max_rating\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:652: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=3.\n",
      "  % (min_groups, self.n_splits)), Warning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal number of features : 8\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAEFCAYAAADT3YGPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4lNX1wPHvJCELS9j3ffMIhDWggCC4C664VKXuS7W17U/Utlq1atVWrdrS1qVV3DfqvqLgghJ2whrBw5IAsiasCRCyzu+P9w2MMctAMvPOJOfzPDxk7tyZ98wQ5sx773vP9fn9fowxxphwifE6AGOMMfWLJR5jjDFhZYnHGGNMWFniMcYYE1ZxXgcQ6dLT0xOAYcBWoMTjcIwxJlrEAu2BhampqQWBd1jiqd4wYJbXQRhjTJQaDaQFNljiqd5WgGOOOYb4+HivYzkkIyODlJQUr8MISjTFCtEVbzTFCtEVbzTFCpEXb2FhIatXrwb3MzSQJZ7qlQDEx8eTkJDgdSw/EmnxVCWaYoXoijeaYoXoijeaYoWIjfcnUxR2cYExxpiwssRjjDEmrCzxGGOMCStLPMYYY8LKEo8xxpiwssRjjDEmrDy/nFpEfMAmYI3bNFdV7xSRc4A/AcXA86r6rIgkAa8CbYA84CpVzalp3/C9WmNqR0lpKW8sWc+cFTkMHFRKXKx9hzTRw/PEA/QEFqvqOWUNItIA+DtO1YD9wGwR+QiYCKxQ1ftE5FLgbhG5vaZ9VXVb2F6tMTXg9/t5P+MH/jRtKSu37wWg9Yzl3H/mII8jMyZ4kfA1KRXoKCJfi8inIiJAH2Ctqu5W1UKccgujgVHAZ+7jpgGn1lJfYyKa3+9nhm5h+ORpXPTiN3yfncs1x/WkQ6MGPPTFCmboFq9DNCZoYT3jEZHrgEnlmm8G/qqqb4nIKJzhsUnA3oA+eUBTIDmgvaK2o+1brYyMjGC6hVV6errXIQQtmmKFyIp3ec4Bnl6WTXr2AQBO65LMLwa0pmtyAmOad+L6GVlc9tJMXh3Xg9YNG3gcbfUi6b2tTjTFCrUTr9/vZ/O+IpbmHGBnfjETj21Jg1hfLUR3WFgTj6pOAaYEtolIQ5z5FlQ1TUQ64iSEJgHdmgB7gNyA9orajrZvtVJSUiKqHEV6ejqpqalehxGUaIoVIife5Vt2c8+0pXy8chMA4/p05MFxgxjUscXhTunpPH7uMP7v/YU8uiKX6TeeGtHzPZHy3gYjmmKFo4+3uKSUpVt2Mzsrm7SsbGZnZbM97yAAPh/cePrx9GvX7Iift6CgoNIv7JEwx3MvsBN4VEQGAhuBlUBvEWkB7ANOBB4DugLjgQXAOJyq0atqoa8xEWNNTi73fb6MqUvX4/fD6B5teHDcYEb1aFNh/5tHCTPXbee9FRt5wOZ7TDX2FRQxb8MOZrtJZt6GHewvLD50f4fkJC4e2JVR3dtwcu929D2KpFOdSEg8DwOvishZOGc+V6tqkYjcCnyOMw/1vKpuFpGngZdEJA0oBCbWRt9wv2BjKrJpz34enLGC5xespaTUz5BOLXhw3GBOl/b4fJUPdfh8Pp67ZARLN+/ioS9WMKp7G06TDmGM3ESyrbkHSMvKYXZWNnOyslm6ZTclpf5D9/dr15QTurfhhO5tGNW9DV2bN6ry9602eJ54VHU3cFYF7R8BH5VrOwBcXNt9jfFSzr6DPPJVBk/NVgqKSzm2TTL3nzmICwd0CfoDoFlSPG9eeSKj/vUZV7yexuJbz6ZD04YhjtxEGr/fz/fZuYeGzGZnZZO5c9+h++NjYxjepZWTaHq0YWS31rRoGP4pBM8TjzH1Ve7BQp6YuYq/f7uSfQXFdGneiHtPH8jlqd2Pap5maOeW/O2cIdzy/iIufy0t4ud7TO0oLfXz3Pw1vDF3I9+9v46dBw5v9tk8KZ6z+nZklHtGk9qpJYkNYj2M1mGJx5gwyy8q5qk05eGvMth1oJA2jRN5aNxgbhjRm4S4mn0o/HrUscxct533V/xg8z31xEuL1vHLt+cD0K1FI87s0+HQsFmfNk2JiQntsNnRsMRjTJgUlZQyZf5aHpqxnC25+TRLiueh8YP4zahjaZRQO5dB+3w+plwykqWbP7b5nnqgoLiEP09fTkJcDC+f0Z2LTh7pdUhBsfNwY0LM7/fzWnomfR/5gJvfmc+eg4XceUoKa/94Pnec0r/Wkk6ZZknxvHnFicTFxHDF62ls2XugVp/fRI7/zl3Nxt37+dUJQvemkbPcozqWeIwJsbs+XcKVr8/mhz0H+PUoYc2dE3hw/GCah3BSd1iXVvztnCHk7Cvg8tfSKC4pDdmxjDf2FRTx0BcraJLQgDtOTvE6nCNiiceYEHpzSRaPfPUdvVs14fs7zmPyhONol5wUlmP/etSxnN+/M9+s284DM5aH5ZgmfCZ/u4qcfQXcOqYPrRoneh3OEbHEY0yILNm0i+unzqVJQgPeu/YkurVoHNbjl833dGvRyOq51TE79xfw2MyVtGqUwKQxfb0O54hZ4jEmBHL2HeSCF2eSX1TCyxNPoE/boEoC1jqb76mbHv0qg9yDRdxxSgpNEiO/Pl95lniMqWVFJaVc+vK3bNy9n/vOGMi5KZ09jcfme+qWzXsP8O80pVPThvxypHgdzlGxxGNMLbv9w0XMXLed8/t35q5T+3sdDmDzPXXJQzNWcLC4hHtOHxARi0GPhiUeY2rRCwvW8u80pV+7prx46QkRs3jP5nvqhnU78pgyfw3HtE7m6mE9vQ7nqFniMaaWzNuQw6/enk/zpHjeu+akiBt7t/me6HfvZ0spLvVz/5kDo7ocUvRGbkwE2bL3ABe9+A3FpX7euGI0PVs1qf5BHrD5nui1fMtu3ly6nsEdW3DRgK5eh1MjlniMqaGC4hIufukbtubm88jZQyK+RI3N90Snu6ctwe+HB8YNipgh3KNliceYGvD7/dz8znzmbdjBxCHdmTSmj9chVcvme6LPnKxsPlm5mdE92nDmsZH9xSYYlniMqYGnZisvLFjHkE4t+O/Phod8A63aUn6+Z2uuzfdEKr/fz93TlgLw0PjBUfM7VhVLPMYcpZlrtzHpg0W0bpzAO1ePJalBdBV7/9F8z6tplJTafE8kmq5b+Wbddsb36cgJ3Sve/jzaWOIx5ihs2LWPS17+Fh/w1lVj6NK8kdchHZWy+Z6Z67bzwPQVXodjyikt9XP3tCUAPDi+7uytZInHRJXsvHwe/nIFvf/yHgP+9iGfrNwU9hgOFBZzwQsz2bG/gH9MGMboHm3DHkNtCZzvefCL5XyxeqvXIZkA76zYyOJNu7hkUDcGdmjhdTi1xhKPiXh+v585Wdlc8VoaXR94l7s+Xcr2vIN8n53LuVO+5uznvmJ1Tm7YYrl+6lyWbtnN9cN7cdOIY8Jy3FBqlhTPG2XzPa/ZfE+kKC4p5d5pS4mN8XH/mQO9DqdWWeIxEetAYTHPzVvD0Cc+YfS/P+f1xVn0bNWEf04YxqZ7L2TxrWdxcq92TFu1mQF/+4g/fJRO7sHCkMb02Ncrmbp0PSO7teZfE46rExO9AMd1acWjZw8he99Bm++JEC8vykRzcrn2uF70bp3sdTi1yvPZUBG5AzjTvdkMaKeq7UTkVuA6IMe970ZgI/Aq0AbIA65S1RwROQf4E1AMPK+qz4pIUrB9w/JCTdBW5+TyzBzlpYWZ7MkvJDbGxwUDuvCrE4SxPdse+rBPaR/P9JtO5b0VP/C7jxbx2MyVvJKeyV/PGsIVqT1qfa3DZ99v5s5PF9MhOYm3rhpDfFx01smqzG9GH8vMddv5IOMHJrwwk2GdW9E+OYkOTRvSvkkSHZom0bpRYtSvIYkGB4tK+PP0ZSTGxXLP6QO8DqfWeZ54VPVh4GEAEfkY+IN71xDgSlVNL+vrJqMVqnqfiFwK3C0itwN/B4YB+4HZIvIRMDHYvqq6LSwv1lSqpLSUj1du5qnZemieoV2TJO45bQDXD+9Fp2YVT977fE5SGtenA4/PXMnDX2Zw7ZtzeGaOMnnCcRzXpVWtxLcmJ5eJr8wiPjaGd64ZG7bN3MLJme8ZgWbv5ZOVm/lk5eaf9ImL8dHOTULtkxvSITmJ9snuz02T6OC2tWyUUGfOBr3wn7mr+WHPAW4b25eOTRt6HU6t8zzxlBGRC4Ddqvq525QK3Cki7YBPVPWvwCjgUff+acA9QB9grarudp8nDRh9hH3fCvHLM5XIzsvn+QVr+c/cNWzcvR+AE3u04aaRwoT+nYM+q0hqEMfdpw3gyqE9+cPH6fxv6QZGTJ7GVcN68pfxg2uUKHIPFjLhhZnsPVjE85eOrLVkFomaN0xg8W1ns3ZHHltz89mSe4Cte52/t+Tms3VvPlvzDrB0824WbNxZ6fPEx8bQLjnJTUyHE9Tu7F0sLlpDQlwMiXGxJMbFkhAXS2KDWBLjYpyf42Kd+xscvj8hLobYmPoxM5B3sIi/uFta/yHKtrQOVlgTj4hcB0wq13yNqi4E7gQuC2h/E3gSyAXeE5GzgWRgr3t/HtC0XFtl7dX1rVZGRkYw3cIqPT29+k4RIjBWv9/Pih35vL1mN19uzKWo1E9SnI8Lezfnwt7N6dUsEUp3smJZ5R9sVbm9byNObtWVxxdt46WF63h7SRbX9W/FJce0pEFscN/Cy+It9fv5/bc/sGr7Pi6VFqTE7Im49z1U8TQHmvugXzOgWSzQ2P3j/BvuLSxhR34xOQeK2XmwmJz8InYcKCYnv9hpzy9g4cb9lPjLB3x0AwxxMRAfE0N8rM/5E+MjPjaGxg1iGNmhMWd0a0rHxvE1eMU/5cW/9XMrctixv4Bf9G/N+u8zWH8Ej420383KhDXxqOoUYEr5dhHpC+xR1bXubR/wD1Xd697+BBiMk4TKqi82AfaUa6usvbq+1UpJSSEhISGYrmGRnp5Oamqq12EEpSzW/QVFvLFkPU/PVpZu2Q1An7ZN+eXIY7hiaA+SE2vvQyMVuPaMUp6dt5Z7pi3hn0uy+WzTQZ44byjj+nQMKl6A+z5bxreb93Fyr3a8dN0pEVcRONJ/D0pL/ew8UOCcOeXms2LVajp06crB4hIKi0s5WFzCwaISCsp+Li6hwG1zfnbaCyrps6eohA15B1m+I59nlucwsltrJg7pzsUDu9KqcWKNYvfivd25v4A33nmP1o0TePTSU46ownmk/S4UFBRU+oU9UobaTsUZDiuTDGSISB+cuZiTgeeBA8B4YAEwDpgFrAJ6i0gLYB9wIvAY0PUI+poQ2pBbwGsfLOTFBevYe7Co0osFaltsTAw3jTyGnw3qyn2fLeOZuas5+7mvGN+nI0+cN7TaK4XeXb6RB2Ysp1uLRrxxxeiISzrRICbGR+vGibRunMjADtB6/zZSU3vU6jH25Bfy7vKNvLE4i6/XbWPO+hxueX8hp0sHJg7pzrn9OtEoIbK2qKjMI19lkFdQxP1nDo24bTVqU6QkHgFmlN1Q1b0i8kfga6AA+FJVPxWRmcBL7txMITBRVYvciw4+x7k8/HlV3SwiTwfbN4yvs17Zk1/ILe8v5JVFmYBzscBvR/ep8mKBUGjRMIF/XnAcN4zozaT3F/Lpqs3MWL2VW07sw12n9q/wP3jG1t1c/cZsGsbH8t41J9X427MJnWZJ8Vx7fC+uPb4Xm/ceYOqS9by+OItPV23m01WbaRQfx/n9OzNxSHdO7d0+Yr9AbN57gCfTlM7NGnJjHVgfVhWf319+ANYESk9P7wZk2VDbkZmhW7h+6lw27T2ANE/k3rOGHdHFAqHi9/t5d8VGfvdhOht276ddkyT+evZgLh9y+PLrr+Ys4MaZm8ncuY83rzyRiwdG7t4nkf57UF444121fS+vL87kjcXrydq1D4DWjRP42cBuTEztzvFdWlV5th3u9/amt+bx7Lw1PPuzEVx7fK8jfnyk/S4EDLV1T01NXR94X6Sc8Zg6Yn9BEXd8soSnZitx7orr05oVcvzgbl6HBjiXDF84oCvj+3Tksa9X8shXGVzzxhyemb2ayROGMbhjC+6evYnMnfu585SUiE46pmp92jblgXGD+fOZg5i3YQevL87if0vX8+Rs5cnZSs+WTbhsSDcmDumOtAnqGqOQWZOTy/ML1iKtk7lyaO0ORUYiSzym1sxdn8PVb8xm7Y48+rZtyksTT2BIp5YReaVNUoM47jl9AFcN68nvP0rnrWUbGD55GkM6tWDxtv2M69OxzpUpqa98Ph8jurVmRLfWPHHeUGas3srr6Zl88N0PPDhjBQ/OWEFqpxZMHNKdSwZ3o31y+NfN3PvZMkpK/dw/blDEDgXWJks8psYKikv48/TlPPrVd/jxc9vYvvz5zEEkNoj8lf1dmjfizStP5JfrtnPLewtZvGkXXZrE89rPR9WbdSP1SYPYGMb36cj4Ph3ZV1DEBxk/8PriLGas3kr6pl387qPFnNSrLROH9KB7aUlYYlq2ZRdTl65nSKcWXNi/S1iO6TVLPKZGlm/ZzVWvz2b51t10b9GYFy4bGZXVmsf0bMvCSeP5ZNVmEvdspmlS7a4HMZGncUIDfp7ag5+n9iA7L5+3lm3g9cVZfLlmG1+u2UZSnI/rt/m45cQ+dGvROGRx3P2ps8nbg+MG15tyRJZ4zFEpLinlsZnfcd/nyykqKeWG4b352zmpUX0JaFxsDOeldCY9PdvrUEyYtWmSxM2jjuXmUceybkcebyzJ4slvv+Nfs77nyTTlooFduG1sP4Z2blmrx52dlc2nqzYzpmdbTpf2tfrckcwSjzlia3JyufqN2czbsIP2yUk8+7MR1S7KNCZa9GzVhLtPG8BpzQpZE9OCJ2au5H9LN/C/pRsY27Mtt47ty7hjO9b47MTv93PXp+4mb+MG1avadpZ4TNBKS/08PUf5w8eLyS8q4dLB3fjXBcfRomHkXGZuTG2Ji/FxeWoPfj6kO1+s3srjM1cyY/VWZq7bTt+2TZk0pi8/T+1OwlEuEfjs+y3MyszmrL4dGVlHtrQOliUeE5Qfdu/nuqlz+HLNNlo0jOf5S0fys0HdvA7LmJDz+XycJh04TTqwbMsunpi5ijeXZHHD/+Zyz7Sl/Ga0cOOIY2h+BF/ASkv93DNtKT6fM7dT39hlO6ZKfr+flxetY+BjH/Hlmm2M79OR5b87x5KOqZcGdmjBSxNPYN1dE7htbF/2FxZz16dL6frAu0x6fyHr3YWq1Xl7+QaWbN7FpYO6MaBD8xBHHXks8ZhKZeflc+GL33DNG3Mo8fv578+G8+F1J3myzsGYSNKpWSMePSeVDfdcwKNnD6FZUjz/nPU9vf/yPpe98i3pP1ReWb24pJR7P1vmLrAeFMaoI4cNtZkKvbdiI798ex45+woY07MtUy4ZQfeWTap/oDH1SNOkeG47qR+/GX0sU5du+MmFCLed1I8zpcOPLkR4ceE6VufkcuOIY+jZqn7+n6o28YhIPPA7nEKevwZuAR5W1dBubm88EVjYMyEuhsfPTeW3o/vUm/UFxhyN+LhYrhjag8tTuzPDvRDhiwouRPD74YHpy0mMi+Wu0/p7HbZngjnjeRLIwdmKuhjohbNFweUhjMt44IvVW7nuzTls2nuAoZ1b8uJlJ9Cnrbc1rIyJJj6fj9OlA6dLB5Zu3sUT36xk6pL1hy5EGNq5JZv2HuD2OrqldbCCmeNJVdU/AkWqegC4CqifA5N12Jert3LGf75gW14+954+gLTfnGlJx5gaGNSxBS9PHMXaP07g1jHOhQgfr9xEcmIDfl9Ht7QOVjBnPH53uK1s/4RWAT+bOuLJ2QrApzecwinH1J8V1MaEWufmjfjbuancfVp/XkvPQtok07JR/V77Fkzi+QfwBdBORP4BTADuD2lUJqyy8/L5ZOUmBnZobknHmBBpmhTPr0aJ12FEhGASzzQgHTgJiAXOUdXlIY3KhNUbS9ZTXOrnqmE9vQ7FGFMPBJN4ZqlqH2BlqIMx4ef3+3lxwTriYnxMHNLd63CMMfVAMIlnmYhcASwA8ssaVXVjyKIyYbN0826Wb93NeSmdad040etwjDH1QDCJ53j3TyA/UPf3Z60HXly4FoCrbZjNGBMm1SYeVbXxlzqqoLiE1xdn0aZxom1rYIwJm2AqF7QG/g2c4vb/Cvilqm4PcWwmxD5euYldBwqZNKYPDerBPu/GmMgQzFDbf4A5wA04C05/AUwBzj7ag4rIBOBiVZ3o3h4OTMapjDBdVe8XkRjgKWAgUABcr6prQ9X3aF9LNHtp4ToAu5rNGBNWwSSeHqp6QcDtR92LDY6KiEwGzgCWBjQ/A1wIZAKfiMgQoBuQqKoj3ATyOHBeCPvWK9ty8/ns+y2kdmpB//b1ryy7McY7wVYu6KyqPwCISBegqAbHnAO8D9zoPl8ykKCq69zbn+MM67UHPgNQ1XkiMjRUfYMJOiMjowYvOTTS09OP+rGvrtpBSamfk9vF1+h5ghWOY9SmaIo3mmKF6Io3mmKF6Ik3mMRzDzBXROYDPpwr3H5R3YNE5DpgUrnma1R1qoiMDWhLBnIDbufhXDGXDOwNaC8JVV8RiVPV4qpeT0pKCgkJkVPmIj09ndTU1KN6rN/v5+ovPyI+Nobfn3diyLeurkmsXoimeKMpVoiueKMpVoi8eAsKCir9wh7MVW0fi8hg4DicOZ4bVTUniMdNwZkLqk4uELgpRRNgD9CwXHtMqPpWl3TqmkU/7GTl9r1cNLBryJOOMcaUV+2lTCJyEvC+qn4CrAbmi8jI2gpAVXOBQhHpKSI+nPmfWcBsYLwbw3BgRaj61tZriRYv2kUFxhgPBXMN7eO48zGqqjgf2pNrOY6bgNdwqiMsUdX5wHvAQRGZA/ydw8N2oepbLxwsKuHNJetpn5zE6VYQ1BjjgWDmeBJV9dBAnap+LyINanJQVZ0JzAy4PQ8YXq5PKU7iKP/YkPStLz7I+IE9+YXcMLwfcbZ2xxjjgWASz/ci8gjwCk6pnIk4Q24mCr20yIbZjDHeCuYr73VAI+ANnOTTEGcxqYkym/ceYIZu5fgurWx3UWOMZ6pNPKq6G7hVVfsDl+BsCpcX6sBM7Xtl0TpK/X6uOs7Odowx3gnmqrY/AS+5C0e/AW7BmZQ3UcTv9/PSwkwS42K5ZFA3r8MxxtRjwQy1nQdcizO385qqngacENKoTK2buz6H1Tm5nN+/M82S4r0OxxhTjwWTeGJUNR+nKOgnbpHNRqENy9Q2u6jAGBMpgkk8X4hIBhAPfIsz3PZhSKMytepAYTFTl2ygU9OGnNK7ndfhGGPquWAuLvgdzqLREe4amN+o6h9CHpmpNe+t2EheQRFXDO1BbIyt3THGeCuYdTyo6saAn5dW1ddEHtt3xxgTSezrbx23Ydc+vlq7jVHd29C7dbLX4RhjjCWeuu6V9Ez8frhyWA+vQzHGGCCIoTYR6Qr8GmiBsx8PAKp6bQjjMrXAWbuzjobxsfxsYDevwzHGGCC4OZ7/4WwnMAunVpuJErMys8ncuY/LU3vQJLFGdV2NMabWBJN4Gqjq7SGPxNS6sosKrrYSOcaYCBLMHE+aiJwjIrbcPYrsKyjirWUb6NaiEWN6tPU6HGOMOSSYM56LcOZ4EJGyNr+qxoYqKFNz7yzfyP7CYm4b2peYGF/1DzDGmDCpNvGoaodwBGJqV9kw25VD7Wo2Y0xkCeaqtobAvcApbv+vgHtUdX+IYzNHKXNnHt+s287Ynm3p3rKJ1+EYY8yPBDPH82+coqDXAlfh1Gx7JpRBmZp5eWEmAFdapQJjTAQKZo4nVVUHBtz+tYisDFVApmZKS/28vGgdjRPiuGhAF6/DMcaYnwhqWwQRaVZ2w/25OHQhmZqYuW4bG3bv56IBXWmUYGt3jDGRJ5gznieAhSLyIU7lgnOAv9bkoCIyAbhYVSe6t08BHgSKgGzgSlU94B6zpduer6rjRKQX8CLOYtYM4GZVLRWRe4GzcJLiLaq64Ej61uT1RJKX3GE2W7tjjIlUwWyL8AIwAcgEsoALVPX5oz2giEzGSVyBx34KOF9VTwTWANe77b2AUao6VlXHuW1PAHer6micRHieiAwBxgDHA5cCTx5F36iXe7CQd5ZvoGfLJozq3sbrcIwxpkKVJh4ROdv9+0pgCJAH7AUGu21Haw7wy3JtY1V1u/tzHHBQRNoCzYCPRCStLB4gFWczOoBpwKnAKGC6qvrdLRziRKT1EfaNem8t20B+UQlXDeuBz2drd4wxkamqobZhwMfASRXc5wderuqJReQ6YFK55mtUdaqIjA1sVNWt7mMmuMe7B2gNPA5MxilQOltEFgA+VS2rGZcHNAWSgZ0BT1nWfiR9c6p6PRkZGVXd7Yn09PQf3X7y6yx8wKCEAz+5z2uRFk91oineaIoVoiveaIoVoifeShOPqt7r/vi6qs4IvE9ELqjuiVV1CjAl2EBEZBJOlYQzVfWgiGwDnlHVYiBbRJYAApQGPKwJsAfIdX8u334kfauUkpJCQkJCsC8n5NLT00lNTT10e01OLstyVnJK73acdeJwDyP7qfKxRrpoijeaYoXoijeaYoXIi7egoKDSL+yVJh4RuQRIAP4sIn8q95g/Au/WVoAichfOsNipqprvNp+KU6rnLBFpDKQAq4AlIjJWVWcC44CvgbXAoyLyGNAJiFHVHSISdN/aei1eOVwQtJfHkRhjTNWqGmprApzg/h043FYM3FVbAbhzOfcCi4Fpbj24qar6tIicISLzcM5c/ugmk9uAZ92ipauAt1W1RERmAXNx5q1udp/+SPpGrZLSUl5elElyYgPOT+nsdTjGGFOlqobangOeE5FTVPXL2jyoewYy0/15O041hIr63VJB22qcq9LKt98H3He0faPZl2u2sXnvAW4Y3puG8cFcIW+MMd4J5lNqv4h8ADTGuSQ5Fuiqqt1CGZgJ3osLnGG2q6xEjjEmCgRTueB54H2cJPUksAl4L5RBmeDtyS/k/YyNSOtkhndt5XU4xhhTrWAST4G7iHQmsBu4EjgjlEGZ4L25ZD0FxaVcNaynrd0xxkSFYBLPQRFpASgwXFVLcIbbTAR4eeE6Ynw+Lrd9d4wxUSKYxPMEMBX4CLhCRL4DomOVUh23avte5m9tT65/AAAZVUlEQVTcwWnSno5NG3odjjHGBCWYWm1vAaerah4wFLgc+HmoAzPVO7R2xy4qMMZEkaoWkL6AUxqn7Hb5LteGKCYThOJSP68syqJZUjzn9rO1O8aY6FHVGc9MnAKbTYAOOFteTweaV/M4Ewbzt+5jW14+lw3uRmIDm3IzxkSPqhaQvgQgIr8CRqhqqXv7f8C88IRnKvNRplNeztbuGGOiTTBnLk1xqkOXaYuzmNR4ZOf+AmZt3ke/dk0Z2rml1+EYY8wRCaZywUPAchGZjZOohgO/DWlUpkpvLsmiqNTP1cN62dodY0zUCeaqtldwKke/CbwGDFbVd0IdmKncSwvXEeuDiUO6ex2KMcYcsap2IP2F+/efgBuAfjhbE9xUbpsEE0ZZO/NI37SL4e0b0y45yetwjDHmiFU11OYr97eJAN9mZgMwvH0jjyMxxpijU9VVbf9x/74/fOGY6szK3A7A4DaWeIwx0amqBaSlBCwgDeAD/Kpqi0c8kJaZTXJiA3o2jZxtuI0x5khUdcZji0QjzLbcfNbsyGNcn47ExtgIqDEmOlV7ObWItMapzxa4EVx3Vb0yxLGZcmZlOfM7o7u3AQq8DcYYY45SMGc1U4FBOMmnEXARUBrKoEzFZq1z5ndG92jjcSTGGHP0gkk8HVT1KpxtEd4FTgQGhzQqU6G0rGwS42KtWoExJqoFk3h2u38rMFBVd4YwHlOJPfmFLN+6m+FdWxEfZ9d1GGOiVzAlc74SkbeA24HpIjIEyK/JQUVkAnCxqk50b18A/A34we1yLzALeAoYiDOhcb2qrhWR4cBkoBiYrqr3i0hMTfvW5PWEw+ysbPx+GGXDbMaYKBdMyZy7gDtUdQMwEefM54KjPaCITAb+Wu7YQ4Dfq+pY9883wPlAoqqOAO4AHnf7PuPGMQo43k2EtdE3os1yF46O7tHW40iMMaZmgrmqbSnwioi8rqrp1Hzb6znA+8CNAW2pwGARuQVYAPwBJ1l8BqCq80RkqIgkAwmqus6N7XPgFKB9TfrW8PWERVpmNrExPoZ3beV1KMYYUyPBDLX9HLgM+EZENgCvAO+q6r6qHiQi1wGTyjVfo6pTRWRsufYZOMkoC+cs5SYgGdgb0KfEbcsNaMsDetS0r4jEqWpxVa8nIyOjqrtD6mBxKQs25nBsi0Q0Y/mh9vT0mn4HCJ9oihWiK95oihWiK95oihWiJ95qE4+qfgfcDdwtIqOBfwBP41xaXdXjpgBTgozjeVXdAyAiHwAX4iSHJgF9YnASSWBbE2AP0LAmfatLOgApKSkkJHhTLeDrtdso8X/PGSk9SE1NBZxfsLKfI100xQrRFW80xQrRFW80xQqRF29BQUGlX9irneMRkVgRGS8iLwOv4gy1nVlbwYmID2e/n05u0ynuMWYD490+w4EVqpoLFIpIT/dxZ+BchFCjvrX1WkLF1u8YY+qSYIbaNuFsdf0azhVghbUZgKr6ReR64F0RyQdWAs/iDJedJiJzcComXOM+5CY3llicK9Xmi8jCWugbsdLcigUndLfEY4yJfsEknn6quqs2D6qqM4GZAbenA9Mr6HpTBY+dh7MLamBbaU37RqrC4hLmrM8hpV0zWjaywqDGmOgXzOXUtZp0zJFZvHkX+UUlNsxmjKkzrAJ1hEtz1+/YwlFjTF1xRIlHRJJFpF+ogjE/9W1m2YUFtnDUGFM3BLOA9HqcxZy/A5YAeSLyiqr+JdTB1XelpX5mZ+XQo2VjOjZt6HU4xhhTK4I54/klcCfOItIPgP7UoGSOCV7Gtj3syS+0sx1jTJ0S1FCbqm7FWfvyibvYMimkURkgYH7HLqM2xtQhwSSe70TkY5xyM1+IyFRgYWjDMnB4fufEnpZ4jDF1RzCJ51rgUWC4u3j0FeD6kEZl8Pv9pGVl065JEj1bNqn+AcYYEyWCSTxdgc7AbhH5L85eOQNDGpVh3c48tubmM7pHG3w+n9fhGGNMrQkm8bzg9jsXOAa4FfhXKIMygfvv2DCbMaZuCSbxJKrqK8A5wGuqOguw2i0hZhu/GWPqqmAST4mIXAicDXwsIufhFPA0IZSWmU2zpHhS2jXzOhRjjKlVwSSeXwBnAb9yL6u+DLu4IKS27D3Aup15nNC9NTExNr9jjKlbgikSugL4O9DB3Zr6YVVdXs3DTA0cGmbrbsNsxpi6J5iN4K7A2Za6O84Vbu+KyLWhDqw+m1VWn83W7xhj6qBg9uO5DThOVXcCiMhDOHvpPB/CuOq1tKxskhrEMqRjC69DMcaYWhfMHE9sWdIBUNUdQGnoQqrfdh0oYMXWPYzo2pr4uFivwzHGmFoXzBnPMhH5BzDFvX0dsCx0IdVvs7Ns/x1jTN0WzBnPDUABztDai0Ah8KsQxlSv2cJRY0xdF8wZz1Oqek3IIzGAs34nLsbH8K6tvQ7FGGNCIpgznhQRaRzySAz7CopI37SToZ1b0jA+mO8ExhgTfYL5dCsFNoqIAvlljap68tEeVEQmABer6kT39syAu48FXlTVO0RkCbDXbc9S1WtEZDgwGSgGpqvq/SISAzyFU7y0ALheVdceSd+jfS21ad6GHRSX+m3/HWNMnRZM4vl9bR5QRCYDZwBLy9pUdax7Xw/gf8CDIpIYeF+AZ4ALgUzgExEZAnTDqSk3wk02jwPnHWFfz5Vt/Da6py0cNcbUXVUmHhFpDnznXkKNiIwBVqpqTg2OOQdnQeqNFdz3D+APqrpPRI4HGorIdDfOPwIrgQRVXefG8zlwCtAe+AxAVeeJyFARSQ62bw1eS62albkdnw9O6GbzO8aYuqvSxCMig4FPgWtwP6iB04HXRWRcdWVzROQ6YFK55mtUdaqIjK2g/wAgWVW/dJsOAI8BzwG9gWnAGCA34GF5ODujJnN4SA6cIqbJwfYVkTh3S+9KZWRkVHV3jRWV+Jm7PpueTRPIXBXcsdLT00MaU22KplghuuKNplghuuKNplgheuKt6oznMeAyVZ1Z1qCqd4nIt8ATwKlVPbGqTuHw2p9gXA48G3B7NbBWVf3AahHZCcQCgdtxNgH2AA3LtcfgJJ2g+laXdABSUlJISAjdbhBzsrIpKFnF6f26kZqaWm3/9PT0oPpFgmiKFaIr3miKFaIr3miKFSIv3oKCgkq/sFd1VVvzwKRTRlU/B1rVTmg/cgqHz6zA2XL7cQAR6YBzprIZKBSRniLiw5krmgXMBsa7fYcDK1Q1N9i+IXgtRywty9bvGGPqh6oSTwP3CrAfcdviQxBLu8DSPDhnS81EJA2YClzrnpncBLwGLACWqOp84D3goIjMwamkXTbEdyR9PfWtLRw1xtQTVQ21fQPc6/4JdDewqCYHdc+kZpZr61judiEwsYLHzgOGl2srxUkyR93XSyWlpczJyqZXqya0T27odTjGGBNSVSWeO4FPReQqnEufDwJDgGzg3DDEVm+s2LqHvQeLmNC/i9ehGGNMyFWaeFQ1T0ROBE4CBuMsJH1SVWeFK7j64tD6nR62fscYU/dVuY7HvaLsK/ePCZFvyzZ+s/kdY0w9EEytNhNCfr+ftKxsOiQn0aOllcQzxtR9lng8tmZHHtvzDjKqRxt8Pp/X4RhjTMhZ4vHYLHeY7USb3zHG1BOWeDxmG78ZY+obSzweS8vMpnlSPH3bNvM6FGOMCQtLPB7atGc/Wbv2MapHG2JibH7HGFM/WOLx0KFhNtv4zRhTj1ji8dAs2/jNGFMPWeLxUFrWdhrGxzK4YwuvQzHGmLCxxOORHfsO8t22vYzo2poGsfbPYIypP+wTzyOH99+xYTZjTP1iiccjtvGbMaa+ssTjkVmZ2TSIjeH4rqHYzNUYYyKXJR4P5B0sYsnmXQzr3JKkBlUWCDfGmDrHEo8H5m7IoaTUzyhbv2OMqYcs8XggzdbvGGPqMUs8HpiVuR2fD0Z2a+11KMYYE3aWeMKsoLiE+Rt3MLB9c5olxXsdjjHGhJ0lnjBbuHEnBcWljLLLqI0x9VRYL6kSkabAq0AyEA/cqqpzRWQ4MBkoBqar6v0iEgM8BQwECoDrVXVtqPqG6z1Iy3I2frOFo8aY+ircZzy3Al+q6hjgauBJt/0ZYCIwCjheRIYA5wOJqjoCuAN4PMR9w+Jb2/jNGFPP+fx+f9gOJiLNgAJVzReRfsCzwJnAfFXt4/b5P5yzofbAAlV9023fDPQJRV9V7VhZzOnp6d2ArNp4/SWlfk59W2mZFMfb5/Sqjac0xphI1z01NXV9YEPIhtpE5DpgUrnma1R1oYi0wxlyuwVn2C03oE8e0MNt3xvQXhKqviISp6rFVb2elJQUEhISqupSrcWbdrK/eBWX9OlCampqjZ4rPT29xs8RLtEUK0RXvNEUK0RXvNEUK0RevAUFBWRkZFR4X8gSj6pOAaaUbxeR/sCbwO2q+o2IJANNAro0AfYADcu1x+AkklrvW13SqS2H1+/YMJsxpv4K6xyPiPQF3gImquo0AFXNBQpFpKeI+IAzgFnAbGC8+7jhwIpQ9Q3Pqw+Y37GKBcaYeizchcL+CiQCk0UEYK+qngfcBLwGxOJcfTZfRBYCp4nIHMAHXOM+R6j6hpTf7yctazudmjakW4vG4TikMcZEpLAmHjfJVNQ+Dxherq0UJ3GEpW+oaXYuOfsKuHRwN3w+X7gPb4wxEcMWkIbJLNv4zRhjAEs8YTMrs2zhqM3vGGPqN0s8YZKWmU3Lhgn0adPU61CMMcZTlnjCYMOufWzYvZ8TurcmJsbmd4wx9ZslnjAom9850fbfMcYYSzzhULZw1HYcNcYYSzxhMStzO43i4xjcsYXXoRhjjOcs8YRYdl4+32fnMqJba+Ji7e02xhj7JAyxtKwcAE60y6iNMQawxBNyZRu/jbKFo8YYA1jiCblZmdnEx8ZwXJeWXodijDERwRJPCOUeLGTp5t0c16UVSQ3CXY/VGGMikyWeEJqzPodSv59RNr9jjDGHWOIJoUMbv1niMcaYQyzxhNDc9TnE+HyM7Nba61CMMSZi2MRDCJ3drxOjerQhOTHe61CMMSZiWOIJoUlj+nodgjHGRBwbajPGGBNWlniMMcaElSUeY4wxYWWJxxhjTFiF9eICEWkKvAokA/HArao6V0ROAR4EioBs4EpVPSAiHwIt3fZ8VR0nIr2AFwE/kAHcrKqlInIvcBZQDNyiqguOpG+Y3gJjjKn3wn3GcyvwpaqOAa4GnnTbnwLOV9UTgTXA9W57L2CUqo5V1XFu2xPA3ao6GvAB54nIEGAMcDxwacDzHklfY4wxYRDuxPN34D/uz3HAQffnsaq6PbBdRNoCzYCPRCRNRM52708FvnF/ngacCowCpquqX1U3AnEi0voI+xpjjAmDkA21ich1wKRyzdeo6kIRaYcz5HYLgKpudR8zATgJuAdoDTwOTAZaALNFZAHgU1W/+3x5QFOcobudAccpaz+SvjmVvJRYgIyMjKBfe7ikp6d7HULQoilWiK54oylWiK54oylWiNh4Y8s3hCzxqOoUYEr5dhHpD7wJ3K6q3wS0TwIuAs5U1YMisg14RlWLgWwRWQIIUBrwdE2APUCu+3P59iPpW5n21bxUY4wxlWsPrAtsCPfFBX2Bt4BLVHVZQPtdOMNip6pqvtt8KvBr4CwRaQykAKuAJSIyVlVnAuOAr4G1wKMi8hjQCYhR1R0iEnTfKsJeCIwGtgIltfE+GGNMPRCLk3QWlr/D5/f7f9o9RETkA2AgsN5t2gv8AvgBWMzhOZ+pqvq0iPwDGI5z5vKoqr4vIscAz+JcFbcKuEFVS0TkPpzkEgNMUtW0I+kb2ldujDGmTFgTjzHGGGMLSI0xxoSVJR5jjDFhZYnHGGNMWFniMcYYE1a2EVwUEZEGwPNANyABeFBVP/Q0qCCISBsgHThNVb/3Op7KiMidwLk4V0E+5a5Fi0ju78JLOL8LJThXbEbceysixwOPqOrYymonehlfeeXiHQT8C+f9LcCpIbm9yicIo8BYA9omAr9R1RGeBRYEO+OJLpcDO93ac+OAf3scT7XcD8j/APnV9fWSiIwFRgIn4NTy6+xpQNUbD8Sp6kjgz8BDHsfzEyLye+A5INFt+kntRK9iq0gF8U7G+RAfC7wL/MGj0H6iglhxE+V1OO9tRLPEE13ewiknVKbYq0COwGPAM8AWrwOpxhnACuA94CPgY2/DqdZqnDqDMThloIo8jqci64ALAm5XVDsxkpSP91JVXer+HFhbMhL8KFYRaQk8jFuGLNJZ4okiqrpPVfNEpAnwNnC31zFVRUSuBnJU9XOvYwlCK2AocDFwE/CaiETyN8d9OMNs3+Mskv6np9FUQFXf4ccJsaLaiRGjfLwBNSRH4lRR+btHof1EYKwiEotTnmwSzvsa8SzxRBkR6YxT+ucVVX3d63iqcS1wmojMBAYBL7sFYiPRTuBzVS1UVcX5dhvJVcsn4cR7DE41kJdEJLGax3itotqJEU1ELsE5Yz9LVSsrJOy1VKA38DROHcy+btWXiGUXF0QRd6uI6cCvVfVLr+Opjru/EgBu8rlJVbd5F1GV0oD/E5EncOpLNeLHVcwjzW4OfzvfBTSggirAEaai2okRS0QuB27E2bZll9fxVMbdyLIfgIh0A95U1YgecrPEE13+CDQH7hGRsrmecQGFVc1RUtWPReREYAHOSMDNqhrJRWH/DjwvIrNwrsL7o6ru9zim6twGPCsiZbUT3/Y4nkq5w1f/BDYC74oIwDeqeq+ngdURVqvNGGNMWNkcjzHGmLCyxGOMMSasLPEYY4wJK0s8xhhjwsoSjzHGmLCyxGOihoh0ExG/iJxWrn29u36hps9fK89TzTG6iIiKyFK3AkVZe7KILBKRDHfL9iN93ohcE+P+m60/yse+ICJdw3EsE16WeEy0KcJZC9Kk2p6RaSyQrqqDVDWwvMkgoFBVU1R19VE+b11zElFQ8NIcOVtAaqLNFmAG8Djwi8A73ArT95WViReRF4GZ7p/3ceqa9QMWA3OAq3EW5E5Q1VXu09wnIgNxSubcqKrL3YoR/8GpWF0K3KmqX4jIfcBwoAvwL1V9OiCWY4D/Ai2A/cBvcZLmg0BjEXlGVW9y+7bB2e6inYh8CEwA/oaTTGKBF1X17yISh1MWJQVoCywHLgMecZ9nvqoeLyJ+VfW5bVfjrLy/2j0bmI+T5EYDZ+IUlYzB2bbiZpwtAJ53jwHO9hDPlnufJwK/d/tmAZer6kERuQP4mRvz55Sr5lzF+9gCp9bYsTjbD9wKHAd0AD4VkdFAD5xFsw2BHe6/TZaIDHYfC7AMExXsjMdEo9uAM8oPuVVjAM4H9ECcrQ+6uXuWvMGPE9gaVR0MPICz3w045fGfV9VUnP16/hNwxpWoqn0Dk47rVeCfqjoAp67a2zir9f8EfFiWdABUNRu4HlikqucCN7jtQ3A+gM9zP3xH4pwVjQB6Ac2A8ar6W7f/8UG8D9NUVXDq0N0AjFTVQUA2cLt7jBbue3AWToIq70HgdPf9yAKOFZEzcWqGDQMGAx2Bn5d7XGXv4wPAWlXtA1wBPKSqD+N8yRiPU/jyOWCi+548jlMYFeBl4A9ue2YQr99EAEs8Juqoai7Oh+aRDLltU9Ul7sZjm4CyWncbcM56yjznHuNToKuINMMp3/9nEVmKU86/AdDT7T+//IFEpDHQS1XfdZ9rHk49NQky1lOBc93jzQc6Af1V9VvgKRG5GedDvDfQOMjnLFMW70nu4+e5xzkP54wjw3kJ8jlOpe7fVfAcHwGzReRR4B1364BTgeNxzpwW41T67lfB66rofRwDvAKgqisq2MTsGLffh+5jHwF6iEgroIOqznD7vXiE74XxiCUeE5VUdTqHh9zK+PnxnECDgJ8Lyz1FZXsZBbb7cIbHYoGT3XmZQTgfsCvcPhXVyavo/5WP4Ie2Y4HfBxxvOE5dtnOB14ADwAvAt1QyBxKwpUODcneVxRsL/C/gGMfhFJ/diZMw/oWTKBe7yfcQVf0/4EKcQqWvusU0Y4F/lHuPym9OV9n7WITzb1cW+7HuPkOBj8sMeFwqMIqf/ntHw/5UBks8JrrdhrOBW3v39g6cb8KJ7rxBRcNE1fk5gIhMAFa5hTe/An7ltvfFOStoWNkTuGdkmSJygfuY4UA793HB+Aq4QUQauGdPaTjJ51ScZPECzpYCJ3G4InWJOwcEzvvQz00+51ZyjJnABBFp4/Z7GrjFTW6vAJ/gzEvtI2A3VhGJE5E1wA5V/SvOUNdgN+YrRKSxG8f7wEUVvK6K3sdvceaqEJFjgc9wkkoxTrL+HmjhDjeCs93G626S3CAiZ7ntE6t4T00EscRjolbAkFu8e/s7nA/M73B2a511FE97jDuccytwldv2G2C4iCwHpuJMple34dblwG9FZAXOFuUXqGr5s67KPAOsAZYAi4AX3K0EngUuc5/zLWA20N19zAfAMndPnjtwdlCdC2hFB1DVZcD9OMngO5wE9jDOEFi+27YAeFVVVwQ8rhhnnmqGiCzCSYiPqOpHwDs4Q3kZwFIOz5GVqex9vBfoLSLLcM7ornA3jPsY+BTnIoOLgcfdx16Fs8UzOO/zvSKyhMPDnybCWXVqY4wxYWVnPMYYY8LKEo8xxpiwssRjjDEmrCzxGGOMCStLPMYYY8LKEo8xxpiwssRjjDEmrP4fm3Dg6R7pb84AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rfecv = RFECV(estimator=rf, step=1, cv=StratifiedKFold(3),\n",
    "              scoring='neg_mean_absolute_error')\n",
    "rfecv.fit(rf_data, y)\n",
    "\n",
    "print(\"Optimal number of features : %d\" % rfecv.n_features_)\n",
    "\n",
    "# Plot number of features VS. cross-validation scores\n",
    "plt.figure()\n",
    "plt.xlabel(\"Number of features selected\")\n",
    "plt.ylabel(\"Cross validation score\")\n",
    "plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.04641857, 0.05705897, 0.03053061, 0.02772822, 0.19056988,\n",
       "       0.28937292, 0.09432954, 0.26399128])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv.estimator_.feature_importances_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 1, 1, 2, 1, 1, 1, 1, 6, 8, 7, 1, 5, 1, 3])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv.ranking_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfecv_rankings = list(zip(rf_data.columns, rfecv.ranking_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = [item[0] for item in rfecv_rankings if item[1]==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['languages',\n",
       " 'Funny',\n",
       " 'Courageous',\n",
       " 'Confusing',\n",
       " 'Informative',\n",
       " 'Fascinating',\n",
       " 'OK',\n",
       " 'Inspiring']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_data = rf_data[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\data.py:625: DataConversionWarning: Data with input dtype int64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "C:\\Users\\pjd04\\Anaconda3\\lib\\site-packages\\sklearn\\base.py:462: DataConversionWarning: Data with input dtype int64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    }
   ],
   "source": [
    "sc = StandardScaler()\n",
    "sc_rf_data = sc.fit_transform(rf_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(sc_rf_data, y, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 100 candidates, totalling 300 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:    7.3s\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 300 out of 300 | elapsed:  3.0min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3, error_score='raise-deprecating',\n",
       "          estimator=RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators='warn', n_jobs=None,\n",
       "           oob_score=False, random_state=42, verbose=0, warm_start=False),\n",
       "          fit_params=None, iid='warn', n_iter=100, n_jobs=-1,\n",
       "          param_distributions={'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000], 'max_features': ['auto', 'sqrt'], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None], 'min_samples_split': [2, 5, 10], 'min_samples_leaf': [1, 2, 4], 'bootstrap': [True, False]},\n",
       "          pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
       "          return_train_score='warn', scoring=None, verbose=2)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n",
    "# Fit the random search model\n",
    "rf_random.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 400,\n",
       " 'min_samples_split': 2,\n",
       " 'min_samples_leaf': 1,\n",
       " 'max_features': 'sqrt',\n",
       " 'max_depth': None,\n",
       " 'bootstrap': False}"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_random.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_rf = rf_random.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
       "           max_features='sqrt', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=400, n_jobs=None,\n",
       "           oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "524029.8997874494"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_absolute_error(y_test, best_rf.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>transcript</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sir Ken Robinson makes an entertaining and pro...</td>\n",
       "      <td>Good morning. How are you?(Laughter)It's been ...</td>\n",
       "      <td>Do schools kill creativity?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>With the same humor and humanity he exuded in ...</td>\n",
       "      <td>Thank you so much, Chris. And it's truly a gre...</td>\n",
       "      <td>Averting the climate crisis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>New York Times columnist David Pogue takes aim...</td>\n",
       "      <td>(Music: \"The Sound of Silence,\" Simon &amp; Garfun...</td>\n",
       "      <td>Simplicity sells</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In an emotionally charged talk, MacArthur-winn...</td>\n",
       "      <td>If you're here today — and I'm very happy that...</td>\n",
       "      <td>Greening the ghetto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You've never seen data presented like this. Wi...</td>\n",
       "      <td>About 10 years ago, I took on the task to teac...</td>\n",
       "      <td>The best stats you've ever seen</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         description  \\\n",
       "0  Sir Ken Robinson makes an entertaining and pro...   \n",
       "1  With the same humor and humanity he exuded in ...   \n",
       "2  New York Times columnist David Pogue takes aim...   \n",
       "3  In an emotionally charged talk, MacArthur-winn...   \n",
       "4  You've never seen data presented like this. Wi...   \n",
       "\n",
       "                                          transcript  \\\n",
       "0  Good morning. How are you?(Laughter)It's been ...   \n",
       "1  Thank you so much, Chris. And it's truly a gre...   \n",
       "2  (Music: \"The Sound of Silence,\" Simon & Garfun...   \n",
       "3  If you're here today — and I'm very happy that...   \n",
       "4  About 10 years ago, I took on the task to teac...   \n",
       "\n",
       "                             title  \n",
       "0      Do schools kill creativity?  \n",
       "1      Averting the climate crisis  \n",
       "2                 Simplicity sells  \n",
       "3              Greening the ghetto  \n",
       "4  The best stats you've ever seen  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_data = main[[\"description\", \"transcript\", \"title\"]]\n",
    "text_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "transcript = tfidf.fit_transform(main[\"transcript\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "svr_train, svr_test, svry_train, svry_test = train_test_split(transcript, np.log1p(y), test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_svr = LinearSVR(C=1.49, dual=True, epsilon=0.0, fit_intercept=True,\n",
    "       intercept_scaling=1.0, loss='epsilon_insensitive', max_iter=1000,\n",
    "       random_state=42, tol=0.0001, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVR(C=1.49, dual=True, epsilon=0.0, fit_intercept=True,\n",
       "     intercept_scaling=1.0, loss='epsilon_insensitive', max_iter=1000,\n",
       "     random_state=42, tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_svr.fit(svr_train, svry_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "857127.2257156542"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_absolute_error(np.expm1(svry_test), np.expm1(best_svr.predict(svr_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "blended = [np.mean([r, s]) for r, s in zip(best_rf.predict(X_test), best_svr.predict(svr_test))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "847337.6607608476"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_absolute_error(y_test, blended)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}