classification_plots.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fd751275",
   "metadata": {},
   "source": [
    "# DATA1030 Interactive Classification Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9ba56a00",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/drewsolomon/opt/anaconda3/envs/data1030vis/lib/python3.10/site-packages/xgboost/compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
      "  from pandas import MultiIndex, Int64Index\n"
     ]
    }
   ],
   "source": [
    "# import libraries and packages\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn\n",
    "from sklearn.datasets import make_moons\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "import xgboost\n",
    "import plotly\n",
    "import plotly.graph_objs as go\n",
    "import ipywidgets\n",
    "from ipywidgets import interact\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77133ebc",
   "metadata": {},
   "source": [
    "## Function to generate contour plot data\n",
    "\n",
    "This function fits a given ML classifier on given input data and calculates the predicted classification probabilities within a specified grid.\n",
    "Returns a dictionary of 2d arrays of predicted probabilities within grid for a given ML algorithm, input data, and hyperparameter grid.\n",
    "\n",
    "Inputs: \n",
    "- input data X\n",
    "- input data y\n",
    "- ML algorithm\n",
    "- hyperparameter grid\n",
    "- contour plot grid step size 'h'\n",
    "\n",
    "Output:\n",
    "- dictionary of 2D arrays of predicted probabilities, with hyperparameter values (in tuple) as keys\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0b7f5ae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_all_data(X, y, ml_algo, param_grid, h):\n",
    "    \"\"\"Returns dictionary of 2D array predicted probabilities within grid for given ML algorithm, input data, and hyperparamater grid\"\"\"\n",
    "    \n",
    "    # create dictionary for 2D arrays of predicted probabilities\n",
    "    Z_dict = {}\n",
    "    \n",
    "    # create mesh grid:\n",
    "    # set step size in the mesh\n",
    "    h = h\n",
    "    # set plot range to slightly larger than data range\n",
    "    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5\n",
    "    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n",
    "    # create meshgrid\n",
    "    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                         np.arange(y_min, y_max, h))\n",
    "    \n",
    "    # loop through all hyperparameter combinations\n",
    "    for p in range(len(ParameterGrid(param_grid))):\n",
    "        # set current parameters\n",
    "        params = ParameterGrid(param_grid)[p]\n",
    "        #print('   ',params) \n",
    "        \n",
    "        # set the classifier algorithm - maybe bring this outside for loop\n",
    "        clf = ml_algo\n",
    "        \n",
    "        # set the model hyperparameters\n",
    "        clf.set_params(**params)\n",
    "        \n",
    "        # fit the classifier on the given input data \n",
    "        clf.fit(X,y)\n",
    "\n",
    "        # calculate predicted probabilities in mesh\n",
    "        Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]\n",
    "\n",
    "        # reshape predicted probabilities\n",
    "        Z = Z.reshape(xx.shape) \n",
    "        \n",
    "        # convert parameter values to string key\n",
    "        param_key = str(sorted(params.items()))\n",
    "            \n",
    "        # add 2D array to dict with param values tuple as key \n",
    "        Z_dict[param_key] = Z\n",
    "        \n",
    "    return Z_dict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba0d1943",
   "metadata": {},
   "source": [
    "## Interactive Plot Function\n",
    "\n",
    "\n",
    "This function plots an interactive contour plot of predicted classification probabilities against a scatterplot of input data for a given ML classifier's hyperparameter grid, input data, dictionary of 2D predicted probability arrays, and grid step size.\n",
    "\n",
    "The interactive plot uses Plotly's FigureWidgets to create a variable number of sliders to adjust each of the ML classifier's hyperparameters. As the hyperparameter values are changed, the contour plot changes to reflect the ML model's classification probabilities for the given hyperparameter combination. Thus, the plot interactively shows how different hyperparameter combinations affect each ML model's classification. In addition, the 0.5 probability decision boundary is plotted in a bold black line.\n",
    "\n",
    "Inputs: \n",
    "- hyperparameter grid\n",
    "- input data X\n",
    "- input data y\n",
    "- dictionary of 2D arrays of predicted probabilities, with hyperparameter values (in tuple) as keys\n",
    "- contour plot grid step size 'h'\n",
    "\n",
    "Output:\n",
    "- interactive figure with a slider for each hyperparameter controlling the contour plot of predicted classification probabilities on top of a scatteplot of the input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b315a00c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_clf_contour(param_grid, X, y, Z_dict, h):\n",
    "    \"\"\"Returns an interactive contour plot of predicted classification probabilities \n",
    "    and a scatterplot of input data for a given ML classifier's hyperparameter grid, \n",
    "    input data, dictionary of 2D predicted probability arrays, and grid step size.\"\"\"\n",
    "    # set plot range to slightly larger than data range\n",
    "    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5\n",
    "    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n",
    " \n",
    "    # create figure widget\n",
    "    fig = go.FigureWidget()\n",
    "    # add contour plot\n",
    "    fig.add_contour()\n",
    "    # set plot size\n",
    "    fig.update_layout(\n",
    "        autosize=False,\n",
    "        width=800,\n",
    "        height=640)\n",
    "    # add scatter plot\n",
    "    fig.add_scatter(x=X[:, 0], y=X[:, 1], mode='markers')\n",
    "    # add second contour line for decision boundary\n",
    "    fig.add_contour()\n",
    "\n",
    "    # create dict of sliders \n",
    "    sliders={}\n",
    "\n",
    "    # create second dict of hyperparameters and their initial values\n",
    "    slider_init_val={}\n",
    "\n",
    "    # create sliders for each hyperparameter\n",
    "    for hyperparam in range(len(param_grid)):\n",
    "        # get hyperparameter name\n",
    "        param_name = list(param_grid.keys())[hyperparam]\n",
    "        # get hyperparameter options\n",
    "        param_options = param_grid[list(param_grid.keys())[hyperparam]]\n",
    "        \n",
    "        # add slider with given hyperparameter name and values to dict\n",
    "        # if options are categorical, create dropdown slider\n",
    "        if all(isinstance(item, str) for item in param_options):\n",
    "            sliders[param_name]=ipywidgets.Dropdown(options=param_options,description=param_name)\n",
    "        # otherwise create selection slider\n",
    "        else:\n",
    "            sliders[param_name]=ipywidgets.SelectionSlider(options=param_options,description=param_name, orientation='horizontal')\n",
    "        # add slider initial value to initial value dict\n",
    "        slider_init_val[param_name]=param_grid[list(param_grid.keys())[hyperparam]][0]\n",
    "\n",
    "\n",
    "    # create interactive plot with sliders that update hyperparameter keys and associated 2d arrays\n",
    "    @interact(**sliders)\n",
    "    def update(**slider_init_val):\n",
    "        with fig.batch_update():\n",
    "            # set contour colorscale\n",
    "            fig.data[0].colorscale = \"RdBu\" # set red blue colorscale\n",
    "            fig.data[0].contours=dict(\n",
    "                size=0.05,\n",
    "                start=0,\n",
    "                end=1)\n",
    "            fig.data[0].colorbar={\"title\": 'predicted probability'}\n",
    "            # set contourplot data and axes \n",
    "            # set parameter key (sorted slider values)\n",
    "            param_key = str(sorted(slider_init_val.items()))\n",
    "            fig.data[0].z=Z_dict[param_key] # update Z - predicted prob 2D array \n",
    "            fig.data[0].x=np.arange(x_min, x_max, h)\n",
    "            fig.data[0].y=np.arange(y_min, y_max, h)\n",
    "\n",
    "            # set contour line for decision boundary data\n",
    "            fig.data[2].colorscale = [[0, 'rgb(0,0,0)'], [1, 'rgb(0,0,0)']]\n",
    "            fig.data[2].z=Z_dict[param_key] # update Z - predicted prob 2D array \n",
    "            fig.data[2].x=np.arange(x_min, x_max, h)\n",
    "            fig.data[2].y=np.arange(y_min, y_max, h)\n",
    "            fig.data[2].contours={\"showlabels\":False, \"coloring\":\"lines\",\n",
    "                                 \"start\":0.5, \"end\":0.5}\n",
    "            fig.data[2].line.width=5\n",
    "            fig.data[2].ncontours=1\n",
    "            # remove scale for contour line\n",
    "            fig.data[2].showscale=False\n",
    "\n",
    "            # set scatterplot point color and shape\n",
    "            fig.data[1].marker.color=y\n",
    "            fig.data[1].marker.colorscale=[[0, 'rgb(255,0,0)'], [1, 'rgb(0,0,255)']]\n",
    "            fig.data[1].marker.size=8\n",
    "            fig.data[1].marker.line.width=1\n",
    "\n",
    "\n",
    "            # generate title\n",
    "            title=\"\"\n",
    "            for i, (k, v) in enumerate(slider_init_val.items()):\n",
    "                if i == len(slider_init_val.items())-1: # if last item, omit comma\n",
    "                    title+=(str(k) + \" = \" + str(v))\n",
    "                else:\n",
    "                    title+=(str(k) + \" = \" + str(v)+ \", \")\n",
    "\n",
    "            # axes labels\n",
    "            fig.update_layout(\n",
    "                title={\n",
    "                    'text': title,\n",
    "                    'y':0.9,\n",
    "                    'x':0.41,\n",
    "                    'xanchor': 'center',\n",
    "                    'yanchor': 'top'},\n",
    "                xaxis_title=\"feature 1\",\n",
    "                yaxis_title=\"feature 2\",\n",
    "                font=dict(\n",
    "                    family=\"arial, monospace\",\n",
    "                    size=16)\n",
    "            )\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91ca4774",
   "metadata": {},
   "source": [
    "## Input classification data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dd1d819c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set ML models' random_state\n",
    "random_state = 2\n",
    "\n",
    "# create the data for classification\n",
    "X,y = make_moons(noise=0.2, random_state=1,n_samples=200)\n",
    "\n",
    "# standardize X \n",
    "X = StandardScaler().fit_transform(X)\n",
    "\n",
    "# set grid step size\n",
    "h = 0.02"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66795226",
   "metadata": {},
   "source": [
    "## Logistic Regression "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5e7c3081",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize logistic regression model, with elastic net regularization\n",
    "ML_algo = LogisticRegression(penalty='elasticnet', solver='saga', random_state=random_state)\n",
    "\n",
    "# set parameter grid to tune C and l1 ratio\n",
    "param_grid = {'C': [1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2],\n",
    "             'l1_ratio': np.linspace(0, 1,11)} # l1_ratio between 0 and 1\n",
    "\n",
    "# generate Z dict for example data and ML alg\n",
    "Z_dict = generate_all_data(X, y, ML_algo, param_grid, h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4a546ee9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63c327959868467599d870898142ea9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(SelectionSlider(description='C', options=(0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0), v…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "863158121c9e4dca90b32da4a530785c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureWidget({\n",
       "    'data': [{'colorbar': {'title': {'text': 'predicted probability'}},\n",
       "              'colorsca…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot classification contour plot\n",
    "plot_clf_contour(param_grid, X, y, Z_dict, h)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e197cc5",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5a465816",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize RandomForestClassifier\n",
    "ML_algo = RandomForestClassifier(max_features=None, random_state=random_state) # max_features to None to consider all (both) features\n",
    "\n",
    "# set RF parameter grid\n",
    "param_grid = {'criterion' :[\"gini\", \"entropy\", \"log_loss\"],\n",
    "              'max_depth': [1, 3, 5, 10], \n",
    "              'n_estimators': [1, 3, 10, 30, 100]\n",
    "             #'min_samples_split': [2, 4, 6, 8, 10],\n",
    "             }\n",
    "\n",
    "# generate Z dict for example data and ML alg\n",
    "Z_dict = generate_all_data(X, y, ML_algo, param_grid, h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6cf0fff7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27f9f5f7430d4ff6b77655b1615291ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='criterion', options=('gini', 'entropy', 'log_loss'), value='gini')…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8a28936a6464929b64993bf54ad57f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureWidget({\n",
       "    'data': [{'colorbar': {'title': {'text': 'predicted probability'}},\n",
       "              'colorsca…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot classification contour plot\n",
    "plot_clf_contour(param_grid, X, y, Z_dict, h)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4a063e5",
   "metadata": {},
   "source": [
    "## Support Vector Machine (SVM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "92b167db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize SVC\n",
    "ML_algo = SVC(probability=True, random_state=random_state)\n",
    "\n",
    "# set SVC parameter grid\n",
    "param_grid = {'gamma': [1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2], # kernel coefficient\n",
    "              'C': [1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3] # C - regularization parameter, inverse of strength of regularization\n",
    "             } \n",
    "\n",
    "# generate Z dict for example data and ML alg\n",
    "Z_dict = generate_all_data(X, y, ML_algo, param_grid, 0.04) # higher grid step size to run faster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "49d66ba9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fd684fa34324c6fb1d7d273a1d2444b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(SelectionSlider(description='gamma', options=(0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f89adb4a7275484baff8e7308ddb9f92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureWidget({\n",
       "    'data': [{'colorbar': {'title': {'text': 'predicted probability'}},\n",
       "              'colorsca…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot classification contour plot\n",
    "plot_clf_contour(param_grid, X, y, Z_dict, 0.04)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ebfd820",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "19cbe13d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize XGBoost classifier\n",
    "ML_algo = xgboost.XGBClassifier(random_state=random_state, eval_metric='logloss', use_label_encoder=False)\n",
    "\n",
    "# set parameter grid to tune the maximum depth and regularization parameters\n",
    "param_grid = {\"max_depth\": np.arange(1,6,1),\n",
    "            \"reg_alpha\": [1e-2, 1e-1, 1e0, 1e1, 1e2],\n",
    "             \"reg_lambda\": [1e-1, 1e0, 1e1, 1e2, 1e3, 1e4],\n",
    "             }\n",
    "\n",
    "# generate Z dict for example data and ML alg\n",
    "Z_dict = generate_all_data(X, y, ML_algo, param_grid, h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2132cd7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f071c546118d4560bcd7ae119f227a90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(SelectionSlider(description='max_depth', options=(1, 2, 3, 4, 5), value=1), SelectionSli…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d2c186dbba345d795315032cddbca7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureWidget({\n",
       "    'data': [{'colorbar': {'title': {'text': 'predicted probability'}},\n",
       "              'colorsca…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot classification contour plot\n",
    "plot_clf_contour(param_grid, X, y, Z_dict, h)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c119a65d",
   "metadata": {},
   "source": [
    "## k-Nearest Neighbors (kNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0129b7c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize kNN classifier\n",
    "ML_algo = KNeighborsClassifier()\n",
    "\n",
    "# set parameter grid to tune the number of nearest neighbors and their weighting (e.g. by distance)\n",
    "param_grid = {'weights':['uniform', 'distance'],\n",
    "              'n_neighbors': np.arange(1,30,2)\n",
    "} \n",
    "\n",
    "# generate Z dict for example data and ML alg\n",
    "Z_dict = generate_all_data(X, y, ML_algo, param_grid, h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8d57d8d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8ae102cc0a7400d905cc8cb8713664b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='weights', options=('uniform', 'distance'), value='uniform'), Selec…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35ecd758ece14ce0b97e54acbbd9fbba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureWidget({\n",
       "    'data': [{'colorbar': {'title': {'text': 'predicted probability'}},\n",
       "              'colorsca…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot classification contour plot\n",
    "plot_clf_contour(param_grid, X, y, Z_dict, h)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}