LSTM-FFF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stVLE = pd.read_csv('studentVle.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# only use the data from the course start\n",
    "stVLE_update = stVLE[stVLE['date']>=0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "stAss = pd.read_csv('studentAssessment.csv')\n",
    "stReg = pd.read_csv('studentRegistration.csv')\n",
    "stInfo = pd.read_csv('studentInfo.csv')\n",
    "VLE = pd.read_csv('vle.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract pass, disctinctio and fail students\n",
    "stInfo_update = stInfo[(stInfo['final_result'] =='Pass') | (stInfo['final_result'] =='Distinction')\n",
    "                       | (stInfo['final_result'] =='Fail')] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'resource': 0, 'oucontent': 1, 'url': 2, 'homepage': 3, 'subpage': 4, 'glossary': 5, 'forumng': 6, 'oucollaborate': 7, 'dataplus': 8, 'quiz': 9, 'ouelluminate': 10, 'sharedsubpage': 11, 'questionnaire': 12, 'page': 13, 'externalquiz': 14, 'ouwiki': 15, 'dualpane': 16, 'repeatactivity': 17, 'folder': 18, 'htmlactivity': 19}\n"
     ]
    }
   ],
   "source": [
    "courses = stInfo_update['code_module'].unique()\n",
    "dic_site_acitivity = {v:k for v, k in zip(VLE['id_site'], VLE['activity_type'])}\n",
    "actions_len = len(VLE['activity_type'].unique())\n",
    "\n",
    "actions_dic = VLE['activity_type'].unique().tolist()\n",
    "\n",
    "actions_dic = {v: k for k, v in enumerate(actions_dic)}\n",
    "\n",
    "dic_acitivity_idx = {v:k for v, k in actions_dic.items()}\n",
    "\n",
    "print(dic_acitivity_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract features for the course FFF, 2013 as the training and 2014 as the test\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "labelencoder = LabelEncoder()\n",
    "\n",
    "labelencoder.fit([\"Distinction\", \"Fail\", \"Pass\"])\n",
    "\n",
    "courses = ['FFF'] # we can also add AAA, BBB, CCC\n",
    "\n",
    "for i, course in enumerate(courses):\n",
    "    \n",
    "    stVLE_subset = stVLE_update[stVLE_update['code_module'] == course]\n",
    "    stInfo_update_filtered = stInfo_update[stInfo_update['code_module'] == course]\n",
    "    \n",
    "    # filter multiple attemps\n",
    "    stInfo_update_filtered.drop_duplicates(subset=['id_student'], keep=False, inplace=True)\n",
    "    \n",
    "    stVLE_subset_filtered = stVLE_subset[stVLE_subset['id_student'].isin(stInfo_update_filtered['id_student'])]\n",
    "    \n",
    "    stInfo_update_filtered = stInfo_update_filtered[stInfo_update_filtered['id_student'].isin(stVLE_subset_filtered['id_student'])]\n",
    "    \n",
    "    \n",
    "    stVLE_subset_train = stVLE_subset_filtered[(stVLE_subset_filtered['code_presentation'] == '2013B') | \n",
    "                                              (stVLE_subset_filtered['code_presentation'] == '2013J')]\n",
    "\n",
    "    stVLE_subset_test = stVLE_subset_filtered[(stVLE_subset_filtered['code_presentation'] == '2014B') | \n",
    "                                              (stVLE_subset_filtered['code_presentation'] == '2014J')]\n",
    "  \n",
    "    stinfo_train = stInfo_update_filtered[stInfo_update_filtered['id_student'].isin(stVLE_subset_train['id_student'])]\n",
    "\n",
    "    stinfo_test = stInfo_update_filtered[stInfo_update_filtered['id_student'].isin(stVLE_subset_test['id_student'])]\n",
    "\n",
    "    \n",
    "    studentno_train = len(stVLE_subset_train['id_student'].unique())\n",
    "    \n",
    "     \n",
    "    stinfo_train['arrayIdx'] = np.arange(studentno_train)\n",
    "    \n",
    "     \n",
    "    print('the number of students {} in train course {}'.format(studentno_train, course))\n",
    "    \n",
    "    cc = stVLE_subset_train.replace({'id_site': dic_site_acitivity}, inplace=False) \n",
    "    stVLE_subset_train['activity_type'] = cc['id_site']\n",
    "    \n",
    "    cc = stVLE_subset_train.replace({'activity_type': dic_acitivity_idx}, inplace=False) \n",
    "    stVLE_subset_train['activity_idx'] = cc['activity_type']\n",
    "   \n",
    "    \n",
    "    stVLE_subset_train = pd.merge(stVLE_subset_train, stinfo_train[{'arrayIdx', 'id_student'}], on='id_student', validate='m:1')\n",
    "    \n",
    "    total_days = np.max(stVLE_subset_train['date'].values) + 1\n",
    "    \n",
    "    actionArray_train = np.zeros([studentno_train, total_days, actions_len])\n",
    "    \n",
    "    y_train = np.zeros(studentno_train)\n",
    "         \n",
    "    y_train = labelencoder.transform(stinfo_train['final_result'])\n",
    "   \n",
    "\n",
    "    for day in range(total_days):\n",
    "        \n",
    "        #print('running the course {} and the day : {}'.format(course, day))\n",
    "        \n",
    "        stVLE_days = stVLE_subset_train[stVLE_subset_train['date'] == day]\n",
    "    \n",
    "        yy = stVLE_days.groupby(['arrayIdx', 'activity_idx'])['sum_click'].sum().unstack(fill_value=0)\n",
    "\n",
    "        newyy = yy.reindex(np.arange(actions_len), axis=1, fill_value=0)\n",
    "\n",
    "        newzz = newyy.reindex(np.arange(studentno_train), axis=0, fill_value=0)\n",
    "\n",
    "        newzz = newzz.values\n",
    "\n",
    "        actionArray_train[:,day,:] = newzz\n",
    "        \n",
    "    ####\n",
    "    \n",
    "    studentno_test = len(stVLE_subset_test['id_student'].unique())\n",
    "    \n",
    "     \n",
    "    stinfo_test['arrayIdx'] = np.arange(studentno_test)\n",
    "    \n",
    "     \n",
    "    print('the number of students {} in test course {}'.format(studentno_test, course))\n",
    "    \n",
    "    cc = stVLE_subset_test.replace({'id_site': dic_site_acitivity}, inplace=False) \n",
    "    stVLE_subset_test['activity_type'] = cc['id_site']\n",
    "    \n",
    "    cc = stVLE_subset_test.replace({'activity_type': dic_acitivity_idx}, inplace=False) \n",
    "    stVLE_subset_test['activity_idx'] = cc['activity_type']\n",
    "   \n",
    "    \n",
    "    stVLE_subset_test = pd.merge(stVLE_subset_test, stinfo_test[{'arrayIdx', 'id_student'}], on='id_student', validate='m:1')\n",
    "    \n",
    "    total_days = np.max(stVLE_subset_test['date'].values) + 1\n",
    "    \n",
    "    actionArray_test= np.zeros([studentno_test, total_days, actions_len])\n",
    "    \n",
    "    y_test = np.zeros(studentno_test)\n",
    "         \n",
    "    y_test = labelencoder.transform(stinfo_test['final_result'])\n",
    "   \n",
    "    \n",
    "    for day in range(total_days):\n",
    "        \n",
    "        #print('running the course {} and the day : {}'.format(course, day))\n",
    "        \n",
    "        stVLE_days = stVLE_subset_test[stVLE_subset_test['date'] == day]\n",
    "    \n",
    "        yy = stVLE_days.groupby(['arrayIdx', 'activity_idx'])['sum_click'].sum().unstack(fill_value=0)\n",
    "\n",
    "        newyy = yy.reindex(np.arange(actions_len), axis=1, fill_value=0)\n",
    "\n",
    "        newzz = newyy.reindex(np.arange(studentno_test), axis=0, fill_value=0)\n",
    "\n",
    "        newzz = newzz.values\n",
    "\n",
    "        actionArray_test[:,day,:] = newzz\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# with open(\"saved_click_data_FFF.pkl\", \"wb\") as fp:   #Pickling\n",
    "#     pickle.dump(savedata, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge Distinction and pass\n",
    "y_train[y_train==2] = 0\n",
    "y_test[y_test==2] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1]\n",
      "[1918  952]\n",
      "[0 1]\n",
      "[1759  742]\n"
     ]
    }
   ],
   "source": [
    "# print the number of positive and negative samples\n",
    "\n",
    "[a, b] = np.unique(y_train, return_counts=True)\n",
    "print(a)\n",
    "print(b)\n",
    "\n",
    "[a, b] = np.unique(y_test, return_counts=True)\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_weeks = 37\n",
    "\n",
    "usedweeks = 10  # the weeks' data we used for prediction, e.g 5, 10, 15, 35 etc\n",
    "\n",
    "X = list()\n",
    "\n",
    "for i in range(total_weeks):\n",
    "    seq_x = np.sum(actionArray_train[:, (7*i):(7*i+6), :], axis=1)\n",
    "    X.append(seq_x)\n",
    "X = np.array(X)\n",
    "\n",
    "actionWeeks = np.swapaxes(X, 0, 1)\n",
    "\n",
    "def prepare_data(actionWeeks, weeks=1):\n",
    "    return actionWeeks[:,:weeks,:]\n",
    "\n",
    "X_train = prepare_data(actionWeeks, weeks=usedweeks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scale the training and test data by the log transformation\n",
    "\n",
    "X = list()\n",
    "\n",
    "for i in range(total_weeks):\n",
    "    seq_x = np.sum(actionArray_test[:, (7*i):(7*i+6), :], axis=1)\n",
    "    X.append(seq_x)\n",
    "X = np.array(X)\n",
    "\n",
    "actionWeeks = np.swapaxes(X, 0, 1)\n",
    "\n",
    "def prepare_data(actionWeeks, weeks=1):\n",
    "    return actionWeeks[:,:weeks,:]\n",
    "\n",
    "X_test = prepare_data(actionWeeks, weeks=usedweeks)\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "scaler = MinMaxScaler(feature_range=(0,1))\n",
    "\n",
    "XX = np.reshape(X_train, (X_train.shape[0]*X_train.shape[1], X_train.shape[2]))\n",
    "scaler.fit(np.log(XX + 1))\n",
    "XX = scaler.transform(np.log(XX+1))\n",
    "\n",
    "X_train = np.reshape(XX, (X_train.shape[0], X_train.shape[1], X_train.shape[2]))\n",
    "\n",
    "XX_test = np.reshape(X_test, (X_test.shape[0]*X_test.shape[1], X_test.shape[2]))\n",
    "XX_test = scaler.transform(np.log(XX_test+1))\n",
    "X_test = np.reshape(XX_test, (X_test.shape[0], X_test.shape[1], X_test.shape[2]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, LSTM, BatchNormalization, Bidirectional, LayerNormalization\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras import regularizers\n",
    "import re\n",
    "from keras.layers import Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "early_stopping = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='val_loss', \n",
    "    verbose=1,\n",
    "    patience=10,\n",
    "    mode='min',\n",
    "    restore_best_weights=True)\n",
    "\n",
    "# METRICS = [\n",
    "#       keras.metrics.TruePositives(name='tp'),\n",
    "#       keras.metrics.FalsePositives(name='fp'),\n",
    "#       keras.metrics.TrueNegatives(name='tn'),\n",
    "#       keras.metrics.FalseNegatives(name='fn'), \n",
    "#       keras.metrics.BinaryAccuracy(name='accuracy'),\n",
    "#       keras.metrics.Precision(name='precision'),\n",
    "#       keras.metrics.Recall(name='recall'),\n",
    "#       keras.metrics.AUC(name='auc'),\n",
    "# ]\n",
    "\n",
    "METRICS = [\n",
    "    #keras.metrics.TruePositives(name='tp'),\n",
    "    keras.metrics.AUC(name='accuracy'),\n",
    "    keras.metrics.Recall(name='recall'),\n",
    "    keras.metrics.Precision(name='precision'),\n",
    "    keras.metrics.AUC(name='auc')\n",
    "]\n",
    "\n",
    "\n",
    "n_steps = X_train.shape[1]\n",
    "n_features = X_train.shape[2]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "masking (Masking)            (None, 10, 20)            0         \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 300)               385200    \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1)                 301       \n",
      "=================================================================\n",
      "Total params: 385,501\n",
      "Trainable params: 385,501\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "\n",
    "tf.keras.backend.clear_session()\n",
    "tf.compat.v1.reset_default_graph()\n",
    "\n",
    "from tensorflow.keras import initializers\n",
    "\n",
    "model = Sequential()\n",
    "model.add(tf.keras.layers.Masking(mask_value=0.,\n",
    "                                   input_shape=(n_steps, n_features))) # skip the weeks without data\n",
    "model.add(LSTM(300, return_sequences=False, \n",
    "               dropout=0.1, recurrent_dropout=0.1))\n",
    "\n",
    "model.add(Dense(1,activation='sigmoid'))\n",
    "model.compile(loss = 'binary_crossentropy', optimizer=keras.optimizers.Adam(lr=0.01),metrics = METRICS)\n",
    "print(model.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "29/29 - 2s - loss: 0.6566 - accuracy: 0.6884 - recall: 0.6040 - precision: 0.5057 - auc: 0.6884 - val_loss: 0.5823 - val_accuracy: 0.7695 - val_recall: 0.7035 - val_precision: 0.5283 - val_auc: 0.7695\n",
      "Epoch 2/100\n",
      "29/29 - 2s - loss: 0.6147 - accuracy: 0.7397 - recall: 0.6901 - precision: 0.5281 - auc: 0.7397 - val_loss: 0.6275 - val_accuracy: 0.7945 - val_recall: 0.8315 - val_precision: 0.4646 - val_auc: 0.7945\n",
      "Epoch 3/100\n",
      "29/29 - 2s - loss: 0.5779 - accuracy: 0.7750 - recall: 0.6891 - precision: 0.5739 - auc: 0.7750 - val_loss: 0.5796 - val_accuracy: 0.8027 - val_recall: 0.7911 - val_precision: 0.4949 - val_auc: 0.8027\n",
      "Epoch 4/100\n",
      "29/29 - 1s - loss: 0.5624 - accuracy: 0.7848 - recall: 0.7258 - precision: 0.5524 - auc: 0.7848 - val_loss: 0.4878 - val_accuracy: 0.8121 - val_recall: 0.5714 - val_precision: 0.6347 - val_auc: 0.8121\n",
      "Epoch 5/100\n",
      "29/29 - 2s - loss: 0.5535 - accuracy: 0.7916 - recall: 0.7164 - precision: 0.5641 - auc: 0.7916 - val_loss: 0.6087 - val_accuracy: 0.8092 - val_recall: 0.8410 - val_precision: 0.4720 - val_auc: 0.8092\n",
      "Epoch 6/100\n",
      "29/29 - 2s - loss: 0.5744 - accuracy: 0.7740 - recall: 0.6838 - precision: 0.5578 - auc: 0.7740 - val_loss: 0.6234 - val_accuracy: 0.8085 - val_recall: 0.8585 - val_precision: 0.4524 - val_auc: 0.8085\n",
      "Epoch 7/100\n",
      "29/29 - 2s - loss: 0.5578 - accuracy: 0.7860 - recall: 0.6901 - precision: 0.5611 - auc: 0.7860 - val_loss: 0.5674 - val_accuracy: 0.8077 - val_recall: 0.7898 - val_precision: 0.5043 - val_auc: 0.8077\n",
      "Epoch 8/100\n",
      "29/29 - 2s - loss: 0.5487 - accuracy: 0.7952 - recall: 0.6985 - precision: 0.5728 - auc: 0.7952 - val_loss: 0.6040 - val_accuracy: 0.8104 - val_recall: 0.8356 - val_precision: 0.4715 - val_auc: 0.8104\n",
      "Epoch 9/100\n",
      "29/29 - 2s - loss: 0.5401 - accuracy: 0.8039 - recall: 0.7447 - precision: 0.5690 - auc: 0.8039 - val_loss: 0.5707 - val_accuracy: 0.8038 - val_recall: 0.7978 - val_precision: 0.4937 - val_auc: 0.8038\n",
      "Epoch 10/100\n",
      "29/29 - 2s - loss: 0.5353 - accuracy: 0.8065 - recall: 0.7258 - precision: 0.5773 - auc: 0.8065 - val_loss: 0.5767 - val_accuracy: 0.8114 - val_recall: 0.7857 - val_precision: 0.4966 - val_auc: 0.8114\n",
      "Epoch 11/100\n",
      "29/29 - 2s - loss: 0.5430 - accuracy: 0.7989 - recall: 0.7258 - precision: 0.5720 - auc: 0.7989 - val_loss: 0.6093 - val_accuracy: 0.8021 - val_recall: 0.8410 - val_precision: 0.4602 - val_auc: 0.8021\n",
      "Epoch 12/100\n",
      "29/29 - 2s - loss: 0.5347 - accuracy: 0.8054 - recall: 0.7437 - precision: 0.5628 - auc: 0.8054 - val_loss: 0.5420 - val_accuracy: 0.8104 - val_recall: 0.7601 - val_precision: 0.5193 - val_auc: 0.8104\n",
      "Epoch 13/100\n",
      "29/29 - 2s - loss: 0.5357 - accuracy: 0.8034 - recall: 0.7384 - precision: 0.5711 - auc: 0.8034 - val_loss: 0.5372 - val_accuracy: 0.8084 - val_recall: 0.7453 - val_precision: 0.5197 - val_auc: 0.8084\n",
      "Epoch 14/100\n",
      "Restoring model weights from the end of the best epoch.\n",
      "29/29 - 2s - loss: 0.5300 - accuracy: 0.8100 - recall: 0.7384 - precision: 0.5753 - auc: 0.8100 - val_loss: 0.5788 - val_accuracy: 0.8090 - val_recall: 0.7992 - val_precision: 0.4849 - val_auc: 0.8090\n",
      "Epoch 00014: early stopping\n"
     ]
    }
   ],
   "source": [
    "batch_size = 100\n",
    "my_callbacks = [early_stopping]\n",
    "\n",
    "neg = len(y_train[y_train ==0])\n",
    "pos = len(y_train[y_train ==1])\n",
    "total = neg + pos\n",
    "weight_for_0 = (1 / neg)*(total)/2.0 \n",
    "weight_for_1 = (1 / pos)*(total)/2.0\n",
    "\n",
    "class_weight = {0: weight_for_0, 1: weight_for_1}\n",
    "\n",
    "history = model.fit(X_train, y_train, epochs = 100, batch_size=batch_size, \n",
    "                    validation_data=(X_test, y_test), \n",
    "                    callbacks = my_callbacks, class_weight=class_weight,\n",
    "                    verbose = 2, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79/79 [==============================] - 1s 9ms/step - loss: 0.4878 - accuracy: 0.8121 - recall: 0.5714 - precision: 0.6347 - auc: 0.8121\n"
     ]
    }
   ],
   "source": [
    "results = model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(history, model)\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "y_pred_train=model.predict_classes(X_train)\n",
    "con_mat = tf.math.confusion_matrix(labels=y_train, predictions=y_pred_train).numpy()\n",
    "\n",
    "cm_plot_labels = ['safe','risky']\n",
    "cm = confusion_matrix(y_true=y_train, y_pred=y_pred_train)\n",
    "\n",
    "\n",
    "plot_confusion_matrix(cm=cm, classes=cm_plot_labels, title='training Confusion Matrix')\n",
    "\n",
    "\n",
    "y_pred_test=model.predict_classes(X_test)\n",
    "con_mat = tf.math.confusion_matrix(labels=y_test, predictions=y_pred_test).numpy()\n",
    "\n",
    "cm_plot_labels = ['safe','risky']\n",
    "cm = confusion_matrix(y_true=y_test, y_pred=y_pred_test)\n",
    "\n",
    "\n",
    "plot_confusion_matrix(cm=cm, classes=cm_plot_labels, title='test Confusion Matrix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "\n",
    "def plot_history(history, model):\n",
    "    metrics = model.metrics_names\n",
    "    plt.figure(figsize=(12, 8))\n",
    "    for n, metric in enumerate(metrics):\n",
    "        name = metric.replace(\"_\",\" \").capitalize()\n",
    "        \n",
    "        plt.subplot(3,2,n+1)\n",
    "        plt.plot(history.epoch,  history.history[metric], color='b', label='Train')\n",
    "        plt.plot(history.epoch, history.history['val_'+metric],\n",
    "             color='r', linestyle=\"--\", label='Val')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel(name)\n",
    "        plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                        normalize=False,\n",
    "                        title='Confusion matrix',\n",
    "                        cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    plt.figure()\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "            horizontalalignment=\"center\",\n",
    "            color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}