Bert_Model_colab.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 73,
     "resources": {
      "http://localhost:8080/nbextensions/google.colab/files.js": {
       "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCkgewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwogICAgICBwZXJjZW50LnRleHRDb250ZW50ID0KICAgICAgICAgIGAke01hdGgucm91bmQoKHBvc2l0aW9uIC8gZmlsZURhdGEuYnl0ZUxlbmd0aCkgKiAxMDApfSUgZG9uZWA7CiAgICB9CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
       "headers": [
        [
         "content-type",
         "application/javascript"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      }
     }
    },
    "id": "e7SKafXhqE2f",
    "outputId": "c36670e4-94ec-4917-8107-c5ab150e1351"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "     <input type=\"file\" id=\"files-5ddbad53-9918-459c-84d1-8e8ddbd3ad81\" name=\"files[]\" multiple disabled\n",
       "        style=\"border:none\" />\n",
       "     <output id=\"result-5ddbad53-9918-459c-84d1-8e8ddbd3ad81\">\n",
       "      Upload widget is only available when the cell has been executed in the\n",
       "      current browser session. Please rerun this cell to enable.\n",
       "      </output>\n",
       "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving SemEval2018-T3-train-taskA_emoji.txt to SemEval2018-T3-train-taskA_emoji.txt\n"
     ]
    }
   ],
   "source": [
    "from google.colab import files\n",
    "uploaded = files.upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "6HiC8xZvqNr6"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train  = pd.read_csv('SemEval2018-T3-train-taskA_emoji.txt', sep=\"\\t\")\n",
    "test = pd.read_csv('SemEval2018-T3_gold_test_taskA_emoji.txt', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "HyrjzA86rnx5"
   },
   "outputs": [],
   "source": [
    "combi = train.append(test, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HiCt9ebumrED",
    "outputId": "2244122d-d1ad-4615-ae5d-993b62d0fc83"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tensorflow_text\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b6/c0/c0fed4301f592c3b56638ae7292612c17d91a43891ba1aaf9636d535beae/tensorflow_text-2.4.3-cp37-cp37m-manylinux1_x86_64.whl (3.4MB)\n",
      "\u001b[K     |████████████████████████████████| 3.4MB 8.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: tensorflow<2.5,>=2.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (2.4.1)\n",
      "Requirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (0.11.0)\n",
      "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (0.36.2)\n",
      "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (3.3.0)\n",
      "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (0.3.3)\n",
      "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (0.2.0)\n",
      "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (1.6.3)\n",
      "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (1.19.5)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (3.12.4)\n",
      "Requirement already satisfied: tensorflow-estimator<2.5.0,>=2.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (2.4.0)\n",
      "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (1.15.0)\n",
      "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (1.1.0)\n",
      "Requirement already satisfied: h5py~=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (2.10.0)\n",
      "Requirement already satisfied: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (2.4.1)\n",
      "Requirement already satisfied: grpcio~=1.32.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (1.32.0)\n",
      "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (0.12.0)\n",
      "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (3.7.4.3)\n",
      "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (1.1.2)\n",
      "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (1.12)\n",
      "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.5,>=2.4.0->tensorflow_text) (1.12.1)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.9.2->tensorflow<2.5,>=2.4.0->tensorflow_text) (54.2.0)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (1.28.0)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (1.8.0)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (2.23.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (3.3.4)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (1.0.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (0.4.3)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (4.7.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (0.2.8)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (4.2.1)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (1.24.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (2020.12.5)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (3.0.4)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (3.8.1)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (1.3.0)\n",
      "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.7/dist-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (0.4.8)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (3.4.1)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow<2.5,>=2.4.0->tensorflow_text) (3.1.0)\n",
      "Installing collected packages: tensorflow-text\n",
      "Successfully installed tensorflow-text-2.4.3\n",
      "Collecting bert-for-tf2\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a5/a1/acb891630749c56901e770a34d6bac8a509a367dd74a05daf7306952e910/bert-for-tf2-0.14.9.tar.gz (41kB)\n",
      "\u001b[K     |████████████████████████████████| 51kB 5.4MB/s \n",
      "\u001b[?25hCollecting py-params>=0.9.6\n",
      "  Downloading https://files.pythonhosted.org/packages/aa/e0/4f663d8abf83c8084b75b995bd2ab3a9512ebc5b97206fde38cef906ab07/py-params-0.10.2.tar.gz\n",
      "Collecting params-flow>=0.8.0\n",
      "  Downloading https://files.pythonhosted.org/packages/a9/95/ff49f5ebd501f142a6f0aaf42bcfd1c192dc54909d1d9eb84ab031d46056/params-flow-0.8.2.tar.gz\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (1.19.5)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (4.41.1)\n",
      "Building wheels for collected packages: bert-for-tf2, py-params, params-flow\n",
      "  Building wheel for bert-for-tf2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for bert-for-tf2: filename=bert_for_tf2-0.14.9-cp37-none-any.whl size=30535 sha256=01bcec868ef4f7504e3481fa765e72bd711c4e3e2bbbc9e995c6cef47fa7c8c5\n",
      "  Stored in directory: /root/.cache/pip/wheels/a1/04/ee/347bd9f5b821b637c76411d280271a857aece00358896a230f\n",
      "  Building wheel for py-params (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for py-params: filename=py_params-0.10.2-cp37-none-any.whl size=7912 sha256=2410b832811270e1f7716c8846e5f947b900cd0a5cb3823de8c2c5db039061a5\n",
      "  Stored in directory: /root/.cache/pip/wheels/d0/4a/70/ff12450229ff1955abf01f365051d4faae1c20aef53ab4cf09\n",
      "  Building wheel for params-flow (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for params-flow: filename=params_flow-0.8.2-cp37-none-any.whl size=19472 sha256=c666a76273ca4972b4167895c9dbdbb68347ee1503303e2b50b20cc1453268be\n",
      "  Stored in directory: /root/.cache/pip/wheels/08/c8/7f/81c86b9ff2b86e2c477e3914175be03e679e596067dc630c06\n",
      "Successfully built bert-for-tf2 py-params params-flow\n",
      "Installing collected packages: py-params, params-flow, bert-for-tf2\n",
      "Successfully installed bert-for-tf2-0.14.9 params-flow-0.8.2 py-params-0.10.2\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow_text\n",
    "!pip install bert-for-tf2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "LrSoj9uyrwoJ"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "# import tensorflow.compat.v1 as tf\n",
    "# tf.disable_v2_behavior()\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_text as text\n",
    "# from bert import tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "3Y9k_-Z4sC-r"
   },
   "outputs": [],
   "source": [
    "module_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'\n",
    "# module_url = 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1'\n",
    "bert_layer = hub.KerasLayer(module_url, trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "_m0C3uYLKl8I"
   },
   "outputs": [],
   "source": [
    "import bert\n",
    "from bert import bert_tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "I5rEOcb2tPoK"
   },
   "outputs": [],
   "source": [
    "# from bert import tokenization\n",
    "max_len = 256\n",
    "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
    "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
    "tokenizer = bert.bert_tokenization.FullTokenizer(vocab_file, do_lower_case)\n",
    "# tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n",
    "\n",
    "def bert_encode(texts, tokenizer, max_len=max_len):\n",
    "    all_tokens = []\n",
    "    all_masks = []\n",
    "    all_segments = []\n",
    "    \n",
    "    for text in texts:\n",
    "        text = tokenizer.tokenize(text)\n",
    "            \n",
    "        text = text[:max_len-2]\n",
    "        #Token embeddings: A [CLS] token is added to the input word tokens at the beginning of the first sentence and a [SEP] token is inserted at the end of each sentence.\n",
    "        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n",
    "        pad_len = max_len - len(input_sequence)\n",
    "        \n",
    "        tokens = tokenizer.convert_tokens_to_ids(input_sequence) + [0] * pad_len\n",
    "        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n",
    "        segment_ids = [0] * max_len\n",
    "        \n",
    "        all_tokens.append(tokens)\n",
    "        all_masks.append(pad_masks)\n",
    "        all_segments.append(segment_ids)\n",
    "    \n",
    "    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bWAWXiic2goD",
    "outputId": "22f249e5-f98e-4012-e844-0e563b559ca4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/device:GPU:0\n"
     ]
    }
   ],
   "source": [
    "device_name = tf.test.gpu_device_name()\n",
    "print(device_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "K9Fcc_ZWti8i"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train_df, val_df = train_test_split(combi.iloc[:3816], test_size=0.1, random_state=2020)\n",
    "\n",
    "train_input = bert_encode(train_df['Tweet text'].values, tokenizer, max_len=max_len)\n",
    "train_labels = np.asarray(train_df['Label'].values).astype('float32').reshape((-1,1))\n",
    "\n",
    "val_input = bert_encode(val_df['Tweet text'].values, tokenizer, max_len=max_len)\n",
    "val_labels = np.asarray(val_df['Label'].values).astype('float32').reshape((-1,1))\n",
    "\n",
    "test_input = bert_encode(combi.iloc[3817:]['Tweet text'].values, tokenizer, max_len=max_len)\n",
    "test_labels = np.asarray(combi.iloc[3817:]['Label'].values).astype('float32').reshape((-1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BYJSxUZOuHQF",
    "outputId": "788ec1bd-5998-4f06-f0cc-21bac362dd68"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  101,  3607,  2758, ...,     0,     0,     0],\n",
       "       [  101, 22091,  2860, ...,     0,     0,     0],\n",
       "       [  101,  2358,  2140, ...,     0,     0,     0],\n",
       "       ...,\n",
       "       [  101,  2040,  9541, ...,     0,     0,     0],\n",
       "       [  101,  1045,  2066, ...,     0,     0,     0],\n",
       "       [  101,  1030, 18243, ...,     0,     0,     0]])"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_input[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "esw8PUnGM0cT",
    "outputId": "23f35120-4d6f-4d6f-bd8d-aabf5df3017f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "256"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(np.count_nonzero(train_input[0], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "VN6yLtJSNAa0"
   },
   "outputs": [],
   "source": [
    "def build_model(bert_layer, max_len=max_len):\n",
    "    input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n",
    "    input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n",
    "    segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"input_type_ids\")\n",
    "    \n",
    "#     encoder_inputs = dict(\n",
    "#             input_word_ids=tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\"),\n",
    "#             input_mask=tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\"),\n",
    "#             segment_ids=tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\"),\n",
    "#             )\n",
    "    \n",
    "    pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n",
    "#     pooled_output, sequence_output = bert_layer(encoder_inputs)\n",
    "#    clf_output = sequence_output[:, 0, :]\n",
    "    clf_output = pooled_output\n",
    "    net = tf.keras.layers.Dense(32, activation='relu')(clf_output)\n",
    "    net = tf.keras.layers.Dropout(0.1)(net)\n",
    "#     net = tf.keras.layers.Dense(32, activation='relu')(net)\n",
    "#     net = tf.keras.layers.Dropout(0.2)(net)\n",
    "    out = tf.keras.layers.Dense(1, activation='sigmoid')(net)\n",
    "    \n",
    "    model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)\n",
    "    model.compile(tf.keras.optimizers.Adam(lr=1e-3), loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yFH5OsVXR6A9",
    "outputId": "b27cf09c-573d-411f-a62f-06f9dc324a4b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_word_ids (InputLayer)     [(None, 256)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_mask (InputLayer)         [(None, 256)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_type_ids (InputLayer)     [(None, 256)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "keras_layer (KerasLayer)        [(None, 768), (None, 109482241   input_word_ids[0][0]             \n",
      "                                                                 input_mask[0][0]                 \n",
      "                                                                 input_type_ids[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 32)           24608       keras_layer[2][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 32)           0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            33          dropout_2[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 109,506,882\n",
      "Trainable params: 24,641\n",
      "Non-trainable params: 109,482,241\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = build_model(bert_layer, max_len=max_len)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5fRP3PGqR9dr",
    "outputId": "9d767976-c84f-4c5d-df0a-8e39a9e1e3dd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "14/14 [==============================] - 79s 6s/step - loss: 0.7100 - accuracy: 0.4875 - val_loss: 0.6829 - val_accuracy: 0.5864\n",
      "\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.58639, saving model to model.h5\n",
      "Epoch 2/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6842 - accuracy: 0.5633 - val_loss: 0.6760 - val_accuracy: 0.5890\n",
      "\n",
      "Epoch 00002: val_accuracy improved from 0.58639 to 0.58901, saving model to model.h5\n",
      "Epoch 3/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6780 - accuracy: 0.5793 - val_loss: 0.6702 - val_accuracy: 0.6047\n",
      "\n",
      "Epoch 00003: val_accuracy improved from 0.58901 to 0.60471, saving model to model.h5\n",
      "Epoch 4/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6767 - accuracy: 0.5748 - val_loss: 0.6662 - val_accuracy: 0.6152\n",
      "\n",
      "Epoch 00004: val_accuracy improved from 0.60471 to 0.61518, saving model to model.h5\n",
      "Epoch 5/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6709 - accuracy: 0.5821 - val_loss: 0.6593 - val_accuracy: 0.6204\n",
      "\n",
      "Epoch 00005: val_accuracy improved from 0.61518 to 0.62042, saving model to model.h5\n",
      "Epoch 6/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6694 - accuracy: 0.5911 - val_loss: 0.6768 - val_accuracy: 0.5524\n",
      "\n",
      "Epoch 00006: val_accuracy did not improve from 0.62042\n",
      "\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "Epoch 7/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6818 - accuracy: 0.5731 - val_loss: 0.6559 - val_accuracy: 0.6257\n",
      "\n",
      "Epoch 00007: val_accuracy improved from 0.62042 to 0.62565, saving model to model.h5\n",
      "Epoch 8/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6636 - accuracy: 0.5990 - val_loss: 0.6507 - val_accuracy: 0.6283\n",
      "\n",
      "Epoch 00008: val_accuracy improved from 0.62565 to 0.62827, saving model to model.h5\n",
      "Epoch 9/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6614 - accuracy: 0.5991 - val_loss: 0.6593 - val_accuracy: 0.5916\n",
      "\n",
      "Epoch 00009: val_accuracy did not improve from 0.62827\n",
      "\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "Epoch 10/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6625 - accuracy: 0.6062 - val_loss: 0.6468 - val_accuracy: 0.6309\n",
      "\n",
      "Epoch 00010: val_accuracy improved from 0.62827 to 0.63089, saving model to model.h5\n",
      "Epoch 11/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6532 - accuracy: 0.6146 - val_loss: 0.6518 - val_accuracy: 0.6309\n",
      "\n",
      "Epoch 00011: val_accuracy did not improve from 0.63089\n",
      "\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "Epoch 12/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6499 - accuracy: 0.6378 - val_loss: 0.6483 - val_accuracy: 0.6283\n",
      "\n",
      "Epoch 00012: val_accuracy did not improve from 0.63089\n",
      "\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "Epoch 13/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6552 - accuracy: 0.6134 - val_loss: 0.6458 - val_accuracy: 0.6387\n",
      "\n",
      "Epoch 00013: val_accuracy improved from 0.63089 to 0.63874, saving model to model.h5\n",
      "Epoch 14/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6580 - accuracy: 0.6169 - val_loss: 0.6454 - val_accuracy: 0.6414\n",
      "\n",
      "Epoch 00014: val_accuracy improved from 0.63874 to 0.64136, saving model to model.h5\n",
      "Epoch 15/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6546 - accuracy: 0.6256 - val_loss: 0.6449 - val_accuracy: 0.6387\n",
      "\n",
      "Epoch 00015: val_accuracy did not improve from 0.64136\n",
      "Epoch 16/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6576 - accuracy: 0.6089 - val_loss: 0.6446 - val_accuracy: 0.6387\n",
      "\n",
      "Epoch 00016: val_accuracy did not improve from 0.64136\n",
      "Epoch 17/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6473 - accuracy: 0.6364 - val_loss: 0.6441 - val_accuracy: 0.6414\n",
      "\n",
      "Epoch 00017: val_accuracy did not improve from 0.64136\n",
      "Epoch 18/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6542 - accuracy: 0.6181 - val_loss: 0.6441 - val_accuracy: 0.6414\n",
      "\n",
      "Epoch 00018: val_accuracy did not improve from 0.64136\n",
      "\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "Epoch 19/20\n",
      "14/14 [==============================] - 76s 5s/step - loss: 0.6518 - accuracy: 0.6247 - val_loss: 0.6439 - val_accuracy: 0.6414\n",
      "\n",
      "Epoch 00019: val_accuracy did not improve from 0.64136\n",
      "Epoch 00019: early stopping\n"
     ]
    }
   ],
   "source": [
    "\n",
    "checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)\n",
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=1e-5, verbose=2)\n",
    "earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, verbose=1)\n",
    "\n",
    "\n",
    "train_history = model.fit(\n",
    "    train_input, train_labels, \n",
    "    validation_data=(val_input, val_labels),\n",
    "    epochs=20,\n",
    "    callbacks=[checkpoint, earlystopping, reduce_lr],\n",
    "    batch_size=256,\n",
    "    verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "T4KKZ8V-SC6T"
   },
   "outputs": [],
   "source": [
    "pred_val_y = model.predict([val_input], batch_size=32, verbose=0)\n",
    "pred_test_y = model.predict([test_input], batch_size=32, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "oeUHZ_i0i9x7"
   },
   "outputs": [],
   "source": [
    "# To compute the best F1, we choose a suitable probability threshold\n",
    "from sklearn import metrics\n",
    "def f1_smart(y_true, y_pred):\n",
    "    thresholds = []\n",
    "    for thresh in np.arange(0.1, 0.501, 0.01):\n",
    "        thresh = np.round(thresh, 2)\n",
    "        res = metrics.f1_score(y_true, (y_pred > thresh).astype(int))\n",
    "        thresholds.append([thresh, res])\n",
    "        print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n",
    "\n",
    "    thresholds.sort(key=lambda x: x[1], reverse=True)\n",
    "    best_thresh = thresholds[0][0]\n",
    "    best_f1 = thresholds[0][1]\n",
    "    print(\"Best threshold: \", best_thresh)\n",
    "    return  best_f1, best_thresh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Qddrb3DQlbH7",
    "outputId": "dba818dc-577a-4e89-c428-10476bdc664c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 score at threshold 0.1 is 0.6643356643356644\n",
      "F1 score at threshold 0.11 is 0.6643356643356644\n",
      "F1 score at threshold 0.12 is 0.6643356643356644\n",
      "F1 score at threshold 0.13 is 0.6643356643356644\n",
      "F1 score at threshold 0.14 is 0.6643356643356644\n",
      "F1 score at threshold 0.15 is 0.6643356643356644\n",
      "F1 score at threshold 0.16 is 0.6643356643356644\n",
      "F1 score at threshold 0.17 is 0.6643356643356644\n",
      "F1 score at threshold 0.18 is 0.6643356643356644\n",
      "F1 score at threshold 0.19 is 0.6643356643356644\n",
      "F1 score at threshold 0.2 is 0.6643356643356644\n",
      "F1 score at threshold 0.21 is 0.6643356643356644\n",
      "F1 score at threshold 0.22 is 0.6643356643356644\n",
      "F1 score at threshold 0.23 is 0.6643356643356644\n",
      "F1 score at threshold 0.24 is 0.6643356643356644\n",
      "F1 score at threshold 0.25 is 0.6643356643356644\n",
      "F1 score at threshold 0.26 is 0.6643356643356644\n",
      "F1 score at threshold 0.27 is 0.6643356643356644\n",
      "F1 score at threshold 0.28 is 0.6654991243432574\n",
      "F1 score at threshold 0.29 is 0.6666666666666666\n",
      "F1 score at threshold 0.3 is 0.6678383128295254\n",
      "F1 score at threshold 0.31 is 0.6701940035273368\n",
      "F1 score at threshold 0.32 is 0.6737588652482269\n",
      "F1 score at threshold 0.33 is 0.6761565836298933\n",
      "F1 score at threshold 0.34 is 0.6797853309481218\n",
      "F1 score at threshold 0.35 is 0.6799276672694394\n",
      "F1 score at threshold 0.36 is 0.6812386156648452\n",
      "F1 score at threshold 0.37 is 0.6802973977695167\n",
      "F1 score at threshold 0.38 is 0.6832061068702289\n",
      "F1 score at threshold 0.39 is 0.6770428015564203\n",
      "F1 score at threshold 0.4 is 0.679920477137177\n",
      "F1 score at threshold 0.41 is 0.6762295081967213\n",
      "F1 score at threshold 0.42 is 0.6722689075630252\n",
      "F1 score at threshold 0.43 is 0.6752688172043011\n",
      "F1 score at threshold 0.44 is 0.6622222222222222\n",
      "F1 score at threshold 0.45 is 0.6712962962962963\n",
      "F1 score at threshold 0.46 is 0.657074340527578\n",
      "F1 score at threshold 0.47 is 0.6716417910447762\n",
      "F1 score at threshold 0.48 is 0.6597402597402597\n",
      "F1 score at threshold 0.49 is 0.6344086021505377\n",
      "F1 score at threshold 0.5 is 0.6183844011142061\n",
      "Best threshold:  0.38\n",
      "Optimal F1: 0.6832061068702289 at threshold: 0.38\n"
     ]
    }
   ],
   "source": [
    "f1, threshold = f1_smart(val_labels, pred_val_y)\n",
    "print('Optimal F1: {} at threshold: {}'.format(f1, threshold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "A-ysfD9elfPj",
    "outputId": "747c9eca-af1c-49e2-b014-9eb79dfadb3f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the F1 score of tweet classification:0.5901960784313727\n"
     ]
    }
   ],
   "source": [
    "pred_test_y = (pred_test_y >threshold).astype(int)\n",
    "res = metrics.f1_score(test_labels, pred_test_y)\n",
    "print('the F1 score of tweet classification:{}'.format(res))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hhrhXPaRr74U"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7DhLbI2FdE4N"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Bert_Model.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}