Next_word_predictor.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.7"
    },
    "colab": {
      "name": "Next-word-predictor.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "iKO2QN_mFP9j"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.layers import Embedding, LSTM, Dense\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import pickle\n",
        "import numpy as np\n",
        "import os"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "3SH8VJKGFP9o",
        "outputId": "5b1372f1-2e0a-425f-e5f8-496fabe1f711"
      },
      "source": [
        "from google.colab import files\n",
        "file = files.upload()\n",
        "file = open(\"dataset.txt\",\"r\")\n",
        "lines = []\n",
        "\n",
        "for i in file:\n",
        "    lines.append(i)\n",
        "    \n",
        "print(lines[0])\n",
        "print(lines[1000])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-2e34074b-1d07-4d1b-a8b6-d8380c17c501\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-2e34074b-1d07-4d1b-a8b6-d8380c17c501\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving dataset.txt to dataset (4).txt\n",
            "﻿It was the best of times, it was the worst of times, it was the age of\n",
            "\n",
            "“A likely thing, too!” replied the strong woman. “If it was ever\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KHAwYUVbFP9t",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "acef13bc-cb20-450d-db4c-1ab101d36883"
      },
      "source": [
        "#Cleaning dataset\n",
        "data = \"\"\n",
        "\n",
        "for i in lines:\n",
        "    data = ' '. join(lines)\n",
        "    \n",
        "data = data.replace('\\n', '').replace('\\r', '').replace('\\ufeff', '')\n",
        "data[:360]"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going '"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NY6agaB3FP9w",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "c0a5b1d9-9ed2-4e2e-afe2-d1a457493830"
      },
      "source": [
        "import string\n",
        "\n",
        "translator = str.maketrans(string.punctuation, ' '*len(string.punctuation)) #map punctuation to space\n",
        "new_data = data.translate(translator)\n",
        "\n",
        "new_data[:150]"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'It was the best of times  it was the worst of times  it was the age of wisdom  it was the age of foolishness  it was the epoch of belief  it was the e'"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7SpTEgi2FP9y",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "286e2a8b-ced2-4298-a1e2-daca214c9c64"
      },
      "source": [
        "z = []\n",
        "\n",
        "for i in data.split():\n",
        "    if i not in z:\n",
        "        z.append(i)\n",
        "        \n",
        "data = ' '.join(z)\n",
        "data[:150]"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'It was the best of times, it worst age wisdom, foolishness, epoch belief, incredulity, season Light, Darkness, spring hope, winter despair, we had eve'"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T3ACafzBFP93",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9f738765-c2e7-413e-f606-6e0e092a1b82"
      },
      "source": [
        "#Tokenization\n",
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts([data])\n",
        "\n",
        "# saving the tokenizer for predict function.\n",
        "pickle.dump(tokenizer, open('tokenizer.pkl', 'wb'))\n",
        "\n",
        "sequence_data = tokenizer.texts_to_sequences([data])[0]\n",
        "sequence_data[:10]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[5, 31, 6, 505, 52, 370, 5, 1141, 753, 3916]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-dWhM7TQFP95",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a09d3350-6dd9-444f-951c-11642f421d3e"
      },
      "source": [
        "vocab_size = len(tokenizer.word_index) + 1\n",
        "print(vocab_size)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "10619\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E5g6voYlFP96",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1d1690bc-6f0c-469b-eeae-3618b6db714b"
      },
      "source": [
        "sequences = []\n",
        "\n",
        "for i in range(1, len(sequence_data)):\n",
        "    words = sequence_data[i-1:i+1]\n",
        "    sequences.append(words)\n",
        "    \n",
        "print(\"The Length of sequences are: \", len(sequences))\n",
        "sequences = np.array(sequences)\n",
        "sequences[:10]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The Length of sequences are:  22558\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[   5,   31],\n",
              "       [  31,    6],\n",
              "       [   6,  505],\n",
              "       [ 505,   52],\n",
              "       [  52,  370],\n",
              "       [ 370,    5],\n",
              "       [   5, 1141],\n",
              "       [1141,  753],\n",
              "       [ 753, 3916],\n",
              "       [3916, 3917]])"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B8xI65spFP98"
      },
      "source": [
        "X = []\n",
        "y = []\n",
        "\n",
        "for i in sequences:\n",
        "    X.append(i[0])\n",
        "    y.append(i[1])\n",
        "    \n",
        "X = np.array(X)\n",
        "y = np.array(y)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mJmafvrjFP99",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e335d0aa-cad2-4993-8faa-3a10d5ed61df"
      },
      "source": [
        "print(\"The Data is: \", X[:5])\n",
        "print(\"The responses are: \", y[:5])"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The Data is:  [  5  31   6 505  52]\n",
            "The responses are:  [ 31   6 505  52 370]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xcnQSl5-FP9_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aacfd492-dc17-47e1-a1a7-f806ca0dd194"
      },
      "source": [
        "y = to_categorical(y, num_classes=vocab_size)\n",
        "y[:5]"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xRgqtfAuFP-B"
      },
      "source": [
        "#Creating the model\n",
        "model = Sequential()\n",
        "model.add(Embedding(vocab_size, 10, input_length=1))\n",
        "model.add(LSTM(50, return_sequences=True))\n",
        "model.add(LSTM(50))\n",
        "model.add(Dense(50, activation=\"relu\"))\n",
        "model.add(Dense(vocab_size, activation=\"softmax\"))"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-18yha5FP-C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03641e5d-e296-46a1-f4be-0517f6901414"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_2 (Embedding)     (None, 1, 10)             106190    \n",
            "                                                                 \n",
            " lstm_4 (LSTM)               (None, 1, 50)             12200     \n",
            "                                                                 \n",
            " lstm_5 (LSTM)               (None, 50)                20200     \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 50)                2550      \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 10619)             541569    \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 682,709\n",
            "Trainable params: 682,709\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qGjx9eYgFP-F"
      },
      "source": [
        "#Callbacks\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
        "from tensorflow.keras.callbacks import TensorBoard\n",
        "\n",
        "checkpoint = ModelCheckpoint(\"nextword1.h5\", monitor='loss', verbose=1,\n",
        "    save_best_only=True, mode='auto')\n",
        "\n",
        "reduce = ReduceLROnPlateau(monitor='loss', factor=0.2, patience=3, min_lr=0.0001, verbose = 1)\n",
        "\n",
        "logdir='logsnextword1'\n",
        "tensorboard_Visualization = TensorBoard(log_dir=logdir)"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JcxteaRVFP-G",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8cd0d76d-1bf1-4f88-9938-75ee4b4ec370"
      },
      "source": [
        "#Compile model\n",
        "model.compile(loss=\"categorical_crossentropy\", optimizer=Adam(lr=0.001))"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJ8EL-ChFP-H",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "66216372-b9c8-4705-cc39-fe34e6ee5fdd"
      },
      "source": [
        "#Fitting the Model\n",
        "model.fit(X, y, epochs=20, batch_size=64, callbacks=[checkpoint, reduce, tensorboard_Visualization])"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 8.3472\n",
            "Epoch 00001: loss improved from 8.61569 to 8.34720, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 8.3472 - lr: 0.0010\n",
            "Epoch 2/20\n",
            "351/353 [============================>.] - ETA: 0s - loss: 8.2409\n",
            "Epoch 00002: loss improved from 8.34720 to 8.24102, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 8.2410 - lr: 0.0010\n",
            "Epoch 3/20\n",
            "352/353 [============================>.] - ETA: 0s - loss: 8.1820\n",
            "Epoch 00003: loss improved from 8.24102 to 8.18142, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 8.1814 - lr: 0.0010\n",
            "Epoch 4/20\n",
            "352/353 [============================>.] - ETA: 0s - loss: 8.1350\n",
            "Epoch 00004: loss improved from 8.18142 to 8.13467, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 8.1347 - lr: 0.0010\n",
            "Epoch 5/20\n",
            "351/353 [============================>.] - ETA: 0s - loss: 8.0966\n",
            "Epoch 00005: loss improved from 8.13467 to 8.09620, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 8.0962 - lr: 0.0010\n",
            "Epoch 6/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 8.0638\n",
            "Epoch 00006: loss improved from 8.09620 to 8.06378, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 8.0638 - lr: 0.0010\n",
            "Epoch 7/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 8.0345\n",
            "Epoch 00007: loss improved from 8.06378 to 8.03448, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 8.0345 - lr: 0.0010\n",
            "Epoch 8/20\n",
            "351/353 [============================>.] - ETA: 0s - loss: 8.0109\n",
            "Epoch 00008: loss improved from 8.03448 to 8.01057, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 8.0106 - lr: 0.0010\n",
            "Epoch 9/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 7.9892\n",
            "Epoch 00009: loss improved from 8.01057 to 7.98919, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.9892 - lr: 0.0010\n",
            "Epoch 10/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 7.9683\n",
            "Epoch 00010: loss improved from 7.98919 to 7.96829, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.9683 - lr: 0.0010\n",
            "Epoch 11/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 7.9471\n",
            "Epoch 00011: loss improved from 7.96829 to 7.94715, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.9471 - lr: 0.0010\n",
            "Epoch 12/20\n",
            "351/353 [============================>.] - ETA: 0s - loss: 7.9250\n",
            "Epoch 00012: loss improved from 7.94715 to 7.92541, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.9254 - lr: 0.0010\n",
            "Epoch 13/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 7.9026\n",
            "Epoch 00013: loss improved from 7.92541 to 7.90262, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.9026 - lr: 0.0010\n",
            "Epoch 14/20\n",
            "352/353 [============================>.] - ETA: 0s - loss: 7.8787\n",
            "Epoch 00014: loss improved from 7.90262 to 7.87851, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.8785 - lr: 0.0010\n",
            "Epoch 15/20\n",
            "351/353 [============================>.] - ETA: 0s - loss: 7.8494\n",
            "Epoch 00015: loss improved from 7.87851 to 7.84957, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.8496 - lr: 0.0010\n",
            "Epoch 16/20\n",
            "352/353 [============================>.] - ETA: 0s - loss: 7.8104\n",
            "Epoch 00016: loss improved from 7.84957 to 7.80999, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.8100 - lr: 0.0010\n",
            "Epoch 17/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 7.7503\n",
            "Epoch 00017: loss improved from 7.80999 to 7.75030, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.7503 - lr: 0.0010\n",
            "Epoch 18/20\n",
            "351/353 [============================>.] - ETA: 0s - loss: 7.6824\n",
            "Epoch 00018: loss improved from 7.75030 to 7.68278, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.6828 - lr: 0.0010\n",
            "Epoch 19/20\n",
            "351/353 [============================>.] - ETA: 0s - loss: 7.6183\n",
            "Epoch 00019: loss improved from 7.68278 to 7.61779, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.6178 - lr: 0.0010\n",
            "Epoch 20/20\n",
            "353/353 [==============================] - ETA: 0s - loss: 7.5594\n",
            "Epoch 00020: loss improved from 7.61779 to 7.55942, saving model to nextword1.h5\n",
            "353/353 [==============================] - 7s 20ms/step - loss: 7.5594 - lr: 0.0010\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f9c8517ccd0>"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4oF1-jm3W9Sv"
      },
      "source": [
        "#Prediction Script\n",
        "# Importing the Libraries\n",
        "\n",
        "from tensorflow.keras.models import load_model\n",
        "import numpy as np\n",
        "import pickle\n",
        "\n",
        "# Load the model and tokenizer\n",
        "model = load_model('nextword1.h5')\n",
        "tokenizer = pickle.load(open('tokenizer.pkl', 'rb'))\n",
        "\n",
        "def Predict_Next_Words(model, tokenizer, text):\n",
        "    for i in range(3):\n",
        "        sequence = tokenizer.texts_to_sequences([text])[0]\n",
        "        sequence = np.array(sequence)\n",
        "        \n",
        "        preds = model.predict_classes(sequence)\n",
        "        predicted_word = \"\"\n",
        "        \n",
        "        for key, value in tokenizer.word_index.items():\n",
        "            if value == preds:\n",
        "                predicted_word = key\n",
        "                break\n",
        "        \n",
        "        print(predicted_word)\n",
        "        return predicted_word"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OtgSFLP_a96m",
        "outputId": "90f6d0b3-e702-4770-9fc0-1477453f8dd1"
      },
      "source": [
        "while(True):\n",
        "\n",
        "    text = input(\"Enter your line: \")\n",
        "    \n",
        "    if text == \"stop the script\":\n",
        "        print(\"Ending The Program.....\")\n",
        "        break\n",
        "    \n",
        "    else:\n",
        "        try:\n",
        "            text = text.split(\" \")\n",
        "            text = text[-1]\n",
        "\n",
        "            text = ''.join(text)\n",
        "            Predict_Next_Words(model, tokenizer, text)\n",
        "            \n",
        "        except:\n",
        "            continue"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your line: at the dull\n",
            "weather\n",
            "Enter your line: collection of textile\n",
            "samples\n",
            "Enter your line: what a strenuous\n",
            "career\n",
            "Enter your line: stop the script\n",
            "Ending The Program.....\n"
          ]
        }
      ]
    }
  ]
}