hw02-working-with-sequences-in-tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "# Homework"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Below is the source code for the homework in [Chapter 3 - Working with sequences in TensorFlow](./ch03-working-with-sequences-in-tensorflow.ipynb).\n",
    "\n",
    "After executing the code, start TensorBoard with `tensorboard --logdir /tmp/tensorflow-workshop/alphabet-predictor`. Once you navigate to `localhost:6006` you should see something similar to the following:\n",
    "- In the `Scalars` tab\n",
    "  ![TensorBoard scalars](./img/hw02-scalars.png)\n",
    "- In the `Graphs` tab, after removing `train` and `rnn` nodes from the main graph:\n",
    "  ![Computational graph](./img/hw02-graph.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "autoscroll": false,
    "collapsed": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "import os.path as path\n",
    "import datetime\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "np.random.seed(2018)\n",
    "\n",
    "alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n",
    "\n",
    "class Encoding():\n",
    "    def __init__(self):\n",
    "        self._char_to_int = dict((c, i) for i, c in enumerate(alphabet))\n",
    "        self._int_to_char = dict((i, c) for i, c in enumerate(alphabet))\n",
    "\n",
    "    def encode_sequence(self, sequence):\n",
    "        return [self._char_to_int[char] for char in sequence]\n",
    "\n",
    "    def encode_letter(self, letter):\n",
    "        return self._char_to_int[letter]\n",
    "\n",
    "    def decode_letter(self, value):\n",
    "        return self._int_to_char[value]\n",
    "\n",
    "class Dataset():\n",
    "    def __init__(self, size=1000, seq_length=5, print_data=True):\n",
    "        self._size = size\n",
    "        self._sequence_length = seq_length\n",
    "        self._inputs = []\n",
    "        self._labels = []\n",
    "        self._print_data = print_data\n",
    "        self._encoding = Encoding()\n",
    "\n",
    "    @property\n",
    "    def inputs(self):\n",
    "        return self._inputs\n",
    "\n",
    "    @property\n",
    "    def labels(self):\n",
    "        return self._labels\n",
    "\n",
    "    @property\n",
    "    def num_classes(self):\n",
    "        return len(alphabet)\n",
    "\n",
    "    @property\n",
    "    def sequence_length(self):\n",
    "        return self._sequence_length\n",
    "\n",
    "    def initialize(self):\n",
    "        self._generate_random_data()\n",
    "        self._reshape_inputs()\n",
    "        self._normalize_inputs()\n",
    "        self._normalize_labels()\n",
    "\n",
    "    def shuffle(self):\n",
    "        perm = np.arange(self._size)\n",
    "        np.random.shuffle(perm)\n",
    "        self._inputs = self._inputs[perm]\n",
    "        self._labels = self._labels[perm]\n",
    "\n",
    "\n",
    "    def _normalize_labels(self):\n",
    "        self._labels = to_categorical(self._labels, num_classes=self.num_classes)\n",
    "        self._labels = np.reshape(self._labels, (self._size, 1, self.num_classes))\n",
    "\n",
    "    def _reshape_inputs(self):\n",
    "        self._inputs = pad_sequences(self._inputs,\n",
    "                                     maxlen=self._sequence_length,\n",
    "                                     dtype='float32')\n",
    "        self._inputs = np.reshape(self._inputs, (self._size, self._sequence_length, 1))\n",
    "\n",
    "    def _normalize_inputs(self):\n",
    "        self._inputs = self._inputs / float(self.num_classes)\n",
    "\n",
    "    def _generate_random_data(self):\n",
    "        for i in range(self._size):\n",
    "            start = np.random.randint(self.num_classes - 2)\n",
    "            end = np.random.randint(start, min(start + self._sequence_length, self.num_classes - 1))\n",
    "            input_seq = alphabet[start:end + 1]\n",
    "            output_seq = alphabet[end + 1]\n",
    "\n",
    "            if(self._print_data):\n",
    "                print(\"{}->{}\".format(input_seq, output_seq))\n",
    "\n",
    "            sample = self._encoding.encode_sequence(input_seq)\n",
    "            label = self._encoding.encode_letter(output_seq)\n",
    "\n",
    "            self._inputs.append(sample)\n",
    "            self._labels.append(label)\n",
    "\n",
    "def build_graph_writer(graph, logs_path='/tmp/tensorflow-workshop/alphabet-predictor'):\n",
    "    current_time = datetime.datetime.now().strftime('%Y-%m-%d-%H%M')\n",
    "    logs_path = path.join(logs_path, current_time)\n",
    "    writer = tf.summary.FileWriter(logs_path, graph)\n",
    "    return writer\n",
    "\n",
    "\n",
    "def build_LSTM(x, sequence_length, num_units=32):\n",
    "    with tf.name_scope(\"LSTM\"):\n",
    "        W = tf.Variable(tf.random_normal([num_units, num_units]), name=\"W\")\n",
    "        b = tf.Variable(tf.random_normal([1, num_units]), name=\"b\")\n",
    "\n",
    "        x = tf.split(x, sequence_length)\n",
    "        inner_cells = [rnn.BasicLSTMCell(num_units=num_units) for _ in range(sequence_length)]\n",
    "        rnn_cell = rnn.MultiRNNCell(inner_cells)\n",
    "        outputs, states = rnn.static_rnn(rnn_cell, x, dtype=tf.float32)\n",
    "        output = tf.nn.embedding_lookup(outputs, sequence_length - 1)\n",
    "        output = tf.reshape(output, [1, num_units])\n",
    "\n",
    "        tf.summary.histogram('weights', W)\n",
    "        tf.summary.histogram('biases', b)\n",
    "\n",
    "        return tf.matmul(output, W) + b\n",
    "\n",
    "def build_fully_connected(x, num_units):\n",
    "    with tf.name_scope(\"dense\"):\n",
    "        return tf.layers.dense(\n",
    "            inputs=x,\n",
    "            units=num_units)\n",
    "\n",
    "dataset = Dataset(size=1000, print_data=False)\n",
    "dataset.initialize()\n",
    "e = Encoding()\n",
    "\n",
    "x = tf.placeholder('float32', shape=(dataset.sequence_length, 1), name='inputs')\n",
    "y = tf.placeholder('float32', shape=(1, len(alphabet)), name=\"labels\")\n",
    "\n",
    "lstm = build_LSTM(x, dataset.sequence_length)\n",
    "dense = build_fully_connected(lstm, num_units=len(alphabet))\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=dense, labels=y))\n",
    "    tf.summary.scalar(\"loss\", cost)\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.AdamOptimizer().minimize(cost)\n",
    "\n",
    "with tf.name_scope(\"accuracy\"):\n",
    "    pred = tf.argmax(dense, 1)\n",
    "    accuracy, accuracy_update = tf.metrics.accuracy(labels=tf.argmax(y, 1), predictions=pred, name='acc')\n",
    "    tf.summary.scalar(\"accuracy\", accuracy)\n",
    "\n",
    "merged = tf.summary.merge_all()\n",
    "init = tf.global_variables_initializer()\n",
    "init_locals = tf.local_variables_initializer()\n",
    "with tf.Session() as session:\n",
    "    session.run(init)\n",
    "    session.run(init_locals)\n",
    "    writer = build_graph_writer(session.graph)\n",
    "    epoch = 0\n",
    "    while epoch < 50:\n",
    "        print(\"Training epoch {:<4d}\".format(epoch), end='\\t')\n",
    "        dataset.shuffle()\n",
    "        # Train the model\n",
    "        for instance, label in zip(dataset.inputs, dataset.labels):\n",
    "            session.run(optimizer, feed_dict={x: instance,\n",
    "                                              y: label})\n",
    "        # Calculate accuracy and loss\n",
    "        for instance, label in zip(dataset.inputs, dataset.labels):\n",
    "            summaries, acc, update_op, loss = session.run([merged, accuracy, accuracy_update, cost],\n",
    "                                                          feed_dict={x: instance,\n",
    "                                                                     y: label})\n",
    "        writer.add_summary(summaries, epoch)\n",
    "        print(\"Accuracy: {:.6f} \\tLoss: {:.6f}\".format(acc, loss))\n",
    "        epoch = epoch + 1\n",
    "    writer.close()\n",
    "\n",
    "    seq = input('Enter a sequence of max 5 consecutive letters:')\n",
    "    seq = seq.upper()\n",
    "    print(\"You entered {}\".format(seq))\n",
    "\n",
    "    seq = e.encode_sequence(seq)\n",
    "    seq = pad_sequences([seq], dataset.sequence_length)\n",
    "    seq = np.reshape(seq, (dataset.sequence_length, 1))\n",
    "    seq = seq / float(dataset.num_classes)\n",
    "\n",
    "    result = session.run(pred, feed_dict={x: seq})\n",
    "    letter = result[0]\n",
    "    print(\"The next letter is: {}\".format(e.decode_letter(letter)))\n",
    "    session.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "name": "hw02-working-with-sequences-in-tensorflow.ipynb"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}