220079842_Office_env.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "220079842_Office_env",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "dDmo5u4Y_JKd"
      },
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "import random"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jdbBUR4fcbmI"
      },
      "source": [
        ""
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YHy8UaAkevEQ"
      },
      "source": [
        "policy = np.zeros([9, 6])\n",
        "policy = [[-1, 1, -1, -1, -3, 10], \n",
        "     [1, 1, -1, 1, 5, -2], \n",
        "     [1, -1, -1, -1, 5, -2], \n",
        "     [-1, 1, -1, -1, 5, -2], \n",
        "     [1, 1, 1, 1, 5, -2], \n",
        "     [1, -1, -1, -1, 5, -2], \n",
        "     [-1, 1, -1, -1, 5, -2], \n",
        "     [1, 1, 1, -1, 5, -2], \n",
        "     [1, -1, -1, -1, 5, -2]]"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HScIjPHN_LXC"
      },
      "source": [
        "class OfficeEnv(gym.Env):\n",
        "\n",
        "    def __init__(self):\n",
        "       \n",
        "        self.action_space = 6\n",
        "        self.state_space = 9\n",
        "        self.observation_space = 72\n",
        "        reward = 0\n",
        "        state = random.randint(0,self.state_space-1)\n",
        "\n",
        "    def step(self, action):\n",
        "\n",
        "        \n",
        "        state = random.randint(0,self.state_space-1)\n",
        "        reward = policy[state][action]\n",
        "            \n",
        "        done = True\n",
        "\n",
        "        info = {}\n",
        "\n",
        "        return state, reward, done, info\n",
        "\n",
        "    def reset(self):\n",
        "        state = random.randint(0,env.state_space-1)\n",
        "        reward = 0\n",
        "        return state"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FC6GL9nGC8SP"
      },
      "source": [
        "env = OfficeEnv()\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTCwCeCGDQi6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "ba6f7422-5cf3-4904-85ae-4cd90b4a1ff0"
      },
      "source": [
        "q_table = np.zeros([9, 6])\n",
        "print(q_table)\n",
        "\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0.]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zkl7_NeHDZ7n"
      },
      "source": [
        "num_of_episodes = 100\n",
        "max_number_of_steps_per_episode = 10 \n",
        "\n",
        "alpha = 0.1\n",
        "gamma = 0.99\n",
        "\n",
        "rate_of_exploration = 1"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nsKf4ZKuESRT",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "470b3a8c-452a-4b48-e66b-a6309929615a"
      },
      "source": [
        "rewards_all_episodes = []\n",
        "import random\n",
        "# CanBot's Q-Learning algorithm\n",
        "for i in range(num_of_episodes):\n",
        "    state = env.reset()\n",
        "    \n",
        "    done = False\n",
        "    rewards = list()\n",
        "    penalties = 0\n",
        "    \n",
        "    for step in range(max_number_of_steps_per_episode):\n",
        "        \n",
        "        \n",
        "        if random.uniform(0,1) > rate_of_exploration: \n",
        "            action = np.argmax(policy[:, state])\n",
        "        else:\n",
        "            action = random.randint(0,env.action_space-1)\n",
        "            #print(\"Action: \", action)\n",
        "            #print(\"State: \", state)\n",
        "        new_state, reward, done, info = env.step(action)\n",
        "        if reward == 10:\n",
        "          done = True\n",
        "        q_table[state, action] = (1 - alpha) * q_table[state, action] + \\\n",
        "            alpha * (reward + gamma * np.max(q_table[new_state,:]))\n",
        "            \n",
        "        state = new_state\n",
        "        rewards.append(reward) \n",
        "        \n",
        "        if done == True: \n",
        "            #print(\"Reward for episode: \", i, \":\", rewards)\n",
        "            break\n",
        "    rewards_all_episodes.append(sum(rewards)/len(rewards))    \n",
        "    \n",
        "print(\"Average rewards foe 100 episodes: \", rewards_all_episodes)    \n",
        "print(\"Q-table\")\n",
        "print(q_table)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Average rewards foe 100 episodes:  [5.0, 5.0, -2.0, -3.0, -2.0, -2.0, -1.0, -1.0, 1.0, 5.0, 1.0, -1.0, -1.0, 1.0, 1.0, 5.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -2.0, 1.0, 1.0, -1.0, -2.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 5.0, 5.0, -2.0, -2.0, -1.0, 1.0, 5.0, -1.0, 5.0, -1.0, -1.0, -1.0, 5.0, -1.0, 1.0, 1.0, 1.0, -1.0, 5.0, -1.0, 1.0, 5.0, 1.0, -1.0, -2.0, 5.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 5.0, 5.0, -2.0, -2.0, 5.0, -2.0, -1.0, 5.0, 1.0, 1.0, -2.0, -1.0, 1.0, 5.0, -1.0, 1.0, 5.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 5.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -2.0]\n",
            "Q-table\n",
            "[[ 0.00000000e+00  4.06956210e-01  1.00000000e-01  3.65373681e-01\n",
            "   1.36656137e+00 -1.49034751e-01]\n",
            " [-1.39561546e-03  3.77348331e-01  1.14800500e-01  3.63656343e-03\n",
            "   1.53052755e+00 -5.32313124e-01]\n",
            " [ 1.94872880e-01  4.33989639e-01  2.00121242e-01  8.55152495e-02\n",
            "   9.74881820e-01 -3.25599500e-01]\n",
            " [ 6.34386154e-02  0.00000000e+00  1.48154259e-01  2.61659581e-01\n",
            "   1.38591739e+00 -1.49034751e-01]\n",
            " [ 2.77737757e-01  2.67693071e-01  0.00000000e+00 -5.61995000e-02\n",
            "   5.59261834e-01 -2.81395615e-01]\n",
            " [-4.69895500e-02  1.16781514e-01  4.36466146e-02 -4.85312948e-02\n",
            "   5.98604385e-01 -1.89420296e-01]\n",
            " [ 0.00000000e+00 -8.50544403e-02  7.06910613e-02 -9.05545000e-02\n",
            "   1.41134906e+00  0.00000000e+00]\n",
            " [ 5.81936154e-02 -4.90347505e-02  0.00000000e+00  2.74335258e-01\n",
            "   1.45709614e+00  0.00000000e+00]\n",
            " [ 1.50965250e-01  0.00000000e+00  1.06865700e-01 -1.49099500e-01\n",
            "   0.00000000e+00 -2.66636416e-01]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g0pyyyCEEVnE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "37956999-3b76-488d-c0fe-d692a65c38dc"
      },
      "source": [
        "!jupyter nbconvert --to PDF \"220079842_Office_env.ipynb\""
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[NbConvertApp] Converting notebook 220079842_Office_env.ipynb to PDF\n",
            "[NbConvertApp] Writing 61758 bytes to ./notebook.tex\n",
            "[NbConvertApp] Building PDF\n",
            "[NbConvertApp] Running xelatex 3 times: [u'xelatex', u'./notebook.tex', '-quiet']\n",
            "[NbConvertApp] Running bibtex 1 time: [u'bibtex', u'./notebook']\n",
            "[NbConvertApp] WARNING | bibtex had problems, most likely because there were no citations\n",
            "[NbConvertApp] PDF successfully created\n",
            "[NbConvertApp] Writing 41077 bytes to 220079842_Office_env.pdf\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HcEC8hNdnXL5"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-P2VAEavnd5Z"
      },
      "source": [
        ""
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zliO-vdfnlaI"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}