couplets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic calling is: Smart\n"
     ]
    }
   ],
   "source": [
    "%autocall 1\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import sys\n",
    "sys.path.append(\"..\")\n",
    "from scienv import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import utils as ut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data import Field, Example, TabularDataset\n",
    "from torchtext.data import BucketIterator, Iterator\n",
    "from torchtext.datasets import TranslationDataset\n",
    "import torch as tc\n",
    "from torchtext import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAD_TOKEN = \"#\"\n",
    "MASK_TOKEN = \"[]\"\n",
    "SRC = Field(sequential=True, tokenize=lambda s:s.strip().split(' '),\n",
    "                     use_vocab=True, batch_first=True,\n",
    "                     eos_token=None, init_token=None,\n",
    "                     include_lengths=True, pad_token=PAD_TOKEN)\n",
    "TRG = Field(sequential=True, tokenize=lambda s:s.strip().split(' '),\n",
    "                     use_vocab=True, batch_first=True,\n",
    "                     eos_token=None, init_token=None,\n",
    "                     include_lengths=True, pad_token=PAD_TOKEN, is_target=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = [(\"src\", SRC), (\"trg\", TRG)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "teDS = TranslationDataset(path='dataset/test/', exts=('in.txt','out.txt'), fields=fields)\n",
    "trDS = TranslationDataset(path='dataset/train/', exts=('in.txt','out.txt'), fields=fields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "teDS_r = TranslationDataset(path='dataset/test/', exts=('out.txt','in.txt'), fields=fields)\n",
    "trDS_r = TranslationDataset(path='dataset/train/', exts=('out.txt','in.txt'), fields=fields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(770491, 4000)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trDS),len(teDS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6745"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SRC.build_vocab(trDS, specials=[MASK_TOKEN], min_freq=2)\n",
    "TRG.vocab = SRC.vocab\n",
    "VOCAB = SRC.vocab\n",
    "n_vocab = len(VOCAB)\n",
    "n_vocab"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import json\n",
    "with open('vocab.json', 'w') as f:\n",
    "    json.dump(VOCAB.itos, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Batch:\n",
    "    def __init__(self, src=None, trg=None, \n",
    "                 src_pad_mask=None, trg_pad_mask=None,\n",
    "                 trg_mask=None, trg_y=None):\n",
    "        self.src=src\n",
    "        self.trg=trg\n",
    "        self.src_pad_mask=src_pad_mask\n",
    "        self.trg_pad_mask=trg_pad_mask\n",
    "        self.trg_mask=trg_mask\n",
    "        self.trg_y=trg_y\n",
    "    def __repr__(self):\n",
    "        return 'Batch{\\n  src: %s\\n  trg: %s}'%(itos(self.src.cpu().numpy()),\n",
    "                                  itos(self.trg.cpu().numpy()))\n",
    "    \n",
    "def format_couplets_str(first, second='', stoi={}, mask_token=\"[]\"):\n",
    "    first = list(first.replace(' ', '').replace(',', '，').replace('.', '。'))\n",
    "    second = [s.replace(' ', mask_token) for s in second]\n",
    "    l = (len(first)-len(second))\n",
    "    assert l >= 0\n",
    "    second = second + [mask_token] * l\n",
    "    return [stoi.get(s, s) for s in first], [stoi.get(s, s) for s in second]\n",
    "\n",
    "def str_to_batch(first, second, stoi, mask_token):\n",
    "    src, trg = format_couplets_str(first, second, stoi, mask_token)\n",
    "    src = tc.LongTensor(src).unsqueeze(0)\n",
    "    trg = tc.LongTensor(trg).unsqueeze(0)\n",
    "    src_pad_mask = tc.ones_like(src, dtype=bool)\n",
    "    trg_pad_mask = tc.ones_like(trg, dtype=bool)\n",
    "    trg_mask = trg != stoi[mask_token]\n",
    "    return Batch(src, trg, src_pad_mask,trg_pad_mask,trg_mask)\n",
    "\n",
    "def itos(i, itos=VOCAB.itos):\n",
    "    return ut.apply(''.join, ut.apply(itos.__getitem__, i), \n",
    "                    at=lambda c:not ut.iscollection(c[0][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyIterator(data.Iterator):\n",
    "    def create_batches(self):\n",
    "        if self.train:\n",
    "            def pool(d, random_shuffler):\n",
    "                for p in data.batch(d, self.batch_size * 100):\n",
    "                    p_batch = data.batch(\n",
    "                        sorted(p, key=self.sort_key),\n",
    "                        self.batch_size, self.batch_size_fn)\n",
    "                    for b in random_shuffler(list(p_batch)):\n",
    "                        yield b\n",
    "            self.batches = pool(self.data(), self.random_shuffler)\n",
    "            \n",
    "        else:\n",
    "            self.batches = []\n",
    "            for b in data.batch(self.data(), self.batch_size,\n",
    "                                          self.batch_size_fn):\n",
    "                self.batches.append(sorted(b, key=self.sort_key))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_trg_mask(trg, lens, r, pad):\n",
    "    size = trg.size()\n",
    "    mask = tc.rand(size) > r\n",
    "    pad_mask = trg != pad\n",
    "    ri = [np.random.randint(i) for i in lens] #至少mask1个\n",
    "    mask[list(range(len(ri))),ri] = False    \n",
    "    return mask, pad_mask\n",
    "\n",
    "def gen_src_mask(src, pad):\n",
    "    pad_mask = src != pad\n",
    "    return pad_mask\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_pybatch(b, mask_ratio='rand'):\n",
    "    mask_ratio = np.random.rand() if mask_ratio=='rand' else mask_ratio\n",
    "    src = b.src[0]\n",
    "    trg = b.trg[0]\n",
    "    trg_mask, trg_pad_mask = gen_trg_mask(trg, b.trg[1], mask_ratio, VOCAB.stoi[PAD_TOKEN])\n",
    "    src_pad_mask = gen_src_mask(src, VOCAB.stoi[PAD_TOKEN])\n",
    "    trg_mask = trg_mask.to(DEVICE)\n",
    "    trg_pad_mask = trg_pad_mask.to(DEVICE)\n",
    "    src_pad_mask = src_pad_mask.to(DEVICE)\n",
    "    trg_m = trg.masked_fill(~trg_mask, VOCAB.stoi[MASK_TOKEN])\n",
    "    trg_m.masked_fill_(~trg_pad_mask, VOCAB.stoi[PAD_TOKEN])\n",
    "    return Batch(src, trg_m, src_pad_mask,trg_pad_mask,trg_mask,trg)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Batch{\n",
       "  src: ['白日依山尽']\n",
       "  trg: ['[]河[][]流']}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str_to_batch('白日依山尽',' 河  流', mask_token=MASK_TOKEN, stoi=VOCAB.stoi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch{\n",
      "  src: ['无尽枫红舒望眼', '痴我风姿应是月', '题到秋深诗句冷', '发情牛马难成对', '好烟孬烟都不好']\n",
      "  trg: ['[]时柳[]展[]眉', '[]伊倩影胜如花', '[]回花落客魂寒', '得志[]狼[]简[]', '非语诽[][][][]']}\n",
      "['几时柳绿展愁眉', '思伊倩影胜如花', '梦回花落客魂寒', '得志虎狼不简单', '非语诽语尽言非']\n"
     ]
    }
   ],
   "source": [
    "IT = MyIterator(teDS, 5, sort_key=lambda x: len(x.src), shuffle=True, device=tc.device('cuda'))\n",
    "\n",
    "for b in IT:\n",
    "    r = parse_pybatch(b)\n",
    "    print(r)\n",
    "    print(itos(r.trg_y.cpu().numpy()))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([5, 5]), torch.Size([5, 5]), torch.Size([5, 5]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.trg_mask.shape,r.trg_pad_mask.shape,r.src_pad_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([5, 5]), torch.Size([5, 5]))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.src.shape,r.trg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.int64"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.src.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[True, True, True, True, True],\n",
       "        [True, True, True, True, True],\n",
       "        [True, True, True, True, True],\n",
       "        [True, True, True, True, True],\n",
       "        [True, True, True, True, True]], device='cuda:0')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.src_pad_mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch as tc\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import math, copy, time\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.tensorboard import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_model = 512\n",
    "em = Embeddings(d_model, n_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = make_model(n_vocab, n_vocab)\n",
    "model.to(DEVICE);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "o = model(r.src, r.trg, r.src_pad_mask.unsqueeze(1), r.trg_pad_mask.unsqueeze(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 7, 512])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/cache/guoyongzhi/.conda/envs/gyz37/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    }
   ],
   "source": [
    "criterion = LabelSmoothing(size=n_vocab, padding_idx=VOCAB.stoi[PAD_TOKEN], smoothing=0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# opt = torch.optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.98), eps=1e-9)\n",
    "opt=None\n",
    "loss_compute = SimpleLossCompute(model.generator, criterion, opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51.37674331665039"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o = model(r.src, r.trg, r.src_pad_mask.unsqueeze(1), r.trg_pad_mask.unsqueeze(1))\n",
    "loss_compute(o, r.trg_y, ignore_mask=r.trg_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainwriter = SummaryWriter('nomem/train')\n",
    "testwriter = SummaryWriter('nomem/test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/cache/guoyongzhi/couplets/model.py:218: TracerWarning: Converting a tensor to a Python index might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  pos = self.pe[:, :x.size(1)]\n",
      "/mnt/cache/guoyongzhi/couplets/model.py:148: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  / math.sqrt(d_k)\n"
     ]
    }
   ],
   "source": [
    "trainwriter.add_graph(model, (r.src, r.trg, r.src_pad_mask.unsqueeze(1), r.trg_pad_mask.unsqueeze(1)))\n",
    "trainwriter.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "firsts = ['天与云与山与水，上下一白',\n",
    "          '两个黄鹂鸣翠柳',\n",
    "          '无边落木萧萧下', \n",
    "          '月落乌啼霜满天', \n",
    "          '提刀上马，江山如画', \n",
    "          '兰亭临帖，行书如行云流水',\n",
    "          '荒烟漫草的年头，就连分手都很沉默',\n",
    "          '一条大河波浪宽，风吹稻花香两岸',\n",
    "          '我欲成仙，快乐齐天',\n",
    "          '只有刚强的人，才有神圣的意志，凡是战斗的人，才能取得胜利',\n",
    "          '？',\n",
    "         ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_epoch(data_iter, model, loss_compute, mask_ratio='rand', step_call_back=lambda l:None):\n",
    "    \"Standard Training and Logging Function\"\n",
    "    start = time.time()\n",
    "    total_tokens = 0\n",
    "    total_loss = 0\n",
    "    tokens = 0\n",
    "    pred_tokens = 0\n",
    "    total_pred_tokens = 0\n",
    "    for i, batch in enumerate(data_iter):\n",
    "        batch = parse_pybatch(batch,mask_ratio=mask_ratio)\n",
    "        out = model.forward(batch.src,batch.trg, batch.src_pad_mask.unsqueeze(1), batch.trg_pad_mask.unsqueeze(1))\n",
    "        ntokens = batch.src_pad_mask.sum().float()\n",
    "        ntokens_pred = (batch.trg_pad_mask&(~batch.trg_mask)).sum().float()\n",
    "        loss = loss_compute(out, batch.trg_y, ntokens, ignore_mask=batch.trg_mask)\n",
    "        total_loss += loss\n",
    "        total_tokens += ntokens.item()\n",
    "        tokens += ntokens.item()\n",
    "        pred_tokens += ntokens_pred.item()\n",
    "        total_pred_tokens += ntokens_pred.item()\n",
    "        step_call_back(loss/ntokens_pred)\n",
    "        if i % 50 == 1:\n",
    "            elapsed = time.time() - start\n",
    "            print(\"Step: %d Loss: %f Tokens per Sec: %f. mask ratio: %f\" %\n",
    "                    (i, loss / ntokens_pred, tokens / elapsed, pred_tokens/tokens))\n",
    "            start = time.time()\n",
    "            tokens = 0\n",
    "            pred_tokens = 0\n",
    "    return total_loss/total_pred_tokens\n",
    "\n",
    "def train_log_cbk(loss):\n",
    "    global step_counter\n",
    "    step_counter += 1\n",
    "    trainwriter.add_scalar('loss', loss, step_counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 GPUs\n"
     ]
    }
   ],
   "source": [
    "print(torch.cuda.device_count(), 'GPUs')\n",
    "devices = list(range(torch.cuda.device_count()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/cache/guoyongzhi/.conda/envs/gyz37/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(52825311, 52825311)"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IT_r = MyIterator(trDS_r, 4000, sort_key=lambda x: len(x.src), shuffle=True, device=torch.device('cuda'))\n",
    "teIT_r = MyIterator(teDS_r, 240, sort_key=lambda x: len(x.src), shuffle=True, device=torch.device('cuda'))\n",
    "IT = MyIterator(trDS, 4000, sort_key=lambda x: len(x.src), shuffle=True, device=torch.device('cuda'))\n",
    "teIT = MyIterator(teDS, 240, sort_key=lambda x: len(x.src), shuffle=True, device=torch.device('cuda'))\n",
    "\n",
    "model = make_model(n_vocab, n_vocab)\n",
    "model.to(DEVICE)\n",
    "criterion = LabelSmoothing(size=n_vocab, padding_idx=VOCAB.stoi[PAD_TOKEN], smoothing=0.7)\n",
    "criterion = criterion.to(DEVICE)\n",
    "criterion = nn.DataParallel(criterion, device_ids=devices)\n",
    "opt = torch.optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.98), eps=1e-9)\n",
    "generator = nn.DataParallel(model.generator, device_ids=devices)\n",
    "loss_compute = SimpleLossCompute(generator, criterion, opt)\n",
    "model_par = nn.DataParallel(model, device_ids=devices)\n",
    "step_counter = 0\n",
    "para_num = sum(p.numel() for p in model_par.parameters())\n",
    "trainable_para_num = sum(p.numel() for p in model_par.parameters() if p.requires_grad)\n",
    "para_num, trainable_para_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = torch.optim.Adam(model.parameters(), lr=0.0005, betas=(0.9, 0.98), eps=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/cache/guoyongzhi/.conda/envs/gyz37/lib/python3.7/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1 Loss: 1.917112 Tokens per Sec: 28459.874830. mask ratio: 0.370994\n",
      "Step: 51 Loss: 1.748450 Tokens per Sec: 32371.496137. mask ratio: 0.576258\n",
      "Step: 101 Loss: 1.799467 Tokens per Sec: 30848.980159. mask ratio: 0.609501\n",
      "Step: 151 Loss: 1.754695 Tokens per Sec: 33107.219298. mask ratio: 0.507312\n",
      "Step: 1 Loss: 1.872898 Tokens per Sec: 10353.018917. mask ratio: 0.257455\n",
      "@ 0 1.7741159352484595\n",
      "Step: 1 Loss: 1.743553 Tokens per Sec: 17459.377144. mask ratio: 0.334083\n",
      "Step: 51 Loss: 1.797242 Tokens per Sec: 33893.259417. mask ratio: 0.539785\n",
      "Step: 101 Loss: 1.771760 Tokens per Sec: 30394.028804. mask ratio: 0.528859\n",
      "Step: 151 Loss: 1.727259 Tokens per Sec: 37029.062249. mask ratio: 0.595890\n",
      "Step: 1 Loss: 1.738872 Tokens per Sec: 13104.386689. mask ratio: 0.749217\n",
      "@ 1 1.746022359752104\n",
      "Step: 1 Loss: 1.702263 Tokens per Sec: 19840.216587. mask ratio: 0.338175\n",
      "Step: 51 Loss: 1.741686 Tokens per Sec: 30984.419383. mask ratio: 0.542386\n",
      "Step: 101 Loss: 1.748284 Tokens per Sec: 34331.351806. mask ratio: 0.562993\n",
      "Step: 151 Loss: 1.697698 Tokens per Sec: 33144.839807. mask ratio: 0.537981\n",
      "Step: 1 Loss: 1.546581 Tokens per Sec: 15279.821597. mask ratio: 0.377921\n",
      "@ 2 1.6832763717789196\n",
      "Step: 1 Loss: 1.692389 Tokens per Sec: 25356.159395. mask ratio: 0.783591\n",
      "Step: 51 Loss: 1.656390 Tokens per Sec: 34112.130307. mask ratio: 0.521598\n",
      "Step: 101 Loss: 1.557410 Tokens per Sec: 30009.249759. mask ratio: 0.565293\n",
      "Step: 151 Loss: 1.643466 Tokens per Sec: 33199.522637. mask ratio: 0.559994\n",
      "Step: 1 Loss: 1.518770 Tokens per Sec: 14371.901412. mask ratio: 0.447523\n",
      "@ 3 1.61624127783956\n",
      "Step: 1 Loss: 1.531229 Tokens per Sec: 15428.786062. mask ratio: 0.616399\n",
      "Step: 51 Loss: 1.503058 Tokens per Sec: 36590.689251. mask ratio: 0.570644\n",
      "Step: 101 Loss: 1.616832 Tokens per Sec: 31498.445849. mask ratio: 0.492468\n",
      "Step: 151 Loss: 1.488768 Tokens per Sec: 30939.388970. mask ratio: 0.576156\n",
      "Step: 1 Loss: 1.593465 Tokens per Sec: 8818.624704. mask ratio: 0.659513\n",
      "@ 4 1.5896758847041785\n",
      "Step: 1 Loss: 1.499582 Tokens per Sec: 29734.260717. mask ratio: 0.426035\n",
      "Step: 51 Loss: 1.460816 Tokens per Sec: 31817.952595. mask ratio: 0.503139\n",
      "Step: 101 Loss: 1.468629 Tokens per Sec: 31908.196548. mask ratio: 0.553207\n",
      "Step: 151 Loss: 1.493416 Tokens per Sec: 30875.638625. mask ratio: 0.521164\n",
      "Step: 1 Loss: 1.446619 Tokens per Sec: 16089.144802. mask ratio: 0.266861\n",
      "@ 5 1.5025650409042193\n",
      "Step: 1 Loss: 1.548306 Tokens per Sec: 18504.621378. mask ratio: 0.809375\n",
      "Step: 51 Loss: 1.461346 Tokens per Sec: 31513.692763. mask ratio: 0.500804\n",
      "Step: 101 Loss: 1.395383 Tokens per Sec: 35574.107804. mask ratio: 0.573590\n",
      "Step: 151 Loss: 1.485394 Tokens per Sec: 33888.304135. mask ratio: 0.529139\n",
      "Step: 1 Loss: 1.425709 Tokens per Sec: 14285.731627. mask ratio: 0.655571\n",
      "@ 6 1.5446833407873466\n",
      "Step: 1 Loss: 1.548928 Tokens per Sec: 13643.349370. mask ratio: 0.683607\n",
      "Step: 51 Loss: 1.405594 Tokens per Sec: 33485.889404. mask ratio: 0.583109\n",
      "Step: 101 Loss: 1.460791 Tokens per Sec: 32920.657632. mask ratio: 0.523381\n",
      "Step: 151 Loss: 1.541305 Tokens per Sec: 31813.328603. mask ratio: 0.606189\n",
      "Step: 1 Loss: 1.583166 Tokens per Sec: 8133.104270. mask ratio: 0.896875\n",
      "@ 7 1.5537942049821973\n",
      "Step: 1 Loss: 1.398223 Tokens per Sec: 18937.241636. mask ratio: 0.562882\n",
      "Step: 51 Loss: 1.456969 Tokens per Sec: 31686.370725. mask ratio: 0.602069\n",
      "Step: 101 Loss: 1.485199 Tokens per Sec: 33151.037377. mask ratio: 0.583680\n",
      "Step: 151 Loss: 1.528810 Tokens per Sec: 31098.579754. mask ratio: 0.583319\n",
      "Step: 1 Loss: 1.544823 Tokens per Sec: 9415.289249. mask ratio: 0.537202\n",
      "@ 8 1.5646780954127966\n",
      "Step: 1 Loss: 1.505820 Tokens per Sec: 13998.861048. mask ratio: 0.599946\n",
      "Step: 51 Loss: 1.505070 Tokens per Sec: 31460.566137. mask ratio: 0.595228\n",
      "Step: 101 Loss: 1.471651 Tokens per Sec: 32770.666189. mask ratio: 0.561790\n",
      "Step: 151 Loss: 1.500027 Tokens per Sec: 30877.662865. mask ratio: 0.589971\n",
      "Step: 1 Loss: 1.371731 Tokens per Sec: 12650.264713. mask ratio: 0.247953\n",
      "@ 9 1.4678185452627428\n",
      "Step: 1 Loss: 1.450709 Tokens per Sec: 12959.678790. mask ratio: 0.404536\n",
      "Step: 51 Loss: 1.341054 Tokens per Sec: 32835.405768. mask ratio: 0.561730\n",
      "Step: 101 Loss: 1.379214 Tokens per Sec: 31436.033095. mask ratio: 0.587565\n",
      "Step: 151 Loss: 1.411659 Tokens per Sec: 32497.159645. mask ratio: 0.653139\n",
      "Step: 1 Loss: 1.380334 Tokens per Sec: 17027.546465. mask ratio: 0.486474\n",
      "@ 10 1.5204921333903694\n",
      "Step: 1 Loss: 1.525119 Tokens per Sec: 12883.807789. mask ratio: 0.618806\n",
      "Step: 51 Loss: 1.400007 Tokens per Sec: 30522.375098. mask ratio: 0.500221\n",
      "Step: 101 Loss: 1.402944 Tokens per Sec: 36114.479546. mask ratio: 0.530676\n",
      "Step: 151 Loss: 1.381160 Tokens per Sec: 32953.343937. mask ratio: 0.556814\n",
      "Step: 1 Loss: 1.609668 Tokens per Sec: 10182.864096. mask ratio: 0.801874\n",
      "@ 11 1.531457343186776\n",
      "Step: 1 Loss: 1.386871 Tokens per Sec: 18932.772989. mask ratio: 0.564132\n",
      "Step: 51 Loss: 1.515564 Tokens per Sec: 31061.862033. mask ratio: 0.562889\n",
      "Step: 101 Loss: 1.397047 Tokens per Sec: 32372.650081. mask ratio: 0.620566\n",
      "Step: 151 Loss: 1.391943 Tokens per Sec: 32758.554774. mask ratio: 0.527094\n",
      "Step: 1 Loss: 1.314866 Tokens per Sec: 13419.260667. mask ratio: 0.360773\n",
      "@ 12 1.5030588107475873\n",
      "Step: 1 Loss: 1.378557 Tokens per Sec: 28097.437840. mask ratio: 0.768710\n",
      "Step: 51 Loss: 1.389511 Tokens per Sec: 33119.239738. mask ratio: 0.573267\n",
      "Step: 101 Loss: 1.421009 Tokens per Sec: 30716.081223. mask ratio: 0.512307\n",
      "Step: 151 Loss: 1.405158 Tokens per Sec: 36757.531578. mask ratio: 0.527794\n",
      "Step: 1 Loss: 1.383532 Tokens per Sec: 9246.313668. mask ratio: 0.440476\n",
      "@ 13 1.451043182866777\n",
      "Step: 1 Loss: 1.406346 Tokens per Sec: 13592.345080. mask ratio: 0.723125\n",
      "Step: 51 Loss: 1.369693 Tokens per Sec: 33182.767639. mask ratio: 0.565147\n",
      "Step: 101 Loss: 1.415591 Tokens per Sec: 30838.175122. mask ratio: 0.603039\n",
      "Step: 151 Loss: 1.407026 Tokens per Sec: 32262.792020. mask ratio: 0.511115\n",
      "Step: 1 Loss: 1.387329 Tokens per Sec: 9088.896668. mask ratio: 0.647024\n",
      "@ 14 1.5030331529467411\n",
      "Step: 1 Loss: 1.350199 Tokens per Sec: 29744.053215. mask ratio: 0.462572\n",
      "Step: 51 Loss: 1.318127 Tokens per Sec: 32675.577696. mask ratio: 0.540234\n",
      "Step: 101 Loss: 1.332443 Tokens per Sec: 31326.872270. mask ratio: 0.530243\n",
      "Step: 151 Loss: 1.271398 Tokens per Sec: 33204.077542. mask ratio: 0.534193\n",
      "Step: 1 Loss: 1.286945 Tokens per Sec: 9190.183460. mask ratio: 0.327976\n",
      "@ 15 1.4712358935011283\n",
      "Step: 1 Loss: 1.263941 Tokens per Sec: 15576.412555. mask ratio: 0.319571\n",
      "Step: 51 Loss: 1.316923 Tokens per Sec: 36130.045811. mask ratio: 0.517541\n",
      "Step: 101 Loss: 1.328857 Tokens per Sec: 32407.844120. mask ratio: 0.581363\n",
      "Step: 151 Loss: 1.355429 Tokens per Sec: 32843.970901. mask ratio: 0.546365\n",
      "Step: 1 Loss: 1.234748 Tokens per Sec: 12493.011835. mask ratio: 0.760664\n",
      "@ 16 1.54075864482986\n",
      "Step: 1 Loss: 1.310527 Tokens per Sec: 21120.522593. mask ratio: 0.317827\n",
      "Step: 51 Loss: 1.396553 Tokens per Sec: 33581.719223. mask ratio: 0.549706\n",
      "Step: 101 Loss: 1.302210 Tokens per Sec: 31509.762076. mask ratio: 0.577324\n",
      "Step: 151 Loss: 1.432140 Tokens per Sec: 34919.042057. mask ratio: 0.576097\n",
      "Step: 1 Loss: 1.537149 Tokens per Sec: 9644.262241. mask ratio: 0.854464\n",
      "@ 17 1.4034556109840992\n",
      "Step: 1 Loss: 1.200500 Tokens per Sec: 16855.058277. mask ratio: 0.239306\n",
      "Step: 51 Loss: 1.229463 Tokens per Sec: 31435.919815. mask ratio: 0.522132\n",
      "Step: 101 Loss: 1.255946 Tokens per Sec: 35661.450851. mask ratio: 0.518976\n",
      "Step: 151 Loss: 1.189237 Tokens per Sec: 32975.798765. mask ratio: 0.571907\n",
      "Step: 1 Loss: 1.240790 Tokens per Sec: 14123.682881. mask ratio: 0.280481\n",
      "@ 18 1.4081937164136438\n",
      "Step: 1 Loss: 1.190199 Tokens per Sec: 15044.845075. mask ratio: 0.469214\n",
      "Step: 51 Loss: 1.292579 Tokens per Sec: 31678.976167. mask ratio: 0.562780\n",
      "Step: 101 Loss: 1.413250 Tokens per Sec: 32489.806143. mask ratio: 0.574304\n",
      "Step: 151 Loss: 1.174645 Tokens per Sec: 34498.876663. mask ratio: 0.566309\n",
      "Step: 1 Loss: 1.286853 Tokens per Sec: 12706.694143. mask ratio: 0.391426\n",
      "@ 19 1.3897098888798118\n"
     ]
    }
   ],
   "source": [
    "mr = 'rand'\n",
    "# mr = 0.2\n",
    "for epoch in range(20):\n",
    "    model_par.train()\n",
    "    run_epoch(IT, model_par, loss_compute, mask_ratio=mr, step_call_back=train_log_cbk)\n",
    "    model_par.eval()\n",
    "    loss = run_epoch(teIT, model_par, SimpleLossCompute(generator, criterion, None), mask_ratio=mr)\n",
    "    testwriter.add_scalar('loss', loss, step_counter)\n",
    "    testwriter.flush()\n",
    "    trainwriter.flush()\n",
    "    print('@', epoch, loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import StringIO\n",
    "sio = StringIO()\n",
    "for first in firsts:\n",
    "    print_match_all(first, beamsize=5, file=sio)\n",
    "    print('\\n', file=sio)\n",
    "trainwriter.add_text(str(step_counter), sio.getvalue().replace('\\n','\\n\\n').replace('\\n\\n\\n\\n','\\n\\n'), step_counter)\n",
    "trainwriter.flush()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "mr = 'rand'\n",
    "# mr = 0.2\n",
    "for epoch in range(20):\n",
    "    model_par.train()\n",
    "    run_epoch(IT_r, model_par, loss_compute, mask_ratio=mr, step_call_back=train_log_cbk)\n",
    "    model_par.eval()\n",
    "    loss = run_epoch(teIT, model_par, SimpleLossCompute(generator, criterion, None), mask_ratio=mr)\n",
    "    testwriter.add_scalar('loss', loss, step_counter)\n",
    "    trainwriter.flush()\n",
    "    testwriter.flush()\n",
    "    print('@', epoch, loss)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "from io import StringIO\n",
    "sio = StringIO()\n",
    "for first in firsts:\n",
    "    print_match_all(first, beamsize=5, file=sio)\n",
    "    print('\\n', file=sio)\n",
    "trainwriter.add_text(str(step_counter), sio.getvalue().replace('\\n','\\n\\n').replace('\\n\\n\\n\\n','\\n\\n'), step_counter)\n",
    "trainwriter.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = torch.optim.Adam(model.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1 Loss: 1.380375 Tokens per Sec: 19632.087375. mask ratio: 0.587318\n",
      "Step: 51 Loss: 1.337274 Tokens per Sec: 31871.944031. mask ratio: 0.562394\n",
      "Step: 101 Loss: 1.264298 Tokens per Sec: 32138.316232. mask ratio: 0.542487\n",
      "Step: 151 Loss: 1.334353 Tokens per Sec: 33221.042192. mask ratio: 0.526440\n",
      "Step: 1 Loss: 1.809319 Tokens per Sec: 9246.010353. mask ratio: 0.733333\n",
      "@ 0 1.3579267184841002\n",
      "Step: 1 Loss: 1.204264 Tokens per Sec: 6528.335306. mask ratio: 0.545571\n",
      "Step: 51 Loss: 1.268560 Tokens per Sec: 35963.018655. mask ratio: 0.535755\n",
      "Step: 101 Loss: 1.149702 Tokens per Sec: 32367.308957. mask ratio: 0.559831\n",
      "Step: 151 Loss: 1.140520 Tokens per Sec: 33690.493131. mask ratio: 0.492375\n",
      "Step: 1 Loss: 1.453105 Tokens per Sec: 8191.655570. mask ratio: 0.773264\n",
      "@ 1 1.378246960909849\n",
      "Step: 1 Loss: 1.304817 Tokens per Sec: 15618.425469. mask ratio: 0.598109\n",
      "Step: 51 Loss: 1.257810 Tokens per Sec: 33323.477318. mask ratio: 0.537961\n",
      "Step: 101 Loss: 1.498854 Tokens per Sec: 31902.294178. mask ratio: 0.527406\n",
      "Step: 151 Loss: 1.392484 Tokens per Sec: 33788.814037. mask ratio: 0.610402\n",
      "Step: 1 Loss: 1.190204 Tokens per Sec: 10620.621995. mask ratio: 0.373226\n",
      "@ 2 1.2884878780701907\n",
      "Step: 1 Loss: 1.098435 Tokens per Sec: 15875.151236. mask ratio: 0.241334\n",
      "Step: 51 Loss: 1.109728 Tokens per Sec: 31480.513232. mask ratio: 0.562254\n",
      "Step: 101 Loss: 1.408647 Tokens per Sec: 35536.718637. mask ratio: 0.635071\n",
      "Step: 151 Loss: 1.382061 Tokens per Sec: 34095.266143. mask ratio: 0.571107\n",
      "Step: 1 Loss: 1.346711 Tokens per Sec: 11743.863171. mask ratio: 0.599955\n",
      "@ 3 1.2956634758319485\n",
      "Step: 1 Loss: 1.245244 Tokens per Sec: 12824.737531. mask ratio: 0.388301\n",
      "Step: 51 Loss: 1.221043 Tokens per Sec: 33237.785389. mask ratio: 0.588383\n",
      "Step: 101 Loss: 1.263912 Tokens per Sec: 31651.978275. mask ratio: 0.566494\n",
      "Step: 151 Loss: 1.139950 Tokens per Sec: 33479.768797. mask ratio: 0.510679\n",
      "Step: 1 Loss: 1.032876 Tokens per Sec: 14222.746945. mask ratio: 0.685329\n",
      "@ 4 1.248757085740618\n",
      "Step: 1 Loss: 1.250250 Tokens per Sec: 25686.643769. mask ratio: 0.587122\n",
      "Step: 51 Loss: 1.320765 Tokens per Sec: 31437.124477. mask ratio: 0.475374\n",
      "Step: 101 Loss: 1.243474 Tokens per Sec: 32465.559374. mask ratio: 0.487250\n",
      "Step: 151 Loss: 1.023130 Tokens per Sec: 32476.741018. mask ratio: 0.519215\n",
      "Step: 1 Loss: 1.138930 Tokens per Sec: 12881.992345. mask ratio: 0.238906\n",
      "@ 5 1.2826945309536468\n",
      "Step: 1 Loss: 1.088011 Tokens per Sec: 14544.692907. mask ratio: 0.301429\n",
      "Step: 51 Loss: 1.287169 Tokens per Sec: 33151.026532. mask ratio: 0.532667\n",
      "Step: 101 Loss: 1.325251 Tokens per Sec: 33067.189172. mask ratio: 0.552806\n",
      "Step: 151 Loss: 1.209534 Tokens per Sec: 33483.309920. mask ratio: 0.521687\n",
      "Step: 1 Loss: 1.026106 Tokens per Sec: 13297.499333. mask ratio: 0.339293\n",
      "@ 6 1.1897494187267168\n",
      "Step: 1 Loss: 1.293960 Tokens per Sec: 19368.958967. mask ratio: 0.665525\n",
      "Step: 51 Loss: 1.161216 Tokens per Sec: 32303.520248. mask ratio: 0.589907\n",
      "Step: 101 Loss: 1.252213 Tokens per Sec: 33155.320156. mask ratio: 0.511344\n",
      "Step: 151 Loss: 1.126097 Tokens per Sec: 33831.361902. mask ratio: 0.450930\n",
      "Step: 1 Loss: 1.035472 Tokens per Sec: 16385.545886. mask ratio: 0.228784\n",
      "@ 7 1.2172161462981788\n",
      "Step: 1 Loss: 1.168285 Tokens per Sec: 19301.283438. mask ratio: 0.756066\n",
      "Step: 51 Loss: 1.414865 Tokens per Sec: 32618.483522. mask ratio: 0.508714\n",
      "Step: 101 Loss: 1.236943 Tokens per Sec: 35114.528622. mask ratio: 0.551545\n",
      "Step: 151 Loss: 1.332831 Tokens per Sec: 31684.499950. mask ratio: 0.631698\n",
      "Step: 1 Loss: 1.166491 Tokens per Sec: 9316.220775. mask ratio: 0.466369\n",
      "@ 8 1.2200555520175778\n",
      "Step: 1 Loss: 1.078236 Tokens per Sec: 17719.012278. mask ratio: 0.705750\n",
      "Step: 51 Loss: 1.166857 Tokens per Sec: 34193.282025. mask ratio: 0.545561\n",
      "Step: 101 Loss: 1.159243 Tokens per Sec: 30825.265824. mask ratio: 0.584175\n",
      "Step: 151 Loss: 1.098248 Tokens per Sec: 33955.723893. mask ratio: 0.549083\n",
      "Step: 1 Loss: 1.007427 Tokens per Sec: 13006.692811. mask ratio: 0.179161\n",
      "@ 9 1.2255470293366344\n",
      "Step: 1 Loss: 0.978947 Tokens per Sec: 22775.797311. mask ratio: 0.299917\n",
      "Step: 51 Loss: 1.343933 Tokens per Sec: 34315.849910. mask ratio: 0.570519\n",
      "Step: 101 Loss: 1.225229 Tokens per Sec: 29363.559189. mask ratio: 0.594176\n",
      "Step: 151 Loss: 1.234325 Tokens per Sec: 34033.735625. mask ratio: 0.531650\n",
      "Step: 1 Loss: 1.283263 Tokens per Sec: 11171.690239. mask ratio: 0.864248\n",
      "@ 10 1.226218406238187\n",
      "Step: 1 Loss: 1.125717 Tokens per Sec: 14165.784063. mask ratio: 0.357286\n",
      "Step: 51 Loss: 1.108942 Tokens per Sec: 32700.275488. mask ratio: 0.526361\n",
      "Step: 101 Loss: 1.365852 Tokens per Sec: 31949.485351. mask ratio: 0.608290\n",
      "Step: 151 Loss: 1.141237 Tokens per Sec: 32698.806485. mask ratio: 0.578498\n",
      "Step: 1 Loss: 1.245988 Tokens per Sec: 16086.586508. mask ratio: 0.806948\n",
      "@ 11 1.1604184462849227\n",
      "Step: 1 Loss: 1.255980 Tokens per Sec: 31132.202787. mask ratio: 0.591823\n",
      "Step: 51 Loss: 1.002656 Tokens per Sec: 31679.997519. mask ratio: 0.581265\n",
      "Step: 101 Loss: 1.232811 Tokens per Sec: 31743.282841. mask ratio: 0.593948\n",
      "Step: 151 Loss: 1.291542 Tokens per Sec: 32634.428383. mask ratio: 0.526153\n",
      "Step: 1 Loss: 1.204482 Tokens per Sec: 14506.753010. mask ratio: 0.551242\n",
      "@ 12 1.1882082002291643\n",
      "Step: 1 Loss: 1.177301 Tokens per Sec: 22019.539012. mask ratio: 0.776070\n",
      "Step: 51 Loss: 1.125330 Tokens per Sec: 37907.115763. mask ratio: 0.488821\n",
      "Step: 101 Loss: 1.214021 Tokens per Sec: 30633.466937. mask ratio: 0.626114\n",
      "Step: 151 Loss: 1.137603 Tokens per Sec: 30644.330477. mask ratio: 0.531592\n",
      "Step: 1 Loss: 1.115144 Tokens per Sec: 11613.994552. mask ratio: 0.553580\n",
      "@ 13 1.1349206740428976\n",
      "Step: 1 Loss: 1.334834 Tokens per Sec: 19325.104388. mask ratio: 0.515854\n",
      "Step: 51 Loss: 0.943925 Tokens per Sec: 34759.528673. mask ratio: 0.499752\n",
      "Step: 101 Loss: 1.050117 Tokens per Sec: 29466.291368. mask ratio: 0.600425\n",
      "Step: 151 Loss: 1.291056 Tokens per Sec: 33991.145642. mask ratio: 0.572701\n",
      "Step: 1 Loss: 0.925146 Tokens per Sec: 9311.979448. mask ratio: 0.336905\n",
      "@ 14 1.150933404330409\n",
      "Step: 1 Loss: 1.129130 Tokens per Sec: 14473.620420. mask ratio: 0.340054\n",
      "Step: 51 Loss: 1.351360 Tokens per Sec: 31872.286559. mask ratio: 0.639295\n",
      "Step: 101 Loss: 1.168512 Tokens per Sec: 35851.071463. mask ratio: 0.573601\n",
      "Step: 151 Loss: 1.004312 Tokens per Sec: 33966.381668. mask ratio: 0.523153\n",
      "Step: 1 Loss: 0.996414 Tokens per Sec: 12286.990585. mask ratio: 0.680009\n",
      "@ 15 1.1908108616458135\n",
      "Step: 1 Loss: 1.304749 Tokens per Sec: 14070.051354. mask ratio: 0.633958\n",
      "Step: 51 Loss: 1.105385 Tokens per Sec: 33861.899216. mask ratio: 0.590778\n",
      "Step: 101 Loss: 0.971309 Tokens per Sec: 31632.753045. mask ratio: 0.517890\n",
      "Step: 151 Loss: 1.346894 Tokens per Sec: 33482.859671. mask ratio: 0.612136\n",
      "Step: 1 Loss: 1.021765 Tokens per Sec: 15937.905634. mask ratio: 0.517479\n",
      "@ 16 1.1735610949948119\n",
      "Step: 1 Loss: 1.358709 Tokens per Sec: 16054.393042. mask ratio: 0.755563\n",
      "Step: 51 Loss: 1.091922 Tokens per Sec: 31376.424534. mask ratio: 0.541443\n",
      "Step: 101 Loss: 1.234564 Tokens per Sec: 33607.093420. mask ratio: 0.523266\n",
      "Step: 151 Loss: 1.267353 Tokens per Sec: 31707.153507. mask ratio: 0.582343\n",
      "Step: 1 Loss: 0.957121 Tokens per Sec: 6889.013912. mask ratio: 0.411075\n",
      "@ 17 1.1717712818303292\n",
      "Step: 1 Loss: 1.039054 Tokens per Sec: 23360.009577. mask ratio: 0.618632\n",
      "Step: 51 Loss: 1.274025 Tokens per Sec: 33654.221368. mask ratio: 0.546472\n",
      "Step: 101 Loss: 1.041302 Tokens per Sec: 29795.626887. mask ratio: 0.501535\n",
      "Step: 151 Loss: 1.006979 Tokens per Sec: 32908.156588. mask ratio: 0.557197\n",
      "Step: 1 Loss: 1.038577 Tokens per Sec: 9278.504996. mask ratio: 0.337202\n",
      "@ 18 1.151958743553692\n",
      "Step: 1 Loss: 1.167965 Tokens per Sec: 24376.923633. mask ratio: 0.507376\n",
      "Step: 51 Loss: 0.974428 Tokens per Sec: 32892.518661. mask ratio: 0.527815\n",
      "Step: 101 Loss: 0.959192 Tokens per Sec: 30670.964795. mask ratio: 0.548243\n",
      "Step: 151 Loss: 1.267905 Tokens per Sec: 32872.553888. mask ratio: 0.489993\n",
      "Step: 1 Loss: 1.262255 Tokens per Sec: 12201.029703. mask ratio: 0.417006\n",
      "@ 19 1.117809404509138\n"
     ]
    }
   ],
   "source": [
    "mr = 'rand'\n",
    "# mr = 0.2\n",
    "for epoch in range(20):\n",
    "    model_par.train()\n",
    "    run_epoch(IT, model_par, loss_compute, mask_ratio=mr, step_call_back=train_log_cbk)\n",
    "    model_par.eval()\n",
    "    loss = run_epoch(teIT, model_par, SimpleLossCompute(generator, criterion, None), mask_ratio=mr)\n",
    "    testwriter.add_scalar('loss', loss, step_counter)\n",
    "    testwriter.flush()\n",
    "    trainwriter.flush()\n",
    "    print('@', epoch, loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import StringIO\n",
    "sio = StringIO()\n",
    "for first in firsts:\n",
    "    print_match_all(first, beamsize=5, file=sio)\n",
    "    print('\\n', file=sio)\n",
    "trainwriter.add_text(str(step_counter), sio.getvalue().replace('\\n','\\n\\n').replace('\\n\\n\\n\\n','\\n\\n'), step_counter)\n",
    "trainwriter.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1 Loss: 1.282584 Tokens per Sec: 22140.243308. mask ratio: 0.491714\n",
      "Step: 51 Loss: 1.227419 Tokens per Sec: 30843.124966. mask ratio: 0.596885\n",
      "Step: 101 Loss: 1.339271 Tokens per Sec: 36835.752519. mask ratio: 0.599932\n",
      "Step: 151 Loss: 1.025975 Tokens per Sec: 32734.072416. mask ratio: 0.559325\n",
      "Step: 1 Loss: 1.235315 Tokens per Sec: 9275.237849. mask ratio: 0.616233\n",
      "@ 0 1.1539560507919164\n",
      "Step: 1 Loss: 1.180294 Tokens per Sec: 14047.080600. mask ratio: 0.665393\n",
      "Step: 51 Loss: 1.253999 Tokens per Sec: 32310.493901. mask ratio: 0.568468\n",
      "Step: 101 Loss: 1.206821 Tokens per Sec: 33830.257453. mask ratio: 0.609964\n",
      "Step: 151 Loss: 1.117793 Tokens per Sec: 32905.980958. mask ratio: 0.499624\n",
      "Step: 1 Loss: 0.915152 Tokens per Sec: 8221.574676. mask ratio: 0.346252\n",
      "@ 1 1.1615135005018462\n",
      "Step: 1 Loss: 1.311983 Tokens per Sec: 14079.563433. mask ratio: 0.536333\n",
      "Step: 51 Loss: 1.088267 Tokens per Sec: 31736.411061. mask ratio: 0.574631\n",
      "Step: 101 Loss: 1.055392 Tokens per Sec: 32429.407183. mask ratio: 0.516777\n",
      "Step: 151 Loss: 1.293319 Tokens per Sec: 36003.105943. mask ratio: 0.523019\n",
      "Step: 1 Loss: 1.022559 Tokens per Sec: 6904.194928. mask ratio: 0.580032\n",
      "@ 2 1.1752804148182112\n",
      "Step: 1 Loss: 1.043501 Tokens per Sec: 19682.755135. mask ratio: 0.482211\n",
      "Step: 51 Loss: 1.281764 Tokens per Sec: 32748.700151. mask ratio: 0.561238\n",
      "Step: 101 Loss: 1.170958 Tokens per Sec: 32310.837880. mask ratio: 0.519302\n",
      "Step: 151 Loss: 1.041192 Tokens per Sec: 33231.375458. mask ratio: 0.540417\n",
      "Step: 1 Loss: 1.249006 Tokens per Sec: 9321.507833. mask ratio: 0.867560\n",
      "@ 3 1.155626072352808\n",
      "Step: 1 Loss: 1.188228 Tokens per Sec: 18640.495719. mask ratio: 0.718662\n",
      "Step: 51 Loss: 0.883601 Tokens per Sec: 33658.731798. mask ratio: 0.521249\n",
      "Step: 101 Loss: 1.006344 Tokens per Sec: 31400.318271. mask ratio: 0.618733\n",
      "Step: 151 Loss: 1.012780 Tokens per Sec: 32356.129335. mask ratio: 0.500644\n",
      "Step: 1 Loss: 1.148389 Tokens per Sec: 9510.643103. mask ratio: 0.617857\n",
      "@ 4 1.125694721105439\n",
      "Step: 1 Loss: 1.274383 Tokens per Sec: 17295.492008. mask ratio: 0.884676\n",
      "Step: 51 Loss: 1.259888 Tokens per Sec: 34142.007594. mask ratio: 0.594575\n",
      "Step: 101 Loss: 1.181515 Tokens per Sec: 33429.810490. mask ratio: 0.577081\n",
      "Step: 151 Loss: 1.174535 Tokens per Sec: 32525.386620. mask ratio: 0.486862\n",
      "Step: 1 Loss: 0.834137 Tokens per Sec: 6949.811596. mask ratio: 0.288601\n",
      "@ 5 1.0756034502783216\n",
      "Step: 1 Loss: 1.211820 Tokens per Sec: 15960.075225. mask ratio: 0.823779\n",
      "Step: 51 Loss: 0.943138 Tokens per Sec: 30909.136112. mask ratio: 0.474749\n",
      "Step: 101 Loss: 1.227500 Tokens per Sec: 33172.771071. mask ratio: 0.569972\n",
      "Step: 151 Loss: 1.023136 Tokens per Sec: 32494.750037. mask ratio: 0.562867\n",
      "Step: 1 Loss: 1.250599 Tokens per Sec: 676.295559. mask ratio: 0.811398\n",
      "@ 6 1.1250277227809369\n",
      "Step: 1 Loss: 1.220311 Tokens per Sec: 18553.920562. mask ratio: 0.525098\n",
      "Step: 51 Loss: 0.953853 Tokens per Sec: 31750.806716. mask ratio: 0.556784\n",
      "Step: 101 Loss: 1.123566 Tokens per Sec: 32966.576342. mask ratio: 0.614209\n",
      "Step: 151 Loss: 1.280277 Tokens per Sec: 35454.464772. mask ratio: 0.524732\n",
      "Step: 1 Loss: 0.959361 Tokens per Sec: 9263.654827. mask ratio: 0.669432\n",
      "@ 7 1.1544943221102921\n",
      "Step: 1 Loss: 1.111977 Tokens per Sec: 20840.977406. mask ratio: 0.622546\n",
      "Step: 51 Loss: 1.099875 Tokens per Sec: 32718.542205. mask ratio: 0.442951\n",
      "Step: 101 Loss: 0.944297 Tokens per Sec: 31812.593464. mask ratio: 0.551822\n",
      "Step: 151 Loss: 1.016213 Tokens per Sec: 33651.054834. mask ratio: 0.587613\n",
      "Step: 1 Loss: 1.334013 Tokens per Sec: 8071.666180. mask ratio: 0.904514\n",
      "@ 8 1.1339645945252135\n",
      "Step: 1 Loss: 0.998716 Tokens per Sec: 23066.686343. mask ratio: 0.315750\n",
      "Step: 51 Loss: 1.048343 Tokens per Sec: 32928.279968. mask ratio: 0.568025\n",
      "Step: 101 Loss: 1.222551 Tokens per Sec: 31232.590952. mask ratio: 0.515082\n",
      "Step: 151 Loss: 1.252254 Tokens per Sec: 33041.943576. mask ratio: 0.493138\n",
      "Step: 1 Loss: 1.105487 Tokens per Sec: 10344.905867. mask ratio: 0.793039\n",
      "@ 9 1.1225222907765033\n",
      "Step: 1 Loss: 1.241090 Tokens per Sec: 21257.138058. mask ratio: 0.825846\n",
      "Step: 51 Loss: 1.143248 Tokens per Sec: 36665.128763. mask ratio: 0.499421\n",
      "Step: 101 Loss: 1.079787 Tokens per Sec: 30997.086740. mask ratio: 0.498065\n",
      "Step: 151 Loss: 1.014043 Tokens per Sec: 32717.999220. mask ratio: 0.501889\n",
      "Step: 1 Loss: 1.269234 Tokens per Sec: 10947.828052. mask ratio: 0.646119\n",
      "@ 10 1.1023536841192056\n",
      "Step: 1 Loss: 1.081123 Tokens per Sec: 14391.234620. mask ratio: 0.505554\n",
      "Step: 51 Loss: 1.264625 Tokens per Sec: 33590.027180. mask ratio: 0.613025\n",
      "Step: 101 Loss: 1.056084 Tokens per Sec: 31683.916682. mask ratio: 0.507096\n",
      "Step: 151 Loss: 0.970425 Tokens per Sec: 34670.180331. mask ratio: 0.488327\n",
      "Step: 1 Loss: 1.271985 Tokens per Sec: 9691.923666. mask ratio: 0.726786\n",
      "@ 11 1.0758150439236263\n",
      "Step: 1 Loss: 1.129781 Tokens per Sec: 20308.715864. mask ratio: 0.636238\n",
      "Step: 51 Loss: 1.114002 Tokens per Sec: 31796.786028. mask ratio: 0.602243\n",
      "Step: 101 Loss: 0.985786 Tokens per Sec: 37192.085627. mask ratio: 0.597985\n",
      "Step: 151 Loss: 0.947929 Tokens per Sec: 32354.408835. mask ratio: 0.622847\n",
      "Step: 1 Loss: 1.306411 Tokens per Sec: 16490.968792. mask ratio: 0.735980\n",
      "@ 12 1.1707164731321609\n",
      "Step: 1 Loss: 1.283225 Tokens per Sec: 18050.996286. mask ratio: 0.696213\n",
      "Step: 51 Loss: 0.854681 Tokens per Sec: 33713.823523. mask ratio: 0.574225\n",
      "Step: 101 Loss: 1.212068 Tokens per Sec: 33110.321083. mask ratio: 0.622193\n",
      "Step: 151 Loss: 1.159671 Tokens per Sec: 33254.505776. mask ratio: 0.544193\n",
      "Step: 1 Loss: 1.307614 Tokens per Sec: 14186.515517. mask ratio: 0.745538\n",
      "@ 13 1.1053362334720842\n",
      "Step: 1 Loss: 0.808310 Tokens per Sec: 19000.684343. mask ratio: 0.133674\n",
      "Step: 51 Loss: 0.978300 Tokens per Sec: 30556.556242. mask ratio: 0.552141\n",
      "Step: 101 Loss: 1.210599 Tokens per Sec: 34273.963870. mask ratio: 0.493614\n",
      "Step: 151 Loss: 1.017851 Tokens per Sec: 31822.945756. mask ratio: 0.558887\n",
      "Step: 1 Loss: 1.282424 Tokens per Sec: 9077.445640. mask ratio: 0.791369\n",
      "@ 14 1.161643945011289\n",
      "Step: 1 Loss: 0.922014 Tokens per Sec: 7896.173817. mask ratio: 0.331663\n",
      "Step: 51 Loss: 1.212020 Tokens per Sec: 34892.986226. mask ratio: 0.490724\n",
      "Step: 101 Loss: 1.111951 Tokens per Sec: 32281.999100. mask ratio: 0.638997\n",
      "Step: 151 Loss: 0.936736 Tokens per Sec: 32111.083473. mask ratio: 0.610535\n",
      "Step: 1 Loss: 1.096590 Tokens per Sec: 9034.579012. mask ratio: 0.518894\n",
      "@ 15 1.0973817238869106\n",
      "Step: 1 Loss: 1.034635 Tokens per Sec: 13785.255053. mask ratio: 0.344187\n",
      "Step: 51 Loss: 0.997558 Tokens per Sec: 33203.416988. mask ratio: 0.544536\n",
      "Step: 101 Loss: 1.297906 Tokens per Sec: 31411.388973. mask ratio: 0.560126\n",
      "Step: 151 Loss: 1.225534 Tokens per Sec: 32652.668748. mask ratio: 0.669401\n",
      "Step: 1 Loss: 1.188927 Tokens per Sec: 18961.484834. mask ratio: 0.761215\n",
      "@ 16 1.1086351163742376\n",
      "Step: 1 Loss: 1.072019 Tokens per Sec: 14982.238904. mask ratio: 0.776286\n",
      "Step: 51 Loss: 0.974471 Tokens per Sec: 31381.961615. mask ratio: 0.636384\n",
      "Step: 101 Loss: 1.090385 Tokens per Sec: 35965.020064. mask ratio: 0.612272\n",
      "Step: 151 Loss: 1.095044 Tokens per Sec: 31186.510043. mask ratio: 0.503488\n",
      "Step: 1 Loss: 1.080024 Tokens per Sec: 8918.556512. mask ratio: 0.477578\n",
      "@ 17 1.0569421931316978\n",
      "Step: 1 Loss: 0.996568 Tokens per Sec: 18708.265435. mask ratio: 0.239461\n",
      "Step: 51 Loss: 1.043926 Tokens per Sec: 31994.582033. mask ratio: 0.612042\n",
      "Step: 101 Loss: 1.147792 Tokens per Sec: 31314.394798. mask ratio: 0.476541\n",
      "Step: 151 Loss: 0.912126 Tokens per Sec: 33225.865568. mask ratio: 0.462105\n",
      "Step: 1 Loss: 1.176942 Tokens per Sec: 6822.876443. mask ratio: 0.792643\n",
      "@ 18 1.1182152336542706\n",
      "Step: 1 Loss: 1.174769 Tokens per Sec: 25118.654699. mask ratio: 0.334037\n",
      "Step: 51 Loss: 0.987371 Tokens per Sec: 31677.274699. mask ratio: 0.482417\n",
      "Step: 101 Loss: 1.021093 Tokens per Sec: 31537.147277. mask ratio: 0.572543\n",
      "Step: 151 Loss: 1.020839 Tokens per Sec: 35631.225285. mask ratio: 0.600028\n",
      "Step: 1 Loss: 1.240862 Tokens per Sec: 16050.994147. mask ratio: 0.674276\n",
      "@ 19 1.078277801275189\n"
     ]
    }
   ],
   "source": [
    "mr = 'rand'\n",
    "# mr = 0.2\n",
    "for epoch in range(20):\n",
    "    model_par.train()\n",
    "    run_epoch(IT, model_par, loss_compute, mask_ratio=mr, step_call_back=train_log_cbk)\n",
    "    model_par.eval()\n",
    "    loss = run_epoch(teIT, model_par, SimpleLossCompute(generator, criterion, None), mask_ratio=mr)\n",
    "    testwriter.add_scalar('loss', loss, step_counter)\n",
    "    testwriter.flush()\n",
    "    trainwriter.flush()\n",
    "    print('@', epoch, loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import StringIO\n",
    "sio = StringIO()\n",
    "for first in firsts:\n",
    "    print_match_all(first, beamsize=5, file=sio)\n",
    "    print('\\n', file=sio)\n",
    "trainwriter.add_text(str(step_counter), sio.getvalue().replace('\\n','\\n\\n').replace('\\n\\n\\n\\n','\\n\\n'), step_counter)\n",
    "trainwriter.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1 Loss: 4.593791 Tokens per Sec: 16180.098453. mask ratio: 0.813781\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3.5329453869081977"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_epoch(teIT_r, model_par, SimpleLossCompute(generator, criterion, None), mask_ratio=mr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1 Loss: 0.991790 Tokens per Sec: 12601.218376. mask ratio: 0.330149\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.1360541676955822"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_par.eval()\n",
    "run_epoch(teIT, model_par, SimpleLossCompute(generator, criterion, None), mask_ratio=mr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1 Loss: 0.828012 Tokens per Sec: 30504.914129. mask ratio: 0.095151\n",
      "Step: 51 Loss: 1.725160 Tokens per Sec: 45051.411805. mask ratio: 0.521330\n",
      "Step: 101 Loss: 2.006808 Tokens per Sec: 46831.572388. mask ratio: 0.532648\n",
      "Step: 151 Loss: 2.597104 Tokens per Sec: 48085.878949. mask ratio: 0.564236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2.6191998852524843"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_epoch(IT, model_par, SimpleLossCompute(generator, criterion, None), mask_ratio=mr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1 Loss: 3.764093 Tokens per Sec: 17074.665737. mask ratio: 0.787457\n",
      "Step: 51 Loss: 3.234247 Tokens per Sec: 47412.154484. mask ratio: 0.513269\n",
      "Step: 101 Loss: 2.584729 Tokens per Sec: 42086.017133. mask ratio: 0.511079\n",
      "Step: 151 Loss: 3.665455 Tokens per Sec: 45131.426026. mask ratio: 0.554868\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3.102383682614523"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_epoch(IT_r, model_par, SimpleLossCompute(generator, criterion, None), mask_ratio=mr)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "state = {'model':model_par.state_dict(), 'optimizer':opt.state_dict()}\n",
    "torch.save(state, 'checkpoint5.cp')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "torch.save(model.state_dict(), 'model_state_share03.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('model_state_share03.pt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# decode search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from decode_search import *\n",
    "\n",
    "def match_couplet_onepass(first, second=''):\n",
    "    '一次性输出，模型调用1次，效果不好'\n",
    "    batch = str_to_batch(first, second, VOCAB.stoi, MASK_TOKEN)\n",
    "    model.eval()\n",
    "    out = model(batch.src.to(DEVICE), batch.trg.to(DEVICE), \n",
    "                batch.src_pad_mask.unsqueeze(1).to(DEVICE), batch.trg_pad_mask.unsqueeze(1).to(DEVICE))\n",
    "    _, ind = torch.max(model.generator(out), dim = -1)\n",
    "    return itos(ind.cpu().numpy())\n",
    "\n",
    "def match_couplet_greedy(first, second=''):\n",
    "    batch = str_to_batch(first, second, VOCAB.stoi, MASK_TOKEN)\n",
    "    model.eval()\n",
    "    trg_pred = decode_greedy(model, batch.src.to(DEVICE), batch.trg.to(DEVICE)\n",
    "                             , mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])\n",
    "    return itos(trg_pred.cpu().numpy())\n",
    "\n",
    "def match_couplet_beam_1D(first, second='', beamsize=5):\n",
    "    batch = str_to_batch(first, second, VOCAB.stoi, MASK_TOKEN)\n",
    "    model.eval()\n",
    "    results = beam_decode_engine(model, batch.src.to(DEVICE), batch.trg.to(DEVICE), beamsize, decode_1D_step\n",
    "                             , mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])\n",
    "    return itos([r.cpu().numpy() for r in results])\n",
    "\n",
    "def match_couplet_beam_margin(first, second='', beamsize=5):\n",
    "    batch = str_to_batch(first, second, VOCAB.stoi, MASK_TOKEN)\n",
    "    model.eval()\n",
    "    results = beam_decode_engine(model, batch.src.to(DEVICE), batch.trg.to(DEVICE), beamsize, decode_margin_step\n",
    "                             , mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])\n",
    "    return itos([r.cpu().numpy() for r in results])\n",
    "\n",
    "def match_couplet_beam_2D(first, second='', beamsize=5):\n",
    "    '条件概率版本'\n",
    "    batch = str_to_batch(first, second, VOCAB.stoi, MASK_TOKEN)\n",
    "    model.eval()\n",
    "    results = beam_decode_engine(model, batch.src.to(DEVICE), batch.trg.to(DEVICE), beamsize, decode_2D_step\n",
    "                             , mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])\n",
    "    return itos([r.cpu().numpy() for r in results])\n",
    "\n",
    "def match_couplet_beam_2D_2(first, second='', beamsize=5):\n",
    "    '联合分布概率版本'\n",
    "    batch = str_to_batch(first, second, VOCAB.stoi, MASK_TOKEN)\n",
    "    model.eval()\n",
    "    results = decode_beam_2D_2(model, batch.src.to(DEVICE), batch.trg.to(DEVICE), beamsize\n",
    "                             , mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])\n",
    "    return itos([r.cpu().numpy() for r in results])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_margin_step2(model, src, trg, mask, padmask, beamsize):\n",
    "    '''\n",
    "    各候选位置选最可能的字 + 最可能的位置选候选字，伪二维搜索\n",
    "    padmask为True的是有效字，mask为True的是句中的给定字（非mask）\n",
    "    '''\n",
    "    model.eval()\n",
    "#     print(padmask.unsqueeze(0).unsqueeze(1))\n",
    "    assert src.size(0) == 1\n",
    "    out = model(src, trg, padmask.unsqueeze(0).unsqueeze(1), padmask.unsqueeze(0).unsqueeze(1))\n",
    "    g = model.generator(out)\n",
    "    s, ind = torch.max(g, dim = -1)\n",
    "    ignore = mask | (~padmask)\n",
    "    s[0][ignore] = -1e9\n",
    "    snp = s[0].detach().cpu().numpy() \n",
    "    pos = max_n_1D(np.array(snp), min(len(snp), beamsize//2))\n",
    "    result = []\n",
    "    keep_pos = pos[:(~ignore).sum().item()]\n",
    "    for p in keep_pos:\n",
    "        r = tc.clone(trg)\n",
    "        r[0, p] = ind[0][p].item()\n",
    "        result.append(r)\n",
    "    logps = snp[keep_pos]\n",
    "    max_pos = pos[0]\n",
    "    g = g[0, max_pos].detach().cpu().numpy()\n",
    "    ind = max_n_1D(g, min(len(g), beamsize-len(result)))\n",
    "    \n",
    "    for i in ind:\n",
    "        r = tc.clone(trg)\n",
    "        r[0, max_pos] = i\n",
    "        result.append(r)\n",
    "#     print(itos([r.cpu().numpy() for r in result]))\n",
    "    return result\n",
    "\n",
    "def beam_decode_engine2(model, src, trg, beamsize, decodestep, mask_int, pad_int):\n",
    "    '候选的概率由logp_of_trg确定（联合分布）'\n",
    "    padmask = (src!=pad_int).squeeze()\n",
    "    result = [trg]\n",
    "    while True:\n",
    "        candidates = {}\n",
    "        for trg in result:\n",
    "            mask = (trg!=mask_int).squeeze()\n",
    "            if mask.all(): \n",
    "                return result\n",
    "            cands = decodestep(model, src, trg, mask, padmask, beamsize)\n",
    "            for c in cands:\n",
    "                candidates[tuple(c[0].tolist())] = c #去重\n",
    "        cands = list(candidates.values())\n",
    "        logps = [logp_of_trg(model, src, trgi, (trgi!=mask_int).squeeze(0), padmask) for trgi in cands]\n",
    "        maxinds = max_n_1D(np.array(logps), min(len(logps), beamsize))\n",
    "        result = [cands[i] for i in maxinds]\n",
    "#         print(itos([r.cpu().numpy() for r in result]))\n",
    "    return result\n",
    "\n",
    "def match_couplet_beam_margin2(first, second='', beamsize=5):\n",
    "    batch = str_to_batch(first, second, VOCAB.stoi, MASK_TOKEN)\n",
    "    model.eval()\n",
    "    results = beam_decode_engine2(model, batch.src.to(DEVICE), batch.trg.to(DEVICE), beamsize, decode_margin_step2\n",
    "                             , mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])\n",
    "    return itos([r.cpu().numpy() for r in results])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 636,
   "metadata": {},
   "outputs": [],
   "source": [
    "out = model(src, trg, padmask.unsqueeze(0).unsqueeze(1), padmask.unsqueeze(0).unsqueeze(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 738,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7, 6745])"
      ]
     },
     "execution_count": 738,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.generator(out).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_match_all(first, second='', beamsize=5, file=sys.stdout):\n",
    "    print(\"上联:\", first, file=file)\n",
    "    print(\"onepass:\", match_couplet_onepass(first, second), file=file)\n",
    "    print(\"greedy :\", match_couplet_greedy(first, second), file=file)\n",
    "    print(\"beam_1D:\\n\", array(match_couplet_beam_1D(first, second, beamsize)), file=file)\n",
    "    print(\"beam_2D:\\n\", array(match_couplet_beam_2D(first, second, beamsize)), file=file)\n",
    "    print(\"beam_2D_2:\\n\", array(match_couplet_beam_2D_2(first, second, beamsize)), file=file)\n",
    "    print(\"beam_margin:\\n\", array(match_couplet_beam_margin(first, second, beamsize)), file=file)\n",
    "    print(\"beam_margin2:\\n\", array(match_couplet_beam_margin2(first, second, beamsize)), file=file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "上联: 天与云与山与水，上下一白\n",
      "onepass: ['人同月同竹同风，古下同清']\n",
      "greedy : ['人和诗和酒和诗，天地同春']\n",
      "beam_1D:\n",
      " [['人同竹同竹同梅，上下两清']\n",
      " ['人同竹同竹同梅，上下双清']\n",
      " ['人和诗和画和诗，古今同辉']]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-140-513c6003519d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     12\u001b[0m          ]\n\u001b[1;32m     13\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfirst\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfirsts\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m     \u001b[0mprint_match_all\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeamsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-77-d8108646bf35>\u001b[0m in \u001b[0;36mprint_match_all\u001b[0;34m(first, second, beamsize, file)\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"greedy :\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatch_couplet_greedy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecond\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"beam_1D:\\n\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmatch_couplet_beam_1D\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecond\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeamsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"beam_2D:\\n\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmatch_couplet_beam_2D\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecond\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeamsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"beam_2D_2:\\n\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmatch_couplet_beam_2D_2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecond\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeamsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"beam_margin:\\n\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmatch_couplet_beam_margin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecond\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeamsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-75-702cbeee79dc>\u001b[0m in \u001b[0;36mmatch_couplet_beam_2D\u001b[0;34m(first, second, beamsize)\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m     results = beam_decode_engine(model, batch.src.to(DEVICE), batch.trg.to(DEVICE), beamsize, decode_2D_step\n\u001b[0;32m---> 38\u001b[0;31m                              , mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])\n\u001b[0m\u001b[1;32m     39\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mitos\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/decode_search.py\u001b[0m in \u001b[0;36mbeam_decode_engine\u001b[0;34m(model, src, trg, beamsize, decodestep, mask_int, pad_int)\u001b[0m\n\u001b[1;32m     95\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m             \u001b[0mcands\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogpi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecodestep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadmask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeamsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     98\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mli\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcands\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogpi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m                 \u001b[0mcandidates\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrlp\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mli\u001b[0m \u001b[0;31m#去重\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/decode_search.py\u001b[0m in \u001b[0;36mdecode_2D_step\u001b[0;34m(model, src, trg, mask, padmask, beamsize)\u001b[0m\n\u001b[1;32m    131\u001b[0m     '''\n\u001b[1;32m    132\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 133\u001b[0;31m     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    134\u001b[0m     \u001b[0;32massert\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    135\u001b[0m     \u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/gyz37/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, src, tgt, src_mask, tgt_mask)\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;34m\"Take in and process masked src and target sequences.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         return self.decode(self.encode(src, src_mask), src_mask,\n\u001b[0;32m---> 24\u001b[0;31m                             tgt, tgt_mask)\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/model.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, memory, src_mask, tgt, tgt_mask)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtgt_embed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtgt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mGenerator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/gyz37/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, memory, src_mask, tgt_mask)\u001b[0m\n\u001b[1;32m    109\u001b[0m         \u001b[0mx0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    112\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmemory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/gyz37/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, memory, src_mask, tgt_mask)\u001b[0m\n\u001b[1;32m    129\u001b[0m         \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmemory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msublayer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mself_attn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msublayer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msrc_attn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msublayer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeed_forward\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/gyz37/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, sublayer)\u001b[0m\n\u001b[1;32m     81\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msublayer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0;34m\"Apply residual connection to any sublayer with the same size.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msublayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mEncoderLayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/model.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    129\u001b[0m         \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmemory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msublayer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mself_attn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtgt_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msublayer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msrc_attn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    132\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msublayer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeed_forward\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/gyz37/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, query, key, value, mask)\u001b[0m\n\u001b[1;32m    181\u001b[0m         \u001b[0;31m# 2) Apply attention on all the projected vectors in batch.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m         x, self.attn = attention(query, key, value, mask=mask, \n\u001b[0;32m--> 183\u001b[0;31m                                  dropout=self.dropout)\n\u001b[0m\u001b[1;32m    184\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m         \u001b[0;31m# 3) \"Concat\" using a view and apply a final linear.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/couplets/model.py\u001b[0m in \u001b[0;36mattention\u001b[0;34m(query, key, value, mask, dropout)\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;31m# print(scores.shape,value.shape,mask.shape)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasked_fill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmask\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1e9\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    152\u001b[0m     \u001b[0mp_attn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mdropout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "firsts = ['天与云与山与水，上下一白',\n",
    "          '两个黄鹂鸣翠柳',\n",
    "          '无边落木萧萧下', \n",
    "          '月落乌啼霜满天', \n",
    "          '提刀上马，江山如画', \n",
    "          '兰亭临帖，行书如行云流水',\n",
    "          '荒烟漫草的年头，就连分手都很沉默',\n",
    "          '一条大河波浪宽，风吹稻花香两岸',\n",
    "          '我欲成仙，快乐齐天',\n",
    "          '只有刚强的人，才有神圣的意志，凡是战斗的人，才能取得胜利',\n",
    "          '？',\n",
    "         ]\n",
    "for first in firsts:\n",
    "    print_match_all(first, beamsize=5)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "src   : ['莫让浮名耽岁月']\n",
      "true  : ['且留正气满乾坤']\n",
      "mask  : ['〇留正〇〇〇〇']\n",
      "greedy: ['但留正气满乾坤']\n",
      "beam 1D:\n",
      " [['但留正气度人生']\n",
      " ['但留正气满乾坤']]\n",
      "beam 2D:\n",
      " [['但留正气度人生']\n",
      " ['但留正气写人生']\n",
      " ['但留正气满乾坤']\n",
      " ['但留正气壮乾坤']\n",
      " ['长留正气度人生']]\n",
      "beam 2D 2:\n",
      " [['长留正气满乾坤']\n",
      " ['长留正气度人生']\n",
      " ['长留正气写人生']\n",
      " ['常留正气满乾坤']\n",
      " ['长留正道度人生']]\n",
      "beam margin:\n",
      " [['但留正气度人生']\n",
      " ['但留正气满乾坤']\n",
      " ['长留正气满乾坤']\n",
      " ['但留正气写人生']\n",
      " ['但留正气壮乾坤']]\n",
      "========\n",
      "src   : ['风吹羊头夕阳醉']\n",
      "true  : ['雨润丹河霞光美']\n",
      "mask  : ['〇〇〇〇〇〇〇']\n",
      "greedy: ['雨润田间谷穗香']\n",
      "beam 1D:\n",
      " [['雨润田间谷穗香']\n",
      " ['雨润心田谷穗香']]\n",
      "beam 2D:\n",
      " [['雨润田间谷穗香']\n",
      " ['雨润大地谷穗香']\n",
      " ['雨润田园谷穗香']\n",
      " ['雨润心田谷穗香']\n",
      " ['雨润田地谷穗香']]\n",
      "beam 2D 2:\n",
      " [['花满人间春意浓']\n",
      " ['花满人间诗意浓']\n",
      " ['花开满园春意浓']\n",
      " ['花艳满园春意浓']\n",
      " ['花落人间春意浓']]\n",
      "beam margin:\n",
      " [['雨润田间谷穗香']\n",
      " ['雨润田园谷穗香']\n",
      " ['雨润心田谷穗香']\n",
      " ['雨润田地谷穗香']\n",
      " ['雨润田田谷穗香']]\n",
      "========\n"
     ]
    }
   ],
   "source": [
    "for i, batch in enumerate(teIT):\n",
    "    batch = parse_pybatch(batch,0.8)\n",
    "    src = batch.src[:1]\n",
    "    src_pad_mask = batch.src_pad_mask[:1]\n",
    "    trg = batch.trg[:1]\n",
    "    trg_y = batch.trg_y[:1]\n",
    "    trg_mask = batch.trg_mask[:1]\n",
    "    trg_pad_mask = batch.trg_pad_mask[:1]\n",
    "#     out = decode(model, src, src_mask)\n",
    "    print(\"src   :\", itos(src.cpu().numpy()))\n",
    "    print(\"true  :\", itos(trg_y.cpu().numpy()))\n",
    "    print(\"mask  :\", [s.replace(MASK_TOKEN, \"〇\") for s in itos(trg.cpu().numpy())])\n",
    "    print(\"greedy:\", itos(decode_greedy(model, src, trg.clone(), \n",
    "                          mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN]).cpu().numpy()))\n",
    "    print(\"beam 1D:\\n\", array(itos([r.cpu().numpy() for r in beam_decode_engine(model, src, trg.clone(), 5, decode_1D_step, \n",
    "                          mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])])))\n",
    "    print(\"beam 2D:\\n\", array(itos([r.cpu().numpy() for r in beam_decode_engine(model, src, trg.clone(), 5, decode_2D_step, \n",
    "                          mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])])))\n",
    "    print(\"beam 2D 2:\\n\", array(itos([r.cpu().numpy() for r in decode_beam_2D_2(model, src, trg.clone(), 5, \n",
    "                          mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])])))\n",
    "    print(\"beam margin:\\n\", array(itos([r.cpu().numpy() for r in beam_decode_engine(model, src, trg.clone(), 5, decode_margin_step, \n",
    "                          mask_int=VOCAB.stoi[MASK_TOKEN], pad_int=VOCAB.stoi[PAD_TOKEN])])))\n",
    "    print('='*8)\n",
    "    if i == 1:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 800,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "上联: 行书如行云流水\n",
      "onepass: ['明卷若明月阳风']\n",
      "greedy : ['品茗若品月清风']\n",
      "beam_1D:\n",
      " [['品茗似品竹啸风']\n",
      " ['品茗似品竹摇风']\n",
      " ['品茗若品竹摇风']\n",
      " ['明镜若明月清风']]\n",
      "beam_2D:\n",
      " [['品茗似品竹摇风']\n",
      " ['品茗若品竹摇风']\n",
      " ['品茗似品竹啸风']\n",
      " ['品茗若品竹啸风']\n",
      " ['明月似明月清风']]\n",
      "beam_2D_2:\n",
      " [['明镜似明月照人']\n",
      " ['明镜若明月照人']\n",
      " ['明镜自明月照人']\n",
      " ['明镜犹明月照人']\n",
      " ['展卷有展艳阳天']]\n",
      "beam_margin:\n",
      " [['品茗似品竹摇风']\n",
      " ['品茗似品竹啸风']\n",
      " ['品茗若品竹摇风']\n",
      " ['品茗若品竹啸风']\n",
      " ['品德似品竹啸风']]\n"
     ]
    }
   ],
   "source": [
    "print_match_all('行书如行云流水', '')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['人和月和花和月，春秋千红']"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent = '天与云与山与水，上下一白'\n",
    "model.eval()\n",
    "pred = match_couplet_onepass(sent,'    花')\n",
    "tgt_sent = pred[0]\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 8, 12, 12])"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.decoder.layers[0].self_attn.attn.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder Layer 2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAeIUlEQVR4nO3dbYyV5bno8WscBmQX99iInV2K4VSDb9FYrYkn0T2WipjAVjCRJoL2kPgWowXfpaEZccREGl9GiQlgG4kEYwnarcQmvMSms9vkfKDGULQqCeGErR09Hg7IMDEjsM4HtxxeZoZnzdyLdc8zv98nWGu45wJn/g4Xz7OmoVKpVAIAAACAUjql3gMAAAAAUDuWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKj6veu/3dEHKrfu2fE+2+3zk163s7Va5Kel7dTIuLMeg+RiBZRX1o0FFoEuRp5bStLj/JtUeqPqYjh8HEF1eq/RXVc/hyKiIP1e/eMeP/rs08Tn+jjeXjSIupLi/iGFlEu2jZc5dui9B9TEbn+XqEW3PYFAAAAUGKWPwAAAAAlZvkDAAAAUGKFXvNnzZo1sWLFisM/37dvXzQ1NcWpp556+LHHHnssrrnmmvQTAvwXLQJyoEVADrQIqEZDpVKpVPuLHn/88bjsssvi+uuvH8K7/iy8wBb11DBtatLzKhs3Jz0vb40R0VLvIbSIUtCiodAiyNXIa1v9e1T2FqX+mIoYDh9XUK3+WzSo274OHjwYjY2NQ5kIYMi0CMiBFgE50CJgIIWu/GltbT3q519++WWMGTMmxowZc9TjnZ2dVbzrfLfKjAwj71+kUqrPv25pEWWkRUOhRZCrkde2k9+jkdYiV/5AEf23qNBr/hwbjOuuuy4uuOCC6OjoGPJoAEVpEZADLQJyoEVANaq+7WvHjh2xZ8+e+OCDD+K9996rxUwAJ6RFQA60CMiBFgEnUtXy59ChQ7F48eKYO3duLFq0KObPnx9dXV21mg2gT1oE5ECLgBxoEVBE4eXPnj17Yv78+VGpVOLuu++Oq6++Om655ZaYPXt2/PWvf63ljACHaRGQAy0CcqBFQFGFlj8dHR0xbdq0GD9+fLz00kvR1NQUERF33nlnPPjgg3H77bfH/fffX9NBAbQIyIEWATnQIqAahV7wec+ePbF69eo477zzjntu1qxZccUVV8TLL7+cfDiAI2kRkAMtAnKgRUA1Cn2r99rI99sIMjKMvG9BmlJ9vr1ybWgR9aVFQ6FFkKuR17ay9CjfFvlW71BE/y2q+rt9AQAAADB8WP4AAAAAlFih1/zJXeenbyU9r3XCDUnPi4jYtOv3Sc+79qwbk56X+3y1cGjDxnqPAAAAADXnyh8AAACAErP8AQAAACgxyx8AAACAErP8AQAAACgxyx8AAACAErP8AQAAACixwsuftra2ePvtt2s5C8AJaRGQAy0CcqBFQFGu/AEAAAAosVHVvHF7e3ssXbq0z+eam5tj/fr1SYYCGIgWATnQIiAHWgQUUdXyp62tLWbMmFGrWQAK0SIgB1oE5ECLgCLc9gUAAABQYslu+1q4cGFMnz49yVAAA9EiIAdaBORAi4AiCi9/2tvbo729vc/nlixZkmwggIFoEZADLQJyoEVAUUlu+9q9e3c0NzenOApg0LQIyIEWATnQIuBIha78aW1t7fe5tWvXRldXV5xxxhnJhgLoixYBOdAiIAdaBFSj0PKns7PzuMcqlUo0NDRET09P7Ny5MyZNmpR8OIAjaRGQAy0CcqBFQDWqesHnIy1YsCC2bdsWvb29MWfOnBg7dmzKuQAK0SIgB1oE5ECLgP4MevnzwgsvpJwDYFC0CMiBFgE50CKgP0le8BkAAACAPA36yp+ctE64od4jnNC1Z91Y7xEGlPt8tXDKddOSnlfZuDnpeQAAwDd8rU1ZNUybmuysSS0TYufqvj9XXPkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKjirzRli1b4oEHHhjwbZYvXx4XXnhhkqEA+qJFQA60CMiBFgHVKLT8ufzyy6Ozs7PWswAMSIuAHGgRkAMtAqrhti8AAACAEit05U9ra2uhw2yegVrSIiAHWgTkQIuAahRa/hwZjI6Ojti1a1c888wzNRsKoC9aBORAi4AcaBFQjapu++ru7o41a9bEz3/+84iI6Orqij/96U81GQygP1oE5ECLgBxoEVBEVcufVatWxUUXXRSXXHJJRETs378/Fi5cGPv376/JcAB90SIgB1oE5ECLgCIKL3+2bt0aL730Ujz00EOHHzvnnHPikksuiVdffbUmwwEcS4uAHGgRkAMtAooq9Jo/e/fujfvuuy/mzZsXLS0t8dFHH8XOnTvj448/jt27d8e6devijjvuqPWswAinRUAOtAjIgRYB1Sh05U9PT098/vnn8dprr8XPfvaz+OUvfxl/+MMf4tChQ3HrrbfG119/He+++26tZwVGOC0CcqBFQA60CKhGoSt/vv/978e2bdv6fX7y5Mlx9tlnJxsKoC9aBORAi4AcaBFQjULLnxM5//zzUxwDMCRaBORAi4AcaBFwpKq+2xcAAAAAw0uSK3/qrfPTt5Ke1zrhhqTnRURs2vX7pOdde9aNSc/Lfb5aOLRhY71HAAAACmiYNjX5mZWNm5OfCdVK+3HY2O8zrvwBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKLHCy5+2trZ4++23azkLwAlpEZADLQJyoEVAUa78AQAAACixUdW8cXt7eyxdurTP55qbm2P9+vVJhgIYiBYBOdAiIAdaBBRR1fKnra0tZsyYUatZAArRIiAHWgTkQIuAItz2BQAAAFBiyW77WrhwYUyfPj3JUAAD0SIgB1oE5ECLgCIKL3/a29ujvb29z+eWLFmSbCCAgWgRkAMtAnKgRUBRSW772r17dzQ3N6c4CmDQtAjIgRYBOdAi4EiFrvxpbW3t97m1a9dGV1dXnHHGGcmGAuiLFgE50CIgB1oEVKPQ8qezs/O4xyqVSjQ0NERPT0/s3LkzJk2alHw4gCNpEZADLQJyoEVANap6wecjLViwILZt2xa9vb0xZ86cGDt2bMq5AArRIiAHWgTkQIuA/gx6+fPCCy+knANgULQIyIEWATnQIqA/SV7wGQAAAIA8Wf4AAAAAlNigb/vKSeuEG+o9wglde9aN9R5hQLnPVwunXDct6XmVjZuTngcAAAApuPIHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQGtfx58skn44033kg9C0BVtAjIgRYBOdAiYCCu/AEAAAAosULLnw8//DC++OKLfp9///33Y/v27cmGAuiLFgE50CIgB1oEVKPQ8mfVqlXx3nvv9fv8O++8E3/5y1+SDQXQFy0CcqBFQA60CKiG274AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASmxU0TdctGhRtLe3R0TEvn37oqmpKTo6OiIioru7O+bPn1+bCQGOoEVADrQIyIEWAUUVWv489dRTAz6/bNmyJMMADESLgBxoEZADLQKqUfjKn4H84he/SHEMwJBoEZADLQJyoEXAkbzmDwAAAECJWf4AAAAAlFiS277qrfPTt5Ke1zrhhqTnRURs2vX7pOdde9aNSc/Lfb5aOLRhY71HAAAAgJpz5Q8AAABAiVn+AAAAAJSY5Q8AAABAiVn+AAAAAJSY5Q8AAABAiRVe/rS1tcXbb79dy1kATkiLgBxoEZADLQKKcuUPAAAAQImNquaN29vbY+nSpX0+19zcHOvXr08yFMBAtAjIgRYBOdAioIiqlj9tbW0xY8aMWs0CUIgWATnQIiAHWgQU4bYvAAAAgBJLdtvXwoULY/r06UmGAhiIFgE50CIgB1oEFFF4+dPe3h7t7e19PrdkyZJkAwEMRIuAHGgRkAMtAopKctvX7t27o7m5OcVRAIOmRUAOtAjIgRYBRyp05U9ra2u/z61duza6urrijDPOSDYUQF+0CMiBFgE50CKgGoWWP52dncc9VqlUoqGhIXp6emLnzp0xadKk5MMBHEmLgBxoEZADLQKqUdULPh9pwYIFsW3btujt7Y05c+bE2LFjU84FUIgWATnQIiAHWgT0Z9DLnxdeeCHlHACDokVADrQIyIEWAf1J8oLPAAAAAOTJ8gcAAACgxAZ921dOWifcUO8RTujas26s9wgDyn2+WjjlumlJz6ts3Jz0PAAYjD9+8mbS86b8YGbS8yIiOj99K+l5qb8W3LDrjaTnTZt4fdLzIiIaGpqSn5mSr4sA8uLKHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKLFBLX+WLVsWq1atSjwKQHW0CMiBFgE50CJgIEmu/Pn666/jk08+SXEUwKBpEZADLQJyoEXAkQa9/NmxY8fhH7e1tcWvf/3rJAMBVEOLgBxoEZADLQL6M6rIG7W2tkZERG9vb0yZMiUmTJgQb731Vtxzzz3x7LPPxvbt2+OVV16p6aAAWgTkQIuAHGgRUI1Cy5/Ozs6IiFi5cmU0NjZGT09PjB8/PmbMmBHTpk2LNWvWxNixY2s6KIAWATnQIiAHWgRUo6rbvjZs2BA33HBDRETMnj07/vmf/znuuusuUQFOKi0CcqBFQA60CCii8PJn8+bNce6558aZZ54ZERFjxoyJBQsWxCOPPBK9vb01GxDgSFoE5ECLgBxoEVBUoeXP3r174+mnn4777rvvqMdnzpwZLS0t8eijj8aBAwdqMiDAt7QIyIEWATnQIqAahZY/9957b9x2223R0tJy3HNLly6NXbt2xV133ZV8OIAjaRGQAy0CcqBFQDUKLX9aW1tj9uzZ0d3dHRERBw4ciIaGhoiIGDt2bKxatSqamppqNyVAaBGQBy0CcqBFQDUKfbevO+64IyIiOjo6YtOmTfHVV1/FsmXLDj8/bty4WL58eW0mBPgvWgTkQIuAHGgRUI2GSqVSqc+7/iwiDtbnXUNENEybmvS8ysbNSc/LW2NEHH+J8fCkRdSXFg2FFvXlj5+8meScb035wcyk50VEdH76VtLzWifckPS8DbveSHretInXJz0vIqKhwRUdeSlLj/L9uij1/y8jRtr/MxkZ+m9RVd/qHQAAAIDhxfIHAAAAoMQKveYPAADDw4FDh+o9wrDX1JD230dH4i1alcrXSc8biX+Gw90/ev5n0vNWvTAv6XkREV2JZ/yXf/rvSc/b1f3npOedNe6qpOcxvLjyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDECi9/Lr300vjpT39ay1kATkiLgBxoEZADLQKKKrz8ufLKK+P000+v5SwAJ6RFQA60CMiBFgFFVXXb1+LFi2s0BkBxWgTkQIuAHGgRUMSoom+4YMGCmDx5ci1nATghLQJyoEVADrQIKKrwlT+iAuRAi4AcaBGQAy0CivLdvgAAAABKrPBtX1BvDdOmJj2vsnFz0vMAgGIOVir1HmFAh+o9QAk0NDTVewTq7MP/+1nS8/7H+bckPS8i4o+fvJn0vH/5p6THxbb/k/bP8KxxSY9jmHHlDwAAAECJWf4AAAAAlJjlDwAAAECJWf4AAAAAlJjlDwAAAECJWf4AAAAAlFjVy58DBw7ExRdfXItZAArTIiAHWgTkQIuAE3HlDwAAAECJWf4AAAAAlNiowfyi3t7euPzyy4967Oabb44HH3wwyVAARWgRkAMtAnKgRcBABrX8GT16dGzZsiX1LABV0SIgB1oE5ECLgIG47QsAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAEqs0As+H/uq8WPGjDnusYjwAmNATWkRkAMtAnKgRUA1Ci1/BAPIgRYBOdAiIAdaBFTDbV8AAAAAJWb5AwAAAFBihW77AgBgeGhoaKj3CMNepVKp9wgw7DUOgxblPmPvoYP1HoESceUPAAAAQIlZ/gAAAACUmOUPAAAAQIlZ/gAAAACUmOUPAAAAQIlZ/gAAAACUWKFv9d7a2nrUz3fv3h2nn356nHLK0bujzs7OdJMBHEOLgBxoEZADLQKqUWj5c2wwZs6cGS+++GJMnDixJkMB9EWLgBxoEZADLQKq4bYvAAAAgBIb9PJn69atKecAGBQtAnKgRUAOtAjoz6CXP88//3zcf//98cUXX6ScB6AqWgTkQIuAHGgR0J9BL39WrlwZP/zhD2PWrFnx5z//OeVMAIVpEZADLQJyoEVAfwq94HNfGhsbY/78+XHxxRfHunXr4qqrrko5F0AhWgTkQIuAHGgR0J9BL3++NWXKlJgyZUqKWQAGTYuAHGgRkAMtAo7lu30BAAAAlJjlDwAAAECJDeq2rzfffDP1HABV0yIgB1oE5ECLgIG48gcAAACgxCx/AAAAAEqsoVKpVOrzrj+LiIP1edcQEQ3TpiY9r7Jxc9Lz8tYYES31HiIRLaK+tGgotAhyNfLaVpYe5dui1B9TEcPh4wqq1X+LXPkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlNqrIG7W2tp74oFGj4p133hnyQAD90SIgB1oE5ECLgGoUWv50dnbWeg6AE9IiIAdaBORAi4BquO0LAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKbFS9BwAAIJ3N//nvSc+bOnFW0vMiIv7jH+uTnvev378+6Xmbdv0+6XlTJ/5b0vMiIhoampKfmVJl4+Z6j0DJ+JiirBqmTU121qSWCbFzdd+fK678AQAAACgxyx8AAACAErP8AQAAACgxyx8AAACAErP8AQAAACixQS1/VqxYEd3d3alnAaiKFgE50CIgB1oEDKTq5c+2bdvi9ddfj+985zu1mAegEC0CcqBFQA60CDiRUUXe6I033ognnngiTjvttMOPXX311Yd/3N3dHfPnz4958+YlHxDgW1oE5ECLgBxoEVCNQsufiIibbropFi1a1Odzy5YtSzYQwEC0CMiBFgE50CKgKC/4DAAAAFBiha/8WbduXWzYsKHP5769pBCg1rQIyIEWATnQIqAot30Bw4oWATnQIiAHWgQU5bYvAAAAgBKz/AEAAAAoMa/5AwwrWgTkQIuAHGgRUFSh5c+oUaNizpw58fDDD/f5vPtJgZNBi4AcaBGQAy0CqtFQqVQq9XnXn0XEwfq8a4iIhmlTk55X2bg56Xl5a4yIlnoPkYgWUV9aNBRa1JfN//nvSc751tSJs5KeFxHxH/9Yn/S8f/3+9UnP27Tr90nPmzrx35KeFxHR0NCU/EyGoiw98nURnGwpvxac1DIhdq7u+2tBr/kDAAAAUGKFX/MnPXsn6mtSy4TEJzYmPi9nZfr8LdPvheFIi4aiTJ+/6X4vpzaeluysb6T/mBrTOC7xiWlnHA5/hiPrc304KEuPyvL7gOEj5deCE8f3fwViHW/7AgAAAKDWrHYBAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASqxx8eLFi+s9RC21tbVFT09PnHvuufUepU+1nO/JJ5+ML774Ii644IIhnVPLGZctWxZ///vf40c/+lHysxmcFStWxPnnnx+jR4+u9yilokVaRHW0qDZGcosi0vRIi0YWLaoNLdIiqpOiRaMSzjMka9asiRUrVhz++b59+6KpqSlOPfXUw4899thjcc0119RjvIiI2LJlSzzwwAMDvs3y5cvjwgsvPEkTlc/XX38dn3/+efzgBz+o9yhx4MCBuPTSS+Nvf/vboM9obW096ue7d++O008/PU455eiL7jo7Owf9PlLatm1bvP7663HnnXfWe5S6yr1HWlR7ZWtRxPDqkRZ9Q4vQovrSom9oETm1KGLk/T0tVYuyWf7MnTs35s6de/jnjz/+eFx22WVx/fXXD/ns9vb2WLp0aZ/PNTc3x/r16wudc/nll9fkP36q+T788MMYP358jB8/vs/n33///Rg9enRMnjy5bjP2ZceOHYd//O0G+/nnnx/0eTk59uNl5syZ8eKLL8bEiRPrNNHx3njjjXjiiSfitNNOO/zY1VdfffjH3d3dMX/+/Jg3b14dpquPWvVIi76hRfWRe4+06HhadLxqP89r1SMtGjwtGn606HhaNPyNxBZls/w51sGDB6OxsTHJWW1tbTFjxowkZ9VCqvlWrVoVU6dOjalTp/b5/DvvvBOnnXbaoP7ClWrGbzesvb29MWXKlJgwYUK89dZbcc8998Szzz4b27dvj1deeWXI74fq3HTTTbFo0aI+n1u2bNlJniY/qXqkRd/QIvqjRQPTourVqkdaVG5aNDAtqp4WMRipW5TN8ufYy66+/PLL2LBhQzz11FNHPV7Py66OnbE/OVwalqtv/2xWrlwZjY2N0dPTE+PHj48ZM2bEtGnTYs2aNTF27Ng6T/n/9fb2xuWXX37UYzfffHM8+OCDgz5z69at2WyU6VvuPdKiodOib+hR3rSo/LToG1qUNy0qv+HWogh/TxuMbJY/x34yXnfddXHBBRdER0fHkM8e6HK4hQsXxvTp06uesaOjI3bt2hXPPPNMNvPVUuoZN2zYEMuXL4/XXnstZs+eHb/73e/irrvuyi4qo0ePji1btiQ98/nnn49NmzbFokWL+r30cyBF/gc3atSoeOeddwqfuW7dutiwYUOfz317SeFIUqseadHQaVFaQ+mRFtWeFg1tvlrSorS0KG9aNLT5ammktigiv7+nDYcWZbP8OdKOHTtiz5498cEHH8R77703pFcZb29vj/b29j6fW7JkyaDO7O7ujjVr1sRvfvObiIjo6uqKjz766Kh78Oo5X2qpZ9y8eXOce+65ceaZZ0ZExJgxY2LBggXxyCOPxOrVq0v/3RRWrlwZb775ZsyaNSueeuqpuOqqq6r69bX4VwuXN/cvVY+0aOi0KL2h9EiLTi4t0qIy06LhQ4u0qMzK3qJTTvwmJ9ehQ4di8eLFMXfu3Fi0aFHMnz8/urq6avK+du/eHc3NzVX/ulWrVsVFF10Ul1xySURE7N+/PxYuXBj79+/PYr6TqdoZ9+7dG08//XTcd999Rz0+c+bMaGlpiUcffTQOHDiQesysNDY2xvz58+OJJ56IdevW1XscBnCyeqRFQ6dFg6NHw4MWDW2+k0mLBkeLhgctGtp8J5MWDU7ZW5TVlT979uyJX/3qV1GpVOLuu++OpqamuOWWW2L27NnR0dERP/7xj6s6b6BLr9auXRtdXV1xxhlnVHXm1q1b46WXXopXX3318GPnnHNOXHLJJfHqq6/GHXfcUdf5Uks947333hu33XZbtLS0HPfc0qVL49Zbb4277rorfvvb3xY+89JLL43vfve7VV1Cl4MpU6bElClT6j0G/UjZIy0aOi2qLT3KlxYNbb7UtKi2tChfWjS0+VLTotoqa4uyWf50dHTEq6++GtOnT4+nn346mpqaIiLizjvvjO9973tx++23x09+8pN47rnnCp/Z16VXlUolGhoaoqenJ3bu3BmTJk0qfN7evXvjvvvui3nz5kVLS0t89NFHsXPnzvj4449j9+7dsW7duqrCknq+iIhFixYdvvxv37590dTUdPh+3MHcF5h6xtbW1pg9e3Z0d3fHuHHj4sCBA9HQ0BAREWPHjo1Vq1bFQw89VNWMV155ZXz66adV/RqO5t72o6XukRZpEcVo0dG0aOgtikjbIy0aGbToaFqkRVpUH6V9zZ89e/bE6tWr47zzzjvuuVmzZsUVV1wRL7/88pDfz4IFC2Lbtm3R29sbc+bMqerFq3p6euLzzz+P1157LdavXx+nn356nHXWWXH22WfHrbfeGs8991y8++67cdlll9VlvmNfcf9Yqe5RHsqM34a3o6MjNm3aFF999dVRc40bNy6WL19e9UyLFy+u+tf05dhXjB8zZsxxj0VETV7ssF5GjRoVc+bMiYcffrjP50five0no0daNHRapEVlp0VDmy/i5PRIi7So7LRIi+rdooiR16NatKihUqlUhjoY3/jwww/j7LPPHhEvhpWT7du3x+TJk+s9BmRDi+pDi+BoWlQfWgRH06L60KL8WP4AAAAAlFh23+0LAAAAgHQsfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMT+H6FiWPV3o+sgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder Layer 4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAej0lEQVR4nO3da4yV5dno8WucAWVXCw3Y2UUMVIOnaKzWxOzoHotO8Q3EwttIE0G6TTwQowU809A94oiJNB5GSRPQNhIJxBLkVYlNOITuzm6zv1hjLFoPicGg7MFtCApOzAis/cEXymFmeNbMvVj3PPx+n2St4V5XVvXPcPV51jRUKpVKAAAAAFBKp9R7AAAAAABqx/IHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKrKl+L/3/IuJA/V4eMjdh9qzkZ25buSrRSadExJmJzqqvq++ZFZ98vjPJWeneX6CY8rQo5fdF/3vHhiTnHPTfx16f9LyIiL/+341Jz7v6B5OTnvd/ujYlPe+//defJj2PHJWjR74vgqGu7xbVcflzICL21+/lIXMf79xRg1P9N3e0Tz7fmfC99v4CA5Xu+6Kv9+9Jcs6/pG9b7jPmPh/Uiu+LoLzc9gUAAABQYpY/AAAAACVm+QMAAABQYoU+82fVqlWxfPnyQ7/es2dPDBs2LE477bRDjz388MNx3XXXpZ8Q4D9pEZADLQJyoEVANQotf2bNmhWzZv3rJw898sgjcfnll8cNN9xQs8EAjqZFQA60CMiBFgHVGNBtX/v374/GxsbUswBURYuAHGgRkAMtAvpT6MqflpaWI3795ZdfxoYNG+Lxxx8/4vHOzs50kwEcRYuAHGgRkAMtAqpRaPlzdDCuv/76uPDCC6Ojo6MmQwH0RouAHGgRkAMtAqpR9W1fH330UezevTvefffdeOutt2oxE8BxaRGQAy0CcqBFwPFUtfw5cOBALFq0KGbNmhULFy6MuXPnRldXV61mA+iVFgE50CIgB1oEFFF4+bN79+6YO3duVCqVuPPOO+Oaa66Jm2++OWbMmBF///vfazkjwCFaBORAi4AcaBFQVKHlT0dHR0yePDnGjBkTzz//fAwbNiwiIu64446477774rbbbot77rmnpoMCaBGQAy0CcqBFQDUKfeDz7t27Y+XKlXH++ecf89z06dPjyiuvjBdeeCH5cACH0yIgB1oE5ECLgGo0VCqVSn1eemdE7K/PS8MQ0DC5NfmZlY2bE53UGBHNic6qrwmzW+PjnTuSnJXu/QWKKU+LUn5ftGn7fyQ556Cfnv3vSc+LiNj8yStJz2sdNz3peX/+9NWk5006a1rS88hROXrk+yIY6vpuUdU/7QsAAACAocPyBwAAAKDESnHbV+6XDkfkP+OWxPNdW4P3kJyU49Lmb7kFlXJJfcto3pftaxEwcCl7Ob55bGxbmXMvi3HbFwx1bvsCAAAAOClZ/gAAAACUmOUPAAAAQIlZ/gAAAACUmOUPAAAAQIlZ/gAAAACUWOHlT1tbW7z++uu1nAXguLQIyIEWATnQIqAoV/4AAAAAlFhTNV/c3t4eS5Ys6fW5kSNHxvr165MMBdAfLQJyoEVADrQIKKKq5U9bW1tMnTq1VrMAFKJFQA60CMiBFgFFuO0LAAAAoMSS3fa1YMGCmDJlSpKhAPqjRUAOtAjIgRYBRRRe/rS3t0d7e3uvzy1evDjZQAD90SIgB1oE5ECLgKKS3Pa1a9euGDlyZIqjAAZMi4AcaBGQAy0CDlfoyp+WlpY+n1uzZk10dXXF6NGjkw0F0BstAnKgRUAOtAioRqHlT2dn5zGPVSqVaGhoiO7u7ti2bVuMHz8++XAAh9MiIAdaBORAi4BqVPWBz4ebN29ebN26NXp6emLmzJkxYsSIlHMBFKJFQA60CMiBFgF9GfDy59lnn005B8CAaBGQAy0CcqBFQF+SfOAzAAAAAHlqqFQqlfq89M6I2F+fl4YhoGFya/IzKxs3JzqpMSKaE51VXxNmt8bHO3ckOSvd+wsUU54WtT7/P2LHl58lOeut+f8zyTkHDW+ckPS8iIie/duSnpd6xtznI0fl6JHvi2Co67tFrvwBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKLGmIl/0xhtvxL333tvv1yxbtiwuuuiiJEMB9EaLgBxoEZADLQKqUWj5c8UVV0RnZ2etZwHolxYBOdAiIAdaBFTDbV8AAAAAJVboyp+WlpZCh9k8A7WkRUAOtAjIgRYB1Si0/Dk8GB0dHbF9+/Z48sknazYUQG+0CMiBFgE50CKgGlXd9rV3795YtWpV/PKXv4yIiK6urvjLX/5Sk8EA+qJFQA60CMiBFgFFVLX8WbFiRVx88cVx6aWXRkTEV199FQsWLIivvvqqJsMB9EaLgBxoEZADLQKKKLz8efvtt+P555+P+++//9Bj5557blx66aWxevXqmgwHcDQtAnKgRUAOtAgoqtBn/nzxxRcxf/78uOWWW6K5uTnef//92LZtW3zwwQexa9euWLt2bdx+++21nhU4yWkRkAMtAnKgRUA1Cl35093dHZ999lm89NJL8Ytf/CJ+/etfx5/+9Kc4cOBAzJ49O7755pt48803az0rcJLTIiAHWgTkQIuAahS68ucHP/hBbN26tc/nJ06cGOecc06yoQB6o0VADrQIyIEWAdUotPw5ngsuuCDFMQCDokVADrQIyIEWAYer6qd9AQAAADC0NFQqlUp9XnpnROxPctLmT15Jcs5BreOmJz0vIv8ZtySe79oavIfkpDEimus9RCLpWgQ5aJjcmvS8ysbNSc9LS4uAgUvZy/HNY2Pbypx7WcyE2a3x8c4dSc7K+88PKKu+vzdy5Q8AAABAiVn+AAAAAJSY5Q8AAABAiVn+AAAAAJSY5Q8AAABAiRVe/rS1tcXrr79ey1kAjkuLgBxoEZADLQKKcuUPAAAAQIk1VfPF7e3tsWTJkl6fGzlyZKxfvz7JUAD90SIgB1oE5ECLgCKqWv60tbXF1KlTazULQCFaBORAi4AcaBFQhNu+AAAAAEos2W1fCxYsiClTpiQZCqA/WgTkQIuAHGgRUETh5U97e3u0t7f3+tzixYuTDQTQHy0CcqBFQA60CCgqyW1fu3btipEjR6Y4CmDAtAjIgRYBOdAi4HCFrvxpaWnp87k1a9ZEV1dXjB49OtlQAL3RIiAHWgTkQIuAahRa/nR2dh7zWKVSiYaGhuju7o5t27bF+PHjkw8HcDgtAnKgRUAOtAioRlUf+Hy4efPmxdatW6OnpydmzpwZI0aMSDkXQCFaBORAi4AcaBHQlwEvf5599tmUcwAMiBYBOdAiIAdaBPQlyQc+AwAAAJAnyx8AAACAEmuoVCqV+rz0zojYX5+XhiGgYXJr8jMrGzcnOqkxIpoTnVVfE2a3xsc7dyQ5K937CxSjRb3RIqiHcvRIi2Co67tFrvwBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKLEBLX8ee+yxWLduXepZAKqiRUAOtAjIgRYB/XHlDwAAAECJFVr+vPfee/H555/3+fw777wTH374YbKhAHqjRUAOtAjIgRYB1Si0/FmxYkW89dZbfT6/ZcuW+Nvf/pZsKIDeaBGQAy0CcqBFQDXc9gUAAABQYpY/AAAAACVm+QMAAABQYpY/AAAAACVm+QMAAABQYk1Fv3DhwoXR3t4eERF79uyJYcOGRUdHR0RE7N27N+bOnVubCQEOo0VADrQIyIEWAUUVWv48/vjj/T6/dOnSJMMA9EeLgBxoEZADLQKqUfjKn/786le/SnEMwKBoEZADLQJyoEXA4XzmDwAAAECJWf4AAAAAlFhDpVKp1Oeld0bE/iQnbf7klSTnHNQ6bnrS8yLyn3FL4vmurcF7SE4aI6K53kMkkq5FkIOGya1Jz6ts3Jz0vLS0CBi4lL0c3zw2tq3MuZfFTJjdGh/v3JHkrLz//ICy6vt7I1f+AAAAAJSY5Q8AAABAiVn+AAAAAJSY5Q8AAABAiVn+AAAAAJRY4eVPW1tbvP7667WcBeC4tAjIgRYBOdAioChX/gAAAACUWFM1X9ze3h5Llizp9bmRI0fG+vXrkwwF0B8tAnKgRUAOtAgooqrlT1tbW0ydOrVWswAUokVADrQIyIEWAUW47QsAAACgxJLd9rVgwYKYMmVKkqEA+qNFQA60CMiBFgFFFF7+tLe3R3t7e6/PLV68ONlAAP3RIiAHWgTkQIuAopLc9rVr164YOXJkiqMABkyLgBxoEZADLQIOV+jKn5aWlj6fW7NmTXR1dcXo0aOTDQXQGy0CcqBFQA60CKhGoeVPZ2fnMY9VKpVoaGiI7u7u2LZtW4wfPz75cACH0yIgB1oE5ECLgGpU9YHPh5s3b15s3bo1enp6YubMmTFixIiUcwEUokVADrQIyIEWAX0Z8PLn2WefTTkHwIBoEZADLQJyoEVAX5J84DMAAAAAebL8AQAAACixhkqlUqnPS++MiP31eWkYAhomtyY/s7Jxc6KTGiOiOdFZ9TVhdmt8vHNHkrPSvb9AMeVpUcrvizZsX5fknIOuP/vnSc+LiNj8yStJz2sdNz3peX/+9NWk5006a1rS88hROXrk+yIY6vpukSt/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxAa0/Fm6dGmsWLEi8SgA1dEiIAdaBORAi4D+JLny55tvvolPP/00xVEAA6ZFQA60CMiBFgGHG/Dy56OPPjr0z21tbfHb3/42yUAA1dAiIAdaBORAi4C+NBX5opaWloiI6OnpiUmTJsXYsWPjtddei7vuuiueeuqp+PDDD+PFF1+s6aAAWgTkQIuAHGgRUI1Cy5/Ozs6IiHjuueeisbExuru7Y8yYMTF16tSYPHlyrFq1KkaMGFHTQQG0CMiBFgE50CKgGlXd9rVhw4b42c9+FhERM2bMiO9+97sxZ84cUQFOKC0CcqBFQA60CCii8PJn8+bNcd5558WZZ54ZERGnnnpqzJs3Lx588MHo6emp2YAAh9MiIAdaBORAi4CiCi1/vvjii3jiiSdi/vz5Rzw+bdq0aG5ujoceeij27dtXkwEBDtIiIAdaBORAi4BqFFr+3H333XHrrbdGc3PzMc8tWbIktm/fHnPmzEk+HMDhtAjIgRYBOdAioBqFlj8tLS0xY8aM2Lt3b0RE7Nu3LxoaGiIiYsSIEbFixYoYNmxY7aYECC0C8qBFQA60CKhGoZ/2dfvtt0dEREdHR2zatCm+/vrrWLp06aHnTz/99Fi2bFltJgT4T1oE5ECLgBxoEVCNhkqlUqnPS++MiP31eWkYAhomtyY/s7Jxc6KTGiPi2EuMh6IJs1vj4507kpyV7v0FiilPi1J+X7Rh+7ok5xx0/dk/T3peRMTmT15Jel7ruOlJz/vzp68mPW/SWdOSnkeOytEj3xfBUNd3i6r6Ue8AAAAADC2WPwAAAAAlVorbvnK/dDgiYtP2/0h63k/P/vek521J/B5eW4P3kJyU49Lmb7kFlXJJfcto3pftaxEwcCl7Ob55bGxbmXMvi9Ii6E/eH80R4bYvAAAAgJOU5Q8AAABAiVn+AAAAAJSY5Q8AAABAiVn+AAAAAJSY5Q8AAABAiVn+AAAAAJSY5Q8AAABAiVn+AAAAAJSY5Q8AAABAiVn+AAAAAJSY5Q8AAABAiVn+AAAAAJRY4eXPZZddFtdee20tZwE4Li0CcqBFQA60CCiq8PLnqquuilGjRtVyFoDj0iIgB1oE5ECLgKKquu1r0aJFNRoDoDgtAnKgRUAOtAgooqnoF86bNy8mTpxYy1kAjkuLgBxoEZADLQKKKnzlj6gAOdAiIAdaBORAi4Ci/LQvAAAAgBIrfNsXg9PQ0FDvEYAa+187Xk163k/GTkt6XkTE5k9eSXpe67jpSc/bsH1d0vOuP/vnSc8bCiobN9d7BACGqAmzZ8XHO3ckOcufR5TRUP732pU/AAAAACVm+QMAAABQYpY/AAAAACVm+QMAAABQYpY/AAAAACVm+QMAAABQYlUvf/bt2xeXXHJJLWYBKEyLgBxoEZADLQKOx5U/AAAAACVm+QMAAABQYk0D+U09PT1xxRVXHPHYTTfdFPfdd1+SoQCK0CIgB1oE5ECLgP4MaPkzfPjweOONN1LPAlAVLQJyoEVADrQI6I/bvgAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQKfeDz0Z8af+qppx7zWET4gDGgprQIyIEWATnQIqAahZY/ggHkQIuAHGgRkAMtAqrhti8AAACAErP8AQAAACixQrd95a513PR6j3Bcuc94bebzRUQ0TG5Nel5l4+ak5zE0TZg9Kz7euSPJWUPh36ncW3T92T+v9wgAZGj/gd3pDz094V+F/ksp/loFlJgrfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQK/UzClpaWI369a9euGDVqVJxyypG7o87OznSTARxFi4AcaBGQAy0CqlFo+XN0MKZNmxa/+93vYty4cTUZCqA3WgTkQIuAHGgRUA23fQEAAACU2ICXP2+//XbKOQAGRIuAHGgRkAMtAvoy4OXPM888E/fcc098/vnnKecBqIoWATnQIiAHWgT0ZcDLn+eeey5++MMfxvTp0+Ovf/1rypkACtMiIAdaBORAi4C+FPrA5940NjbG3Llz45JLLom1a9fG1VdfnXIugEK0CMiBFgE50CKgLwNe/hw0adKkmDRpUopZAAZMi4AcaBGQAy0CjuanfQEAAACUmOUPAAAAQIkN6LavV199NfUcAFXTIiAHWgTkQIuA/rjyBwAAAKDELH8AAAAASmzQP+0LTpTKxs31HuGEapjcmvzMk+09BADKoenfbkx+ZtrvixoTngWQnit/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxJqKfFFLS8vxD2pqii1btgx6IIC+aBGQAy0CcqBFQDUKLX86OztrPQfAcWkRkAMtAnKgRUA13PYFAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAl1lTvAQAATnYTZs+Kj3fuSHJWZePmJOdATvx7DTA4rvwBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKLEBLX+WL18ee/fuTT0LQFW0CMiBFgE50CKgP1Uvf7Zu3Rovv/xyfOc736nFPACFaBGQAy0CcqBFwPE0FfmidevWxaOPPhpnnHHGoceuueaaQ/+8d+/emDt3btxyyy3JBwQ4SIuAHGgRkAMtAqpRaPkTEXHjjTfGwoULe31u6dKlyQYC6I8WATnQIiAHWgQU5QOfAQAAAEqs8JU/a9eujQ0bNvT63MFLCgFqTYuAHGgRkAMtAopy2xcwpGgRkAMtAnKgRUBRbvsCAAAAKDHLHwAAAIAS85k/wJCiRUAOtAjIgRYBRRVa/jQ1NcXMmTPjgQce6PV595MCJ4IWATnQIiAHWgRUo6FSqVTq89I7I2J/fV4ahoCGya3Jz6xs3JzopMaIaE50Vn1NmN0aH+/ckeSsdO8vUIwW9UaLoB7K0SMtgqGu7xb5zB8AAACAEiv8mT/p2TtBf8Y3j63BqY2JzinPf7/jxqT8f+lSvb9AMVrUOy2CE68cPdIiGOr6blEdb/sCAAAAoNbKsaIGAAAAoFeWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKNixYtWlTvIWqpra0turu747zzzqv3KL2q5XyPPfZYfP7553HhhRcO6pxazrh06dL45z//GT/60Y+Sn83ALF++PC644IIYPnx4vUcpFS3SIqqjRbVxMrcoIk2PtOjkokW1oUVaRHVStKgp4TyDsmrVqli+fPmhX+/ZsyeGDRsWp5122qHHHn744bjuuuvqMV5ERLzxxhtx77339vs1y5Yti4suuugETVQ+33zzTXz22Wdx1lln1XuU2LdvX1x22WXxj3/8Y8BntLS0HPHrXbt2xahRo+KUU4686K6zs3PAr5HS1q1b4+WXX4477rij3qPUVe490qLaK1uLIoZWj7ToW1qEFtWXFn1Li8ipRREn39/TUrUom+XPrFmzYtasWYd+/cgjj8Tll18eN9xww6DPbm9vjyVLlvT63MiRI2P9+vWFzrniiitq8j9+qvnee++9GDNmTIwZM6bX5995550YPnx4TJw4sW4z9uajjz469M8HN9jPPPPMgM/LydH/vkybNi1+97vfxbhx4+o00bHWrVsXjz76aJxxxhmHHrvmmmsO/fPevXtj7ty5ccstt9RhuvqoVY+06FtaVB+590iLjqVFx6r2v/Na9UiLBk6Lhh4tOpYWDX0nY4uyWf4cbf/+/dHY2JjkrLa2tpg6dWqSs2oh1XwrVqyI1tbWaG1t7fX5LVu2xBlnnDGgv3ClmvHghrWnpycmTZoUY8eOjddeey3uuuuueOqpp+LDDz+MF198cdCvQ3VuvPHGWLhwYa/PLV269ARPk59UPdKib2kRfdGi/mlR9WrVIy0qNy3qnxZVT4sYiNQtymb5c/RlV19++WVs2LAhHn/88SMer+dlV0fP2JccLg3L1cH35rnnnovGxsbo7u6OMWPGxNSpU2Py5MmxatWqGDFiRJ2n/Jeenp644oorjnjspptuivvuu2/AZ7799tvZbJTpXe490qLB06Jv6VHetKj8tOhbWpQ3LSq/odaiCH9PG4hslj9H/8d4/fXXx4UXXhgdHR2DPru/y+EWLFgQU6ZMqXrGjo6O2L59ezz55JPZzFdLqWfcsGFDLFu2LF566aWYMWNG/PGPf4w5c+ZkF5Xhw4fHG2+8kfTMZ555JjZt2hQLFy7s89LP/hT5A66pqSm2bNlS+My1a9fGhg0ben3u4CWFJ5Na9UiLBk+L0hpMj7So9rRocPPVkhalpUV506LBzVdLJ2uLIvL7e9pQaFE2y5/DffTRR7F79+54991346233hrUp4y3t7dHe3t7r88tXrx4QGfu3bs3Vq1aFb///e8jIqKrqyvef//9I+7Bq+d8qaWecfPmzXHeeefFmWeeGRERp556asybNy8efPDBWLlyZel/msJzzz0Xr776akyfPj0ef/zxuPrqq6v6/bX4fy1c3ty3VD3SosHTovQG0yMtOrG0SIvKTIuGDi3SojIre4tOOf6XnFgHDhyIRYsWxaxZs2LhwoUxd+7c6Orqqslr7dq1K0aOHFn171uxYkVcfPHFcemll0ZExFdffRULFiyIr776Kov5TqRqZ/ziiy/iiSeeiPnz5x/x+LRp06K5uTkeeuih2LdvX+oxs9LY2Bhz586NRx99NNauXVvvcejHieqRFg2eFg2MHg0NWjS4+U4kLRoYLRoatGhw851IWjQwZW9RVlf+7N69O37zm99EpVKJO++8M4YNGxY333xzzJgxIzo6OuLHP/5xVef1d+nVmjVroqurK0aPHl3VmW+//XY8//zzsXr16kOPnXvuuXHppZfG6tWr4/bbb6/rfKmlnvHuu++OW2+9NZqbm495bsmSJTF79uyYM2dO/OEPfyh85mWXXRbf+973qrqELgeTJk2KSZMm1XsM+pCyR1o0eFpUW3qULy0a3HypaVFtaVG+tGhw86WmRbVV1hZls/zp6OiI1atXx5QpU+KJJ56IYcOGRUTEHXfcEd///vfjtttui5/85Cfx9NNPFz6zt0uvKpVKNDQ0RHd3d2zbti3Gjx9f+Lwvvvgi5s+fH7fccks0NzfH+++/H9u2bYsPPvggdu3aFWvXrq0qLKnni4hYuHDhocv/9uzZE8OGDTt0P+5A7gtMPWNLS0vMmDEj9u7dG6effnrs27cvGhoaIiJixIgRsWLFirj//vurmvGqq66KHTt2VPV7OJJ724+UukdapEUUo0VH0qLBtygibY+06OSgRUfSIi3Sovoo7Wf+7N69O1auXBnnn3/+Mc9Nnz49rrzyynjhhRcG/Trz5s2LrVu3Rk9PT8ycObOqD6/q7u6Ozz77LF566aVYv359jBo1Ks4+++w455xzYvbs2fH000/Hm2++GZdffnld5jv6E/ePluoe5cHMeDC8HR0dsWnTpvj666+PmOv000+PZcuWVT3TokWLqv49vTn6E+NPPfXUYx6LiJp82GG9NDU1xcyZM+OBBx7o9fmT8d72E9EjLRo8LdKistOiwc0XcWJ6pEVaVHZapEX1blHEydejWrSooVKpVAY7GN9677334pxzzjkpPgwrJx9++GFMnDix3mNANrSoPrQIjqRF9aFFcCQtqg8tyo/lDwAAAECJZffTvgAAAABIx/IHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABKzPIHAAAAoMQsfwAAAABK7P8DMrLMJwnrY7MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder Layer 6\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfZBV9Zkn8OfS3TSdCI0DhlVxXbXwbeISjbXWrG4TxNIqWCNbG2Y3EGfdNeqmYsBEE5nBbbHVLUiZpJVJFeJMycaCMg46a1hTxeqQSVeyM7NDUo7BjMaJy5ZlpuOaHhDoYENz9w8Hwku/nNv9u9wfh8/nL/qey8PXpvtr34dzzq1Uq9VqAAAAAFBKExodAAAAAID6sfwBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASa27cH/3/IuJgkkk/+Pv/mWTOIdeceX3SeRH5Z8w9H7mZEBFnNDpEEtXq30fEYJJZlUpLkjmHHKzuTzovImJC8owDSedNqExMOu+C//Kfk86LiPj5g2uTz8xZtZrm++NI+6vvJJo0MSZOuDTRrEZL93PRrvd/kmTOIe2tlyWdF5F/xtzz1cNPfvXnSedNbW1LOu+c065KOi+9svxs5DXaeKTO+L8S5/uXXqOdAobvogYufw5Gqhdc+wZ3J5nzG+l/0M09Y+75oH4GI93Xa+qTKQ8knheRf8ampNP+b19v0nkfONX6rR7/vWmXhuWQ7ueig/HrJHN+I/3XQO4Zc89XDwMH9ySdd6CaZoHwG/l/DsvBa7Tx8RqNfLnsCwAAAKDELH8AAAAASszyBwAAAKDECt3zZ8OGDfH4448f/nj37t3R0tISkyZNOvzY/fffH/PmzUufEOAf6SIgB7oIyIEuAmpRaPmzZMmSWLJkyeGPH3jggbjiiivixhtvrFswgGPpIiAHugjIgS4CajGmy74GBwejqSntO7IA1EoXATnQRUAOdBEwkkJn/nR0dBz18XvvvRdbtmyJVatWHfV4T09PumQAx9BFQA50EZADXQTUotDy59jCuOGGG+KSSy6J7u7uuoQCGIouAnKgi4Ac6CKgFjVf9vXmm2/Gzp0746c//Wm8/PLL9cgEMCpdBORAFwE50EXAaGpa/hw8eDBWrlwZS5YsiRUrVsTSpUujt7e3XtkAhqSLgBzoIiAHuggoovDyZ+fOnbF06dKoVqvxuc99LubMmROf+cxnYtGiRfGjH/2onhkBDtNFQA50EZADXQQUVWj5093dHddff31Mnz49nnjiiWhpaYmIiNtvvz3uvvvu+OxnPxtf/OIX6xoUQBcBOdBFQA50EVCLQjd83rlzZzz11FNx0UUXHXds4cKFcdVVV8WTTz6ZPBzAkXQRkANdBORAFwG1KLT8Wbly5YjHzzzzzPiDP/iDFHkAhqWLgBzoIiAHugioRc3v9gUAAADAycPyBwAAAKDECl32BVBeTckm/fLXf5VsVkTEjLarks6LyD9j6nwDf7gx6byIiMHqe0nn/d2uv04677wpFyadN3HCOUnnRUQ0xdREk/wYM5QpE4+//0ducs+Ye76IiMHq7sTzqknn/df//ZdJ5z0xb07SeRERB6v9Cac1x4RKwnEl8FuTJjU6wqhyzzg183wREe8P/jzpvNamC5LOS/1zW1NlStJ5ERHV6mDSeZVhusiZPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKWPwAAAAAlZvkDAAAAUGKFlz+dnZ3xwgsv1DMLwKh0EZADXQTkQBcBRTnzBwAAAKDEmmt5cldXV6xevXrIY+3t7bF58+YkoQBGoouAHOgiIAe6CCiipuVPZ2dnLFiwoF5ZAArRRUAOdBGQA10EFOGyLwAAAIASS3bZ1/Lly2P+/PlJQgGMRBcBOdBFQA50EVBE4eVPV1dXdHV1DXnsoYceShYIYCS6CMiBLgJyoIuAopJc9tXX1xft7e0pRgGMmS4CcqCLgBzoIuBIhc786ejoGPbYM888E729vTFt2rRkoQCGoouAHOgiIAe6CKhFoeVPT0/PcY9Vq9WoVCrR398fO3bsiHPPPTd5OIAj6SIgB7oIyIEuAmpR0w2fj7Rs2bLYvn17DAwMxOLFi6OtrS1lLoBCdBGQA10E5EAXAcMZ8/LnscceS5kDYEx0EZADXQTkQBcBw0lyw2cAAAAA8jTmM38AymCw+l5E7E8ya0bbVUnmHFKtHkw6LyJ9xtRS53vi1f+WdF5ExG2//R+Szrto6ryk86rVwaTz6mHFXzyZZM7prVPj3o/fm2RWmfx9/98knTfzw1cnnReRf8bc80VENFUmJ513emvay4MenTM/6bx6mFD5UMJpTQlnlUPfvn2NjjCq3DPuzDxfRERr0wWNjjCipsqURkcYVaWSsj+Gn+XMHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASay7ypG3btsWXvvSlEZ+zdu3auPTSS5OEAhiKLgJyoIuAHOgioBaFlj9XXnll9PT01DsLwIh0EZADXQTkQBcBtXDZFwAAAECJFTrzp6Ojo9Awm2egnnQRkANdBORAFwG1KLT8ObIwuru746233oqvfe1rdQsFMBRdBORAFwE50EVALWq67GvPnj2xYcOG+L3f+72IiOjt7Y3vf//7dQkGMBxdBORAFwE50EVAETUtf9avXx8f/ehHY/bs2RERsXfv3li+fHns3bu3LuEAhqKLgBzoIiAHuggoovDy55VXXoknnngi7rnnnsOPXXDBBTF79uzYuHFjXcIBHEsXATnQRUAOdBFQVKF7/uzatSvuuuuuuOWWW2LGjBnx+uuvx44dO+JnP/tZ9PX1xaZNm+K2226rd1bgFKeLgBzoIiAHugioRaEzf/r7++Odd96Jp59+On73d383fv/3fz+++93vxsGDB+Pmm2+O/fv3x49//ON6ZwVOcboIyIEuAnKgi4BaFDrz58wzz4zt27cPe3zWrFlx/vnnJwsFMBRdBORAFwE50EVALQotf0Zz8cUXpxgDMC66CMiBLgJyoIuAI9X0bl8AAAAAnFySnPkDcLJqqkyJiMEks375679KMueQGW1XJZ0XkX/G1PluvfTfJ50XETFYfS/pvL/b9ddJ55035cKk8yZWzkk6LyLi4d/5j4km+TFmKGd+aHajI4wq94y554uIGKzuTjrvH97/ddJ5X9323aTznph3WdJ5EREHq/0JpzXHhErCcSXwW5MmNTrCqHLPODXzfBER7w/+POm81qYLks5L/XPbB68d0qpW07wWOaQyTBc58wcAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxAovfzo7O+OFF16oZxaAUekiIAe6CMiBLgKKcuYPAAAAQIk11/Lkrq6uWL169ZDH2tvbY/PmzUlCAYxEFwE50EVADnQRUERNy5/Ozs5YsGBBvbIAFKKLgBzoIiAHuggowmVfAAAAACWW7LKv5cuXx/z585OEAhiJLgJyoIuAHOgioIjCy5+urq7o6uoa8thDDz2ULBDASHQRkANdBORAFwFFJbnsq6+vL9rb21OMAhgzXQTkQBcBOdBFwJEKnfnT0dEx7LFnnnkment7Y9q0aclCAQxFFwE50EVADnQRUItCy5+enp7jHqtWq1GpVKK/vz927NgR5557bvJwAEfSRUAOdBGQA10E1KKmGz4fadmyZbF9+/YYGBiIxYsXR1tbW8pcAIXoIiAHugjIgS4ChjPm5c9jjz2WMgfAmOgiIAe6CMiBLgKGk+SGzwAAAADkyfIHAAAAoMTGfNkXQBm8vXdbDFb7k8w6d/K/SjLnkGp1f9J5EREz2q5KOq9aPZh0Xup8//Z/PJB0XkTEs//6/qTzLpo6L+m81H8n9bDz/deSzJlQaYvTW89PMqtM3t33atJ5qb8vI/LPmHu+iIimyuSk8/7h/X1J5/3hJxYlnVcPEyofSjitKeGscujbl/Zrqh5yz7gz83wREa1NFzQ6woiaKlMaHWFUlUrK/hh+ljN/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAEpsTMufhx9+OJ577rnUWQBqoouAHOgiIAe6CBiJM38AAAAASqzQ8ue1116Ld999d9jjr776arzxxhvJQgEMRRcBOdBFQA50EVCLQsuf9evXx8svvzzs8a1bt8YPf/jDZKEAhqKLgBzoIiAHugiohcu+AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAEqsuegTV6xYEV1dXRERsXv37mhpaYnu7u6IiNizZ08sXbq0PgkBjqCLgBzoIiAHuggoqtDyZ9WqVSMeX7NmTZIwACPRRUAOdBGQA10E1KLwmT8j+cIXvpBiDMC46CIgB7oIyIEuAo7knj8AAAAAJWb5AwAAAFBiSS77AjhZnf3hKyNiMMmsX/76r5LMOWRG21VJ50XknzF1vj9ZsDzpvIiIwep7Sef9/L1tSef9s8mzks6bWDkn6byIiKmtFyea5MeYoUyf9NuNjjCq3DPmni8iYrC6O+m801snJZ1355//SdJ5T8y7N+m8iIiD1f6E05pjQiXhuBL4rUlpv6bqIfeMUzPPFxHx/uDPk85rbbog6bzUP7c1VaYknRcRUa2meS1ySGWYLnLmDwAAAECJWf4AAAAAlJjlDwAAAECJWf4AAAAAlJjlDwAAAECJFV7+dHZ2xgsvvFDPLACj0kVADnQRkANdBBTlzB8AAACAEmuu5cldXV2xevXqIY+1t7fH5s2bk4QCGIkuAnKgi4Ac6CKgiJqWP52dnbFgwYJ6ZQEoRBcBOdBFQA50EVCEy74AAAAASizZZV/Lly+P+fPnJwkFMBJdBORAFwE50EVAEYWXP11dXdHV1TXksYceeihZIICR6CIgB7oIyIEuAopKctlXX19ftLe3pxgFMGa6CMiBLgJyoIuAIxU686ejo2PYY88880z09vbGtGnTkoUCGIouAnKgi4Ac6CKgFoWWPz09Pcc9Vq1Wo1KpRH9/f+zYsSPOPffc5OEAjqSLgBzoIiAHugioRU03fD7SsmXLYvv27TEwMBCLFy+Otra2lLkACtFFQA50EZADXQQMZ8zLn8ceeyxlDoAx0UVADnQRkANdBAwnyQ2fAQAAAMiT5Q8AAABAiY35si+AMqjGYEQcSDJr+qTfTjLnkGp1MOm8iPQZB6t7ks5Lna8SLUnnRURMqLQmnXdh+7VJ51WrB7OeFxFx+j3/Kcmcf3r6P4nt921KMqtMvvf2S0nnXTdzYdJ5Efln/H7ifNfW4XPYVJmcdN6/+Mj8tPPmpZ1XDxMqH0o4rSnhrHLo27ev0RFGlXvGnZnni4hobbqg0RFG1FSZ0ugIo6pUUvbH8LOc+QMAAABQYpY/AAAAACVm+QMAAABQYpY/AAAAACVm+QMAAABQYpY/AAAAACU2puXPmjVrYv369YmjANRGFwE50EVADnQRMJIkZ/7s378/3n777RSjAMZMFwE50EVADnQRcKQxL3/efPPNw7/u7OyMr371q0kCAdRCFwE50EVADnQRMJzmIk/q6OiIiIiBgYGYO3dunHXWWfGd73wnPv/5z8fXv/71eOONN+Jb3/pWXYMC6CIgB7oIyIEuAmpRaPnT09MTERHr1q2Lpqam6O/vj+nTp8eCBQvi+uuvjw0bNkRbW1tdgwLoIiAHugjIgS4CalHTZV9btmyJT37ykxERsWjRopgyZUrccccdSgU4oXQRkANdBORAFwFFFF7+vPTSS3HhhRfGGWecERERra2tsWzZsvjKV74SAwMDdQsIcCRdBORAFwE50EVAUYWWP7t27YpHHnkk7rrrrqMev+mmm2LGjBlx7733xoEDB+oSEOAQXQTkQBcBOdBFQC0KLX/uvPPOuPXWW2PGjBnHHVu9enW89dZbcccddyQPB3AkXQTkQBcBOdBFQC0KLX86Ojpi0aJFsWfPnoiIOHDgQFQqlYiIaGtri/Xr10dLS0v9UgKELgLyoIuAHOgioBaF3u3rtttui4iI7u7uePHFF2Pfvn2xZs2aw8dPO+20WLt2bX0SAvwjXQTkQBcBOdBFQC0KLX8Oue++++K+++6rVxaAQnQRkANdBORAFwFF1PRW7wAAAACcXCx/AAAAAEqspsu+AMpmQqUlUu3B/+ZX30sy55DZ025IOi8iYvuvfph0XuqMf/OrLUnn1eNzuHv/y0nnndb8z5POS61SSf/vRLse2ZBoUlOiOeUy7+wbGx1hVLlnnJt5voiIyo1p+626OW3/wtyzr210hFHlnnHO2fMaHWFUldv/TdJ51XV/mnQev+HMHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASK7z8ufzyy+Paa6+tZxaAUekiIAe6CMiBLgKKKrz8ufrqq2Pq1Kn1zAIwKl0E5EAXATnQRUBRNV32tXLlyjrFAChOFwE50EVADnQRUERz0ScuW7YsZs2aVc8sAKPSRUAOdBGQA10EFFX4zB+lAuRAFwE50EVADnQRUJR3+wIAAAAoscKXfQEwsjPapiSd13/glaTzItJnrFYHk85Ln29/0nkREZNbPpZ8Zs6q1YPJZ/7dez9IMqe58uE4b8rMJLPK5M/e3px03nUzFyadF5F/xu8lzndtHT6H1c1bks+ElL739tak8+rRRbln/P7bf5Z0Xl26aN2fJp9JfTjzBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASszyBwAAAKDELH8AAAAASqzm5c+BAwfisssuq0cWgMJ0EZADXQTkQBcBo3HmDwAAAECJWf4AAAAAlFjzWH7TwMBAXHnllUc99ulPfzruvvvuJKEAitBFQA50EZADXQSMZEzLn4kTJ8a2bdtSZwGoiS4CcqCLgBzoImAkLvsCAAAAKDHLHwAAAIASs/wBAAAAKDHLHwAAAIASK3TD52PvGt/a2nrcYxHhBmNAXekiIAe6CMiBLgJqUWj5ozCAHOgiIAe6CMiBLgJq4bIvAAAAgBKz/AEAAAAosUKXfQEwutf6fpl03rUzFyadFxHxl73/Pem8s2Y2JZ2X+nN41syWpPNORZVK+n8nmtU+N9GktF9/AABl5cwfAAAAgBKz/AEAAAAoMcsfAAAAgBKz/AEAAAAoMcsfAAAAgBKz/AEAAAAosUJv9d7R0XHUx319fTF16tSYMOHo3VFPT0+6ZADH0EVADnQRkANdBNSi0PLn2MK46aab4pvf/GbMnDmzLqEAhqKLgBzoIiAHugiohcu+AAAAAEpszMufV155JWUOgDHRRUAOdBGQA10EDGfMy59HH300vvjFL8a7776bMg9ATXQRkANdBORAFwHDGfPyZ926dXHeeefFwoUL4wc/+EHKTACF6SIgB7oIyIEuAoZT6IbPQ2lqaoqlS5fGZZddFps2bYprrrkmZS6AQnQRkANdBORAFwHDGfPy55C5c+fG3LlzU2QBGDNdBORAFwE50EXAsbzbFwAAAECJWf4AAAAAlNiYLvt6/vnnU+cAqJkuAnKgi4Ac6CJgJM78AQAAACgxyx8AAACAEhv3u30BnMz+z3t/EQeqe5PMunbmwiRzDqlWDyadF5F/xtT5/t13u5LOi4j49vzO5DNTSv13Uqmk/3eif3j/R0nmTIi2aG+dkWRWmcw9+9pGRxhV7hnnZJ6vHi5adUvSedvuvivpvMktH0s6D+BU48wfAAAAgBKz/AEAAAAoMcsfAAAAgBKz/AEAAAAoMcsfAAAAgBKz/AEAAAAoMcsfAAAAgBKz/AEAAAAoMcsfAAAAgBKz/AEAAAAoseYiT+ro6Bh9UHNzbN26ddyBAIaji4Ac6CIgB7oIqEWh5U9PT0+9cwCMShcBOdBFQA50EVALl30BAAAAlJjlDwAAAECJWf4AAAAAlJjlDwAAAECJWf4AAAAAlJjlDwAAAECJWf4AAAAAlJjlDwAAAECJNTc6AEAjnTfldyJiMMmsP//F80nmHDLnzBuTzotIn/ETZ92UdF7qfN+e35l0Xj1Uq/uTzqtUWpLOq4fTWz+eaFJTojnl8r23tyadd93MhUnnReSf8fuJ811bh89haq8vX9/oCADUkTN/AAAAAErM8gcAAACgxCx/AAAAAErM8gcAAACgxCx/AAAAAEpsTMufxx9/PPbs2ZM6C0BNdBGQA10E5EAXASOpefmzffv2ePbZZ+PDH/5wPfIAFKKLgBzoIiAHuggYTXORJz333HPx4IMPxuTJkw8/NmfOnMO/3rNnTyxdujRuueWW5AEBDtFFQA50EZADXQTUotDyJyLiU5/6VKxYsWLIY2vWrEkWCGAkugjIgS4CcqCLgKLc8BkAAACgxAqf+bNp06bYsmXLkMcOnVIIUG+6CMiBLgJyoIuAolz2BZxUdBGQA10E5EAXAUW57AsAAACgxCx/AAAAAErMPX+Ak4ouAnKgi4Ac6CKgqELLn+bm5li8eHF8+ctfHvK460mBE0EXATnQRUAOdBFQi0LLn09+8pMjHv/CF76QJAzASHQRkANdBORAFwG1cM8fAAAAgBIrfM+f9NLtnSY1TU426wNNieflnzH3fOSmTHtjXTQ+umj8DiaedzL8N6eii4ZyMnwf5Z4x93zkqCx9pIvGRxfRaMN/D1eq1Wr1BCYBAAAA4AQqy4oaAAAAgCFY/gAAAACUmOUPAAAAQIlZ/gAAAACUmOUPAAAAQIlZ/gAAAACUmOUPAAAAQIlZ/gAAAACUmOUPAAAAQIk1rVy5cmWjQ9RTZ2dn9Pf3x4UXXtjoKEOqZ76HH3443n333bjkkkvGNaeeGdesWRN/+7d/Gx/72MeSz2ZsHn/88bj44otj4sSJjY5SKrpIF1EbXVQfp3IXRaTpI110atFF9aGLdBG1SdFFzQnzjMuGDRvi8ccfP/zx7t27o6WlJSZNmnT4sfvvvz/mzZvXiHgREbFt27b40pe+NOJz1q5dG5deeukJSlQ++/fvj3feeSfOPvvsRkeJAwcOxOWXXx4/+clPxjyjo6PjqI/7+vpi6tSpMWHC0Sfd9fT0jPnPSGn79u3x7LPPxu23397oKA2Vex/povorWxdFnFx9pIs+oIvQRY2liz6gi8ipiyJOvddpqboom+XPkiVLYsmSJYc/fuCBB+KKK66IG2+8cdyzu7q6YvXq1UMea29vj82bNxeac+WVV9blLz9Vvtdeey2mT58e06dPH/L4q6++GhMnToxZs2Y1LONQ3nzzzcO/PrTBfvTRR8c8LyfHfr3cdNNN8c1vfjNmzpzZoETHe+655+LBBx+MyZMnH35szpw5h3+9Z8+eWLp0adxyyy0NSNcY9eojXfQBXdQYufeRLjqeLjperd/n9eojXTR2uujko4uOp4tOfqdiF2Wz/DnW4OBgNDU1JZnV2dkZCxYsSDKrHlLlW79+fVx33XVx3XXXDXl869atMXny5DG94EqV8dCGdWBgIObOnRtnnXVWfOc734nPf/7z8fWvfz3eeOON+Na3vjXuP4fafOpTn4oVK1YMeWzNmjUnOE1+UvWRLvqALmI4umhkuqh29eojXVRuumhkuqh2uoixSN1F2Sx/jj3t6r333ostW7bEqlWrjnq8kaddHZtxODmcGparQ5+bdevWRVNTU/T398f06dNjwYIFcf3118eGDRuira2twSl/Y2BgIK688sqjHvv0pz8dd99995hnvvLKK9lslBla7n2ki8ZPF31AH+VNF5WfLvqALsqbLiq/k62LIrxOG4tslj/HfjPecMMNcckll0R3d/e4Z490Otzy5ctj/vz5NWfs7u6Ot956K772ta9lk6+eUmfcsmVLrF27Np5++ulYtGhRfPvb34477rgju1KZOHFibNu2LenMRx99NF588cVYsWLFsKd+jqTI/+Cam5tj69athWdu2rQptmzZMuSxQ6cUnkrq1Ue6aPx0UVrj6SNdVH+6aHz56kkXpaWL8qaLxpevnk7VLorI73XaydBF2Sx/jvTmm2/Gzp0746c//Wm8/PLL47rLeFdXV3R1dQ157KGHHhrTzD179sSGDRvij/7ojyIiore3N15//fWjrsFrZL7UUmd86aWX4sILL4wzzjgjIiJaW1tj2bJl8ZWvfCWeeuqp0r+bwrp16+L555+PhQsXxqpVq+Kaa66p6ffX418tnN48vFR9pIvGTxelN54+0kUnli7SRWWmi04eukgXlVnZu2jC6E85sQ4ePBgrV66MJUuWxIoVK2Lp0qXR29tblz+rr68v2tvba/5969evj49+9KMxe/bsiIjYu3dvLF++PPbu3ZtFvhOp1oy7du2KRx55JO66666jHr/ppptixowZce+998aBAwdSx8xKU1NTLF26NB588MHYtGlTo+MwghPVR7po/HTR2Oijk4MuGl++E0kXjY0uOjnoovHlO5F00diUvYuyOvNn586dcd9990W1Wo3Pfe5z0dLSEp/5zGdi0aJF0d3dHR//+MdrmjfSqVfPPPNM9Pb2xrRp02qa+corr8QTTzwRGzduPPzYBRdcELNnz46NGzfGbbfd1tB8qaXOeOedd8att94aM2bMOO7Y6tWr4+abb4477rgj/viP/7jwzMsvvzxOP/30mk6hy8HcuXNj7ty5jY7BMFL2kS4aP11UX/ooX7pofPlS00X1pYvypYvGly81XVRfZe2ibJY/3d3dsXHjxpg/f3488sgj0dLSEhERt99+e3zkIx+Jz372s/GJT3wivvGNbxSeOdSpV9VqNSqVSvT398eOHTvi3HPPLTxv165dcdddd8Utt9wSM2bMiNdffz127NgRP/vZz6Kvry82bdpUU7GkzhcRsWLFisOn/+3evTtaWloOX487lusCU2fs6OiIRYsWxZ49e+K0006LAwcORKVSiYiItra2WL9+fdxzzz01Zbz66qvjF7/4RU2/h6O5tv1oqftIF+kiitFFR9NF4++iiLR9pItODbroaLpIF+mixijtPX927kpHOmAAAAFKSURBVNwZTz31VFx00UXHHVu4cGFcddVV8eSTT477z1m2bFls3749BgYGYvHixTXdvKq/vz/eeeedePrpp2Pz5s0xderUOOecc+L888+Pm2++Ob7xjW/Ej3/847jiiisaku/YO+4fK9U1yuPJeKh4u7u748UXX4x9+/Ydleu0006LtWvX1pxp5cqVNf+eoRx7x/jW1tbjHouIutzssFGam5tj8eLF8eUvf3nI46fite0noo900fjpIl1UdrpofPkiTkwf6SJdVHa6SBc1uosiTr0+qkcXVarVanW8wfjAa6+9Fueff/4pcTOsnLzxxhsxa9asRseAbOiixtBFcDRd1Bi6CI6mixpDF+XH8gcAAACgxLJ7ty8AAAAA0rH8AQAAACgxyx8AAACAErP8AQAAACgxyx8AAACAErP8AQAAACgxyx8AAACAEvv/fsojQb7vyfcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder Self Layer 2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfXTU9Z33/9d3JjO54SaoKAi41rRStkR/oVp7rG3UHliIPa60yNUjv27rarVbqyLdtsLSVduVrtg7uxWqVLq2l0V6qFxVziK0l3B+UerPC2843qCNUHFFIUZuQxImc/O5/kgTSbmZ+Sbvb+aTyfNxDufIJL7yZjLzmu985vOdCZxzTgAAAAAAAChJsWIPAAAAAAAAgOiw+AMAAAAAAFDCWPwBAAAAAAAoYSz+AAAAAAAAlDAWfwAAAAAAAEoYiz8AAAAAAAAljMUfAAAAAACAElZWrB+cdW9KyhTrx5+Qc53FHiGv//G/lpjm/fZz3zTNc8qa5klSYL5WGTfOC4zz7NlNGFcQnG6WVkwLnlqsPYf3mWQt/PiFJjndxg+bZJonSfFgmGleECRM85xL2+ZF0EXWjxEx89+JdbfZcy5nlBRXEJxmlFVc/7TudrW07zXJ+p+XzzLJ6VYWH2WaJ0llgW1mLKg0zcu6NtO8QLZd2Z3qb5pkfZwVBPavWdt1kVQqfdSWflpOh02yfvXaFpOcbheOHWuaJ0nnnPL/mObFg5NM86K43VtzzvZYazAcx/gtJunUY36laIs/XQs/Vos/ziinm+2TjyjsbmsxTrReiLN/wmW/Uc36duP/4s/gmHFg7Tm8T+927DHJyhg/WYimi6zvm9b3S+v5oniRwfr3EkVf+s7yCVdpaGnfa/bY7tRhkvO+KuM8yf/jDuv5onj8tc70/RjB+rhNoouO5nTYrEMOdO43yenWmRtumtfF98f0KG731obicczg5P9SIgAAAAAAAPqMxR8AAAAAAIASxuIPAAAAAABACevT4s/ChQu1evVq61kAIBS6CIAP6CIAPqCLAJwIO38AAAAAAABKWJ8Wf5xzyuV4d3wAxUUXAfABXQTAB3QRgBPp0+LPvn379NZbb1nPAgCh0EUAfEAXAfABXQTgREIv/uRyOb300kt64oknopgHAApCFwHwAV0EwAd0EYB8Qi/+bNiwQRMnTtSYMWP09NNPRzETAORFFwHwAV0EwAd0EYB8Qi3+HDp0SHfffbe+/OUv68tf/rIWL16sTCYT1WwAcEx0EQAf0EUAfEAXAShEwYs/nZ2duuWWW3TRRRfpE5/4hC688ELV1tbqtttuk3MuyhkBoAddBMAHdBEAH9BFAApV0OJPS0uLrr76alVVVWnBggU9ly9cuFBvv/22vva1r+nAgQORDQkAEl0EwA90EQAf0EUAwsi7+PPss8/qiiuuUF1dnX784x8rmUz2fK2yslLLli3T8OHDNX36dL3yyiuRDgtg6KKLAPiALgLgA7oIQFhl+b7hzDPP1NKlS1VXV3fMr5eXl+vuu+/Wpk2bVFNTYz4gAEh0EQA/0EUAfEAXAQgr7+LPqaeeqlNPPTVv0EUXXWQyEAAcC10EwAd0EQAf0EUAwgr9Ue8AAAAAAAAYPPLu/IlKoLgkq3egD4xyuuOM8yTlXJtpXjadNc1zss2LRtFurgWy/kSFnHGe5MzWe2PW97qi+R+TJqoj02qSdTibNsl5P+8t0zxJqoifYZoXV7VpnpPtdegiuR/Z3tedOk3z5JL5vyeUUrm3+23JjAZlXbtJ1u6OfSY53Toy75rmSdLE6rNN8xKx8aZ51o/p1t3WFWr7Ud4usD3OiqncNM//T67yfb7CJGMjJaPf3bmjTzHJ6bblvRbTPEmaMtr2OVo8sP03DwbmzyOd7XFHELDfpRvXBAAAAAAAQAlj8QcAAAAAAKCEsfgDAAAAAABQwlj8AQAAAAAAKGEs/gAAAAAAAJQwFn8AAAAAAABKWOjFH+ecWlttPhYZAPqKLgLgA7oIgA/oIgD5lOX7ht27d+u5557T9u3btXXrVr388stqaGjQ3LlztWfPnp7vi8fjmjBhQqTDAhi66CIAPqCLAPiALgIQVkGLP5s3b1ZNTY22bNmixsZGJZNJrV69Wvfdd5/OOussSdKrr76qxsbGyAcGMDTRRQB8QBcB8AFdBCCsvIs/dXV1qqurUyqV0v33369kMtnztYaGBs2bN0+SVF9fH92UAIY8ugiAD+giAD6giwCElfc9fx5//HHV19dr2rRp2rt3r+rr63X55ZcPxGwA0IMuAuADugiAD+giAGHlXfxpaGhQY2Oj5s+fr/Lycq1YsUJr1qwZiNkAoAddBMAHdBEAH9BFAMIq+NO+VqxYoUwmo2uvvVbvvfdelDMBwHHRRQB8QBcB8AFdBKBQBS3+PP/880okEqqurtasWbN00003RT0XAByFLgLgA7oIgA/oIgBh5F38aW9v14IFC3rK5LrrrtPVV18d9VwA0AtdBMAHdBEAH9BFAMLKu/izatUqTZ06VZMnT5YkBUGg6dOnRz4YAByJLgLgA7oIgA/oIgBh5f2o91mzZimZTMo5d9TXVq5cqfXr10cyGAAciS4C4AO6CIAP6CIAYeVd/Bk+fLgkKZVK9bp88uTJWrJkic4//3xJ0urVqyMYDwC60EUAfEAXAfABXQQgrMAda7l4AOTcDkkZo7TAKKeLM5vrfTnXZpr3yf/8nmneH6+xzYtCoGSxR8jD+q6UM86TQnzAXx5ligVnGGUV1//e+Qt1ZFpNsmqqR5nkdDtz+GmmeZJUEbf9vcWDatM8p1T+bwqVZ38/ci5tmhcLKkzz7LvS9jHWVlxBcHqxhzCxu32tsq7dJOtwttMkp1tHxvY2L0kTq882zUvExpvm5XTYNC+S+5EzPl4N8r4mHEpM5aZ5dscwUYkrCMYVe4h+68w+Jxk9Fv9h57MmOd12HjpkmidJ/zjpEtO8ROxvTPOCwPfbvZRzto85Qf79KeHyBsF1aCsuacwxv2J7zYbglJWUNckyv4EobprXlWk7Y1nSNs/6yUwQxUFOYPuExnpG+1XUKNZlbe5zfj8ZDOeDI09WxtkcoJbHEyY53falbBaljnRKxdumeZVx48Uf4wMI6yczkhQYZzqz+2U32yeE1o9ftory+lUkhicqzf41HR22i6g7D9l30dnVxvd1Y/6/4CS5IIoXiXwWxbFH6XSIFcvnaOecMtYkp9uplQdN8yTpz63bTfMmVhu/cOdsjy2jYP3CnTln2x1B4P/v5HiG2jIYAAAAAADAkMLiDwAAAAAAQAlj8QcAAAAAAKCEsfgDAAAAAABQwlj8AQAAAAAAKGEs/gAAAAAAAJQwFn8AAAAAAABKWFmh3zh//nxt3LhR5eXlvS53zimRSGjDhg3mwwHAX6OLAPiALgLgA7oIQKEKXvyRpEWLFmnq1Km9Ljt48KBmzpxpOhQAnAhdBMAHdBEAH9BFAAoRavFn4cKF+u53v9vrsu5VZQAYKHQRAB/QRQB8QBcBKAQ7fwAMOnQRAB/QRQB8QBcBKETBb/icTqdZPQZQdHQRAB/QRQB8QBcBKFTBO3/279+vW2+9VclkstflbCkEMJDoIgA+oIsA+IAuAlCoghZ/nHPasWOHGhsbjyoWthQCGCh0EQAf0EUAfEAXAQijoNO+Nm3apEmTJh1VKgAwkOgiAD6giwD4gC4CEEbexR/nnJYsWaIrr7yy1+XpdFqZTEbt7e0KgiCyAQFAoosA+IEuAuADughAWHlP+2pqalJVVZUuvfTSXpe3tLRo9uzZam9v14wZMyIbEAAkugiAH+giAD6giwCEFTjnXDF+cNZtl5QxyQrCfWJ9AeyvkpxrN827+NffM837//7ffzXNC2T/SkMQVNnmGc/ozG83NvePaJQpFnyg2EOYeOPg75RxbSZZ5XHbN1aM4vW6UyqqTfMq4x8xzcu5VtM8BdaPD5L9Y4TtbzqQ9e0wiuvQSlxBMK7YQ5g4lN4opw6TrOaOfSY53bbtt82TpEvH15rmJWNnmeY55UzzouCUMk4s+EOAC0wrN82T4sZ5km2fl0YfpbL/RzK6bTV37DLJ6ba7/aBpniSNTNreTidWX2iaZ/2YHoWcDpvmBcbdYf2cLwh8/53EJY055ldsWx4AAAAAAABeYfEHAAAAAACghBVtL3fOHZbUaZJlv600CrZbVc8/8wzTPOutw/anQEkx87VK69uN9Uk69if9OGWNkvzfDl+occPGyBltVw2M7+dtmQOmeZK0q32Pad4HhtueppV2tlvEy46z7bU/YkGFaZ4zPsUz52y3xceCkaZ51krl7UyT8VGSbE5vPrXC9jG4arTtbV6yfDzqzrP9N+eMTgc+ItE4T8q6/aZ51qeXuGCYaV4QRHG8b9kgZZGcmDbw0nJGz9FOqzzZJKfbqKTtbUqSDmVsT1nKuL2meXK2XRkYH8N0sT2Oicn29+wC41NaXRT7ZywfI3I63nu9s/MHAAAAAACghLH4AwAAAAAAUMJY/AEAAAAAAChhLP4AAAAAAACUMBZ/AAAAAAAASljoxR/nnFpbbT/dBQDCoosA+IAuAuADughAPnk/6n337t167rnntH37dm3dulUvv/yyGhoaNHfuXO3Z8/5HBsfjcU2YMCHSYQEMXXQRAB/QRQB8QBcBCKugxZ/NmzerpqZGW7ZsUWNjo5LJpFavXq377rtPZ511liTp1VdfVWNjY+QDAxia6CIAPqCLAPiALgIQVt7Fn7q6OtXV1SmVSun+++9XMpns+VpDQ4PmzZsnSaqvr49uSgBDHl0EwAd0EQAf0EUAwsr7nj+PP/646uvrNW3aNO3du1f19fW6/PLLB2I2AOhBFwHwAV0EwAd0EYCw8i7+NDQ0qLGxUfPnz1d5eblWrFihNWvWDMRsANCDLgLgA7oIgA/oIgBhFfxpXytWrFAmk9G1116r9957L8qZAOC46CIAPqCLAPiALgJQqIIWf55//nklEglVV1dr1qxZuummm6KeCwCOQhcB8AFdBMAHdBGAMPIu/rS3t2vBggU9ZXLdddfp6quvjnouAOiFLgLgA7oIgA/oIgBh5V38WbVqlaZOnarJkydLkoIg0PTp0yMfDACORBcB8AFdBMAHdBGAsPJ+1PusWbOUTCblnDvqaytXrtT69esjGQwAjkQXAfABXQTAB3QRgLDyLv4MHz5ckpRKpXpdPnnyZC1ZskTnn3++JGn16tURjAcAXegiAD6giwD4gC4CEFbgjrVcPADSuVckdZpkxVRukhMlJ9ur+Z+fetA07wefvMo0T8b/XkmKBcNM8wLz201gnJc1zpOcWWaZ4kGNUVZxpbJPy+mwSVaguElOt7bMAdM8SdqXajXN+8DwOtO8tNtlmlcWjDHNk6RYUGGa55SxzXOp/N8UQiwYaZpnq0yx4IxiD2GiM/eCrI6LOjK2n/jTlrHpyCOdXD7KNC8Z+6BpXs4dMs2TcsZ5UtbtN80LlDDNMz9uC6I43rc8diuNY6NUdpPZcZH184HOrO3jmyQdMu630RXGj0nO9vlAYHwM08X2OCYm2+5QUPAHnBfEfD5Jto8RcQXBhGN+xfaaAAAAAAAAgFeKtvMnk2uSlDbJslud7hJTpWmeFMWrH7avzmSd7W6Ag51NpnmS1LR/t2neGSNONs0bU/G3pnlBkDTNk6Sc2Y6AhMpiE42yiivndsr6FQsrTvavcDV3vGyaN7byPNO8Gzb+xDTvp5d8wTRPksqC0caJtq/DZN1B07x4BDt/LHchxo7z6tZg49xOWe34dBHsMrHmjI4Bu1kfu3XmdpjmWe8M7QrN++4NRZVzbaZ573a0mOZJUs7Z3VfKgmGaMPxys7ziaVYUu8+HinTO9vnKtN/+yDTvt38/2zRPkk4ptz0WDIx36jjD+7kUzfF5xlnu2E0qGZtyzK+w8wcAAAAAAKCEsfgDAAAAAABQwlj8AQAAAAAAKGEs/gAAAAAAAJQwFn8AAAAAAABKWKjFn6VLl+qnP/1pz9+XL1+upUuXmg8FACdCFwHwAV0EwAd0EYBC5P2MyBdeeEFz586VJLW1tck5p1WrVkmS2tvb5ZzTypUrJUmLFy/WhRdeGOG4AIYqugiAD+giAD6giwCElXfxZ8qUKWpsbJQkPfzww8pms/rCF74gSXrkkUfU2dmpq666KtopAQx5dBEAH9BFAHxAFwEIK+/iz5Hq6urU1NQkSVq2bJk+//nPq7q6OpLBAOB46CIAPqCLAPiALgJQiILf86ezs1Pf/va3VVFRIUlatWqVWltbJUkPPvigfvzjH0czIQAcgS4C4AO6CIAP6CIAhSpo549zTrfffrvGjx+v6dOn91y+e/du3XnnndqzZw9vKgYgcnQRAB/QRQB8QBcBCCPvzp90Oq1/+Zd/0fr163XWWWdJkvbt26eOjg599atfVW1trX7961/r1FNPjXxYAEMXXQTAB3QRAB/QRQDCyrvz58CBA8pkMvrGN76h5uZm/elPf9KcOXOUSqW0cuVK1dbWDsScAIY4ugiAD+giAD6giwCElXfnz+jRo/X973+/5zzSD3/4w/rDH/6g008/XaNGjer5vrffflvZbDa6SQEMaXQRAB/QRQB8QBcBCKvgN3w+0sknnyxJPUXinNNtt92mjRs32k0GAHnQRQB8QBcB8AFdBOBEQn3U+5E++tGPas6cOSorK1M6nda4ceP08Y9/3HI2AMiLLgLgA7oIgA/oIgDHEzjnXDF+cCbXJCltkuV02CSnW0yVpnmSFAuGmeYFQcI0L+taTfMOdjaZ5klS0/7dpnlnjDjZNG9Mxd+a5gVB0jRPknIuZZSUUFlsolFWceXcTkmZYo9xTE5Wv6/3NXe8bJo3tvI807wbNv7ENO+nl3zBNE+SyoLRxol92oR7XFl30DQvHow0zZMkJ6tTEMoUCyYYZRWXczslo+vFKWeSEyVndAzYzfrYrTO3wzQvUNw0ryu0z6/hDoicazPNe7ejxTRPknLO7r5SFgzThOGXm+UVT7OsumgoSudsn69M++2PTPN++/ezTfMk6ZRy22PBILA9LnKG93MpmuPzjHvPMC2pZGzKMb9ie80CAAAAAADAKyz+AAAAAAAAlLAinva1VVanfeWMt14FCkzzJCkWVJvmPf7mOtO8hjNnmOZl3T7TPEnK5GwzE7FTTfPixr/jfrwl1wAonVMtUtnNklGHxIJyk5xugWzzJPutqjnXYZq3v3OPaV5F3P70yYr4CNO8eHCKaZ71dv1YBKd92c1YplhwplFWcdmeglqUQ7tQfD/tK5V7zTTP+rQDyf70cOvj36zZqebdefanIlk+DQpUqRHJT5vlFUvO7ZBdF0VwuqMx6+Oi3e22p9ePSNh228t73zbNk6TzTv2IaV4iNtY0z/5UaPvHWNtTg8sUCz5wzK+w8wcAAAAAAKCEsfgDAAAAAABQwlj8AQAAAAAAKGEs/gAAAAAAAJQwFn8AAAAAAABKWOjFH+ecWltbo5gFAApGFwHwAV0EwAd0EYB88n6W9O7du/Xcc89p+/bt2rp1q15++WU1NDRo7ty52rPn/Y/kjcfjmjChND76GYB/6CIAPqCLAPiALgIQVkGLP5s3b1ZNTY22bNmixsZGJZNJrV69Wvfdd5/OOussSdKrr76qxsbGyAcGMDTRRQB8QBcB8AFdBCCsvIs/dXV1qqurUyqV0v33369kMtnztYaGBs2bN0+SVF9fH92UAIY8ugiAD+giAD6giwCElfc9fx5//HHV19dr2rRp2rt3r+rr63X55ZcPxGwA0IMuAuADugiAD+giAGHlXfxpaGhQY2Oj5s+fr/Lycq1YsUJr1qwZiNkAoAddBMAHdBEAH9BFAMIq+NO+VqxYoUwmo2uvvVbvvfdelDMBwHHRRQB8QBcB8AFdBKBQBS3+PP/880okEqqurtasWbN00003RT0XAByFLgLgA7oIgA/oIgBh5F38aW9v14IFC3rK5LrrrtPVV18d9VwA0AtdBMAHdBEAH9BFAMLKu/izatUqTZ06VZMnT5YkBUGg6dOnRz4YAByJLgLgA7oIgA/oIgBh5f2o91mzZimZTMo5d9TXVq5cqfXr10cyGAAciS4C4AO6CIAP6CIAYeVd/Bk+fLgkKZVK9bp88uTJWrJkic4//3xJ0urVqyMYDwC60EUAfEAXAfABXQQgrMAda7l4AGRyWyWlTbJySuX/phACBaZ5khQLqk3zHn9znWlew5kzTPOybp9pniRlcraZidippnlx499xAWuzRVSmWDCh2EOYSGU3S0YdEgvKTXK6BbLNkyRn3Jc512Gat79zj2leRTxpmteVOcI0Lx6cYponZU3TYsFI07wuVjOWKRacaZRVXDm3U1LGKK0oh3ahOKNjwG4xVZrmpXKvmeY5lzPNk6QgsO036+PfrLN9vMk6226TdMxdM30VqFIjkp82yyuWnNshuy6KG+VEx/q4aHf7y6Z5IxK23fby3rdN8yTpvFM/YpqXiI01zXOy7l/7x9jA9L5SpljwgWN+peCPegcAAAAAAMDgU7StBbFghKxWlQN1muR0c85qtft9gRKmeW+3HTLNC2T76lFMw0zzJCkWtJrmpXPvmeYFMdu7U2C8i0SSAq93ExVHWTBKVrsQrV/JDoIK0zxJ5i9WlMVsd8FUJ21f2W1L2/aGJL11+C3TvL8ZZny/DGxfaQ0Uwe1wEOxMGWixICGr1+Si2GViz3aXSWB8uy8LxpjmucB2d4EkpXN7zTMtHUrb7gztyNhfhyOSVWZZMfPdBcVi95hku5tBcsY7WyX7Y+NTKkaZ5uWMd7ylsvbXYSZne6xlvZPeesnD+nmzJAWB5YzHf3xl5w8AAAAAAEAJY/EHAAAAAACghLH4AwAAAAAAUMJY/AEAAAAAAChhLP4AAAAAAACUMBZ/AAAAAAAAShiLPwAAAAAAACWs4A+Unz9/vjZu3Kjy8vJelzvnlEgktGHDBvPhAOCv0UUAfEAXAfABXQSgUAUv/kjSokWLNHXq1F6XHTx4UDNnzjQdCgBOhC4C4AO6CIAP6CIAhQi1+LNw4UJ997vf7XVZ96oyAAwUugiAD+giAD6giwAUgp0/AAYdugiAD+giAD6giwAUouA3fE6n06weAyg6ugiAD+giAD6giwAUquCdP/v379ett96qZDLZ63K2FAIYSHQRAB/QRQB8QBcBKFRBiz/OOe3YsUONjY1HFQtbCgEMFLoIgA/oIgA+oIsAhFHQaV+bNm3SpEmTjioVABhIdBEAH9BFAHxAFwEII+/ij3NOS5Ys0ZVXXtnr8nQ6rUwmo/b2dgVBENmAACDRRQD8QBcB8AFdBCCsvKd9NTU1qaqqSpdeemmvy1taWjR79my1t7drxowZkQ0IABJdBMAPdBEAH9BFAMIKnHOuGD84596SlDHJcuo0yenJczZzHSkWVJnm/Xzrw6Z5131kjmlezh0yzZOkjNttmpdzWdO8ZGysaV4QlJvmSVJQ+Hu851GmWPABo6ziyuZel5Q2yXJGOd2CoNI0T5KcO2yaFwS2W83TuV2meW3pVtM8SdrXaZv5N8POMs1TEDeNiwcnmeZ1sTr0KFM8+KBRVrE1S7J5XHIuZ5ITJWd0DNgtZtxFmdx7pnlOKdM8SUrn9ppnWjqU7jDN68jYX4cjknbH5zFValT51Pzf6Lmc2ymr52iBbB+PnFFH9mbbRZ25HaZ51s9Xnmn+s2meJF1w2gdM85Lx0aZ5IT7jqiCB7E+zjAUVhmlligVnHvvnGP4UAAAAAAAAeIbFHwAAAAAAgBJmuwcqlOAvfyySjLdeRfDmaNbbfZ9rftc077qP2G4pDCJYVywLTjfNc4Hx6YIy3mbv7Lc32922S+cNBHPqlIxOHbU+7Svm7Cvaesu09X09Hgw3zasqs72fS1J7xvbUuayzPY0spmGmedHc3emiv9Z1qpbN44j1KVV2p+kdmWjbl864L6270kVwHR7O2vZbPLDt82TM9ndSXp4wzZOkRMzutKRA9vMVQ9dzFqu35rC93WdyLaZ5kv3p64nYBNM8GZ/Ge+FY+2PLvYf3meblZPv2IadWjDHNS8aOfUpV/1ieInn8LHb+AAAAAAAAlDAWfwAAAAAAAEoYiz8AAAAAAAAljMUfAAAAAACAEsbiDwAAAAAAQAlj8QcAAAAAAKCEsfgDAAAAAABQwlj8AQAAAAAAKGEs/gAAAAAAAJQwFn8AAAAAAABKGIs/AAAAAAAAJYzFHwAAAAAAgBLG4g8AAAAAAEAJK2jxZ8qUKfr0pz8d9SwAcEJ0EQAf0EUAfEAXAQijoMWfiy66SKNGjYp6FgA4IboIgA/oIgA+oIsAhFHwaV933HFHhGMAQGHoIgA+oIsA+IAuAlCoskK+ae7cuTr77LOjngUAToguAuADugiAD+giAGEUtPOHUgHgA7oIgA/oIgA+oIsAhMGnfQEAAAAAAJSwgk778l1gvIYVqNw0r4vtjBv//1dM83Sp8TpgELfNkxQzv7lWGOfZcsoWe4QTCIo9gJl4MEwyus/nXLtJTrScaVqgpGleLBhhmheP5UzzJCkZt/097+9sM82rTtr+jmPO/jHR7nE2UUJ15GR3/8wY5XRxsr8fOZe2DQxsb/fWXWmfJ8WMb/xlge2xYMb4dmM9nyQFhtdhqVRRTOWye4po+zsri9nfBpw7bJoXKGGal9Ee0zzr4zZJSsRtn6O9dWivaV7c+HnpmMq/Mc2TBq4/2PkDAAAAAABQwlj8AQAAAAAAKGEs/gAAAAAAAJQwFn8AAAAAAABKGIs/AAAAAAAAJaxPiz/f/OY3lUqlrGcBgFDoIgA+oIsA+IAuAnAiBX0u2y233KI33nhDkpTNZvXmm2+qqamp1/eMHj1ay5cvt58QAP6CLgLgA7oIgA/oIgBhFLT4c8899/T899q1a9XY2Ki77rorsqEA4FjoIgA+oIsA+IAuAhBGQYs/DzzwgNauXStJeu211/ShD31In/vc53p9z5VXXqk5c+bYTwgAf0EXAfABXQTAB3QRgDDyLv68/vrrCoJAn/nMZ7Rr1y7t3LlTV1xxxVHf19HRoXfeeUfjxo2LZFAAQxtdBMAHdBEAH9BFANbsZ/UAABo3SURBVMLK+4bPo0aNUm1trZqamtTY2Kh77rlHtbW1qq2t1eTJk7Vq1SodOnRItbW1GjZs2EDMDGAIoosA+IAuAuADughAWHl3/lRXV2vDhg169NFHNXr0aC1evLjX11taWvTLX/5S5eXlOu+88yIbFMDQRhcB8AFdBMAHdBGAsPIu/jQ3N2vbtm269tprNWzYMN1www29vn7DDTdo6tSpeuihhzRz5kyNHTs2smEBDF10EQAf0EUAfEAXAQgrcM65Qr5x+fLl+s///E+dcsopvS7fuXOnli5dqgsuuEBBEBT8g3Nup6RMqGGPJ1DhP7d48p5hF8rEf7/GNO9PC5aZ5jl1muZJUlDY+5OXDKdssUc4gTLFg5qi/GT7Ltohqy7KuXaTnPfFjfMkp5RpXjw42TTP6bBpXtYdMM2TpH2pd0zzcoU9DBesOlllmpeInZL/m0IKVG6UlFBZ7G+NssKx7iLn3pGMet/6fu6UM82TJOfSpnnxYKRpXsa1mOY52f57Jak9bTtjImb7mJNxtrebssD2WFqSYoaZgSpUUXaxWV6h7Ltop6y6yPr5T04dpnmS5JztcUcssD3FLuP2mOY5Z/v4IEkH07YzvnVor2ne6VWjTPPGVH7UNE+yfp4bVxCcfsyvhPopc+bMOeaqsqRQpQIA/UEXAfABXQTAB3QRgEKEWvz51a9+pccee6zXZc3NzfrSl75kOhQAnAhdBMAHdBEAH9BFAApR8Glf1jjtq3847av0cdrXwOC0r/7htK/+47Sv/ijeaV/WOO2rfzjtq/847at/inXalzVO++ofTvvqP0776q/jn/Zl36IAAAAAAADwBos/AAAAAAAAJaxo59E41yFZbYENkjY5PexPtZBst/V/7KNnm+ZZb0eOYkuh3RbUvzDePmx9Wpr1lnjJ8nS8pOKD4WzLggR/+WOQFFSY5PRwNqejHSnrDprmxYNq0zzr7giUMM2TpJPLzzTNy+RsT93Ym2o1zTup3P7OHg8qjZLKS+iE4Nxf/liw/Z0FEbxW6IxOt42K+WlaxqdASVJlmXH/Gp+y35qyPe12eMKqN96XNfy9BMbH+sVj9zwoMD7Wjmu4aZ4k5WT9PNL2dvD6gVdN8yYMG22aJ0nViTGmeSNOsv09H0q3meZljU/Fs5dUgtO+AAAAAAAAhh4WfwAAAAAAAEoYiz8AAAAAAAAljMUfAAAAAACAEsbiDwAAAAAAQAlj8QcAAAAAAKCEFbT4s2vXLj322GO64YYbNG/evJ7Ln376ad18882RDQcAR6KLAPiALgLgA7oIQBhl+b7h9ddf19e//nV1dnZq9uzZuuaaa3q+lk6nVVFREemAACDRRQD8QBcB8AFdBCCsvDt/zj77bK1Zs0b19fWaMGGCYrGYDh06pBdffFGdnZ2qqKhQW1ublixZMhDzAhii6CIAPqCLAPiALgIQVt7Fn3Xr1mnGjBl69NFHtWjRIs2YMUMvvfSSHnjgAR04cEAVFRW666671NzcPBDzAhii6CIAPqCLAPiALgIQVt7FnxkzZmjt2rWqqqrSjBkztG7dOm3dulUTJ07Unj179Nhjj6mpqUkLFiwYiHkBDFF0EQAf0EUAfEAXAQiroDd8fuqpp7Rr1y6tWbNGa9asUVNTk6ZNm6Y33nhDBw8e1M9+9jNVVlZGPSuAIY4uAuADugiAD+giAGHkfcNnSfr5z3+umpoaXXXVVVq7dq2WLl2qIAi0ZcsWnXvuudq2bZsuuOCCqGcFMMTRRQB8QBcB8AFdBCCMvDt/fve736mtrU2f/OQnddppp+nee+9VEATaunWrysvL9cUvflErV64ciFkBDGF0EQAf0EUAfEAXAQgr7+LPSSedpO985zs9f4/H43LO6e6779b111+vadOmacuWLXrllVciHRTA0EYXAfABXQTAB3QRgLDyLv5cfPHFOuecc3pddtddd2nkyJG67LLLlEgkdPPNN+tb3/qWWltbIxsUwNBGFwHwAV0EwAd0EYCwCnrD5yO99dZbevPNN7V48eKey2bOnKlzzjlHr776qulwAHA8dBEAH9BFAHxAFwHIJ3DOuWL84GzudUlpk6wgSJrkvC9unCdJgWnaF9bfbZr3P6ffYprn3GHTPEkKCnt/8hCBodc+TxxnPF/OpUzzJMmp0ygpqURsslFWceXcm5IyJllOWZOc9wNt5jpSxr1rmpeInWGal3OHTPPMfycRyORaTPP2pmxf4T2pfIRpniTFA6tPnylXefzjRlnF5dxOyej26ow67chEaznXZpoXD042zUu7d0zz5HK2ebL/PdsdI3TZnzpgmjc84fenVgWq1LDEJcUeo9+c2yWrLgqMj7WjkHO2t3vrvnx1f6Np3oRho03zJKkybtu/Odk+PhxK2+ZVJyeY5tlLKhE755hf8f8eCQAAAAAAgD4z3kpRuM7cO3LqMMmKB1UmOe/nnWKaJ0mxYJhp3pdra03zAuN1QBfFq4TGq8CBs90xFgQJ4zz7HWh2G/2KsmEwIjFZrYMHxteLi+AVM+vNnjnjXX7WO3WCCHZyBkGFaV48sL0OEzGbx9Zuh7PWr4pK5TGbXRCBctFs1i2CrsdN+90hNmx3L3fx+xfnzHff2l+HQVBuG2j80F4etz0uSufsd8NaipnvuCuOrt06pXScd2Ix4zNInLM9jjmtsto0r7LMfudPIhhrnmkpkbR9Drmz7SXTPEn6rx3bzLJGJKr1D5PY+QMAAAAAADDksPgDAAAAAABQwlj8AQAAAAAAKGEs/gAAAAAAAJQwFn8AAAAAAABKWL8Wfzo7O7V9+3Zt2rTJah4ACI0uAuADugiAD+giAMdS0Ee9P/PMM/rjH/+offv2qaWlRe+++66am5vV2tqqcePGqaamRmPHjtUHP/jBqOcFMITRRQB8QBcB8AFdBCCMghZ/xo8fr6qqKo0bN04nnXSSGhsbdckll+jGG29UEARRzwgAkugiAH6giwD4gC4CEEZBiz8TJkzQV77ylZ6/79ixQ5IoFQADii4C4AO6CIAP6CIAYfCGzwAAAAAAACUs786f1157Tddff32vy9ra2iRJDz30UK/Lf/KTn2jKlCmG4wFAF7oIgA/oIgA+oIsAhJV38WfSpElqbGzsddmyZcsk6ajCAYCo0EUAfEAXAfABXQQgLE77AgAAAAAAKGEs/gAAAAAAAJQwFn8AAAAAAABKGIs/AAAAAAAAJSzvGz4fC28iBsAHdBEAH9BFAHxAFwE4EXb+AAAAAAAAlDAWfwAAAAAAAEoYiz8AAAAAAAAlrE/v+WMh63JyyhmldRjldIkFKdO8LhWmaYl43DRPCozTEqZ5kuSUNc/0m+3vpCsxaZRk//stlpw7JCld7DEGjP19M2Oa5txh2zzTtC4x674MbH8nI5PDTPOiOVSw6fOgeIcx5pw6ZXd/sn78sL8nOc97N5VtNc1Lxq0ef98X03DbQOObTTJme/+MBfavWWed1XOR0uoj+GNUcnyxR8jL7jl9l8B6f0pg+7y5ucP28UGSDnbaPSYGJziWYOcPAAAAAABACWPxBwAAAAAAoISx+AMAAAAAAFDCWPwBAAAAAAAoYSz+AAAAAAAAlLCCF3/Wrl2rpqamKGcBgLzoIgA+oIsA+IAuAlCogj6TMJfL6Uc/+pEeeughSdKNN96oF198UalUSqlUSiNHjpQkrVixQhMmTIhuWgBDGl0EwAd0EQAf0EUAwiho8eeJJ55QbW2tDh48qN///ve69957JUnr1q3Tk08+qUWLFkU6JABIdBEAP9BFAHxAFwEII+9pX+3t7frBD36gz3/+8/rhD3+ocePGDcRcANALXQTAB3QRAB/QRQDCyrvzp7W1VSNGjND3vvc9VVVVaerUqQMxFwD0QhcB8AFdBMAHdBGAsPLu/BkzZox+8YtfKJVK6c477xyImQDgKHQRAB/QRQB8QBcBCCvv4k8mk9G8efNUV1enRx99VBs3bhyIuQCgF7oIgA/oIgA+oIsAhJV38Wfbtm3asWOHstmsJk6cqI997GMDMRcA9EIXAfABXQTAB3QRgLDyvufPpEmT9MQTT6i9vV3PPvusnn32WV1yySUDMBoAvI8uAuADugiAD+giAGHl3fnzyiuv6LOf/aw+9alP6Te/+Y3i8fhAzAUAvdBFAHxAFwHwAV0EIKy8O39qamo0f/58TZkyRclkUi+++KLq6+slSalUSqlUSk8++aQkacGCBWpoaIh2YgBDEl0EwAd0EQAf0EUAwgqcc64YP/hQeqOcOkyy4oHtSncidqppniTFg2rTvE27/7dp3kVjbT8e0rmUaZ4k5XTYNC+Q7e0mFgw3zXPKmOZJknNpo6SEymIfNsoqrkzuVUlW14v/su6gaV48GGGal3NtpnlRsL+v2/Zl1u03zSvgdaI+yJqkBKpQefyTJlnFlnXbJbPeD4xyutkfKuZcu2leWTDaNK81/aJpXjKeNM2TpLjxv9kZPxams3tM82JB3hMWQsu6nFlWoEoNS1xillc8zbLq6KHIOdvrLuN2m+ZFwbqLgvwnJ4Vi/RzyuZYnTfMkacNbO82yqpOj9NVzbjnm1+xbFAAAAAAAAN6I4uW8ggSqMMyyXsOyf3XG+qouj9u+8mx/U7B7JSW6TOvbTdHuTkVQSv/WRLEHGGDlxnnW15/1fFGw/jfb7qqwfHzt4u/On8FxeymU5fXs/84f+2Mt2928gSqN86J4rLG+Dm1vN/bXofXtWgoMb9v23Vss7A3wSxTPS61ZHydY39dt50vGrJ+Hd+3WsTIiMfK4XyvaaV8AAAAAAACIHku7AAAAAAAAJYzFHwAAAAAAgBLG4g8AAAAAAEAJY/EHAAAAAACghLH4AwAAAAAAUMJY/AEAAAAAAChhLP4AAAAAAACUMBZ/AAAAAAAAShiLPwAAAAAAACVsyCz+OOfU2tpa7DGOy3K+pUuX6qc//WnP35cvX66lS5f2K9P36y8q3/zmN5VKpYo9RkE6Ozu1fft2bdq0KfT/u3btWjU1NUUwFf6a7/cl37tI8v86jAJdBGu+34+s5+PYyAZdBGu+34/oIn8Nlj7yqYsC55wzSzO0cOFCnXfeefrc5z7Xp/9/9+7deu6557R9+3Zt3bpVL7/8shoaGjR37lzt2bOn5/vi8bgmTJhQcO78+fO1ceNGlZeX97rcOadEIqENGzYUZb4XXnhBc+fOlSS1tbXJOafhw4dLktrb2+Wc07BhwyRJixcv1oUXXjig8x3J6jqMwi233KI33nhDkpTNZvXmm2+qpqam1/eMHj1ay5cvLyhv165d2rx5s9atW6fy8nL9+Mc/liQ9/fTTevjhh/Uf//EfoWd85pln9Mc//lH79u1TS0uL3n33XTU3N6u1tVXjxo1TTU2Nvv71r+uDH/xgQXm5XE5/93d/p4ceekhjx47VjTfeqBdffFGpVEqpVEojR46UJK1YsSL077pU9KeP6KL+dVEUM3aji+iiwYYuCjffYDk2GkpdJNn3EV008OgiuqgYfD82GgxdVBbqXzSI7N69W5s3b1ZNTY22bNmixsZGJZNJrV69Wvfdd5/OOussSdKrr76qxsbGUNmLFi3S1KlTe1128OBBzZw5s2jzTZkypef7Hn74YWWzWX3hC1+QJD3yyCPq7OzUVVddVbT5/prFdRiFe+65p+e/165dq8bGRt111119ynr99df19a9/XZ2dnZo9e7auueaanq+l02lVVFT0KXf8+PGqqqrSuHHjdNJJJ6mxsVGXXHKJbrzxRgVBEDrviSeeUG1trQ4ePKjf//73uvfeeyVJ69at05NPPqlFixb1aU50oYv610VRzHgkuoguGip8vx9FMd9gOjYaCl0kRdNHdNHg4vv9iC7ys4sk/4+NBkMXebv445xTLpfr8/9fV1enuro6pVIp3X///Uomkz1fa2ho0Lx58yRJ9fX1obMXLlyo7373u0fNm0gkvJivrq6uZ3vYsmXL9PnPf17V1dWhM6KaT7K5DqPwwAMPaO3atZKk1157TR/60IeOelXjyiuv1Jw5c/JmnX322VqzZo0WLVqkCRMmKBaL6dChQ/rzn/+szs5OVVRUqK2tTQ8++KC+9rWvFTzjhAkT9JWvfKXn7zt27JCkPpVKe3u7fvCDH+iOO+7QD3/4Q82ePTt0xlDQnz6ii/rXRVHPSBfRRYMJXdT34w7fj42GQhdJ0fQRXTTw6CK6qBh8PzYaDF3k7eLPvn379NZbb/X5/3/88cf17//+75KkvXv3qr6+XtXV1frHf/zHfs9msSIa1XydnZ369re/reuvv16StGrVKl122WWqrq7Wgw8+qH379vUUQjHm6+bjqvLrr7+uIAj0mc98Rrt27dLOnTt1xRVXHPV9HR0deueddzRu3LgT5q1bt0733HOP9u7d2/Pft99+ux5++GFdfPHFqqio0F133dWnQrDS2tqqESNG6Hvf+56qqqqO+p2gS3/6iC7qXxdFOaNEF9FFgwtd1DeD4dhoKHSR5H8f0UWFoYv6hi7qu6F2bBRVF3n5hs+5XE4vvfSSnnjiiT5nNDQ0qLGxUfPnz1d5eblWrFihNWvW9Hu2dDptsvIZxXzOOd1+++0aP368pk+f3nP57t279U//9E/6r//6r54thsWYr5vVdWht1KhRqq2tVVNTkxobG3XPPfeotrZWtbW1mjx5slatWqVDhw6ptra257zcE5kxY4bWrl2rqqoqzZgxQ+vWrdPWrVs1ceJE7dmzR4899piampq0YMGCgmd87bXXVF9f3+vP/fffr/vvv/+oy1944YW8eWPGjNEvfvELpVIp3XnnnQXPMZT0t4/oov51UVQzSnQRXTS40EV9MxiOjYZKF0n2fUQXDTy6qG/oov7x/dhosHSRlzt/NmzYoIkTJyoIAj399NMFvSHo8axYsUKZTEbXXnutfv3rX/d7tv379+vWW2/ttb1O6vt2OKv50um0brvtNq1fv17/8A//IKlrVb6jo0Nf/epX9aUvfUnXX3/9UXMP1HxHsr4OrVRXV2vDhg169NFHNXr0aC1evLjX11taWvTLX/5S5eXlOu+88wrKfOqpp7Rr1y6tWbNG5557rpqamnTNNdfowQcf1MGDB/Wzn/1MlZWVBc84adKko87dXbZsmST1vIoQRiaT0bx581RXV6dHH31U5513ni699NLQOaXMqo/oov51keWM3egiumgwoYvCGyzHRkOpiyTbPqKLBh5dFB5d1H++HxsNli7ybvHn0KFDuvvuu3XHHXcoCAItXrxYv/3tb1VWFn7U559/XolEQtXV1Zo1a5Zuuummfp0v55zTjh07et5U60h92Q5nOd+BAweUyWT0jW98Q83NzfrTn/6kOXPmKJVKaeXKlaqtrQ2daX39SfbXoaXm5mZt27ZN1157rYYNG6Ybbrih19dvuOEGTZ06VQ899JBmzpypsWPH5s38+c9/rpqaGl111VVau3atli5dqiAItGXLFp177rnatm2bLrjggqj+SXlt27ZNO3bs0KhRozRx4kR97GMfK9osPrLqI7qof11kPaNEF9FFgwtd1DeD4dhoqHWR5Hcf0UUnRhf1DV3Uf0Pt2CiqLvLqtK/Ozk7dcsstuuiii/SJT3xCF154oWpra3Xbbbcp7CfSt7e3a8GCBbrpppskSdddd52uvvrqfs23adMmTZo0qU+vWEc93+jRo/X973+/553JP/zhD+sPf/iDTj/9dI0aNarn+95++21ls9kBn6+b5XUodb17/qc//WmTrDPOOEPLly/XySefrBUrVuiKK67o9eeZZ57R+PHj9cgjjxRUKL/73e/U1tamT37ykzrttNN07733KggCbd26VeXl5friF7+olStXmszeV5MmTdITTzyhf/u3f9OoUaP07LPPFnUen1j1EV3Uvy6KYkaJLqKLBg+6qO8Gw7HRUOoiyf8+oouOjy7qO7qo/4basVFUXeTN4k9LS4uuvvpqVVVV9Tq3buHChXr77bf1ta99TQcOHCg4b9WqVZo6daomT54sqetdto88vzIs55yWLFmiK6+8stfl6XRamUxG7e3tod4Qynq+Yzn55JMlqadEnHO67bbbtHHjxqLMZ30dStJFF13UqzStzJkzR48++mivPx//+MclFf6O7SeddJK+853v9Pw9Ho/LOae7775b119/vaZNm6YtW7bolVdeMZ+/UK+88oo++9nP6lOf+pR+85vfKB6PF20Wn1j2EV3Uvy6KYka6iC4aLOgi2y6S/Do2GmpdJPnfR3TRsdFFdJEvXSQNjWOjqLrIi9O+nn32Wd18882aOXOm/vmf/7nXP66yslLLli3Tv/7rv2r69Olavnx5zw39RGbNmqVkMnnMleiVK1dq/fr1oWZsampSVVXVUefatbS0aPbs2Wpvb9eMGTMKzrOe73g++tGPas6cOSorK1M6nda4ceN67hwDPZ/1ddjtjjvuCP3/5POrX/1Kjz32WK/Lmpub9aUvfangjIsvvliSeuXcddddGjlypC677DJJ0s0336xvfetbWrlypUaMGGEweTg1NTWaP3++pkyZomQyqRdffLHnYyFTqZRSqZSefPJJSdKCBQvU0NAw4DMONOs+oou69LWLopiRLqKLBgO6qH/znYgvx0ZDrYsk//uILjoaXdS/+U6ELuqboXBsFFkXOQ+8++677oUXXsj7fU899ZRrb28PlX348GH3iU98oufvr732mtu8eXPP3x955JFQedaYr3+ampqK+vMLceedd7rHH3/c/fd//7f7yle+ctRt+NZbb3XPPPNMkabDX4uqj3y/L/k+n3N+z0gXwRpd1MW3+Zzze8bB0EXO0UeDCV3Uxbf5nPN7RrrIT4FzId9MBwAAAAAAAIOGN+/5AwAAAAAAAHss/gAAAAAAAJQwFn8AAAAAAABKGIs/AAAAAAAAJYzFHwAAAAAAgBLG4g8AAAAAAEAJY/EHAAAAAACghP1fR1V+Uj2ZXKMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder Src Layer 2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3df6xddZ03+s/a+/zoEcqptlhUnDoYEIg8DMjVTDTFqhdnaBzIfay5tjIPiQrXq9Oi+KOTTioWvKET1EKvCT9mYiNPeRhSuVHCPNOLYTK9Tu7z5EFDmIr8uEM6QWcqQi309LQ9P/a6f5w5tYXTdu9zPrt7dfF6JSR079P3+Xbvvd57rc9Za5+iLMsyAAAAAKilRq8XAAAAAED3GP4AAAAA1JjhDwAAAECNGf4AAAAA1JjhDwAAAECNGf4AAAAA1JjhDwAAAECN9fXqG0+W/xIRE7369sdVlmO9XsIJfeL/+m5q3rb/5SupeWVMpuZFRBTps8pmcl6RnJcvb4XNKIq3pKX10p//ZGO8dPC3KVnr3veHKTnT3nba+al5ERHN4rTUvKLoT80ry/HcvC50UfZ7RCP9Ocnutnxl2UpKakZRvDkpq7f+t7/7evxmdE9K1r0f+48pOdP6mgtS8yIi+orczEYxlJo3We5PzSsityunU6ubFpG9n1UU+T+zzuuiiLr00f7x/zfKOJiS9f2nHk/JmfaHZ52VmhcRcdHCi1PzmsUbU/O68brPVpa5+1qnwn5MtTUi4swZ7+nZ8Gdq8JM1/CmTcqblHnx0w+79v0lOzB7E5R9w5Z+olv26qf7w59RY48n10sHfxgsHXkrJmkg+WOhOF2Vvm9nbZfb6uvFDhuznpRt9WXWZB1z18JvRPWnv7WUcSMn5nTck50VUf78je33deP/Nzqz6PkL2fluELnqtMg6mdcjLY3tTcqaNtU5PzZtS9ff0brzus70e92NOTdUfJQIAAAAwa4Y/AAAAADVm+AMAAABQY2195s/WrVvjrrvuOvznffv2RX9/f8ybN+/wbV//+tfjwx/+cP4KAf6dLgKqQBcBVaCLgE60NfxZtWpVrFq16vCfv/GNb8Sll14aH/vYx7q2MIBX00VAFegioAp0EdCJWV32NTk5Gc2mX8EG9JYuAqpAFwFVoIuA42nrzJ+lS5ce9edXXnkltm/fHrfeeutRt+/YsSNvZQCvoouAKtBFQBXoIqATbQ1/Xl0YH/3oR+OCCy6ITZs2dWVRADPRRUAV6CKgCnQR0ImOL/t67rnnYu/evfHkk0/G448/3o01AZyQLgKqQBcBVaCLgBPpaPjTarXipptuilWrVsW6deti9erVsXv37m6tDWBGugioAl0EVIEuAtrR9vBn7969sXr16ijLMj73uc/F5ZdfHp/61KdixYoV8dOf/rSbawQ4TBcBVaCLgCrQRUC72hr+bNq0Ka644opYtGhR3HPPPdHf3x8REdddd13ceOON8ZnPfCa++MUvdnWhALoIqAJdBFSBLgI60dYHPu/duzfuvffeeNe73vWa+66++up43/veF9/73vfSFwdwJF0EVIEuAqpAFwGdKMqyLHvxjSfLf46IiaS03H9CWY6l5nXD0v/8zdS8n1zzf6TmlTGZmhcRUXT++eQn0EzOK5Lz8hVpa2xGUZydlNVb1/14bbxw4KWUrG9dviwlZ9qS0/9Dal5ERLOYn5pXFP2peWU5npuX9j5zRGZ5KDWvkf6cZHdbvrJsJSU1oyjekpTVW//xwT+L3ft/k5L1f/+v/yklZ1p/c2FqXkREX/Gm1LxG8YbUvMlyJDWviNyunE7NTcvej8ntoqLI3g/M7KKIuvTRyPjfRxkHUrL+zyf+e0rOtA+dnb/v+Z4z35ua1yxy+7Ibr/tsZZl73Hcq7MdUWzMiFs94T/VfTQAAAADMmuEPAAAAQI219Zk/3VBEM/Iu10o+TbXIv3ynVe5PzZsczz29rhuXaeXr2cu1TdlXUGaeijylTJv3Nk6Bi9za84nzz4sDE/tSsg5O5l6ydHDy+dS8iIh5zben5jVjODWvjOzLvrqxHSVfahzJlxqXA7l5tdnaq+27f/THMVmOpmTtPvDblJxpByZeSM2LiDhv+NzUvP7G21Lz0j9SILnbpkJzL2sti9z9rEYMpub16JMqOlD19bVnoHFGRNJz9x8W5V4C9fiLOZfGHumSRbnHaNmXfZ0K0o8jy+RLWk+BS+dOFo8EAAAAQI0Z/gAAAADUmOEPAAAAQI0Z/gAAAADUmOEPAAAAQI0Z/gAAAADUWNvDn/Xr18fDDz/czbUAnJAuAqpAFwFVoIuAdjnzBwAAAKDG+jr54g0bNsTGjRtnvG94eDgeeuihlEUBHI8uAqpAFwFVoIuAdnQ0/Fm/fn0sX768W2sBaIsuAqpAFwFVoIuAdrjsCwAAAKDG0i77Wrt2bVx55ZUpiwI4Hl0EVIEuAqpAFwHtaHv4s2HDhtiwYcOM991yyy1pCwI4Hl0EVIEuAqpAFwHtSrnsa8+ePTE8PJwRBTBrugioAl0EVIEuAo7U1pk/S5cuPeZ9DzzwQOzevTsWLlyYtiiAmegioAp0EVAFugjoRFvDnx07drzmtrIsoyiKGB0djV27dsWSJUvSFwdwJF0EVIEuAqpAFwGd6OgDn4+0Zs2a2LlzZ4yNjcXKlStjaGgoc10AbdFFQBXoIqAKdBFwLLMe/txxxx2Z6wCYFV0EVIEuAqpAFwHHkvKBzwAAAABU06zP/JmrMiYjYjIlq0j+ZxTRTM2bysxdY99Abl5ZjqfmFVGk5k2FDuTGJa+xTE3rTmLWNhfdeH575J1nvCkmysGUrMFmf0rOtN8e2peaFxGxcN6vUvOGmrm/RaQsx1Lzosh/myuSM8u07XLaRGpa9vtXrm70ZG+c3j+U9q85cOBQUtKUX47kd9G5w8nberIicvc5uqEsWr1ewknWjX2P+nRIlsxjtIsWnpWSM+3MoVdS8yIintv3z6l55w2/OTUvytx9y24oI/c9J12Z2x1FUf3n5Fic+QMAAABQY4Y/AAAAADVm+AMAAABQY4Y/AAAAADVm+AMAAABQY4Y/AAAAADVm+AMAAABQY33tfNFjjz0WX/rSl477NXfeeWdceOGFKYsCmIkuAqpAFwFVoIuATrQ1/Lnssstix44d3V4LwHHpIqAKdBFQBboI6ITLvgAAAABqrK0zf5YuXdpWmMkz0E26CKgCXQRUgS4COtHW8OfIwti0aVM8//zz8a1vfatriwKYiS4CqkAXAVWgi4BOdHTZ18jISGzdujX+9E//NCIidu/eHf/wD//QlYUBHIsuAqpAFwFVoIuAdnQ0/NmyZUu8+93vjosvvjgiIvbv3x9r166N/fv3d2VxADPRRUAV6CKgCnQR0I62hz9PPPFE3HPPPfHlL3/58G3vfOc74+KLL4777ruvK4sDeDVdBFSBLgKqQBcB7WrrM39efvnluOGGG+Laa6+NxYsXx9NPPx27du2KZ555Jvbs2RPbtm2Lz372s91eK/A6p4uAKtBFQBXoIqATbZ35Mzo6Gi+88ELcf//98YlPfCL+/M//PP72b/82Wq1WXHPNNTE+Ph4/+9nPur1W4HVOFwFVoIuAKtBFQCfaOvPnLW95S+zcufOY95977rlxzjnnpC0KYCa6CKgCXQRUgS4COtHW8OdEzj///IwYgDnRRUAV6CKgCnQRcKSOftsXAAAAAKeWlDN/ZqNVHoyIsZSsRgym5HRXMzXtsiVvT80r41ByXpmaFxHRSJ9VZr9uiornRZQxmZTUSsrpvbeetjjKOJiSVSRv5/snXk7Ni4j4t9GXUvPecfq+1Lzx8t9S8/picWpeRESjmJeaV8ZEal6rfCU1r1GckZqXLb8pe2OguSAi3pCSdea83PfgNyzKfc1HZL4fTefl/ptbZfavyM5/35ws96bmFdGfmlcWp6XmFUU39vczG6QveS+gV8ajTDpGe/PQm1Jypi0YyH1NRUSMTOTsA06bKPek5kWZ25VF8j7MlNz9mEbkPs9lkXsM2Si7cf5M5ntEK4pjVJszfwAAAABqzPAHAAAAoMYMfwAAAABqzPAHAAAAoMYMfwAAAABqrO3hz/r16+Phhx/u5loATkgXAVWgi4Aq0EVAu5z5AwAAAFBjfZ188YYNG2Ljxo0z3jc8PBwPPfRQyqIAjkcXAVWgi4Aq0EVAOzoa/qxfvz6WL1/erbUAtEUXAVWgi4Aq0EVAO1z2BQAAAFBjaZd9rV27Nq688sqURQEcjy4CqkAXAVWgi4B2tD382bBhQ2zYsGHG+2655Za0BQEcjy4CqkAXAVWgi4B2pVz2tWfPnhgeHs6IApg1XQRUgS4CqkAXAUdq68yfpUuXHvO+Bx54IHbv3h0LFy5MWxTATHQRUAW6CKgCXQR0oq3hz44dO15zW1mWURRFjI6Oxq5du2LJkiXpiwM4ki4CqkAXAVWgi4BOdPSBz0das2ZN7Ny5M8bGxmLlypUxNDSUuS6AtugioAp0EVAFugg4llkPf+64447MdQDMii4CqkAXAVWgi4BjSfnAZwAAAACqyfAHAAAAoMaKsizLXnzjidYzETGeklXGwZScaY3Ivza2UZyWmlcU/al5k+W+1LxXxp5JzYuIeGbv7tS8t89/U2re4nkXpOYVxUBqXkREqzyUlNQffY3zkrJ6q1X+MiImer2MGZWR9Xz9zq8P7EzNO2voPal5//vf356at/mDn0rNi4joKxYlJ+b+HGayfCU1r1mckZoXEVHGZFJSXzSKs5OyeqssfxmR9LiU0UrJ6aYyaR9wWva+21hrV2peEc3UvKnQWX96w0nRKven5r1w4DepeRERrTJvW+krTouzT/9YWl7v/Dqyuuj1aLyVe7zyP2/7dmretj9ZkZoXEbFwMHdfsChy94vKxO08ojv75xPli4lpAzHQuGTGe5z5AwAAAFBjhj8AAAAANWb4AwAAAFBjhj8AAAAANWb4AwAAAFBjsxr+fPOb34wHH3wwey0AHdFFQBXoIqAKdBFwPM78AQAAAKixtoY/Tz31VLz44rF/9/zPf/7zePbZZ9MWBTATXQRUgS4CqkAXAZ1oa/izZcuWePzxx495/6OPPhr/+I//mLYogJnoIqAKdBFQBboI6ITLvgAAAABqzPAHAAAAoMYMfwAAAABqzPAHAAAAoMYMfwAAAABqrK/dL1y3bl1s2LAhIiL27dsX/f39sWnTpoiIGBkZidWrV3dnhQBH0EVAFegioAp0EdCutoY/t95663Hv37x5c8piAI5HFwFVoIuAKtBFQCfaPvPneP7sz/4sIwZgTnQRUAW6CKgCXQQcyWf+AAAAANSY4Q8AAABAjRVlWZa9+MYTrScjYjwlqxWHUnKmFVGk5kVENIrh1Lz/+i9/l5r3x0v+KDVvsvxtal5ExEQrN7O/cWZqXjP5OU66KrNL+qJRnN3rRaQ4NPk/IpI6pFEMpuRMKyI3LyKiTO7LVnkgNW/v2EupefOaA6l5U5nzU/OaxcLUvIjJ1LRGcUZq3pSsNfZFo1iSlNVbrfKXETGRlNaTXbuOlEn7gNMaMZSad6j1VGpeWbZS8yIiiiK337L3fyfL3PebyTK32yIiMg+DihiK+QMfSsvrlVa5K/K6qJmU0z3Z+0W7R3em5s3vz+22nXt+lZoXEfGeMy9MzetvnJWaV0Z2/+a/xxap20pfNIp3zHiPM38AAAAAaszwBwAAAKDGDH8AAAAAaszwBwAAAKDGDH8AAAAAaqzt4c/69evj4Ycf7uZaAE5IFwFVoIuAKtBFQLuc+QMAAABQY32dfPGGDRti48aNM943PDwcDz30UMqiAI5HFwFVoIuAKtBFQDs6Gv6sX78+li9f3q21ALRFFwFVoIuAKtBFQDtc9gUAAABQY2mXfa1duzauvPLKlEUBHI8uAqpAFwFVoIuAdhRlWZZzDbnlllvi0ksv7ahYJlpPRsT4XL91RES04lBKzrQiitS8iIhGMZya91//5e9S8/54yR+l5k2Wv03Ni4iYaOVm9jfOTM1rJj/HHc5mT7K+aBRn93oRrzGbLjo0+T8ikjqkUQym5EwrIjcvIqJM7stWeSA1b+/YS6l585oDqXlTmfNT85rFwtS8iMnUtEZxRmrelKw19kWjWJKUlWc2XdQqfxkRE0krmPOuXdeVSfuA0xoxlJp3qPVUal5ZtlLzIiKKIrffsvd/J8vc95vJMrfbIiISDoMOK2Io5g98KC0vw+y6aFfkdVEzKad7sveLdo/uTM2b35/bbTv3/Co1LyLiPWdemJrX3zgrNa+M7P7Nf48tUreVvmgU75jxnpTLvvbs2RPDw9kHvgCd0UVAFegioAp0EXCktk4tWLp06THve+CBB2L37t2xcGH2Ty4BjqaLgCrQRUAV6CKgE20Nf3bs2PGa28qyjKIoYnR0NHbt2hVLllTvtGugXnQRUAW6CKgCXQR0YtYfKrJmzZrYuXNnjI2NxcqVK2NoKPd6RIB26CKgCnQRUAW6CDiWWQ9/7rjjjsx1AMyKLgKqQBcBVaCLgGNJ+cBnAAAAAKrJ8AcAAACgxmZ92ddcNYr5ETGRklXEWErOtLLMWdeRiuhPzfvV/pHUvCIGUvMacVpqXkREo9iXmjfeejE1r2jkbk5FMZiaFxFR9G6Tr6y+YkFEjKdklUk504piXmpeRESUuXF9jfmpecMDk6l5+8dzeyMi4vmDz6fm/d5pydtl0cyNiy68DrNfiDXQKPoj62dyZdlKyemuIjct+XXfVyxOzSuLQ6l5ERHjrT3pmZlGxg+k5h2YyH8M5w+8IS2rEafCdteOvPekInK3yzJy9xEi8veNF85bkJrXKnP/zYcm8x/DiVbuvlYj/Rgo+Rgt+bg5IqIoMtd47PdXZ/4AAAAA1JjhDwAAAECNGf4AAAAA1JjhDwAAAECNGf4AAAAA1JjhDwAAAECNzWr4s3nz5tiyZUvyUgA6o4uAKtBFQBXoIuB4Us78GR8fj1/96lcZUQCzpouAKtBFQBXoIuBIsx7+PPfcc4f/f/369fGXf/mXKQsC6IQuAqpAFwFVoIuAY+lr54uWLl0aERFjY2OxbNmyeOtb3xo/+tGP4vOf/3x8+9vfjmeffTa+//3vd3WhALoIqAJdBFSBLgI60dbwZ8eOHRERcffdd0ez2YzR0dFYtGhRLF++PK644orYunVrDA0NdXWhALoIqAJdBFSBLgI60dFlX9u3b48/+ZM/iYiIFStWxBlnnBHXX3+9UgFOKl0EVIEuAqpAFwHtaHv48+Mf/zjOO++8OPPMMyMiYnBwMNasWRNf/epXY2xsrGsLBDiSLgKqQBcBVaCLgHa1Nfx5+eWX47bbbosbbrjhqNuvuuqqWLx4cXzta1+LiYmJriwQYJouAqpAFwFVoIuATrQ1/PnCF74Qn/70p2Px4sWvuW/jxo3x/PPPx/XXX5++OIAj6SKgCnQRUAW6COhEW8OfpUuXxooVK2JkZCQiIiYmJqIoioiIGBoaii1btkR/f3/3VgkQugioBl0EVIEuAjpRlGVZtvvFt9xySzzyyCNx8ODB2Lx5c7z3ve+d9Tdulc9HRM5piGXkXs9alvmnRzaKN6Tm3fPkf0nN++yFK1PzWuVIal5ExES5OzWvVU6m5g00zkrNK4rB1LyIiKK9X/DXhr5oFO9IyupcZhdNtp6NiPGUdZVJOdOKIv+DGsvyYGpeUQyk5o23/i01b//4vtS8iIjfjuVm/t5pv5+aF0UzNa5ZvDE1b0rbux4n0BfN4p1JWZ3L7KKIX0dEzvtSWbZScrqpTNoHnNZI7qKJ1oupeWUcSs2LiBhv7UnPzDQyfiA178BE/mM4fyBv/7wRQ7Fg8CNpeZ3IPUb7ZWQdoxWR+35UJnXk0XK7aKy1KzUv+3jlv//6udS8iIj3vvkdqXkDzUWpeW3+gvO2FZH7fhMR0SjmJab1RaNYMuM9HQ1/Mhn+zI3hz9wZ/sxFb4c/mQx/5sbwZ+4Mf+ait8OfXIY/c2H4Uz2GP6cmw5+5MfyZO8OfuTr28KejX/UOAAAAwKnF8AcAAACgxnLPgepI8e//ZSQln3pV5KzrSNmn+/701y+k5n32wtxTCosuzBX7irek5pVF8uWCkXyafZl/enPeazt/G+mVVoxFJF06mn3ZV6PMr+jsU6azt/VmcXpq3hv6crfziIjRidxL5ybL3MvIGnFaal53Nndd9GpTl2rlvI9kX1KVd5nekYm5fVkm92V2V5ZdeAwPTub2W7PI7fOBRu5zMjiY/8HF/Y28y5KKqMcHK08ds2R9NEfu636i9ZvUvIj8y9f7G2en5kXyZbx/eFb+vuWeg79NzWtF7seHnDnvtb8Nby4GGjNfUjU3mZdIHjvLmT8AAAAANWb4AwAAAFBjhj8AAAAANWb4AwAAAFBjhj8AAAAANWb4AwAAAFBjhj8AAAAANWb4AwAAAFBjhj8AAAAANWb4AwAAAFBjhj8AAAAANWb4AwAAAFBjhj8AAAAANdb28OeSSy6JD33oQ91cC8AJ6SKgCnQRUAW6CGhX28Of97///bFgwYJurgXghHQRUAW6CKgCXQS0q6PLvm666aYuLQOgfboIqAJdBFSBLgLa0dfuF65ZsybOPffcbq4F4IR0EVAFugioAl0EtKvtM3+UClAFugioAl0EVIEuAtrlt30BAAAA1Fjbl31VWZE8wypiMDVvSu4a//6//Tw1L5YlzwGLZm5eRDTSX67zkvNylTHZ6yUcR9HrBaRpFqdFJG3zrXI0Jae7ytS0IgZS8xrF/NS8ZqOVmhcRMdDMfZ73ju1PzRseyH2OG2X+e2Le+2x/jeqojLztcyIpZ0oZ+dtRWY7nBha5r/vsrszPi2gkv/j7itx9wYnk1032+iIiisTHsC5V1IjByDtEzH3O+hr5r4GyPJiaV0R/at5EvJSal73fFhHR38w9Rnt+ZE9qXjP5uHTx0O+l5kWcvP5w5g8AAABAjRn+AAAAANSY4Q8AAABAjRn+AAAAANSY4Q8AAABAjRn+AAAAANRYx8OfiYmJuOiii7qxFoC26SKgCnQRUAW6CDgRZ/4AAAAA1JjhDwAAAECN9c3mL42NjcVll1121G2f/OQn48Ybb0xZFEA7dBFQBboIqAJdBBzPrIY/AwMD8dhjj2WvBaAjugioAl0EVIEuAo7HZV8AAAAANWb4AwAAAFBjhj8AAAAANWb4AwAAAFBjbX3g86s/NX5wcPA1t0WEDxgDukoXAVWgi4Aq0EVAJ9oa/igMoAp0EVAFugioAl0EdMJlXwAAAAA1ZvgDAAAAUGNtXfbVDWV5ICLGc8KKgZycw5rJeRERZWra/3Tpual5ZdZzMZ1XHkrNmzKZG1fkzj6L5M2pLHOfk4iIMsaSkgaiWSRF9Vzx7/8lJBXzUnIOKydy8yJisnwlNa9ZDKfmZXdHEf2peRERbxpckpo30fpNat6eQ/tS8944mL+xN4uhpKTB3u3IpGv9+38Zcp+zogs/Kywjv98yZe8XRZn13P7OUF9y/6btI0zZd+jl1LzT+7N643cmE5+XInlfv3fyjoOK5H3tZpyemhcR0Yrs48jc18GzL/8iNe/s0xal5kVEDPcvTs2b/8bc53lkfH9q3mT5UmpevoHoL94y4z3O/AEAAACoMcMfAAAAgBoz/AEAAACoMcMfAAAAgBoz/AEAAACoMcMfAAAAgBpr6zekLl269Kg/79mzJxYsWBCNxtGzox07duStDOBVdBFQBboIqAJdBHSireHPqwvjqquuiu9+97tx9tlnd2VRADPRRUAV6CKgCnQR0AmXfQEAAADU2KyHP0888UTmOgBmRRcBVaCLgCrQRcCxzHr4c/vtt8cXv/jFePHFFzPXA9ARXQRUgS4CqkAXAccy6+HP3XffHb//+78fV199dfzkJz/JXBNA23QRUAW6CKgCXQQcS1sf+DyTZrMZq1evjosuuii2bdsWH/jABzLXBdAWXQRUgS4CqkAXAccy6+HPtGXLlsWyZcsy1gIwa7oIqAJdBFSBLgJezW/7AgAAAKgxwx8AAACAGpvVZV8//OEPs9cB0DFdBFSBLgKqQBcBx+PMHwAAAIAaM/wBAAAAqLE5/7av2Rpr/WuUcSAlq1m8ISXnd3kLU/MiIhrFaal5n3n3u1PziuQ5YBllal5ERCv2p+YV5UBuXtGfnNdMzYuIKMus5yX/+e2dRmTNwYvkx6Us8ufzea+BKa3yYGpeGZOpeUXkb0dFMS81r1nkPob9jZz31mkHJ8dS8yIiBhutlJwiWtGFp7gnpt43cx6XfEUXMqv9xJXloeTE/MewKAZzA5Pf2gebuftF462J1Lxsjaj2+tpVFI2o137e8TWK3OOBsszdj3nz0HBq3lDfotS8iIj+4qz0zEz9A7nHkL/c/0+peRERD+/6/9Ky5vcPxzXnXzTjfc78AQAAAKgxwx8AAACAGjP8AQAAAKgxwx8AAACAGjP8AQAAAKgxwx8AAACAGjP8AQAAAKgxwx8AAACAGjP8AQAAAKgxwx8AAACAGutr54uWLl164qC+vnj00UfnvCCAY9FFQBXoIqAKdBHQibaGPzt27Oj2OgBOSBcBVaCLgCrQRUAnXPYFAAAAUGOGPwAAAAA1ZvgDAAAAUGOGPwAAAAA1ZvgDAAAAUGOGPwAAAAA1ZvgDAAAAUGOGPwAAAAA11terbzxZtqKMVlLagaScKY3iUGrelHmpaf3NZmpeRJGc1p+aFxFRxmR6ZrXlPidTiQNJSfnPb6+0ypGIGO/1Mk6a/G1zIjWtLA/m5qWmTWlk92WR+5ycMXBaal53dhVy+rzo3W5MujLGIm97yn7/yN+Syor37qHJfal5A82s99/facTpuYHJL5uBRu722Sjyf2Y9WWYdi9Srj6iOBQNv6/USTijvmH5KkX1+SpF73PzrA7nvDxERr4zlvScWx9mXcOYPAAAAQI0Z/gAAAADUmOEPAAAAQI0Z/gAAAADUmOEPAAAAQI3Navhz1113xcjISPZaADqii4Aq0EVAFegi4Hg6Hv7s3LkzfvCDH8Rpp2X/KlmA9ukioAp0EVAFugg4kb52vujBBx+Mm2++OebPn3/4tssvv/zw/4+MjEKXKvAAAAqRSURBVMTq1avj2muvTV8gwDRdBFSBLgKqQBcBnWhr+BMR8fGPfzzWrVs3432bN29OWxDA8egioAp0EVAFugholw98BgAAAKixts/82bZtW2zfvn3G+6ZPKQToNl0EVIEuAqpAFwHtctkXcErRRUAV6CKgCnQR0C6XfQEAAADUmOEPAAAAQI35zB/glKKLgCrQRUAV6CKgXW0Nf/r6+mLlypXxla98Zcb7XU8KnAy6CKgCXQRUgS4COlGUZVn24huPjP99lHEgJatZNFNypvU3zkzNi4hoFsOpef+4+8epee8/6yOpeWV5KDUvIqIVB1Pzish93TSK01PzyphIzYuIKMvxpKT+6Gu8KymrtyZav4iIrMel+ibLV1LzmsX81LxWuT81rxvyt/Xcvpws96bmdXCScAcmU1KKmBeDzQ+kZPXaZPnPEWm9XyTlTMvfVWyVo6l5fcWi1Lx940+k5g00B1LzIiKayf/mMvm9cHzypdS8RpH/aRWTZSstq4ihOK3/g2l5vfPryOro16OyzH3sJsrdqXndkN1FRfIn02QfQ/70N/9Pal5ExKPP/zIta3hgQXzuohtmvM9n/gAAAADUWDd+nNeWIuYlZmXPsPJ/OpP9UA82c3/ynP9SyPtJSvcys183PduceqBO/9b+Xi/gJBtMzst+/LLX1w3Z/+bcsyoy31+nVPfMn1Pj9dKuzMe5+mf+5O9r5Z7NW8RQcl433muyH8Pc103+Y5j9uo4oEl/b+d3bK84NqJZuHJdmy95PyN7Wc9c30Mg+Dp86WyfL/P4zjnlfzy77AgAAAKD7jHYBAAAAaszwBwAAAKDGDH8AAAAAaszwBwAAAKDGDH8AAAAAaszwBwAAAKDGDH8AAAAAaszwBwAAAKDGDH8AAAAAaqx500033dTrRXTT+vXrY3R0NM4777xeL2VG3VzfN7/5zXjxxRfjggsumFNON9e4efPm+MUvfhF/8Ad/kJ7N7Nx1111x/vnnx8DAQK+XUiu6SBfRGV3UHa/nLorI6SNd9Pqii7pDF+kiOpPRRX2J65mTrVu3xl133XX4z/v27Yv+/v6YN2/e4du+/vWvx4c//OFeLC8iIh577LH40pe+dNyvufPOO+PCCy88SSuqn/Hx8XjhhRfibW97W6+XEhMTE3HJJZfEP/3TP806Y+nSpUf9ec+ePbFgwYJoNI4+6W7Hjh2z/h6Zdu7cGT/4wQ/iuuuu6/VSeqrqfaSLuq9uXRRxavWRLpqii9BFvaWLpugiqtRFEa+/47SsLqrM8GfVqlWxatWqw3/+xje+EZdeeml87GMfm3P2hg0bYuPGjTPeNzw8HA899FBbOZdddllXnvys9T311FOxaNGiWLRo0Yz3//znP4+BgYE499xze7bGmTz33HOH/396gn377bfPOq9KXv16ueqqq+K73/1unH322T1a0Ws9+OCDcfPNN8f8+fMP33b55Zcf/v+RkZFYvXp1XHvttT1YXW90q4900RRd1BtV7yNd9Fq66LU63c671Ue6aPZ00alHF72WLjr1vR67qDLDn1ebnJyMZrOZkrV+/fpYvnx5SlY3ZK1vy5Yt8ZGPfCQ+8pGPzHj/o48+GvPnz5/VAVfWGqcnrGNjY7Fs2bJ461vfGj/60Y/i85//fHz729+OZ599Nr7//e/P+fvQmY9//OOxbt26Ge/bvHnzSV5N9WT1kS6aoos4Fl10fLqoc93qI11Ub7ro+HRR53QRs5HdRZUZ/rz6tKtXXnkltm/fHrfeeutRt/fytKtXr/FYqnBqWFVNPzZ33313NJvNGB0djUWLFsXy5cvjiiuuiK1bt8bQ0FCPV/k7Y2Njcdlllx112yc/+cm48cYbZ535xBNPVGaizMyq3ke6aO500RR9VG26qP500RRdVG26qP5OtS6KcJw2G5UZ/rx6Y/zoRz8aF1xwQWzatGnO2cc7HW7t2rVx5ZVXdrzGTZs2xfPPPx/f+ta3KrO+bspe4/bt2+POO++M+++/P1asWBF/8zd/E9dff33lSmVgYCAee+yx1Mzbb789HnnkkVi3bt0xT/08nnbe4Pr6+uLRRx9tO3Pbtm2xffv2Ge+bPqXw9aRbfaSL5k4X5ZpLH+mi7tNFc1tfN+miXLqo2nTR3NbXTa/XLoqo3nHaqdBFlRn+HOm5556LvXv3xpNPPhmPP/74nD5lfMOGDbFhw4YZ77vllltmlTkyMhJbt26Nv/qrv4qIiN27d8fTTz991DV4vVxftuw1/vjHP47zzjsvzjzzzIiIGBwcjDVr1sRXv/rVuPfee2v/2xTuvvvu+OEPfxhXX3113HrrrfGBD3ygo7/fjZ9aOL352LL6SBfNnS7KN5c+0kUnly7SRXWmi04dukgX1Vndu6hx4i85uVqtVtx0002xatWqWLduXaxevTp2797dle+1Z8+eGB4e7vjvbdmyJd797nfHxRdfHBER+/fvj7Vr18b+/fsrsb6TqdM1vvzyy3HbbbfFDTfccNTtV111VSxevDi+9rWvxcTERPYyK6XZbMbq1avj5ptvjm3btvV6ORzHyeojXTR3umh29NGpQRfNbX0nky6aHV10atBFc1vfyaSLZqfuXVSpM3/27t0bf/EXfxFlWcbnPve56O/vj0996lOxYsWK2LRpU7znPe/pKO94p1498MADsXv37li4cGFHmU888UTcc889cd999x2+7Z3vfGdcfPHFcd9998VnP/vZnq4vW/Yav/CFL8SnP/3pWLx48Wvu27hxY1xzzTVx/fXXx1//9V+3nXnJJZfEG9/4xo5OoauCZcuWxbJly3q9DI4hs4900dzpou7SR9Wli+a2vmy6qLt0UXXpormtL5su6q66dlFlhj+bNm2K++67L6688sq47bbbor+/PyIirrvuunjzm98cn/nMZ+KDH/xgfOc732k7c6ZTr8qyjKIoYnR0NHbt2hVLlixpO+/ll1+OG264Ia699tpYvHhxPP3007Fr16545plnYs+ePbFt27aOiiV7fRER69atO3z63759+6K/v//w9bizuS4we41Lly6NFStWxMjISJx++ukxMTERRVFERMTQ0FBs2bIlvvzlL3e0xve///3xr//6rx39HY7m2vajZfeRLtJFtEcXHU0Xzb2LInL7SBe9Puiio+kiXaSLeqO2n/mzd+/euPfee+Nd73rXa+67+uqr433ve19873vfm/P3WbNmTezcuTPGxsZi5cqVHX141ejoaLzwwgtx//33x0MPPRQLFiyIt7/97XHOOefENddcE9/5znfiZz/7WVx66aU9Wd+rP3H/1bKuUZ7LGqeLd9OmTfHII4/EwYMHj1rX6aefHnfeeWfHa7rppps6/jszefUnxg8ODr7mtojoyocd9kpfX1+sXLkyvvKVr8x4/+vx2vaT0Ue6aO50kS6qO100t/VFnJw+0kW6qO50kS7qdRdFvP76qBtdVJRlWc51YUx56qmn4pxzznldfBhWlTz77LNx7rnn9noZUBm6qDd0ERxNF/WGLoKj6aLe0EXVY/gDAAAAUGOV+21fAAAAAOQx/AEAAACoMcMfAAAAgBoz/AEAAACoMcMfAAAAgBoz/AEAAACoMcMfAAAAgBr7/wG28E3qxwCU/gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder Self Layer 4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfXSU9Z338c81k8yE8BBQFAxo11iBLtE71KfjQ6P2gBBbb6nIcWVtZcWHFkXE2gpLq9SVFmy72lVQUFrbpRFL4RbYBrAH2DOKrgsqBwVshIr1AWIEhEBwkslc9x80KSkPM0O+V+aXyft1jueYmeGTLyH55Mo31zXj+b7vCwAAAAAAADkplO0BAAAAAAAAEByWPwAAAAAAADmM5Q8AAAAAAEAOY/kDAAAAAACQw1j+AAAAAAAA5DCWPwAAAAAAADmM5Q8AAAAAAEAOy8vWO/7m7+5Tzf5dJlkv3jLDJKfZnvhG0zxJ6hU91zzTUiL5WbZHSCkv1DPbI7Qr30+aZ3qe1b43JOkUo6zsije9Kakh22Mc1Wdxm448XO3n+03zBvQcYJoXVlfTvIRv3235odOMExOmaV72vrWnzTdLCivkWf97ZEeT/xdZfS5sr9tgktPsC92+ZJonSdPX/T/TvAcvGmeaF2963zQvEjrdNK9jaDJN87x80zzJ+lgrLM871TAvOxqa3pbVcVHI62KS06wx+YlpniRFwv1N8xZte9E0b9RZw03zEskdpnmSlGd8XOQp7HSezPOkpOoN0/IU9r5wjHuypGb/Ln28z+oL2PabS9I/aJp3iO2M9hqzPUAaXP8YWrNf/lj+yJU7GiTFsz3EUTX5lt8IDmlI2i5/7BdnEeO8IP5tbZc19nmecV4Q6KIjJWT1uZBIHjDJ+Rv7Y4Td8T3GidbHCNbd0dmOYST7v3MQFywEcazV0VkeF9n+qOkriJ/RbPttf+Ne0zzrYwRfn5vmHWJ9HNMZjxHa52dxLvsCAAAAAADIYSx/AAAAAAAAchjLHwAAAAAAgBx2QsufqVOnavHixdazAEBG6CIALqCLALiALgJwPJz5AwAAAAAAkMNOaPnj+76SSZ4dH0B20UUAXEAXAXABXQTgeE5o+bNnzx598MEH1rMAQEboIgAuoIsAuIAuAnA8GS9/ksmk3nrrLa1atSqIeQAgLXQRABfQRQBcQBcBSCXj5c/q1as1YMAA9enTR6+++moQMwFASnQRABfQRQBcQBcBSCWj5c/+/fv1yCOP6NZbb9Wtt96qmTNnKpFIBDUbABwVXQTABXQRABfQRQDSkfbyp6GhQffcc48uvfRSXXLJJbr44otVWlqqBx54QL7vBzkjALSgiwC4gC4C4AK6CEC60lr+1NbWauzYsSosLNSUKVNabp86dao++ugj3Xnnndq7d29gQwKARBcBcANdBMAFdBGATKRc/qxfv17XXnutysrK9OijjyoSibTc16VLF82dO1fdunXT8OHDtWnTpkCHBdB50UUAXEAXAXABXQQgU3mpHvCFL3xBs2fPVllZ2VHvj0ajeuSRR7R27VqVlJSYDwgAEl0EwA10EQAX0EUAMpVy+XPKKafolFNOSRl06aWXmgwEAEdDFwFwAV0EwAV0EYBMZfxS7wAAAAAAAOg4Up75E5QXb5khqSlb7/64Ti443zyzIfmhaV4k1N80L//OW03z/CcXm+Z1BL6ftM3T56Z5kuSp0Dyzo8sLnSLJ5uVQfb/RJKdZny69TfMk6RsLHjDN+59brjLN82X7Mcz3+prmSVLCrzHNC3u9TPMk2y4KeVHTPEnyzJLCZknZlkh+KilukvUP3c80yWnmeQWmeZL0zqe1pnlJ3+Zj18zL3iFy2jzP7d/h+r7tcX7SbzDNkyTPtENy45W1PjiwXY3JAyZZZxddYJLTbP0n203zJOmy0842zfvnAZeY5nmKpH5QRnn23zeDyLTkGx8X2R3DHMb058jkMYd0+7sGAAAAAAAA2oTlDwAAAAAAQA5j+QMAAAAAAJDDWP4AAAAAAADkMJY/AAAAAAAAOYzlDwAAAAAAQA7LePnj+77q6uqCmAUA0kYXAXABXQTABXQRgFTyUj1g586dev3117Vt2zZt3rxZb7/9tioqKjRx4kTt2rWr5XHhcFj9+/cPdFgAnRddBMAFdBEAF9BFADKV1vJn3bp1Kikp0YYNGxSLxRSJRLR48WI99dRTOvPMMyVJW7ZsUSwWC3xgAJ0TXQTABXQRABfQRQAylXL5U1ZWprKyMsXjcc2ZM0eRSKTlvoqKCk2aNEmSVF5eHtyUADo9ugiAC+giAC6giwBkKuVz/ixfvlzl5eUaNmyYdu/erfLycl1zzTXtMRsAtKCLALiALgLgAroIQKZSLn8qKioUi8U0efJkRaNRVVZWatmyZe0xGwC0oIsAuIAuAuACughAptJ+ta/KykolEgmNGzdOn376aZAzAcAx0UUAXEAXAXABXQQgXWktf9544w3l5+erqKhIo0aN0oQJE4KeCwCOQBcBcAFdBMAFdBGATKRc/tTX12vKlCktZXLbbbdp7NixQc8FAK3QRQBcQBcBcAFdBCBTKZc/Cxcu1NChQzV48GBJkud5Gj58eOCDAcDh6CIALqCLALiALgKQqZQv9T5q1ChFIhH5vn/EfQsWLNDKlSsDGQwADkcXAXABXQTABXQRgEylXP5069ZNkhSPx1vdPnjwYM2aNUvnn3++JGnx4sUBjAcAh9BFAFxAFwFwAV0EIFMplz/NotGo1q5d2/L2wIEDW91/3XXX2U0FAMdAFwFwAV0EwAV0EYB0pb38sZZI7pbUaJKVF+ptktPsjGk3meZJ0l+mzTfPtOQ/aftbgT3x103zJKlX9DzzTJeFvELzTN9PGiV58jyjqCwLKSIpbJKV9Kw+vsH5n1t+bpr3pZ/dapq38d4HTfPCoZNM8yQpEjrdPNNlSb8hgNQjL1M4MXkK5UgXHTqWSZhkvfgX28s9Rpwx0jRPks4+6WTTvJAXNc3zzPPSeoHdnOJ5+bZ5pmlBsDmWyLZ/6P5/ZNVF8ptscv7q4r72Pwsk/c9N80Yu/ZVpXtXIaaZ5Ia/INE+SEv6npnl5Xh/TPM945RFEn4e9npZpx7yn830nAgAAAAAA6ERY/gAAAAAAAOQwlj8AAAAAAAA5jOUPAAAAAABADmP5AwAAAAAAkMNY/gAAAAAAAOQwlj8AAAAAAAA5LO0XvZ88ebLWrFmjaDTa6nbf95Wfn6/Vq1ebDwcAf48uAuACugiAC+giAOlKe/kjSdOnT9fQoUNb3bZv3z6NHDnSdCgAOB66CIAL6CIALqCLAKQjo+XP1KlT9dBDD7W6rXmrDADthS4C4AK6CIAL6CIA6eDMHwAdDl0EwAV0EQAX0EUA0pH2Ez43NjayPQaQdXQRABfQRQBcQBcBSFfaZ/589tlnuv/++xWJRFrdzimFANoTXQTABXQRABfQRQDSldbyx/d9bd++XbFY7Ihi4ZRCAO2FLgLgAroIgAvoIgCZSOuyr7Vr12rQoEFHlAoAtCe6CIAL6CIALqCLAGQi5fLH933NmjVL119/favbGxsblUgkVF9fL8/zAhsQACS6CIAb6CIALqCLAGQq5WVf1dXVKiws1JVXXtnq9traWo0ePVr19fUaMWJEYAMCgEQXAXADXQTABXQRgEylXP4MHDhQ8+bNO+L24uJirV27NpChAODv0UUAXEAXAXABXQQgU2m/1DsAAAAAAAA6HpY/AAAAAAAAOSytl3oPQtgrktSUrXd/XH+ZNt88c9u+/zbNO6vHFaZ5Fzx9l2neutueMM3rCJKqtw30beMkKex1M0pib3x0th+XkLqY5h2SNE3bdN8vTPN633+rad6nM582zZMk37f9GHqe219PIc/+VWSSvlVf5s6TmXqKyKpDNu/ZY5LT7KozDpjmSdIZPbqbZ1rK807O9ggp+X6jaZ7n5ZvmoWOqb3xfvg6aZHXPLzXJafaj/600zZOkaReOM8175qqvmeZZH1t6sv86z/NOMc3zlTDNs945+AH8jGY7Y1LHeq53t484AQAAAAAA0CYsfwAAAAAAAHIYyx8AAAAAAIAcxvIHAAAAAAAgh7H8AQAAAAAAyGEZL39831ddXV0QswBA2ugiAC6giwC4gC4CkErKl3rfuXOnXn/9dW3btk2bN2/W22+/rYqKCk2cOFG7du1qeVw4HFb//v0DHRZA50UXAXABXQTABXQRgEyltfxZt26dSkpKtGHDBsViMUUiES1evFhPPfWUzjzzTEnSli1bFIvFAh8YQOdEFwFwAV0EwAV0EYBMpVz+lJWVqaysTPF4XHPmzFEkEmm5r6KiQpMmTZIklZeXBzclgE6PLgLgAroIgAvoIgCZSvmcP8uXL1d5ebmGDRum3bt3q7y8XNdcc017zAYALegiAC6giwC4gC4CkKmUy5+KigrFYjFNnjxZ0WhUlZWVWrZsWXvMBgAt6CIALqCLALiALgKQqbRf7auyslKJRELjxo3Tp59+GuRMAHBMdBEAF9BFAFxAFwFIV1rLnzfeeEP5+fkqKirSqFGjNGHChKDnAoAj0EUAXEAXAXABXQQgEymXP/X19ZoyZUpLmdx2220aO3Zs0HMBQCt0EQAX0EUAXEAXAchUyuXPwoULNXToUA0ePFiS5Hmehg8fHvhgAHA4ugiAC+giAC6giwBkKuVLvY8aNUqRSES+7x9x34IFC7Ry5cpABgOAw9FFAFxAFwFwAV0EIFMplz/dunWTJMXj8Va3Dx48WLNmzdL5558vSVq8eHEA4wHAIXQRABfQRQBcQBcByFTK5U+zaDSqtWvXtrw9cODAVvdfd911dlMBwDHQRQBcQBcBcAFdBCBdab/UOwAAAAAAADoezz/ahaLtoDG5SVKDSVZ+qI9JTrOPDrximidJxYUXmOYl/QOmeSGvh2le7efrTfMkqXdBqWleY3Knad7BxGemeT0ig0zzJGlX/G2TnJBXqJOjV5lkZV+NpKZsD4G/8v2kaV7o6yNM8yQp+V/LTfM8L2ya17mEJdkeA2SL7++QVRcl/B0mOc3C3immeZIUvnW0aZ4/b4ltnnEXeR6/b819udFHTf42SY0mWUm/ziSn2bt7t5jmSdLZRV8yzQt7vUzzPjm40TQvGs43zZOknpEy0zzf+Ljck2eaF4R48s9mWZ4KFA1ffNT7+E4EAAAAAACQw1j+AAAAAAAA5DCWPwAAAAAAADmM5Q8AAAAAAEAOY/kDAAAAAACQwzJa/syePVuPP/54y9vz5s3T7NmzzYcCgOOhiwC4gC4C4AK6CEA68lI94M0339TEiRMlSQcOHJDv+1q4cKEkqb6+Xr7va8GCBZKkmTNn6uKLj/6yYgDQFnQRABfQRQBcQBcByFTK5c+QIUMUi8UkSc8995yampp00003SZIWLVqkhoYG3XjjjcFOCaDTo4sAuIAuAuACughAplIufw5XVlam6upqSdLcuXN1ww03qKioKJDBAOBY6CIALqCLALiALgKQjrSf86ehoUE/+MEPVFBQIElauHCh6urqJEnPPvusHn300WAmBIDD0EUAXEAXAXABXQQgXWmd+eP7vh588EH169dPw4cPb7l9586devjhh7Vr1y6eVAxA4OgiAC6giwC4gC4CkImUZ/40NjbqX//1X7Vy5UqdeeaZkqQ9e/bo4MGD+s53vqPS0lL99re/1SmnnBL4sAA6L7oIgAvoIgAuoIsAZCrlmT979+5VIpHQfffdp5qaGv3pT3/SmDFjFI/HtWDBApWWlrbHnAA6OboIgAvoIgAuoIsAZCrlmT+9e/fWT3/605brSAcOHKg//vGPOu2009SzZ8+Wx3300UdqamoKblIAnRpdBMAFdBEAF9BFADKV9hM+H+6kk06SpJYi8X1fDzzwgNasWWM3GQCkQBcBcAFdBMAFdBGA48nopd4P9+Uvf1ljxoxRXl6eGhsbVVxcrIsuushyNgBIiS4C4AK6CIAL6CIAx+L5vu9n4x03JjdJajDJyg/1Mclp9tGBV0zzJKm48ALTvKR/wDQv5PUwzav9fL1pniT1LrC9drkxudM072DiM9O8HpFBpnmStCv+tklOyCvUydGrTLKyr0YSp0O7wveTpnmhr48wzZOk5H8tN83zvLBpXucSlmR7DJAtvr9DVl2U8HeY5DQLe/ZPGBu+dbRpnj9viW2ecRd53gmdbI8OJTf6qMnfJqnRJCvp15nkNHt37xbTPEk6u+hLpnlhr5dp3icHN5rmRcP5pnmS1DNSZprnGx+Xe/JM84IQT/7ZLMtTgaLhi496H9+JAAAAAAAAchjLHwAAAAAAgBx2ws/501b5od6yOr3Z921PDevX9RLTPEnaZXwZ1MkF55vmXfjMBNO8/731cdO8IETDJaZ5kZDt5+HnTZtN8yTplIILjZK4TKWjamh63zQvEv6CaZ6vuGle47IFpnmS1OdfbzLN2/njX5rmhbwupnloH0kdlJQwyQp53U1ymnkBdP6pfQpN8xLJPaZ5vj43zcv3TjPNA4KSSO4w+/yPhs42yWnWp4vtJa2SlOedapr3zOZK07xx//hPpnm747aXkR3i/mVVtuzPn4mEzjBMO/aKhzN/AAAAAAAAchjLHwAAAAAAgBzG8gcAAAAAACCHsfwBAAAAAADIYSx/AAAAAAAAcljGyx/f91VXVxfELACQNroIgAvoIgAuoIsApJLypd537typ119/Xdu2bdPmzZv19ttvq6KiQhMnTtSuXbtaHhcOh9W/f/9AhwXQedFFAFxAFwFwAV0EIFNpLX/WrVunkpISbdiwQbFYTJFIRIsXL9ZTTz2lM888U5K0ZcsWxWKxwAcG0DnRRQBcQBcBcAFdBCBTKZc/ZWVlKisrUzwe15w5cxSJRFruq6io0KRJkyRJ5eXlwU0JoNOjiwC4gC4C4AK6CECmUj7nz/Lly1VeXq5hw4Zp9+7dKi8v1zXXXNMeswFAC7oIgAvoIgAuoIsAZCrl8qeiokKxWEyTJ09WNBpVZWWlli1b1h6zAUALugiAC+giAC6giwBkKu1X+6qsrFQikdC4ceP06aefBjkTABwTXQTABXQRABfQRQDSldby54033lB+fr6Kioo0atQoTZgwIei5AOAIdBEAF9BFAFxAFwHIRMrlT319vaZMmdJSJrfddpvGjh0b9FwA0ApdBMAFdBEAF9BFADKVcvmzcOFCDR06VIMHD5YkeZ6n4cOHBz4YAByOLgLgAroIgAvoIgCZSvlS76NGjVIkEpHv+0fct2DBAq1cuTKQwQDgcHQRABfQRQBcQBcByFTK5U+3bt0kSfF4vNXtgwcP1qxZs3T++edLkhYvXhzAeABwCF0EwAV0EQAX0EUAMpVy+dMsGo1q7dq1LW8PHDiw1f3XXXed3VQAcAx0EQAX0EUAXEAXAUhX2i/1DgAAAAAAgI7H8492oWi7qJHUlJ13nYLvJ80zPa9z7dny7xplntn4xCLTPN+3/fxL+DtN8/K8vqZ5kpRUvVFSnsLeWUZZ2eZuF8FN3qgK07zE739nmhdSoWme54VN82yFJfXJ9hBG7LrI+jgmiGMY6xl9JUzzwtdeY5rnL+X5V3JfrvQRx0Vt4V09zDTPr/qjbZ7faJonSZ6Xb56Jtjh2F3WujQQAAAAAAEAnw/IHAAAAAAAgh7H8AQAAAAAAyGEsfwAAAAAAAHIYyx8AAAAAAIAcxvIHAAAAAAAgh7H8AQAAAAAAyGF56T5w8uTJWrNmjaLRaKvbfd9Xfn6+Vq9ebT4cAPw9ugiAC+giAC6giwCkK+3ljyRNnz5dQ4cObXXbvn37NHLkSNOhAOB46CIALqCLALiALgKQjoyWP1OnTtVDDz3U6rbmrTIAtBe6CIAL6CIALqCLAKSDM38AdDh0EQAX0EUAXEAXAUhH2k/43NjYyPYYQNbRRQBcQBcBcAFdBCBdaZ/589lnn+n+++9XJBJpdTunFAJoT3QRABfQRQBcQBcBSFdayx/f97V9+3bFYrEjioVTCgG0F7oIgAvoIgAuoIsAZCKty77Wrl2rQYMGHVEqANCe6CIALqCLALiALgKQiZTLH9/3NWvWLF1//fWtbm9sbFQikVB9fb08zwtsQACQ6CIAbqCLALiALgKQqZSXfVVXV6uwsFBXXnllq9tra2s1evRo1dfXa8SIEYENCAASXQTADXQRABfQRQAy5fm+72fnXddIasrOu07B95PmmZ6X9gur5YT8u0aZZzY+scg0z/dtP/8S/k7TvDyvr2meJCVVb5SUp7B3llFWtrnbRXCTN6rCNC/x+9+Z5oVUaJrneWHTPFthSX2yPYQRuy6yPo4J4hjGekZfCdO88LXXmOb5S1ea5sFFudJHHBe1hXf1MNM8v+qPtnl+o2meJHkeTyzulmN3UefaSAAAAAAAAHQyLH8AAAAAAAByWFov9R6EpN8oGZ2i68n2lHS7S2P+pqlpt2lefuh007zd8TdM8+KP/6dpniQl/QbTvCa/1jTvhfdeMs0bVfJ/TfMkKd70nkmOpy7qkpcrl33Zsf4cDXn2r97xedNm07yC8D+a5nUEn//ucdO8eNN207x/qlpomvfCNVNN8yQp5EXNM/E31pdAeQrilYRsn3XAsz6k7V1gGteU/Mw0T5I8z/YSzyC+56DjSSR3SbK5NCjsnWyS0yzp7zPNk6RwqKdpXtWT3zbNs75M6/G37H9Gm3DOzaZ51j+Le7I95jD/fiPJ/HviMZ7rnTN/AAAAAAAAchjLHwAAAAAAgBzG8gcAAAAAACCHsfwBAAAAAADIYSx/AAAAAAAAchjLHwAAAAAAgBzG8gcAAAAAACCHsfwBAAAAAADIYSx/AAAAAAAAchjLHwAAAAAAgBzG8gcAAAAAACCHsfwBAAAAAADIYSx/AAAAAAAAclhay58hQ4boq1/9atCzAMBx0UUAXEAXAXABXQQgE2ktfy699FL17Nkz6FkA4LjoIgAuoIsAuIAuApCJtC/7mjZtWoBjAEB66CIALqCLALiALgKQrrx0HjRx4kSdffbZQc8CAMdFFwFwAV0EwAV0EYBMpHXmD6UCwAV0EQAX0EUAXEAXAcgEr/YFAAAAAACQw9K67CsIIS9fru6ewupunxm2z7R0csH5pnm+32iaJ0mel2+a56vINO/bT79gmjd6xj+Z5klS0vdNcjzZ5Ljg0Odqk02W4iY5fxMxzpPyQ2eY5vl+0jRPxp9bnhc2zZOkaPiL5pmWFn+9v2ne4J+PN82TpDcnfdckx1OBouE+Jlm5xMve4V3agvjatJSc93vjRPu/b2hkhWle8oUq0zzX/41xdGGvh6yOi8x59t3m+7Z/13N7n2aa5xsfF53apcA0T5ISfq1pXrypxjQvGu5tmhf2bPMO8UyzjpXm5vYFAAAAAAAAJlj+AAAAAAAA5DCWPwAAAAAAADmM5Q8AAAAAAEAOY/kDAAAAAACQw05o+fO9731P8bj1q9oAQGboIgAuoIsAuIAuAnA8ab1e3j333KP33ntPktTU1KT3339f1dXVrR7Tu3dvzZs3z35CAPgrugiAC+giAC6giwBkIq3lz2OPPdby/1VVVYrFYpoxY0ZgQwHA0dBFAFxAFwFwAV0EIBNpLX+eeeYZVVVVSZLeeecdffGLX9R1113X6jHXX3+9xowZYz8hAPwVXQTABXQRABfQRQAykXL58+6778rzPH3ta1/Tjh079OGHH+raa6894nEHDx7Uxx9/rOLi4kAGBdC50UUAXEAXAXABXQQgUymf8Llnz54qLS1VdXW1YrGYHnvsMZWWlqq0tFSDBw/WwoULtX//fpWWlqpr167tMTOAToguAuACugiAC+giAJlKeeZPUVGRVq9erSVLlqh3796aOXNmq/tra2v161//WtFoVOedd15ggwLo3OgiAC6giwC4gC4CkKmUy5+amhpt3bpV48aNU9euXTV+/PhW948fP15Dhw7V/PnzNXLkSPXt2zewYQF0XnQRABfQRQBcQBcByFTK5c/pp5+uefPmad68efrVr36llStXtrr/ww8/1M0336xFixbJ87zABgXQudFFAFxAFwFwAV0EIFNpvdpXszFjxhx1qyyJUgHQbugiAC6giwC4gC4CkI6Mlj+/+c1vtHTp0la31dTU6OabbzYdCgCOhy4C4AK6CIAL6CIA6Uh7+TNu3DiNGzcuyFkAICW6CIAL6CIALqCLAKQr5Uu9AwAAAAAAoONi+QMAAAAAAJDDMnrOHyB94WwPkFJIXUzzdn90wDQvCNFwH6OkiFGOC+w+V0PqapYVlLDXLdsjHFfSrzfN81RomtcR5IV6mea9Oele0zxJOuV700xyzujVV5t+eJlJFnA4z8s3zfP9pGmeJBUN6Gmatz/xlmle17wvmeaFvKhpHo4lJMnP9hAd1sGmuGmeJ9sn7B7Y62TTPEnyjH8uCBk/SXlD06emedGw/bGl51muZZqOeQ9n/gAAAAAAAOQwlj8AAAAAAAA5jOUPAAAAAABADmP5AwAAAAAAkMNY/gAAAAAAAOQwlj8AAAAAAAA5LK3lz44dO7R06VKNHz9ekyZNarn91Vdf1d133x3YcABwOLoIgAvoIgAuoIsAZCLlC8q/++67uvfee9XQ0KDRo0frlltuabmvsbFRBQUFgQ4IABJdBMANdBEAF9BFADKV8syfs88+W8uWLVN5ebn69++vUCik/fv3a+PGjWpoaFBBQYEOHDigWbNmtce8ADopugiAC+giAC6giwBkKuXyZ8WKFRoxYoSWLFmi6dOna8SIEXrrrbf0zDPPaO/evSooKNCMGTNUU1PTHvMC6KToIgAuoIsAuIAuApCplMufESNGqKqqSoWFhRoxYoRWrFihzZs3a8CAAdq1a5eWLl2q6upqTZkypT3mBdBJ0UUAXEAXAXABXQQgU2k94fPLL7+sHTt2aNmyZVq2bJmqq6s1bNgwvffee9q3b5+efPJJdenSJehZAXRydBEAF9BFAFxAFwHIRMonfJakp59+WiUlJbrxxhtVVVWl2bNny/M8bdiwQeeee662bt2qCy+8MOhZAXRydBEAF9BFAFxAFwHIRMozf1544QUdOHBAl112mU499VQ98cQT8p7I+d4AABPySURBVDxPmzdvVjQa1be+9S0tWLCgPWYF0InRRQBcQBcBcAFdBCBTKZc/vXr10o9+9KOWt8PhsHzf1yOPPKLbb79dw4YN04YNG7Rp06ZABwXQudFFAFxAFwFwAV0EIFMplz+XX365zjnnnFa3zZgxQz169NDVV1+t/Px83X333fr+97+vurq6wAYF0LnRRQBcQBcBcAFdBCBTaT3h8+E++OADvf/++5o5c2bLbSNHjtQ555yjLVu2mA4HAMdCFwFwAV0EwAV0EYBUPN/3/ey86xpJTdl51wic7yfNMz0v413lcfm+7edf6FsjTfP8/1xmmidJjcmdRkkR5YdKjbKyy/d3yNUusv6c7wiSfr1pXsgrNM3rjD5vsr9k4JTv/Sj1g9JwRq++2vTD35tkZZ/dcZH19+DO2EXWgjgu6vX9G03zPvix7UuCd837kmleyIua5tkLS+qT7SHazPc/lt1xkWeUc0hStscIkhSS7auhbauLmead1f0y07wNu1ab5knSOSddYJrXkPzQNM963RENn2GaJ0mel9brcKUpT2HvrKPew3dzAAAAAACAHGa5YgJadITfEnpe2DQv+Zslpnne164yzZMk/w8vGiXZfuyy6bOGt5X0D5pkFUVKTHKaNSU/Nc2TpPzQF0zzrH8T6/sNpnkJ/3PTPEnKC51knumyaMj2t/eS9M9ft/kt4ckFvUxy0P6serdZyLP97b21II6LPvvp86Z5ieRu07zTH7zFNO/PDz5omidJ+aF/MEzLU8j2RJccYH2Bif0ZdL7x2d9f6Gb7PdNXo2me9Vk6khT2iozzbI9/m2R7LJjwra6k+Jvnq183y+qa30PXn8WZPwAAAAAAAJ0Oyx8AAAAAAIAcxvIHAAAAAAAgh7H8AQAAAAAAyGEsfwAAAAAAAHJYm5Y/DQ0N2rZtm9auXWs1DwBkjC4C4AK6CIAL6CIAR5PWS72/9tpreuWVV7Rnzx7V1tbqk08+UU1Njerq6lRcXKySkhL17dtXZx3jJcUAwAJdBMAFdBEAF9BFADKR1vKnX79+KiwsVHFxsXr16qVYLKYrrrhCd911lzzPC3pGAJBEFwFwA10EwAV0EYBMpLX86d+/v+64446Wt7dv3y5JlAqAdkUXAXABXQTABXQRgEzwhM8AAAAAAAA5LOWZP++8845uv/32VrcdOHBAkjR//vxWt//iF7/QkCFDDMcDgEPoIgAuoIsAuIAuApCplMufQYMGKRaLtbpt7ty5knRE4QBAUOgiAC6giwC4gC4CkCku+wIAAAAAAMhhLH8AAAAAAAByGMsfAAAAAACAHMbyBwAAAAAAIIelfMLno+FJxAC4gC4C4AK6CIAL6CIAx8OZPwAAAAAAADmM5Q8AAAAAAEAOY/kDAAAAAACQwzzf9/3svOsaSU3ZedcInO83mmd6Xr5pXtJvMM2b/MpTpnmPXHq3aZ4k/eT1OSY5RZGeGn/OJJOsbPvvD5/VwaY6k6wL+pxhktOsyU+a5klSnhc2zTu54HzTPN/47+zrc9M8SfIUNc3zlTDN82TblUFI+vuMkvIVDp1tlJVtHBcht8Sbtpnm/eNPppnmSdLG++8zy/LURYX5V5jlZYvvfyyrLvKMjzl8P4iO9EzTrlw42TRvzeifmOZ9XP+aaZ4kFRdeZJpnfVxk/W8cxLGl7Tk5eQp7JYG/FwAAAAAAADiG5Q8AAAAAAEAOY/kDAAAAAACQw1j+AAAAAAAA5DCWPwAAAAAAADks7eVPVVWVqqurg5wFAFKiiwC4gC4C4AK6CEC68tJ5UDKZ1L//+79r/vz5kqS77rpLGzduVDweVzweV48ePSRJlZWV6t+/f3DTAujU6CIALqCLALiALgKQibSWP6tWrVJpaan27dunF198UU888YQkacWKFXrppZc0ffr0QIcEAIkuAuAGugiAC+giAJlIedlXfX29fvazn+mGG27Qz3/+cxUXF7fHXADQCl0EwAV0EQAX0EUAMpXyzJ+6ujp1795dP/7xj1VYWKihQ4e2x1wA0ApdBMAFdBEAF9BFADKV8syfPn366Je//KXi8bgefvjh9pgJAI5AFwFwAV0EwAV0EYBMpVz+JBIJTZo0SWVlZVqyZInWrFnTHnMBQCt0EQAX0EUAXEAXAchUyuXP1q1btX37djU1NWnAgAG64IIL2mMuAGiFLgLgAroIgAvoIgCZSvmcP4MGDdKqVatUX1+v9evXa/369briiivaYTQA+Bu6CIAL6CIALqCLAGQq5Zk/mzZt0je+8Q195Stf0fPPP69wONwecwFAK3QRABfQRQBcQBcByFTKM39KSko0efJkDRkyRJFIRBs3blR5ebkkKR6PKx6P66WXXpIkTZkyRRUVFcFODKBToosAuIAuAuACughAplIuf7p06aKLLrqo5e1zzz1XsVgs0KEA4O/RRQBcQBcBcAFdBCBTKS/7AgAAAAAAQMeV8syf4LB3ym3JADKtr2W2/fTvFe1lmmf/95WKIj1Ncrrn9zDJcUFBuJtZVsjrYpYlSb580zxJCnnW3Wv9eeoZ5wXxbc7151VwfT5JyjfKyeJhjDmOi5BroqZp/Yr6mOZJkie779ueCsyyssvye0hH+H5ke9zRt7C3aZ71xzDsFZrm/TU1gExL1seWVscwh7Oc8djHRp7v+/Y/XQAAAAAAAMAJ/JoJAAAAAAAgh7H8AQAAAAAAyGEsfwAAAAAAAHIYyx8AAAAAAIAcxvIHAAAAAAAgh7H8AQAAAAAAyGEsfwAAAAAAAHIYyx8AAAAAAIAcxvIHAAAAAAAgh3Wa5Y/v+6qrq8v2GMdkOd/s2bP1+OOPt7w9b948zZ49u02Zrn/8gvK9731P8Xg822OkpaGhQdu2bdPatWsz/rNVVVWqrq4OYCr8Pde/llzvIsn9j2EQ6CJYc/3ryHo+jo1s0EWw5vrXEV3kro7SRy51kef7vm+WZmjq1Kk677zzdN11153Qn9+5c6def/11bdu2TZs3b9bbb7+tiooKTZw4Ubt27Wp5XDgcVv/+/dPOnTx5stasWaNoNNrqdt/3lZ+fr9WrV2dlvjfffFMTJ06UJB04cEC+76tbt26SpPr6evm+r65du0qSZs6cqYsvvrhd5zuc1ccwCPfcc4/ee+89SVJTU5Pef/99lZSUtHpM7969NW/evLTyduzYoXXr1mnFihWKRqN69NFHJUmvvvqqnnvuOf3Hf/xHxjO+9tpreuWVV7Rnzx7V1tbqk08+UU1Njerq6lRcXKySkhLde++9Ouuss9LKSyaTuuqqqzR//nz17dtXd911lzZu3Kh4PK54PK4ePXpIkiorKzP+t84VbekjuqhtXRTEjM3oIrqoo6GLMpuvoxwbdaYukuz7iC5qf3QRXZQNrh8bdYQuysvob9SB7Ny5U+vWrVNJSYk2bNigWCymSCSixYsX66mnntKZZ54pSdqyZYtisVhG2dOnT9fQoUNb3bZv3z6NHDkya/MNGTKk5XHPPfecmpqadNNNN0mSFi1apIaGBt14441Zm+/vWXwMg/DYY4+1/H9VVZVisZhmzJhxQlnvvvuu7r33XjU0NGj06NG65ZZbWu5rbGxUQUHBCeX269dPhYWFKi4uVq9evRSLxXTFFVforrvukud5GeetWrVKpaWl2rdvn1588UU98cQTkqQVK1bopZde0vTp009oThxCF7Wti4KY8XB0EV3UWbj+dRTEfB3p2KgzdJEUTB/RRR2L619HdJGbXSS5f2zUEbrI2eWP7/tKJpMn/OfLyspUVlameDyuOXPmKBKJtNxXUVGhSZMmSZLKy8szzp46daoeeuihI+bNz893Yr6ysrKW08Pmzp2rG264QUVFRRlnBDWfZPMxDMIzzzyjqqoqSdI777yjL37xi0f8VuP666/XmDFjUmadffbZWrZsmaZPn67+/fsrFApp//79+vOf/6yGhgYVFBTowIEDevbZZ3XnnXemPWP//v11xx13tLy9fft2STqhUqmvr9fPfvYzTZs2TT//+c81evTojDM6g7b0EV3Uti4Keka6iC7qSOiiEz/ucP3YqDN0kRRMH9FF7Y8uoouywfVjo47QRc4uf/bs2aMPPvjghP/88uXL9ZOf/ESStHv3bpWXl6uoqEj/8i//0ubZLDaiQc3X0NCgH/zgB7r99tslSQsXLtTVV1+toqIiPfvss9qzZ09LIWRjvmYubpXfffddeZ6nr33ta9qxY4c+/PBDXXvttUc87uDBg/r4449VXFx83LwVK1boscce0+7du1v+/8EHH9Rzzz2nyy+/XAUFBZoxY8YJFYKVuro6de/eXT/+8Y9VWFh4xL8JDmlLH9FFbeuiIGeU6CK6qGOhi05MRzg26gxdJLnfR3RReuiiE0MXnbjOdmwUVBc5+YTPyWRSb731llatWnXCGRUVFYrFYpo8ebKi0agqKyu1bNmyNs/W2NhosvkMYj7f9/Xggw+qX79+Gj58eMvtO3fu1Le//W394Q9/aDnFMBvzNbP6GFrr2bOnSktLVV1drVgspscee0ylpaUqLS3V4MGDtXDhQu3fv1+lpaUt1+Uez4gRI1RVVaXCwkKNGDFCK1as0ObNmzVgwADt2rVLS5cuVXV1taZMmZL2jO+8847Ky8tb/TdnzhzNmTPniNvffPPNlHl9+vTRL3/5S8XjcT388MNpz9GZtLWP6KK2dVFQM0p0EV3UsdBFJ6YjHBt1li6S7PuILmp/dNGJoYvaxvVjo47SRU6e+bN69WoNGDBAnufp1VdfTesJQY+lsrJSiURC48aN029/+9s2z/bZZ5/p/vvvb3V6nXTip8NZzdfY2KgHHnhAK1eu1De/+U1Jh7byBw8e1He+8x3dfPPNuv3224+Yu73mO5z1x9BKUVGRVq9erSVLlqh3796aOXNmq/tra2v161//WtFoVOedd15amS+//LJ27NihZcuW6dxzz1V1dbVuueUWPfvss9q3b5+efPJJdenSJe0ZBw0adMS1u3PnzpWklt8iZCKRSGjSpEkqKyvTkiVLdN555+nKK6/MOCeXWfURXdS2LrKcsRldRBd1JHRR5jrKsVFn6iLJto/oovZHF2WOLmo714+NOkoXObf82b9/vx555BFNmzZNnudp5syZ+v3vf6+8vMxHfeONN5Sfn6+ioiKNGjVKEyZMaNP1cr7va/v27S1PqnW4EzkdznK+vXv3KpFI6L777lNNTY3+9Kc/acyYMYrH41qwYIFKS0szzrT++En2H0NLNTU12rp1q8aNG6euXbtq/Pjxre4fP368hg4dqvnz52vkyJHq27dvysynn35aJSUluvHGG1VVVaXZs2fL8zxt2LBB5557rrZu3aoLL7wwqL9SSlu3btX27dvVs2dPDRgwQBdccEHWZnGRVR/RRW3rIusZJbqILupY6KIT0xGOjTpbF0lu9xFddHx00Ymhi9qusx0bBdVFTl321dDQoHvuuUeXXnqpLrnkEl188cUqLS3VAw88oExfkb6+vl5TpkzRhAkTJEm33Xabxo4d26b51q5dq0GDBp3Qb6yDnq9379766U9/2vLM5AMHDtQf//hHnXbaaerZs2fL4z766CM1NTW1+3zNLD+G0qFnz//qV79qknX66adr3rx5Oumkk1RZWalrr7221X+vvfaa+vXrp0WLFqVVKC+88IIOHDigyy67TKeeeqqeeOIJeZ6nzZs3KxqN6lvf+pYWLFhgMvuJGjRokFatWqV/+7d/U8+ePbV+/fqszuMSqz6ii9rWRUHMKNFFdFHHQReduI5wbNSZukhyv4/oomOji04cXdR2ne3YKKgucmb5U1tbq7Fjx6qwsLDVtXVTp07VRx99pDvvvFN79+5NO2/hwoUaOnSoBg8eLOnQs2wffn1lpnzf16xZs3T99de3ur2xsVGJREL19fUZPSGU9XxHc9JJJ0lSS4n4vq8HHnhAa9asycp81h9DSbr00ktblaaVMWPGaMmSJa3+u+iiiySl/4ztvXr10o9+9KOWt8PhsHzf1yOPPKLbb79dw4YN04YNG7Rp0ybz+dO1adMmfeMb39BXvvIVPf/88wqHw1mbxSWWfUQXta2LgpiRLqKLOgq6yLaLJLeOjTpbF0nu9xFddHR0EV3kShdJnePYKKgucuKyr/Xr1+vuu+/WyJEj9d3vfrfVX65Lly6aO3eufvjDH2r48OGaN29eyyf68YwaNUqRSOSom+gFCxZo5cqVGc1YXV2twsLCI661q62t1ejRo1VfX68RI0aknWc937F8+ctf1pgxY5SXl6fGxkYVFxe3fHG093zWH8Nm06ZNy/jPpPKb3/xGS5cubXVbTU2Nbr755rQzLr/8cklqlTNjxgz16NFDV199tSTp7rvv1ve//30tWLBA3bt3N5g8MyUlJZo8ebKGDBmiSCSijRs3trwsZDweVzwe10svvSRJmjJliioqKtp9xvZm3Ud00SEn2kVBzEgX0UUdAV3UtvmOx5Vjo87WRZL7fUQXHYkuatt8x0MXnZjOcGwUWBf5Dvjkk0/8N998M+XjXn75Zb++vj6j7M8//9y/5JJLWt5+5513/HXr1rW8vWjRoozyrDFf21RXV2f1/afj4Ycf9pcvX+7/5S9/8e+4444jPofvv/9+/7XXXsvSdPh7QfWR619Lrs/n+27PSBfBGl10iGvz+b7bM3aELvJ9+qgjoYsOcW0+33d7RrrITZ7vZ/hkOgAAAAAAAOgwnHnOHwAAAAAAANhj+QMAAAAAAJDDWP4AAAAAAADkMJY/AAAAAAAAOYzlDwAAAAAAQA5j+QMAAAAAAJDDWP4AAAAAAADksP8PKhCZ3q+yElYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder Src Layer 4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3df5SdBZ0f/s8zdyaT8GuCCcYfaHahQeELZWFRT6snmNXFbrII/Wr2rIlsqSDUigkIarbZhjhgl2xRA1nP8kNrKg0HaeCrUOiJePh2092ebosuJwYWpNLsUjUixgDJkExm7tM/2GACM8m9M5+b++TJ6/VXcp/Jez5JJu/73E+e505RlmUZAAAAANRST7cHAAAAAKBzLH8AAAAAaszyBwAAAKDGLH8AAAAAaszyBwAAAKDGLH8AAAAAaszyBwAAAKDGerv1iS+6+5r42Y5fpGR952M3pOTs9cvdm1LzIiKO7/+H6ZmZRprbuz3CQfX2TO/2CIdUWTbTM4sia9/bExEnJGV11+7Rv46I4W6PMabtu3M6cl8/37UjNe+U6aek5jXi6NS8kTK/2/p63picOJKaVnTvqb1lZVpSI3qK7L+P7hgt/y6yvha2vPhoSs5es485NTUvIuIL//P/S8279l2XpObtHv3b1LwpPW9JzTs8jKamFUVfal5E9rlWI4ri9Yl53TE8ujmyzot6imkpOXvtaT6bmhcRMaVxYmrePT/6Tmreh07+QGreSPOnqXkREb3J50VFNCqdF+l5Ec0YSkzrjUYxe5wjXfKzHb+In7yQ9Q8498mlWb6Umvey3Bnz7en2AC2o+p9htvzlT+ZLrvoYjojd3R5iTKNl5hPBy4abucuf/MXZlOS8Tvzd5i5r8vOK5LxO0EWvNRJZXwsjzZ0pOb+Sf46wbfcvkxOzzxGyu+NIO4eJyP89d+KGhU6cax3uMs+Lcl9qltGJ12i5/bZjz/OpednnCGXsSs17WfZ5zJF4jnBoXou77QsAAACgxix/AAAAAGrM8gcAAACgxlq6EXPdunVx6623vvLzF198Mfr6+mLq1KmvPHbttdfG+973vvwJAf6eLgKqQBcBVaCLgHa0tPxZvHhxLF68+JWff/7zn4+zzz47zj///I4NBvBqugioAl0EVIEuAtoxodu+RkdHo9HI/xZnAO3QRUAV6CKgCnQRcCAtXfkzd+7c/X7+wgsvxIYNG+KGG27Y7/GNGzfmTQbwKroIqAJdBFSBLgLa0dLy59WF8YEPfCBOPfXUWL16dUeGAhiLLgKqQBcBVaCLgHa0fdvX008/Hdu3b4/HH388Hn300U7MBHBQugioAl0EVIEuAg6mreVPs9mMlStXxuLFi2P58uWxZMmS2Lp1a6dmAxiTLgKqQBcBVaCLgFa0vPzZvn17LFmyJMqyjE984hNx7rnnxkc/+tFYuHBhfO973+vkjACv0EVAFegioAp0EdCqlpY/q1evjvPOOy9mzpwZt99+e/T19UVExGWXXRZXX311XHrppXHVVVd1dFAAXQRUgS4CqkAXAe1o6Q2ft2/fHnfccUe87W1ve82xCy+8MN71rnfF17/+9fThAPali4Aq0EVAFegioB1FWZZlNz7xef/un8dPXng2JWvzlben5Oz1i12PpOZFRMyYek56ZqaR5rZuj3BQvT2v6/YIh1RZNtMzi6Lt93gfRyMiZiVlddfu0b+KiN3dHmNMv9iV05H72vrSC6l5p7/u9NS8RhyTmjdS5ndbX8+JyYkjqWlF9KXmdUIZWacevdFTZP99dMdo+XRkfS38r+f/KiVnr5OPOzM1LyLi6r+4IzXvprm5VzfsHv1Rat6Unl9PzTs8jKamFUV+t+WeazWiKN6YmNcdw6Pfj6zzop7i6JScvYabP0nNi4job/xaat6/f+JbqXn/7O3/b2renuYzqXkREX09b0nNK6KRnNfS9S5tyJ0vIqIZOxLT+qJRnDzmkaxXggAAAABUkOUPAAAAQI1lXwPVsu987IbIvhw0Sydu0Rpu/p/UvCnJtx30ffLS1Lzyz+5NzTscZN+mVcau1LyIiCKOSs883PX2nBBZt1qU5Z6UnL1mTZuZmhcR8U/vWpGa998/dl5qXhm5f4Z9xRtS8yIiRsqfpeY1iuNT8yJyu6in6E/Ni4go0pLyL73ulpHmc5F1q8WvHZt7i1FRTE3Ni4h44rmfp+Y1y9zbd/NvE8iXdyt3Z5Rl7nl+sxxOzYvIvr2kK++kke6ZnVtiT3NnStacgXek5Oz1yLNbUvMiIt7zxjmpeYtP+cepeUVMSc7Lf97sRGamMvm8KO8cZh+pryOb4w5Z7WcNAAAAACbF8gcAAACgxix/AAAAAGrM8gcAAACgxix/AAAAAGrM8gcAAACgxlpe/qxYsSIeeOCBTs4CcFC6CKgCXQRUgS4CWuXKHwAAAIAa623ngwcHB2PVqlVjHhsYGIj7778/ZSiAA9FFQBXoIqAKdBHQiraWPytWrIgFCxZ0ahaAlugioAp0EVAFughohdu+AAAAAGos7bavZcuWxfz581OGAjgQXQRUgS4CqkAXAa1oefkzODgYg4ODYx67/vrr0wYCOBBdBFSBLgKqQBcBrUq57Wvbtm0xMDCQEQUwYboIqAJdBFSBLgL21dKVP3Pnzh332N133x1bt26NGTNmpA0FMBZdBFSBLgKqQBcB7Whp+bNx48bXPFaWZRRFEUNDQ7Fly5aYPXt2+nAA+9JFQBXoIqAKdBHQjrbe8HlfS5cujc2bN8fw8HAsWrQopk2bljkXQEt0EVAFugioAl0EjGfCy5+bb745cw6ACdFFQBXoIqAKdBEwnpQ3fAYAAACgmiZ85c9kjTS3RcSelKzenpkpOXu9deVHU/MiIv5u5X9Iz8xU/tm9qXm/3P291LyIiOP7fzM9s8p6iqPSM8uymZRURFEkRXVZT0yJiEZKVrPI+vPtnP/+sS+m5p1646WpeZs+fW1qXqPndal5ERFTet6SnlllzXK4A6llUk5v9NSki14+lxlJyfrO321Iydnrn7z1wtS8iIg5r8t9E9qeoj81r0jPO/L+v7Uo+nLzUtM6Iedcott+7dgzI6uLohzNyfl7/+gN+a8FmuWu1LwL7/t6at6DF65Mzesp8r/720j5XGpebzErNa9IXnl0os8bxfTMtHGPHHnPRAAAAABHEMsfAAAAgBqz/AEAAACoMcsfAAAAgBqz/AEAAACoMcsfAAAAgBqz/AEAAACosZa+6f0jjzwSn/70pw/4MbfcckucdtppKUMBjEUXAVWgi4Aq0EVAO1pa/pxzzjmxcePGTs8CcEC6CKgCXQRUgS4C2uG2LwAAAIAaa+nKn7lz57YUZvMMdJIuAqpAFwFVoIuAdrS0/Nm3MFavXh3PPPNMfPGLX+zYUABj0UVAFegioAp0EdCOtm772rFjR6xbty7+4A/+ICIitm7dGn/+53/ekcEAxqOLgCrQRUAV6CKgFW0tf9auXRunn356nHnmmRERsXPnzli2bFns3LmzI8MBjEUXAVWgi4Aq0EVAK1pe/mzatCluv/32uOaaa1557OSTT44zzzwz7rzzzo4MB/BqugioAl0EVIEuAlrV0nv+PP/883HllVfGxRdfHLNmzYonn3wytmzZEj/84Q9j27ZtsX79+vj4xz/e6VmBI5wuAqpAFwFVoIuAdrR05c/Q0FA8++yzcdddd8Xv/d7vxR/+4R/Ggw8+GM1mMy666KLYs2dPfP/73+/0rMARThcBVaCLgCrQRUA7Wrry541vfGNs3rx53ONz5syJk046KW0ogLHoIqAKdBFQBboIaEdLy5+Defvb354RAzApugioAl0EVIEuAvbV1nf7AgAAAODwknLlz0Q0ioGIGO3Wpz+gv1v5H9Izf/TCf0nNO/m496bmveP2K1Lz/ufH/zQ173DQjKHcwDI3LiKiURyTlGRvPLbcP5eemJaa97Jmatpj19yUmjfzc5em5j236vbUvIiIssz9MyyKav976immpGc2y6y+LJJyuq+IKZHVIY//8pcpOXud99b8bxf91uOOTc/M1FvM6PYIB1WWe1LziqIvNY/D09Cev40yXkrJOrbv9JScvT7/P/K/e9nKd16SmvfV8xak5mWfWxaR/++8tzghNa+MkdS87J1D2YHXaLkzNqMY5/So2mecAAAAAEyK5Q8AAABAjVn+AAAAANSY5Q8AAABAjVn+AAAAANRYy8ufFStWxAMPPNDJWQAOShcBVaCLgCrQRUCrXPkDAAAAUGO97Xzw4OBgrFq1asxjAwMDcf/996cMBXAgugioAl0EVIEuAlrR1vJnxYoVsWDBgk7NAtASXQRUgS4CqkAXAa1w2xcAAABAjaXd9rVs2bKYP39+ylAAB6KLgCrQRUAV6CKgFS0vfwYHB2NwcHDMY9dff33aQAAHoouAKtBFQBXoIqBVKbd9bdu2LQYGBjKiACZMFwFVoIuAKtBFwL5auvJn7ty54x67++67Y+vWrTFjxoy0oQDGoouAKtBFQBXoIqAdLS1/Nm7c+JrHyrKMoihiaGgotmzZErNnz04fDmBfugioAl0EVIEuAtrR1hs+72vp0qWxefPmGB4ejkWLFsW0adMy5wJoiS4CqkAXAVWgi4DxTHj5c/PNN2fOATAhugioAl0EVIEuAsaT8obPAAAAAFST5Q8AAABAjRVlWZbd+MR7mo9FxHBKVl/PrJScvX6887+l5kVEvOmod6TmNcudqXk9xXGpeT/f9UhqXkTEzKmnp+btaW5NzXtpZHtq3nFT3p6aFxHxi92bU3J6iqNiRv95KVnd97OIGO32EPy9smym5vX87j9JzYuIaP6n/5yaVxSN1LwjSyMics8BuqUsfxpZXTRS/jQlZ69GcUJqXkRE49KFqXnl176dm5fcRUXh/1vrrx59NFr+KCL2pGQ1yxdTcvZ66vm/Sc2LiJgzcGpqXqM4PjXv2Zc2peb1N/pS8yIipk/5jdS8Mvm8vIgiNa8TdjefTssqYmr0N/7RmMc8EwEAAADUmOUPAAAAQI1Z/gAAAADUmOUPAAAAQI1Z/gAAAADU2ISWP1/4whfi3nvvzZ4FoC26CKgCXQRUgS4CDsSVPwAAAAA11tLy54knnojnnntu3OOPPfZYPPXUU2lDAYxFFwFVoIuAKtBFQDtaWv6sXbs2Hn300XGPP/zww/GXf/mXaUMBjEUXAVWgi4Aq0EVAO9z2BQAAAFBjlj8AAAAANWb5AwAAAFBjlj8AAAAANWb5AwAAAFBjva1+4PLly2NwcDAiIl588cXo6+uL1atXR0TEjh07YsmSJZ2ZEGAfugioAl0EVIEuAlrV0vLnhhtuOODxNWvWpAwDcCC6CKgCXQRUgS4C2tHylT8H8qlPfSojBmBSdBFQBboIqAJdBOzLe/4AAAAA1JjlDwAAAECNpdz2NRF9PTMjYjQlqyxzcvZ689H/ODUvIuIXux5JzZsx9ZzUvHd+Nfey0P9xafXvMe5vnJSaN6Un9+tw1+jjqXkRESdMfWdSUiMph0NtePRvU/OmNGan5pWxOzVvz/13peZFRMz6Vx9Nzdv6b/5dal5PMS01j0OjGS9FxEhKVk9xbErOXkUHOv/1s45KzRtp/jI1r4xdqXl9xRtT86BTRpo/Tfv67++Zk5Kz16xpP03Ni4joLV6fmvfVx+9MzbvktN9Pzdu2e1Nq3suKDmRWWf71M1N63pqYNv6Kx5U/AAAAADVm+QMAAABQY5Y/AAAAADVm+QMAAABQY5Y/AAAAADXW8vJnxYoV8cADD3RyFoCD0kVAFegioAp0EdAqV/4AAAAA1Nj43wR+DIODg7Fq1aoxjw0MDMT999+fMhTAgegioAp0EVAFughoRVvLnxUrVsSCBQs6NQtAS3QRUAW6CKgCXQS0wm1fAAAAADWWdtvXsmXLYv78+SlDARyILgKqQBcBVaCLgFa0vPwZHByMwcHBMY9df/31aQMBHIguAqpAFwFVoIuAVqXc9rVt27YYGBjIiAKYMF0EVIEuAqpAFwH7aunKn7lz54577O67746tW7fGjBkz0oYCGIsuAqpAFwFVoIuAdrS0/Nm4ceNrHivLMoqiiKGhodiyZUvMnj07fTiAfekioAp0EVAFughoR1tv+LyvpUuXxubNm2N4eDgWLVoU06ZNy5wLoCW6CKgCXQRUgS4CxjPh5c/NN9+cOQfAhOgioAp0EVAFuggYT8obPgMAAABQTZY/AAAAADVWlGVZdudT/ywiRrvzqQ+iLJvpmUVxZO3Z+q74UHrmnj+9JzWvLHO//kbKral5vcUbUvMiIpoxlJTUG43i5KSsbqtuF1FNxYd+JzVvZP3dqXk9cVRqXlE0UvNyNSJiVreHSJLXRdnnMZ04h8mesYyR1LzGBeen5pX3bUjNo4rq0kfOiyajmP/bqXnlgw/l5pV7UvMiIoqiLz2TyRi/i46sjQQAAADAEcbyBwAAAKDGLH8AAAAAaszyBwAAAKDGLH8AAAAAaszyBwAAAKDGJrT8WbNmTaxduzZ5FID26CKgCnQRUAW6CDiQlCt/9uzZEz/+8Y8zogAmTBcBVaCLgCrQRcC+Jrz8efrpp1/58YoVK+JP/uRPUgYCaIcuAqpAFwFVoIuA8fS28kFz586NiIjh4eGYN29evOlNb4r77rsvPvnJT8aXvvSleOqpp+Ib3/hGRwcF0EVAFegioAp0EdCOlpY/GzdujIiI2267LRqNRgwNDcXMmTNjwYIFcd5558W6deti2rRpHR0UQBcBVaCLgCrQRUA72rrta8OGDfHBD34wIiIWLlwYxx13XFx++eVKBTikdBFQBboIqAJdBLSi5eXPd7/73TjllFPihBNOiIiI/v7+WLp0aXz2s5+N4eHhjg0IsC9dBFSBLgKqQBcBrWpp+fP888/HjTfeGFdeeeV+j19wwQUxa9as+NznPhcjIyMdGRBgL10EVIEuAqpAFwHtaGn5c8UVV8Qll1wSs2bNes2xVatWxTPPPBOXX355+nAA+9JFQBXoIqAKdBHQjpaWP3Pnzo2FCxfGjh07IiJiZGQkiqKIiIhp06bF2rVro6+vr3NTAoQuAqpBFwFVoIuAdhRlWZatfvD1118fDz30UOzatSvWrFkT73znOyfxqX8WEaOT+PWdU5bN9MyiaOu9tQ97fVd8KD1zz5/ek5pXlrlffyPl1tS83uINqXkREc0YSkrqjUZxclJW+46ULqKaig/9TmreyPq7U/N64qjUvKJopOblakTEa//H+1Cpahdln8d04hwme8Yycm9taVxwfmpeed+G1DyqqHt9VNUuOhIV8387Na988KHcvHJPal5ERFFYMFbL+F3U1vInV3WLxfJn8ix/Js/y51CpbhdRTZY/VdLd5U8uy59J5Vn+0HV16SPnRZNh+UP3jd9FR9ZGAgAAAOAIY/kDAAAAUGO93frEzXJPRNIlukXkXpKed2vMr4yObkvN6+t5S2rett3fT83bveaO1LyIiGY5nJo3Wv48Ne9b//u/puZ96KQPpuZFROwe/d8pOUVMi2m9dbntK0/212hPMSU1LyJi1+jjqXlTG6el5h0Odt29JjVv9+iW1Lzff/A/puZ96/zlqXkRET1Ff3omv5J9C1QR+V0UkfuuA0X2Ke3Mqalxo83tqXkREUWRe4tnJ55zOPyMNH8RETm3BjWKGSk5ezXLF1LzIiIaPdNT8x78s3+Rmpd9m9aaH+S/RvvUGf8sNS/7tXgRuecc6c83EZH+nFiM/bgrfwAAAABqzPIHAAAAoMYsfwAAAABqzPIHAAAAoMYsfwAAAABqzPIHAAAAoMYsfwAAAABqzPIHAAAAoMYsfwAAAABqzPIHAAAAoMYsfwAAAABqzPIHAAAAoMYsfwAAAABqrOXlz1lnnRW/9Vu/1clZAA5KFwFVoIuAKtBFQKtaXv68+93vjunTp3dyFoCD0kVAFegioAp0EdCqtm77WrlyZYfGAGidLgKqQBcBVaCLgFb0tvqBS5cujTlz5nRyFoCD0kVAFegioAp0EdCqlq/8USpAFegioAp0EVAFughole/2BQAAAFBjLd/2la2n6Iuq7p4acWx+ZiM/M9OMqeek5pXlntS8iIii6EvNK2MgNe9f3P6t1LyFN/x+al5ERLMsU3KKyMmpgpe/VkdzsmJ3Ss6vTEnOi+jreWtqXlk2U/Mi+WurKBqpeRER/Y1/kJ6Z6d7fPTE17//54r9MzYuI+Ourrk7JKWJq9DdmpWTVSdG907uWdeLfZqbm19YnJ+b/fnsu/J3UvOa3HkzNq/rfMWNrFMdF1nlRuiK/28oy9/f6D2e+MTWvTD4vev20qal5EREj5c9T83aP/iw1r78xMzWvUeTmvaxIzRovrZrbFwAAAABSWP4AAAAA1JjlDwAAAECNWf4AAAAA1JjlDwAAAECNWf4AAAAA1Fjby5+RkZE444wzOjELQMt0EVAFugioAl0EHIwrfwAAAABqzPIHAAAAoMZ6J/KLhoeH45xzztnvsY985CNx9dVXpwwF0ApdBFSBLgKqQBcBBzKh5c+UKVPikUceyZ4FoC26CKgCXQRUgS4CDsRtXwAAAAA1ZvkDAAAAUGOWPwAAAAA1ZvkDAAAAUGMtveHzq981vr+//zWPRYQ3GAM6ShcBVaCLgCrQRUA7Wlr+KAygCnQRUAW6CKgCXQS0w21fAAAAADVm+QMAAABQYy3d9gXta3R7gIPqiWmpedt+vDM1rxP6G7OSkqYk5VRB3tdqTxydltUpjeKYbo9wQM1yKDWviKNS8w4HvT3Hp+b99VWfTs2LiDjhMytTct56/BvisX/9npQs2FdR9KXmlWUzNS8iYuCU6al5O0Z+kJp3dO+pqXk9RX9qHuPpiYiy20Mctl4a3Z2aV0SRmve242ek5kVEFMmvC3qK3N/z8OhzqXn9jfxzy6LIXMuMjnvElT8AAAAANWb5AwAAAFBjlj8AAAAANWb5AwAAAFBjlj8AAAAANWb5AwAAAFBjLX1Psblz5+73823btsX06dOjp2f/3dHGjRvzJgN4FV0EVIEuAqpAFwHtaGn58+rCuOCCC+IrX/lKnHjiiR0ZCmAsugioAl0EVIEuAtrhti8AAACAGpvw8mfTpk2ZcwBMiC4CqkAXAVWgi4DxTHj5c9NNN8VVV10Vzz33XOY8AG3RRUAV6CKgCnQRMJ4JL39uu+22+PVf//W48MIL4y/+4i8yZwJomS4CqkAXAVWgi4DxtPSGz2NpNBqxZMmSOOOMM2L9+vXxnve8J3MugJboIqAKdBFQBboIGM+Elz97zZs3L+bNm5cxC8CE6SKgCnQRUAW6CHg13+0LAAAAoMYsfwAAAABqbEK3fX3729/OngOgbboIqAJdBFSBLgIOxJU/AAAAADVm+QMAAABQY5P+bl8wlqKo/l6xKBqpec1v5F5qWyw4LzUvIqJ84DtJSbl/dt20fXhzNMuXUrIGppyUkrPXaPO51LyIiL6e2al5PUV/al5ZDqfmjZS7UvMiInp7XpeeWWX9PaemZy7+3Xek5MyYenxKDodeVu/u1VNMS83L1onzou3/9pupeSPNbal5b7n2Y6l5T197bWpeRERfz68lpvVGT5EYVwtlcl4zOS+ijNHUvNnH5D5nlrEnNe+M1+U8/+6rUQwk5+We/45G7rngSLk1NS8i4ps//F5a1tF9x8WHTz55zGPVf4UOAAAAwIRZ/gAAAADUmOUPAAAAQI1Z/gAAAADUmOUPAAAAQI1Z/gAAAADUmOUPAAAAQI1Z/gAAAADUmOUPAAAAQI1Z/gAAAADUWG8rHzR37tyDB/X2xsMPPzzpgQDGo4uAKtBFQBXoIqAdLS1/Nm7c2Ok5AA5KFwFVoIuAKtBFQDvc9gUAAABQY5Y/AAAAADVm+QMAAABQY5Y/AAAAADVm+QMAAABQY5Y/AAAAADVm+QMAAABQY5Y/AAAAADVWlGVZdudT/ywiRrvzqem4styTnlkUfal5zXI4NW/Zf7slNe9P3r0kNS8i4o+/d2tKzsCU6fEvz7gqJavb/sv/WRsvjb6YkvWOWW9NydlrtGym5kVE9BaN1LwZU89JzSuTf89l7ErNi4gooj81r4yR1LwicruyE5rlC0lJfdHomZOU1W3Oi6iX3aM/Ss077Y9XpuZFRGz63DVpWUVMi6P63puW1y1l+ZPI6qIi+ZyjLDvRkUVq2rz/uCw17/9f+MepeT8Z+qvUvIiINx31rtS87POi7L/jTpxb5l6T0xuN4qSOfxYAAAAAKsbyBwAAAKDGLH8AAAAAaszyBwAAAKDGLH8AAAAAamxCy59bb701duzYkT0LQFt0EVAFugioAl0EHEjby5/NmzfHPffcE0cffXQn5gFoiS4CqkAXAVWgi4CD6W3lg+6999647rrr4thjj33lsXPPPfeVH+/YsSOWLFkSF198cfqAAHvpIqAKdBFQBboIaEdLy5+IiA9/+MOxfPnyMY+tWbMmbSCAA9FFQBXoIqAKdBHQKm/4DAAAAFBjLV/5s379+tiwYcOYx/ZeUgjQaboIqAJdBFSBLgJa5bYv4LCii4Aq0EVAFegioFVu+wIAAACoMcsfAAAAgBrznj/AYUUXAVWgi4Aq0EVAq1pa/vT29saiRYviM5/5zJjH3U8KHAq6CKgCXQRUgS4C2tHS8ueDH/zgAY9/6lOfShkG4EB0EVAFugioAl0EtMN7/gAAAADUWMvv+ZPP3qnemh3IbCTn5X75H99/fGpe/u83YmDK9JScY/uOS8mpgqmNY9KyeoppaVkREWWUqXkRET1Fdvdmf50WyXmdeJrL/7eZq+rzRUT0JeV08TQmnfMi6qY/Ne3NA7NS8yIiish73i5ialpWd2U+hxwOz0e55x1vOGpmal72n2GjOCo17+9TO5CZKfvcMuscZl+ZM45/blSUZZn/6gIAAACASvDfTAAAAAA1ZvkDAAAAUGOWPwAAAAA1ZvkDAAAAUGOWPwAAAAA1ZvkDAAAAUGOWPwAAAAA1ZvkDAAAAUGOWPwAAAAA11li5cuXKbg/RSStWrIihoaE45ZRTuj3KmDo53xe+8IV47rnn4tRTT51UTidnXLNmTfzN3/xN/MZv/EZ6NhNz6623xuC5GvQAAAeoSURBVNvf/vaYMmVKt0epFV2ki2iPLuqMI7mLInL6SBcdWXRRZ+giXUR7MrqoN3GeSVm3bl3ceuutr/z8xRdfjL6+vpg6deorj1177bXxvve9rxvjRUTEI488Ep/+9KcP+DG33HJLnHbaaYdoovrZs2dPPPvss/HmN7+526PEyMhInHXWWfGDH/xgwhlz587d7+fbtm2L6dOnR0/P/hfdbdy4ccKfI9PmzZvjnnvuicsuu6zbo3RV1ftIF3Ve3boo4vDqI130Ml2ELuouXfQyXUSVuijiyHudltVFlVn+LF68OBYvXvzKzz//+c/H2WefHeeff/6kswcHB2PVqlVjHhsYGIj777+/pZxzzjmnI3/5WfM98cQTMXPmzJg5c+aYxx977LGYMmVKzJkzp2szjuXpp59+5cd7N9g33XTThPOq5NVfLxdccEF85StfiRNPPLFLE73WvffeG9ddd10ce+yxrzx27rnnvvLjHTt2xJIlS+Liiy/uwnTd0ak+0kUv00XdUfU+0kWvpYteq91/553qI100cbro8KOLXksXHf6OxC6qzPLn1UZHR6PRaKRkrVixIhYsWJCS1QlZ861duzbe//73x/vf//4xjz/88MNx7LHHTugFV9aMezesw8PDMW/evHjTm94U9913X3zyk5+ML33pS/HUU0/FN77xjUl/Htrz4Q9/OJYvXz7msTVr1hziaaonq4900ct0EePRRQemi9rXqT7SRfWmiw5MF7VPFzER2V1UmeXPqy+7euGFF2LDhg1xww037Pd4Ny+7evWM46nCpWFVtffP5rbbbotGoxFDQ0Mxc+bMWLBgQZx33nmxbt26mDZtWpen/JXh4eE455xz9nvsIx/5SFx99dUTzty0aVNlNsqMrep9pIsmTxe9TB9Vmy6qP130Ml1Ubbqo/g63LorwOm0iKrP8efU/xg984ANx6qmnxurVqyedfaDL4ZYtWxbz589ve8bVq1fHM888E1/84hcrM18nZc+4YcOGuOWWW+Kuu+6KhQsXxje/+c24/PLLK1cqU6ZMiUceeSQ186abboqHHnooli9fPu6lnwfSyhNcb29vPPzwwy1nrl+/PjZs2DDmsb2XFB5JOtVHumjydFGuyfSRLuo8XTS5+TpJF+XSRdWmiyY3XycdqV0UUb3XaYdDF1Vm+bOvp59+OrZv3x6PP/54PProo5N6l/HBwcEYHBwc89j1118/ocwdO3bEunXr4qtf/WpERGzdujWefPLJ/e7B6+Z82bJn/O53vxunnHJKnHDCCRER0d/fH0uXLo3Pfvazcccdd9T+uyncdttt8e1vfzsuvPDCuOGGG+I973lPW7++E/9r4fLm8WX1kS6aPF2UbzJ9pIsOLV2ki+pMFx0+dJEuqrO6d1HPwT/k0Go2m7Fy5cpYvHhxLF++PJYsWRJbt27tyOfatm1bDAwMtP3r1q5dG6effnqceeaZERGxc+fOWLZsWezcubMS8x1K7c74/PPPx4033hhXXnnlfo9fcMEFMWvWrPjc5z4XIyMj2WNWSqPRiCVLlsR1110X69ev7/Y4HMCh6iNdNHm6aGL00eFBF01uvkNJF02MLjo86KLJzXco6aKJqXsXVerKn+3bt8cf/dEfRVmW8YlPfCL6+vriox/9aCxcuDBWr14dv/mbv9lW3oEuvbr77rtj69atMWPGjLYyN23aFLfffnvceeedrzx28sknx5lnnhl33nlnfPzjH+/qfNmyZ7ziiivikksuiVmzZr3m2KpVq+Kiiy6Kyy+/PL72ta+1nHnWWWfF8ccf39YldFUwb968mDdvXrfHYByZfaSLJk8XdZY+qi5dNLn5sumiztJF1aWLJjdfNl3UWXXtososf1avXh133nlnzJ8/P2688cbo6+uLiIjLLrssXv/618ell14a733ve+PLX/5yy5ljXXpVlmUURRFDQ0OxZcuWmD17dst5zz//fFx55ZVx8cUXx6xZs+LJJ5+MLVu2xA9/+MPYtm1brF+/vq1iyZ4vImL58uWvXP734osvRl9f3yv3407kvsDsGefOnRsLFy6MHTt2xDHHHBMjIyNRFEVEREybNi3Wrl0b11xzTVszvvvd746f/OQnbf0a9ufe9v1l95Eu0kW0RhftTxdNvosicvtIFx0ZdNH+dJEu0kXdUdv3/Nm+fXvccccd8ba3ve01xy688MJ417veFV//+tcn/XmWLl0amzdvjuHh4Vi0aFFbb141NDQUzz77bNx1111x//33x/Tp0+Mtb3lLnHTSSXHRRRfFl7/85fj+978fZ599dlfme/U77r9a1j3Kk5lxb/GuXr06Hnroodi1a9d+cx1zzDFxyy23tD3TypUr2/41Y3n1O8b39/e/5rGI6MibHXZLb29vLFq0KD7zmc+MefxIvLf9UPSRLpo8XaSL6k4XTW6+iEPTR7pIF9WdLtJF3e6iiCOvjzrRRUVZluVkB+NlTzzxRJx00klHxJthVclTTz0Vc+bM6fYYUBm6qDt0EexPF3WHLoL96aLu0EXVY/kDAAAAUGOV+25fAAAAAOSx/AEAAACoMcsfAAAAgBqz/AEAAACoMcsfAAAAgBqz/AEAAACoMcsfAAAAgBr7vxWMZSY+nNgoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder Self Layer 6\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3SU9b3v8c+TSWYwIEFF0UCPNa3A2UQbqtUqNmpPkMRuFwiy3XLaSsXL9gKItQql9VbpBloruwoKlZZ2Y0wXlS2k5abAOaORY/HCQgWMUHErQoyAEAhOLvM7f7CTTcpl5kl+k/nlmfdrrXaZmeGTL7l88uTL88x4xhgjAAAAAAAABFJWugcAAAAAAABA6rD8AQAAAAAACDCWPwAAAAAAAAHG8gcAAAAAACDAWP4AAAAAAAAEGMsfAAAAAACAAGP5AwAAAAAAEGDZ6XrHl0363/r4sxorWdv//VkrOQCSlSXp9HQPYUXc7JTUbCXLU8hKTosm86nVPEn65xeesZpXMeI2q3lZXq7VPNufE0nK8rpZzTMmbjUvbg5YzZNn/2NoT7ZC3v9I9xCW1Eqy87XwWs1LVnJaXNynxGpeV2CMnZ8LLbwUfB8Nr3jQat6Sf37Yal7mCcaxkTE1snZc5PTPj8wUNw3WM8f85TGrec99536reZ7n/vku/6/mRWtZkazuGnz6tce8L23Ln48/q9GHNZ9YSrP7AxpAJmmW1JTuIY7D/g/oXQdrLSfanjFsOc9YzpPs/8yxu/yRGi3npeJjiKPFZetrK9ZcZyXnv2XicZb7f+edB2z3uft/Z3SGZvG1EGT2j3l31X9mOdH215/7xzFfWP+5fWzur8EAAAAAAADQbix/AAAAAAAAAozlDwAAAAAAQIC1a/kzdepULV682PYsAOALXQTABXQRABfQRQBOhDN/AAAAAAAAAqxdyx9jjOJx269OAgD+0EUAXEAXAXABXQTgRNq1/Nm7d68++ugj27MAgC90EQAX0EUAXEAXATgR38ufeDyut99+W6tXr07FPACQFLoIgAvoIgAuoIsAJOJ7+bNmzRr1799fffr00bp161IxEwAkRBcBcAFdBMAFdBGARHwtfw4cOKCZM2fq5ptv1s0336wZM2aoqakpVbMBwDHRRQBcQBcBcAFdBCAZSS9/GhoadPfdd2vIkCG69NJLdckll6iwsFAPPPCAjDGpnBEAWtFFAFxAFwFwAV0EIFlJLX9qa2s1duxY5ebmasqUKa23T506VTt27NCdd96pffv2pWxIAJDoIgBuoIsAuIAuAuBHwuXP66+/ruHDh6uoqEiPP/64wuFw630nnXSS5s2bpx49emjYsGF69913UzosgMxFFwFwAV0EwAV0EQC/shM94Oyzz9acOXNUVFR0zPsjkYhmzpypqqoqFRQUWB8QACS6CIAb6CIALqCLAPiVcPlz+umn6/TTT08YNGTIECsDAcCx0EUAXEAXAXABXQTAL98v9Q4AAAAAAICuI+GZPwAQZN4R/99xdl9VI8vLs5onSQtHDLeal+0l/ldHf2x9Lg4zilnNk6S4sZvZbD63mjfuxflW8xZcda/VPEnyrB1+hCzlpN9vN/1RdY12nph1wvk/sJLT4ovmTVbzJKlb6B+sZ9rkeXa/tmLNf7OaJ0k/L/1f1jOBG1f+SjX1n1nJWnHtw1ZyUsvu97rn2T23wphmq3m2jzkkaemIGywn2j2eNiZuN0+NVvMk6ZI+51lMixz3Hs78AQAAAAAACDCWPwAAAAAAAAHG8gcAAAAAACDAWP4AAAAAAAAEGMsfAAAAAACAAGP5AwAAAAAAEGC+lz/GGNXV1aViFgBIGl0EwAV0EQAX0EUAEslO9IBdu3bpjTfe0LZt27Rp0ya98847Kisr08SJE7V79+7Wx4VCIfXr1y+lwwLIXHQRABfQRQBcQBcB8Cup5c/69etVUFCgDRs2KBqNKhwOa/HixXr66ad1zjnnSJI2b96saDSa8oEBZCa6CIAL6CIALqCLAPiVcPlTVFSkoqIixWIxzZ07V+FwuPW+srIyTZo0SZJUXFycuikBZDy6CIAL6CIALqCLAPiV8Dl/li9fruLiYg0dOlR79uxRcXGxrrnmms6YDQBa0UUAXEAXAXABXQTAr4TLn7KyMkWjUU2ePFmRSETl5eWqrKzsjNkAoBVdBMAFdBEAF9BFAPxK+tW+ysvL1dTUpHHjxumzzz5L5UwAcFx0EQAX0EUAXEAXAUhWUsufN998Uzk5OcrLy9OoUaM0fvz4VM8FAEehiwC4gC4C4AK6CIAfCZc/9fX1mjJlSmuZ3HLLLRo7dmyq5wKANugiAC6giwC4gC4C4FfC5c+iRYtUUlKiQYMGSZI8z9OwYcNSPhgAHIkuAuACugiAC+giAH4lfKn3UaNGKRwOyxhz1H0VFRVauXJlSgYDgCPRRQBcQBcBcAFdBMCvhMufHj16SJJisVib2wcNGqTZs2frwgsvlCQtXrw4BeMBwGF0EQAX0EUAXEAXAfAr4fKnRSQSUVVVVevbAwYMaHP/yJEj7U0FAMdBFwFwAV0EwAV0EYBkeeZY5wp2ihpJzVaSvKtKrOS0MKtespoHBE9IUp90D2HF+/ueV2P8oJWs6X/9q5WcFguummQ1T5Li5pDVvJDXy2pek9lpNS/bO8tqXip4XlIvvJk2xsTTPcIJhOR1gc9xcuwdF8Wa/2Ylp0U468tW8yTp/+6stJp3+VnXWM2Lq95qXpZOsponSb/euMBq3oTzx1rNazJ2X3Y8J8v1445gHBv9bvMTqmvcZyXr8r59reS0GHTKRVbzJCnL6245L2w174vmTVbzQt5pVvMOZ9o9FszyIlbzugJj7Pz8Pywkz8s/5j1uH3ECAAAAAACgQ1j+AAAAAAAABBjLHwAAAAAAgABj+QMAAAAAABBgLH8AAAAAAAACjOUPAAAAAABAgLH8AQAAAAAACLDsZB84efJkrV27VpFIpM3txhjl5ORozZo11ocDgL9HFwFwAV0EwAV0EYBkJb38kaRp06appKSkzW379+/XiBEjrA4FACdCFwFwAV0EwAV0EYBk+Fr+TJ06VY888kib21q2ygDQWegiAC6giwC4gC4CkAzO/AHQ5dBFAFxAFwFwAV0EIBlJP+FzY2Mj22MAaUcXAXABXQTABXQRgGQlfebP559/rvvvv1/hcLjN7ZxSCKAz0UUAXEAXAXABXQQgWUktf4wx2r59u6LR6FHFwimFADoLXQTABXQRABfQRQD8SOqyr6qqKg0cOPCoUgGAzkQXAXABXQTABXQRAD8SLn+MMZo9e7auu+66Nrc3NjaqqalJ9fX18jwvZQMCgEQXAXADXQTABXQRAL8SXvZVXV2t3NxcXXnllW1ur62t1ejRo1VfX6/S0tKUDQgAEl0EwA10EQAX0EUA/Eq4/BkwYIDmz59/1O35+fmqqqpKyVAA8PfoIgAuoIsAuIAuAuBX0i/1DgAAAAAAgK6H5Q8AAAAAAECAecYYk553XSOpOT3vOgHvqhLrmWbVS9YzgfQJSeqT7iGsMOYT2eqi0v/4qZWcFsuunWI1T5I8hazmZXm5VvO6gu11Uat5/br/g9U8258Ttz/HQeqij2XvuMju9/nF8ydazZOkv978hPVMl9Uces16Zl64p9W8bqH/aTUv8wSjj2x20RfNW6zktNjy+SdW8yQpy/JzYn/ttGFW84yx/fuy/ScBP9j0ttW87Cy7r14Xzvqy1bws7ySrefYdv4s48wcAAAAAACDAWP4AAAAAAAAEGMsfAAAAAACAAGP5AwAAAAAAEGAsfwAAAAAAAALM9/LHGKO6urpUzAIASaOLALiALgLgAroIQCLZiR6wa9cuvfHGG9q2bZs2bdqkd955R2VlZZo4caJ2797d+rhQKKR+/fqldFgAmYsuAuACugiAC+giAH4ltfxZv369CgoKtGHDBkWjUYXDYS1evFhPP/20zjnnHEnS5s2bFY1GUz4wgMxEFwFwAV0EwAV0EQC/Ei5/ioqKVFRUpFgsprlz5yocDrfeV1ZWpkmTJkmSiouLUzclgIxHFwFwAV0EwAV0EQC/Ej7nz/Lly1VcXKyhQ4dqz549Ki4u1jXXXNMZswFAK7oIgAvoIgAuoIsA+JVw+VNWVqZoNKrJkycrEomovLxclZWVnTEbALSiiwC4gC4C4AK6CIBfSb/aV3l5uZqamjRu3Dh99tlnqZwJAI6LLgLgAroIgAvoIgDJSmr58+abbyonJ0d5eXkaNWqUxo8fn+q5AOAodBEAF9BFAFxAFwHwI+Hyp76+XlOmTGktk1tuuUVjx45N9VwA0AZdBMAFdBEAF9BFAPxKuPxZtGiRSkpKNGjQIEmS53kaNmxYygcDgCPRRQBcQBcBcAFdBMCvhC/1PmrUKIXDYRljjrqvoqJCK1euTMlgAHAkugiAC+giAC6giwD4lXD506NHD0lSLBZrc/ugQYM0e/ZsXXjhhZKkxYsXp2A8ADiMLgLgAroIgAvoIgB+JVz+tIhEIqqqqmp9e8CAAW3uHzlypL2pAOA46CIALqCLALiALgKQrKRf6h0AAAAAAABdj2eOdaFop6iR1Jyed50G3lUlVvPMqpes5gH+hCT1SfcQVsTNR5KaLGXVWclpMeR3j1jNk6RXf2A3c3es2mre6d0utJonpelHnC+e3TQvk/5dJzhdZPO4qCm+10pOi+ysU6zmSVLc1FvNy/JyreaNrHzIal751ddbzZOk2kO7reZ9qcdlVvMyTzD6KG62y9Zx0f6GLVZyWvQMD7SaJ0mX/ftUq3lV35tpNS+uQ1bzstTDal4qGH1hOS9uNS/kuf4xPH4XZdIRIgAAAAAAQMZh+QMAAAAAABBgLH8AAAAAAAACjOUPAAAAAABAgLH8AQAAAAAACDBfy585c+boiSeeaH17/vz5mjNnjvWhAOBE6CIALqCLALiALgKQjOxED3jrrbc0ceJESdLBgwdljNGiRYskSfX19TLGqKKiQpI0Y8YMXXLJJSkcF0CmoosAuIAuAuACugiAXwmXP4MHD1Y0GpUkPffcc2pubtZ3v/tdSdLzzz+vhoYG3XDDDamdEkDGo4sAuIAuAuACugiAXwmXP0cqKipSdXW1JGnevHm6/vrrlZeXl5LBAOB46CIALqCLALiALgKQjKSf86ehoUE/+clP1K1bN0nSokWLVFdXJ0lasGCBHn/88dRMCABHoIsAuIAuAuACughAspI688cYowcffFB9+/bVsGHDWm/ftWuXHn30Ue3evZsnFQOQcnQRABfQRQBcQBcB8CPhmT+NjY368Y9/rJUrV+qcc86RJO3du1eHDh3S7bffrsLCQj377LM6/fTTUz4sgMxFFwFwAV0EwAV0EQC/Ep75s2/fPjU1Nenee+9VTU2N3nvvPY0ZM0axWEwVFRUqLCzsjDkBZDi6CIAL6CIALqCLAPiV8Myf3r176xe/+EXrdaQDBgzQiy++qLPOOku9evVqfdyOHTvU3NycukkBZDS6CIAL6CIALqCLAPiV9BM+H+nUU0+VpNYiMcbogQce0Nq1a+1NBgAJ0EUAXEAXAXABXQTgRHy91PuRvv71r2vMmDHKzs5WY2Oj8vPzdfHFF9ucDQASoosAuIAuAuACugjA8SS9/Bk5cmSbt2fMmGF9GABIhC4C4AK6CIAL6CIAyWrXZV8AAAAAAADoGlj+AAAAAAAABJhnjDHpedc1knjm+fbyriqxmmdWvWQ1D0EXktQn3UNYYq+LjIlbyWnRZGqs5knS15940GrexglPW83zvMz7NwnbXze7Dv3Vat5Zud+0mmdXcLqovvH/yOiQlazuOV+zktOiMb7Tap4k5WSdZT3TZd9ccLf1zLXfHWc176Ts86zmZZ5g9NGe2CrFjZ0uOi1ygZWcFkZfWM2TpNov3raaV/6e3bzx54+ympfl5VrNO5wZsZ7pssb4LuuZ2d5pFtNC8rx+x7wn846yAQAAAAAAMgjLHwAAAAAAgABj+QMAAAAAABBgLH8AAAAAAAACjOUPAAAAAABAgPle/hhjVFdXl4pZACBpdBEAF9BFAFxAFwFIJDvRA3bt2qU33nhD27Zt06ZNm/TOO++orKxMEydO1O7du1sfFwqF1K/fsV9SDAA6ii4C4AK6CIAL6CIAfiW1/Fm/fr0KCgq0YcMGRaNRhcNhLV68WE8//bTOOeccSdLmzZsVjUZTPjCAzEQXAXABXQTABXQRAL8SLn+KiopUVFSkWCymuXPnKhwOt95XVlamSZMmSZKKi4tTNyWAjEcXAXABXQTABXQRAL8SPufP8uXLVVxcrKFDh2rPnj0qLi7WNddc0xmzAUArugiAC+giAC6giwD4lXD5U1ZWpmg0qsmTJysSiai8vFyVlZWdMRsAtKKLALiALgLgAroIgF9Jv9pXeXm5mpqaNG7cOH322WepnAkAjosuAuACugiAC+giAMlKavnz5ptvKicnR3l5eRo1apTGjx+f6rkA4Ch0EQAX0EUAXEAXAfAj4fKnvr5eU6ZMaS2TW265RWPHjk31XADQBl0EwAV0EQAX0EUA/Eq4/Fm0aJFKSko0aNAgSZLneRo2bFjKBwOAI9FFAFxAFwFwAV0EwK+EL/U+atQohcNhGWOOuq+iokIrV65MyWAAcCS6CIAL6CIALqCLAPiVcPnTo0cPSVIsFmtz+6BBgzR79mxdeOGFkqTFixenYDwAOIwuAuACugiAC+giAH4lXP60iEQiqqqqan17wIABbe4fOXKkvakA4DjoIgAuoIsAuIAuApCspF/qHQAAAAAAAF2PZ451oWinqJHUnJ53jaN4V5VYzTOrXrKaB9eEJPVJ9xCW2OsiY+JWclIpbvZbzTtr6u1W87b/7MdW83Kzz7OaB9fQRcfSGN9hJadFTlZfq3mS1BjfZTUv5J1qNe+b839oNW/duGlW8yT7n+dI1oDED/LB8zLt35iD0kfuHhel4mvKmEbLiSGraVkjyqzmxV/4s9U8SfK8HOuZ6Ijjd1GmtTIAAAAAAEBGYfkDAAAAAAAQYCx/AAAAAAAAAozlDwAAAAAAQICx/AEAAAAAAAgwlj8AAAAAAAABxvIHAAAAAAAgwLKTfeDkyZO1du1aRSKRNrcbY5STk6M1a9ZYHw4A/h5dBMAFdBEAF9BFAJKV9PJHkqZNm6aSkpI2t+3fv18jRoywOhQAnAhdBMAFdBEAF9BFAJLha/kzdepUPfLII21ua9kqA0BnoYsAuIAuAuACughAMjjzB0CXQxcBcAFdBMAFdBGAZCT9hM+NjY1sjwGkHV0EwAV0EQAX0EUAkpX0mT+ff/657r//foXD4Ta3c0ohgM5EFwFwAV0EwAV0EYBkJbX8McZo+/btikajRxULpxQC6Cx0EQAX0EUAXEAXAfAjqcu+qqqqNHDgwKNKBQA6E10EwAV0EQAX0EUA/Ei4/DHGaPbs2bruuuva3N7Y2KimpibV19fL87yUDQgAEl0EwA10EQAX0EUA/Ep42Vd1dbVyc3N15ZVXtrm9trZWo0ePVn19vUpLS1M2IABIdBEAN9BFAFxAFwHwyzPGmPS86xpJzel51ziKd1VJ4gf5YFa9ZDUPrglJ6pPuISyx10XGxK3kpFLc7Lead9bU263mbf/Zj63m5WafZzUPrqGLjqUxvsNKToucrL5W8ySpMb7Lal7IO9Vq3jfn/9Bq3rpx06zmSfY/z5GsAVbzPC/pFxUOiKD0kbvHRan4mjKm0XJiyGpa1ogyq3nxF/5sNU+SPI8nFnfL8bso01oZAAAAAAAgo7D8AQAAAAAACDAu+0JKGGP/c5tVNsxqnlnBpWntF5RTmyW6yC1xE7Oal+VFrOZJkvePV1nNM39eZTUvswSni3bHVilu6q1knd7tIis5LeKmwWqeJGV5mfXqRLa7TZK+9OBNVvN2PPKs1bzME4w+ipuPJTVZycq07/OuINb8N+uZ3e60e5msefo/rOZlHi77AgAAAAAAyEgsfwAAAAAAAAKM5Q8AAAAAAECAsfwBAAAAAAAIMJY/AAAAAAAAAcbyBwAAAAAAIMBY/gAAAAAAAAQYyx8AAAAAAIAAY/kDAAAAAAAQYCx/AAAAAAAAAozlDwAAAAAAQICx/AEAAAAAAAgwlj8AAAAAAAABltTyZ/Dgwfr2t7+d6lkA4IToIgAuoIsAuIAuAuBHUsufIUOGqFevXqmeBQBOiC4C4AK6CIAL6CIAfiR92ddDDz2UwjEAIDl0EQAX0EUAXEAXAUhWdjIPmjhxos4999xUzwIAJ0QXAXABXQTABXQRAD+SOvOHUgHgAroIgAvoIgAuoIsA+MGrfQEAAAAAAARYUpd9AX55Xsh+aI7dzLiJWc3L8iJW89D1GBO3mud5mbif96ymxU291TxJUrcU9Bsy3mmRr0lqTvcYx5TlhdM9QpeXimOEHY88azWvoXm71bzIzROs5pnfLbWah2MzapDUZCmN7nDNe59vtZ4Zmz3Lat4/Lvmp1bw/D/+Z1byuLBN/swAAAAAAAMgYLH8AAAAAAAACjOUPAAAAAABAgLH8AQAAAAAACDCWPwAAAAAAAAHWruXPj370I8Vidl8pCQD8oosAuIAuAuACugjAiST1Uu933323PvjgA0lSc3OzPvzwQ1VXV7d5TO/evTV//nz7EwLAf6GLALiALgLgAroIgB9JLX9mzZrV+t/Lli1TNBrV9OnTUzYUABwLXQTABXQRABfQRQD8SGr588wzz2jZsmWSpC1btuirX/2qRo4c2eYx1113ncaMGWN/QgD4L3QRABfQRQBcQBcB8CPh8uf999+X53n6zne+o507d+rjjz/W8OHDj3rcoUOH9Mknnyg/Pz8lgwLIbHQRABfQRQBcQBcB8CvhEz736tVLhYWFqq6uVjQa1axZs1RYWKjCwkINGjRIixYt0oEDB1RYWKju3bt3xswAMhBdBMAFdBEAF9BFAPxKeOZPXl6e1qxZoyVLlqh3796aMWNGm/tra2v1+9//XpFIRBdccEHKBgWQ2egiAC6giwC4gC4C4FfC5U9NTY22bt2qcePGqXv37rrjjjva3H/HHXeopKRECxcu1IgRI3TmmWembFgAmYsuAuACugiAC+giAH55xhiTzAPnz5+v3/3udzrttNPa3P7xxx9rzpw5uuiii+R5no93XSOp2cfjkem8a4ZZzWteutRqXpYXsZrntpCkPml5zy53kTFxKzktPC/hlbmBEzcNlhObLOdJodGjrOaZPy23mpdZ6CLAlobm7VbzIjdPsJpnfmf3uM2+9PSR7S5qNn+TrZ+dIa+HlRzYs3H3KuuZA3sNsJo38s/PWM378/CfWc1z3/G7KKlX+2oxZsyYY26VJfk8wAGA9qOLALiALgLgAroIQDJ8LX/+8Ic/aOnfnS1RU1OjG2+80epQAHAidBEAF9BFAFxAFwFIRtKXfdnH6c3wh8u+XJK+Sy3s47Ivl3DZF/yhiwBbuOyro4LRR1z2FWxc9pUJjt9FmfebBQAAAAAAQAZh+QMAAAAAABBgvp7zB0gnU7nSap53VYnVPLPqJat56IrSdBVtgGR5YcuJtvMkNdq9vC9uYlbzMusSVHSGuKm3npnl5VrPRMecfv99VvPqfzPNap79y4JT8TOn6wt53cUlqMF1/mlXpXuEhP4yZ53dwOF247oyzvwBAAAAAAAIMJY/AAAAAAAAAcbyBwAAAAAAIMBY/gAAAAAAAAQYyx8AAAAAAIAAY/kDAAAAAAAQYEktf3bu3KmlS5fqjjvu0KRJk1pvX7dunSZMmJCy4QDgSHQRABfQRQBcQBcB8CM70QPef/993XPPPWpoaNDo0aN10003td7X2Niobt26pXRAAJDoIgBuoIsAuIAuAuBXwjN/zj33XFVWVqq4uFj9+vVTVlaWDhw4oI0bN6qhoUHdunXTwYMHNXv27M6YF0CGoosAuIAuAuACugiAXwmXPytWrFBpaamWLFmiadOmqbS0VG+//baeeeYZ7du3T926ddP06dNVU1PTGfMCyFB0EQAX0EUAXEAXAfAr4fKntLRUy5YtU25urkpLS7VixQpt2rRJ/fv31+7du7V06VJVV1drypQpnTEvgAxFFwFwAV0EwAV0EQC/knrC51deeUU7d+5UZWWlKisrVV1draFDh+qDDz7Q/v379dRTT+mkk05K9awAMhxdBMAFdBEAF9BFAPxI+ITPkvSb3/xGBQUFuuGGG7Rs2TLNmTNHnudpw4YNOv/887V161ZddNFFqZ4VQIajiwC4gC4C4AK6CIAfCc/8eeGFF3Tw4EFddtllOuOMM/Tkk0/K8zxt2rRJkUhE3//+91VRUdEZswLIYHQRABfQRQBcQBcB8Cvh8ueUU07Rww8/3Pp2KBSSMUYzZ87UrbfeqqFDh2rDhg169913UzoogMxGFwFwAV0EwAV0EQC/Ei5/Lr/8cp133nltbps+fbp69uypq6++Wjk5OZowYYLuu+8+1dXVpWxQAJmNLgLgAroIgAvoIgB+JfWEz0f66KOP9OGHH2rGjBmtt40YMULnnXeeNm/ebHU4ADgeugiAC+giAC6giwAk4hljTHredY2k5vS8a0CSd1WJ1Tyz6iWreW4LSeqT7iEssddFxtjtNM8LWc2DHd7wYVbzml9YajUvy4tYzXMbXdQZ4qbeemaWl2s9Ex2Td+8/Wc3bNf2nVvMioQFW8yQpywtbTAtKH7nbRcgM3jDLv6OtzKTf0aQTdZHvM38AAAAAAADQdXDmD2BJZp1JFJR/3ZLoIvjVbA5YzVu3a63VvPteXGM179XvP241zy66CMgUts+ulaSsq0utZZ19Rr62//5Fa3npY6+LbP+8zJL9MwbX7rB79u23+42wmtcU32M1LzvrVKt5qfDpob9azdu671OreRee8TWreZIUzvqSxTTO/AEAAAAAAMhILH8AAAAAAAACjOUPAAAAAABAgLH8AQAAAAAACDCWPwAAAAAAAAHWoeVPQ0ODtm3bpqqqKlvzAIBvdBEAF9BFAFxAFwE4luxkHvTaa6/p1Vdf1d69e1VbW6tPP/1UNTU1qqurU35+vgoKCnTmmWfqK1/5SqrnBZDB6CIALqCLALiALgLgR84BGqIAABEhSURBVFLLn759+yo3N1f5+fk65ZRTFI1GdcUVV+iuu+6S53mpnhEAJNFFANxAFwFwAV0EwI+klj/9+vXTbbfd1vr29u3bJYlSAdCp6CIALqCLALiALgLgB0/4DAAAAAAAEGAJz/zZsmWLbr311ja3HTx4UJK0cOHCNrf/27/9mwYPHmxxPAA4jC4C4AK6CIAL6CIAfiVc/gwcOFDRaLTNbfPmzZOkowoHAFKFLgLgAroIgAvoIgB+cdkXAAAAAABAgLH8AQAAAAAACDCWPwAAAAAAAAHG8gcAAAAAACDAEj7h87HwJGIAXEAXAXABXQTABXQRgBPhzB8AAAAAAIAAY/kDAAAAAAAQYCx/AAAAAAAAAswzxpj0vOsaSc3pedc4ijFxq3mex16xo7zSEuuZ8eWrLCWF5HlnWcpKN3e7yPb3ZSrwvd5xxjRazcu67Z+s5sWeetxqniSFQ1+2lBSS1MdSVrrZ66Lm+OdWclqEsnpZzZMkY+z2rueFrOY1xndazcvJsv8z0/bHUPIs59n9FaPnD2+wmidJ+x971mJaSJ7Xz2Jeurh7XJQK/A4UfLY/x1nXlFrNk6R45QqLacf/PY2vTgAAAAAAgABj+QMAAAAAABBgLH8AAAAAAAACjOUPAAAAAABAgLH8AQAAAAAACLCklz/Lli1TdXV1KmcBgIToIgAuoIsAuIAuApCs7GQeFI/H9atf/UoLFy6UJN11113auHGjYrGYYrGYevbsKUkqLy9Xv35BeIlDAC6iiwC4gC4C4AK6CIAfSS1/Vq9ercLCQu3fv1+rVq3Sk08+KUlasWKFXn75ZU2bNi2lQwKARBcBcANdBMAFdBEAPxJe9lVfX69f/vKXuv766/XYY48pPz+/M+YCgDboIgAuoIsAuIAuAuBXwjN/6urqdPLJJ+vnP/+5cnNzVVJS0hlzAUAbdBEAF9BFAFxAFwHwK+GZP3369NFvf/tbxWIxPfroo50xEwAchS4C4AK6CIAL6CIAfiVc/jQ1NWnSpEkqKirSkiVLtHbt2s6YCwDaoIsAuIAuAuACugiAXwmXP1u3btX27dvV3Nys/v376xvf+EZnzAUAbdBFAFxAFwFwAV0EwK+Ez/kzcOBArV69WvX19Xr99df1+uuv64orruiE0QDgv9FFAFxAFwFwAV0EwK+EZ/68++67uvbaa/Wtb31Lf/zjHxUKhTpjLgBogy4C4AK6CIAL6CIAfiU886egoECTJ0/W4MGDFQ6HtXHjRhUXF0uSYrGYYrGYXn75ZUnSlClTVFZWltqJAWQkugiAC+giAC6giwD45RljTHredY2k5vS8axzFmLjVPM9LeFIZEvBK7b9kZ3z5KktJIXneWZay0s3dLrL9fZkKfK93nDGNVvOybvsnq3mxpx63midJ4dCXLSWFJPWxlJVu9rqoOf65lZwWoaxeVvMkyRi7vet5ds96aIzvtJqXk2X/Z6btj6HkWc6z+ytGzx/eYDVPkvY/9qzFtJA8r5/FvHRx97goFfgdKPhsf46zrim1midJ8coVFtOO/3saX50AAAAAAAABlvCyr9Rh7+QW2//aw+e3o87uk5+CVFv/Mhqk68pd/lq1/X2ZCi5//LoKu/8idfZpts8wiFjOk+x1SJC+/mz+XXIsZkldo/Ntzxi2nNcVPoZun/nzP04502reYTY/L13hc5yMIPVqMvgdKPjsfo7PPsPl39FOnJXGy74AAAAAAACQaqwmAQAAAAAAAozlDwAAAAAAQICx/AEAAAAAAAgwlj8AAAAAAAABxvIHAAAAAAAgwFj+AAAAAAAABBjLHwAAAAAAgABj+QMAAAAAABBgLH8AAAAAAAACLGOWP8YY1dXVpXuM47I535w5c/TEE0+0vj1//nzNmTOnQ5muf/xS5Uc/+pFisVi6x0hKQ0ODtm3bpqqqKt9/dtmyZaqurk7BVPh7rn8vud5Fkvsfw1Sgi2Cb699Htufj2MgOugi2uf59RBe5q6v0kUtd5BljjLU0i6ZOnaoLLrhAI0eObNef37Vrl9544w1t27ZNmzZt0jvvvKOysjJNnDhRu3fvbn1cKBRSv379ks6dPHmy1q5dq0gk0uZ2Y4xycnK0Zs2atMz31ltvaeLEiZKkgwcPyhijHj16SJLq6+tljFH37t0lSTNmzNAll1zSqfMdydbHMBXuvvtuffDBB5Kk5uZmffjhhyooKGjzmN69e2v+/PlJ5e3cuVPr16/XihUrFIlE9Pjjj0uS1q1bp+eee06//vWvfc/42muv6dVXX9XevXtVW1urTz/9VDU1Naqrq1N+fr4KCgp0zz336Ctf+UpSefF4XFdddZUWLlyoM888U3fddZc2btyoWCymWCymnj17SpLKy8t9f66DoiN9RBd1rItSMWMLuogu6mroIn/zdZVjo0zqIsl+H9FFnY8uoovSwfVjo67QRdm+/kZdyK5du7R+/XoVFBRow4YNikajCofDWrx4sZ5++mmdc845kqTNmzcrGo36yp42bZpKSkra3LZ//36NGDEibfMNHjy49XHPPfecmpub9d3vfleS9Pzzz6uhoUE33HBD2ub7ezY+hqkwa9as1v9etmyZotGopk+f3q6s999/X/fcc48aGho0evRo3XTTTa33NTY2qlu3bu3K7du3r3Jzc5Wfn69TTjlF0WhUV1xxhe666y55nuc7b/Xq1SosLNT+/fu1atUqPfnkk5KkFStW6OWXX9a0adPaNScOo4s61kWpmPFIdBFdlClc/z5KxXxd6dgoE7pISk0f0UVdi+vfR3SRm10kuX9s1BW6yNnljzFG8Xi83X++qKhIRUVFisVimjt3rsLhcOt9ZWVlmjRpkiSpuLjYd/bUqVP1yCOPHDVvTk6OE/MVFRW1nh42b948XX/99crLy/Odkar5JDsfw1R45plntGzZMknSli1b9NWvfvWof9W47rrrNGbMmIRZ5557riorKzVt2jT169dPWVlZOnDggP72t7+poaFB3bp108GDB7VgwQLdeeedSc/Yr18/3Xbbba1vb9++XZLaVSr19fX65S9/qYceekiPPfaYRo8e7TsjE3Skj+iijnVRqmeki+iiroQuav9xh+vHRpnQRVJq+ogu6nx0EV2UDq4fG3WFLnJ2+bN371599NFH7f7zy5cv17/+679Kkvbs2aPi4mLl5eXpBz/4QYdns7ERTdV8DQ0N+slPfqJbb71VkrRo0SJdffXVysvL04IFC7R3797WQkjHfC1c3Cq///778jxP3/nOd7Rz5059/PHHGj58+FGPO3TokD755BPl5+efMG/FihWaNWuW9uzZ0/rfDz74oJ577jldfvnl6tatm6ZPn96uQrClrq5OJ598sn7+858rNzf3qM8JDutIH9FFHeuiVM4o0UV0UddCF7VPVzg2yoQuktzvI7ooOXRR+9BF7Zdpx0ap6iInn/A5Ho/r7bff1urVq9udUVZWpmg0qsmTJysSiai8vFyVlZUdnq2xsdHK5jMV8xlj9OCDD6pv374aNmxY6+27du3Sv/zLv+gvf/lL6ymG6Zivha2PoW29evVSYWGhqqurFY1GNWvWLBUWFqqwsFCDBg3SokWLdODAARUWFrZel3sipaWlWrZsmXJzc1VaWqoVK1Zo06ZN6t+/v3bv3q2lS5equrpaU6ZMSXrGLVu2qLi4uM3/5s6dq7lz5x51+1tvvZUwr0+fPvrtb3+rWCymRx99NOk5MklH+4gu6lgXpWpGiS6ii7oWuqh9usKxUaZ0kWS/j+iizkcXtQ9d1DGuHxt1lS5y8syfNWvWqH///vI8T+vWrUvqCUGPp7y8XE1NTRo3bpyeffbZDs/2+eef6/77729zep3U/tPhbM3X2NioBx54QCtXrtT3vvc9SYe38ocOHdLtt9+uG2+8UbfeeutRc3fWfEey/TG0JS8vT2vWrNGSJUvUu3dvzZgxo839tbW1+v3vf69IJKILLrggqcxXXnlFO3fuVGVlpc4//3xVV1frpptu0oIFC7R//3499dRTOumkk5KeceDAgUdduztv3jxJav1XBD+ampo0adIkFRUVacmSJbrgggt05ZVX+s4JMlt9RBd1rItsztiCLqKLuhK6yL+ucmyUSV0k2e0juqjz0UX+0UUd5/qxUVfpIueWPwcOHNDMmTP10EMPyfM8zZgxQ3/605+Une1/1DfffFM5OTnKy8vTqFGjNH78+A5dL2eM0fbt21ufVOtI7TkdzuZ8+/btU1NTk+69917V1NTovffe05gxYxSLxVRRUaHCwkLfmbY/fpL9j6FNNTU12rp1q8aNG6fu3bvrjjvuaHP/HXfcoZKSEi1cuFAjRozQmWeemTDzN7/5jQoKCnTDDTdo2bJlmjNnjjzP04YNG3T++edr69atuuiii1L1V0po69at2r59u3r16qX+/fvrG9/4RtpmcZGtPqKLOtZFtmeU6CK6qGuhi9qnKxwbZVoXSW73EV10YnRR+9BFHZdpx0ap6iKnLvtqaGjQ3XffrSFDhujSSy/VJZdcosLCQj3wwAPy+4r09fX1mjJlisaPHy9JuuWWWzR27NgOzVdVVaWBAwe261+sUz1f79699Ytf/KL1mckHDBigF198UWeddZZ69erV+rgdO3aoubm50+drYfNjKB1+9vxvf/vbVrK+9KUvaf78+Tr11FNVXl6u4cOHt/nfa6+9pr59++r5559PqlBeeOEFHTx4UJdddpnOOOMMPfnkk/I8T5s2bVIkEtH3v/99VVRUWJm9vQYOHKjVq1frZz/7mXr16qXXX389rfO4xFYf0UUd66JUzCjRRXRR10EXtV9XODbKpC6S3O8juuj46KL2o4s6LtOOjVLVRc4sf2prazV27Fjl5ua2ubZu6tSp2rFjh+68807t27cv6bxFixappKREgwYNknT4WbaPvL7SL2OMZs+ereuuu67N7Y2NjWpqalJ9fb2vJ4SyPd+xnHrqqZLUWiLGGD3wwANau3ZtWuaz/TGUpCFDhrQpTVvGjBmjJUuWtPnfxRdfLCn5Z2w/5ZRT9PDDD7e+HQqFZIzRzJkzdeutt2ro0KHasGGD3n33XevzJ+vdd9/Vtddeq29961v64x//qFAolLZZXGKzj+iijnVRKmaki+iiroIusttFklvHRpnWRZL7fUQXHRtdRBe50kVSZhwbpaqLnLjs6/XXX9eECRM0YsQI/fCHP2zzlzvppJM0b948/fSnP9WwYcM0f/781i/0Exk1apTC4fAxN9EVFRVauXKlrxmrq6uVm5t71LV2tbW1Gj16tOrr61VaWpp0nu35jufrX/+6xowZo+zsbDU2Nio/P7/1m6Oz57P9MWzx0EMP+f4zifzhD3/Q0qVL29xWU1OjG2+8MemMyy+/XJLa5EyfPl09e/bU1VdfLUmaMGGC7rvvPlVUVOjkk0+2MLk/BQUFmjx5sgYPHqxwOKyNGze2vixkLBZTLBbTyy+/LEmaMmWKysrKOn3Gzma7j+iiw9rbRamYkS6ii7oCuqhj852IK8dGmdZFkvt9RBcdjS7q2HwnQhe1TyYcG6Wsi4wDPv30U/PWW28lfNwrr7xi6uvrfWV/8cUX5tJLL219e8uWLWb9+vWtbz///PO+8mxjvo6prq5O6/tPxqOPPmqWL19u/vM//9PcdtttR30N33///ea1115L03T4e6nqI9e/l1yfzxi3Z6SLYBtddJhr8xnj9oxdoYuMoY+6ErroMNfmM8btGekiN3nG+HwyHQAAAAAAAHQZzjznDwAAAAAAAOxj+QMAAAAAABBgLH8AAAAAAAACjOUPAAAAAABAgLH8AQAAAAAACDCWPwAAAAAAAAHG8gcAAAAAACDA/j+Aw0I6wryxWwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder Src Layer 6\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABH8AAAEXCAYAAAA0tOgSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfZBddZkn8Of0S0J4MTgBe0GcOFiAsGIGpAa3pBqjqeAkg8nsGndIwGWWt6LUBEQhTmZCaGCLWCANKXcC6JKVDYtsZAYijoEUjqlx3LEYi8KIIFNUphDoIJNNSGihk75n/2A6k5fu5N7u3+X+cvrz+YMi93S+90nf7m/ffvqc20VZlmUAAAAAUEltrR4AAAAAgOax/AEAAACoMMsfAAAAgAqz/AEAAACoMMsfAAAAgAqz/AEAAACoMMsfAAAAgArraNUdn3P1gvj1a5uTZG26b3WSHKBebRFxbKuHSKJWvhIRg0myimhPkjNkV/lq0ryIiD/5628mzXtg7hVJ89qKw5PmpX5MIiLaisOS5pVlLWlerdyRNC+K9O/DdDqivfjdVg+RyG8iIs3Hwj9sXp8kZ8jZXTOS5h0KyjLN14UhRRM+j+Y8cH3SvIf/5IakeeNPNZ4bleXmSPa8KOuvH+NTrRxInjn/0duS5v3v2dclzSuK/M93+b+bH0+WNbHtiDjj2D8e9ljLlj+/fm1z/PPmlxOlpf0CDYwngxGxq9VDjCD9F+i+N36TODH1jBMS55WJ8yLSf81Ju/yJ2Jk4rxnvQ/ZXi1QfW28Nbk+S82/G4/Os/P/Nr+xI3ef5/5t5JwyGj4UqS/+ct6//tcSJqT/+8n8e82byr9vDy38NBgAAAMCoWf4AAAAAVJjlDwAAAECF1fWaP6tXr4677rpr95+3b98enZ2dcdhh//ail9dff3188pOfTD8hwL/SRUAOdBGQA10ENKKu5c+CBQtiwYIFu/98ww03xJlnnhnnn39+0wYD2JcuAnKgi4Ac6CKgEaO67GtwcDDa2/3qPqC1dBGQA10E5EAXAQdS15k/3d3de/359ddfj3Xr1sUtt9yy1+0bNmxINxnAPnQRkANdBORAFwGNqGv5s29hnHfeeXHqqadGb29vU4YCGI4uAnKgi4Ac6CKgEQ1f9vXCCy/E1q1b45lnnomnnnqqGTMBHJQuAnKgi4Ac6CLgYBpa/tRqtVi2bFksWLAglixZEgsXLoy+vr5mzQYwLF0E5EAXATnQRUA96l7+bN26NRYuXBhlWcaVV14Z5557blx44YUxb968+Md//Mdmzgiwmy4CcqCLgBzoIqBedS1/ent7Y+bMmXHMMcfEPffcE52dnRERcfnll8c111wTl156aVx99dVNHRRAFwE50EVADnQR0Ii6XvB569atcd9998Upp5yy37G5c+fG2WefHffee2/y4QD2pIuAHOgiIAe6CGhEUZZl2Yo7fv9FM+KfN7+cJKt8bH2SHKBe7RHR1eohkqiVv46IXUmyivr26XXbVb6SNC8i4uOrb0ua97cLrk2a11YcmTSviPakeRERbcWkpHllWUuaVytfT5oXRdqP67Q6or04sdVDJLI5IgaTJG14+ZEkOUO6j/900rxDQVmmeSyGFEX6LvqDb34xad5PL12RNG/8qcZzo7J8OVJ1UTM+7hmbWvlW8sxP/J+/SJr3w3m3JM0rioZ/x9U77m9ffjhZ1mHtR8VHuy4c9lj+7wkAAAAARs3yBwAAAKDCcj6XG6Dpij3+O3Zpr6JtKyYnzYuI+F9z5yTN6yiOTZqX7rF4WxnpT29Ofcr0YLk1ad4lj38rad6qmV9OmheR8hLJ6lxS8D+e+U5s37ktSdbCD/9pkpwhbw4+kzQvIuKw9tOSZ6aU+nKVtwZfSJoXEfHfPvXJ5JnwX9Z9PTb3v5Yk6wd/fEOSnOZK+7me+hKj1Jegpn7OERHxyNwLEiemfT6d+vL6MnYmzYuI+A9dpydMmzjiEWf+AAAAAFSY5Q8AAABAhVn+AAAAAFSY5Q8AAABAhVn+AAAAAFSY5Q8AAABAhdW9/Fm6dGk8+uijzZwF4KB0EZADXQTkQBcB9XLmDwAAAECFdTTyxj09PbF8+fJhj02ePDnWrl2bZCiAA9FFQA50EZADXQTUo6Hlz9KlS2P27NnNmgWgLroIyIEuAnKgi4B6uOwLAAAAoMKSXfa1ePHimDVrVpKhAA5EFwE50EVADnQRUI+6lz89PT3R09Mz7LGbbrop2UAAB6KLgBzoIiAHugioV5LLvrZs2RKTJ09OEQUwaroIyIEuAnKgi4A91XXmT3d394jHHnzwwejr64spU6YkGwpgOLoIyIEuAnKgi4BG1LX82bBhw363lWUZRVFEf39/bNq0KaZOnZp8OIA96SIgB7oIyIEuAhrR0As+72nRokWxcePGGBgYiPnz58ekSZNSzgVQF10E5EAXATnQRcBIRr38ufPOO1POATAqugjIgS4CcqCLgJEkecFnAAAAAPJUlGVZtuauN0fEYJKkYuaMJDlDysfWJ82D6mmPiK5WD5HE89u+GztrbyTJuuWnP02SM2TVzKuT5kVE1MrfJs1rL45OmrerfCVpXkdxXNK8ZiiKvH8OU5a1Vo9wAO1RHAKPcX3SPS96a/CFJDlDJrS9P2leRMSPXlmbNO/c485PmleL/qR5bZH+0ps7n16VNG/hhy9OmrerfC1pXmdb7s87qvHc6N5frojtO7clyTr3ve9NkjPk37/7D5LmRUS0FUckzpuQNO/NwWeS5rUX6V8APPVzwbZiYtK8Q0FZpvn6/7b2KIrjhz2S9zNOAAAAAMbE8gcAAACgwix/AAAAACrM8gcAAACgwix/AAAAACrM8gcAAACgwix/AAAAACqso543evLJJ+NLX/rSAd9m5cqVcdpppyUZCmA4ugjIgS4CcqCLgEbUtfw566yzYsOGDc2eBeCAdBGQA10E5EAXAY1w2RcAAABAhdV15k93d3ddYTbPQDPpIiAHugjIgS4CGlHX8mfPwujt7Y0XX3wxbrvttqYNBTAcXQTkQBcBOdBFQCMauuxrx44dsXr16vjc5z4XERF9fX3xox/9qCmDAYxEFwE50EVADnQRUI+Glj+rVq2KD33oQzFt2rSIiHjjjTdi8eLF8cYbbzRlOIDh6CIgB7oIyIEuAupR9/Ln6aefjnvuuSe+/OUv777tAx/4QEybNi3uv//+pgwHsC9dBORAFwE50EVAvep6zZ9t27bFVVddFRdffHF0dXXFc889F5s2bYpf/epXsWXLllizZk1cdtllzZ4VGOd0EZADXQTkQBcBjajrzJ/+/v549dVX44EHHojPfvaz8dWvfjW+//3vR61Wi4suuih27twZP/vZz5o9KzDO6SIgB7oIyIEuAhpR15k/xx13XGzcuHHE4yeddFKceOKJyYYCGI4uAnKgi4Ac6CKgEXUtfw7mgx/8YIoYgDHRRUAOdBGQA10E7Kmh3/YFAAAAwKGlKMuybM1db46Iwdbc9UEUM2ckzywfW588E1qnPSK6Wj1EEmX5cqTqok/91V8kyRny/T/+atK8iIgi2pPmtRWHJ807FGzaviFp3glHnJY0L/VjkvdjXKUu+nWke16U9vP87G8tSpoXEfHTS1ckz8zZ5t/+Q/LMyRPelTTvsPZTk+aNP9Xoo5Rd9Obgs0lyhjy79eWkeRERbUXavGlTzkuaV5apv19O/A+OiDd2/TxpXkfbhKR5E9renzSvrZiUNC+9kbvImT8AAAAAFWb5AwAAAFBhlj8AAAAAFWb5AwAAAFBhlj8AAAAAFVb38mfp0qXx6KOPNnMWgIPSRUAOdBGQA10E1MuZPwAAAAAV1tHIG/f09MTy5cuHPTZ58uRYu3ZtkqEADkQXATnQRUAOdBFQj4aWP0uXLo3Zs2c3axaAuugiIAe6CMiBLgLq4bIvAAAAgApLdtnX4sWLY9asWUmGAjgQXQTkQBcBOdBFQD3qXv709PRET0/PsMduuummZAMBHIguAnKgi4Ac6CKgXkku+9qyZUtMnjw5RRTAqOkiIAe6CMiBLgL2VNeZP93d3SMee/DBB6Ovry+mTJmSbCiA4egiIAe6CMiBLgIaUdfyZ8OGDfvdVpZlFEUR/f39sWnTppg6dWry4QD2pIuAHOgiIAe6CGhEQy/4vKdFixbFxo0bY2BgIObPnx+TJk1KORdAXXQRkANdBORAFwEjGfXy584770w5B8Co6CIgB7oIyIEuAkaS5AWfAQAAAMiT5Q8AAABAhRVlWZatuevNETHYmrtugWLmjKR55WPrk+ZBY9ojoqvVQyRRK1+MiF2JsrYnyRnysXt7kuZFRPz9n6bN/Je3fpU079jDzkqaF9GiL3ENKdKmFePp5zrV6aKUz4t21f5fkpwhHW3vTpoXEVEr+5PmtRWHJ837j2uXJc27f9Z/TpoXEfGb3/5L0rz3HXlO0rzxpxp9VCs3RarnRa8PPJskZ8i7JnwwaV5ExDn3LUma9+OLvpY0rxa/TZrXFkcmzWuGMt5MnFdLmtde5P4+HLmLxtMzRAAAAIBxx/IHAAAAoMIsfwAAAAAqzPIHAAAAoMIsfwAAAAAqbFTLn5tvvjkeeuih1LMANEQXATnQRUAOdBFwIM78AQAAAKiwupY/zz77bLz22msjHv/FL34Rzz//fLKhAIaji4Ac6CIgB7oIaERdy59Vq1bFU089NeLxJ554In784x8nGwpgOLoIyIEuAnKgi4BGuOwLAAAAoMIsfwAAAAAqzPIHAAAAoMIsfwAAAAAqzPIHAAAAoMI66n3DJUuWRE9PT0REbN++PTo7O6O3tzciInbs2BELFy5szoQAe9BFQA50EZADXQTUq67lzy233HLA4ytWrEgyDMCB6CIgB7oIyIEuAhpR95k/B/LFL34xRQzAmOgiIAe6CMiBLgL25DV/AAAAACrM8gcAAACgwoqyLMvW3PXmiBhszV1XQDFzRtK88rH1SfOouvaI6Gr1EImk66KyrCXJGbKr3Jw0LyLizBXXJ817euHKpHlFMf5+JpH646bvtz9Nmnfc4R9NmpdWdbqof+ffRhm/TZJ1ROe0JDlDdtZeSZoXEdHZdlzyzJx9dNVVyTN/eOElSfMmdZyeNG/8qUYfbXnrsaiVabpoysSPJMkZUsabSfMiIn7z5s+T5t3/XNq8L374PyXNaysOT5r3dubE5Jk521nrS57ZUUxJmNYeRXHCsEfG37NsAAAAgHHE8gcAAACgwix/AAAAACrM8gcAAACgwix/AAAAACqs7uXP0qVL49FHH23mLAAHpYuAHOgiIAe6CKiXM38AAAAAKqyjkTfu6emJ5cuXD3ts8uTJsXbt2iRDARyILgJyoIuAHOgioB4NLX+WLl0as2fPbtYsAHXRRUAOdBGQA10E1MNlXwAAAAAVluyyr8WLF8esWbOSDAVwILoIyIEuAnKgi4B61L386enpiZ6enmGP3XTTTckGAjgQXQTkQBcBOdBFQL2SXPa1ZcuWmDx5cooogFHTRUAOdBGQA10E7KmuM3+6u7tHPPbggw9GX19fTJkyJdlQAMPRRUAOdBGQA10ENKKu5c+GDRv2u60syyiKIvr7+2PTpk0xderU5MMB7EkXATnQRUAOdBHQiIZe8HlPixYtio0bN8bAwEDMnz8/Jk2alHIugLroIiAHugjIgS4CRjLq5c+dd96Zcg6AUdFFQA50EZADXQSMJMkLPgMAAACQJ8sfAAAAgAoryrIsW3PXmyNisDV3zX6KmTOS5pWPrU+aR27aI6Kr1UMkkq6LyrKWJKeZauXrSfOOW3Jl0rxNN/5Z0rzDO05PmkdudNFwdtZeSpIzpLPtvUnzIiJ21vqS5rUXv5M076PfuiZp3k8uuTlpXkT6x3li2ylJ84pivP2MuSp9lO/zomZ8TJXlzsSJ7UnT2ub+YdK82l9/L2leRERRdCbPZCxG7qLx1soAAAAA44rlDwAAAECFWf4AAAAAVJjlDwAAAECFWf4AAAAAVJjlDwAAAECFjWr5s2LFili1alXiUQAao4uAHOgiIAe6CDiQJGf+7Ny5M1566aUUUQCjpouAHOgiIAe6CNjTqJc/L7zwwu7/X7p0aXzta19LMhBAI3QRkANdBORAFwEj6ajnjbq7uyMiYmBgIKZPnx7HH398PPLII/H5z38+vv71r8fzzz8f3/72t5s6KIAuAnKgi4Ac6CKgEXUtfzZs2BAREXfffXe0t7dHf39/HHPMMTF79uyYOXNmrF69OiZNmtTUQQF0EZADXQTkQBcBjWjosq9169bFpz/96YiImDdvXrzrXe+KK664QqkA7yhdBORAFwE50EVAPepe/qxfvz5OPvnkOPbYYyMiYuLEibFo0aK49tprY2BgoGkDAuxJFwE50EVADnQRUK+6lj/btm2LW2+9Na666qq9bp8zZ050dXXFddddF7t27WrKgABDdBGQA10E5EAXAY2oa/nzhS98IS655JLo6ura79jy5cvjxRdfjCuuuCL5cAB70kVADnQRkANdBDSiruVPd3d3zJs3L3bs2BEREbt27YqiKCIiYtKkSbFq1aro7Oxs3pQAoYuAPOgiIAe6CGhEXb/t67LLLouIiN7e3nj88cfjzTffjBUrVuw+fuSRR8bKlSubMyHAv9JFQA50EZADXQQ0oijLsmzNXW+OiMHW3DX7KWbOSJpXPrY+aR65aY+I/U8xPjSl66KyrCXJaaZa+XrSvOOWXJk0b9ONf5Y07/CO05PmkRtdNJydtZeS5AzpbHtv0ryIiJ21vqR57cXvJM376LeuSZr3k0tuTpoXkf5xnth2StK8omjolwpXQFX6KN/nRc34mCrLnYkT25Omtc39w6R5tb/+XtK8iIiicHZZXkbuovHWygAAAADjiuUPAAAAQIW57IumKMv0j23bH56XNK/8gUvTRq8qpzZH6KK81Mq3kua1FROT5kVEFH80M2le+b3HkuaNL9Xpon9567Golf1Jso497A+S5AyplQNJ8yIi2ooJyTNzlrrbIiLed/1/TZr3Us/qpHnjTzX6qFb+OiLS/Hr48fZ5fih4a/CF5JmHfT7tZbLlyr9Kmjf+uOwLAAAAYFyy/AEAAACoMMsfAAAAgAqz/AEAAACoMMsfAAAAgAqz/AEAAACoMMsfAAAAgAqz/AEAAACoMMsfAAAAgAqz/AEAAACoMMsfAAAAgAqz/AEAAACoMMsfAAAAgAqre/lzxhlnxCc+8YlmzgJwULoIyIEuAnKgi4B61b38+djHPhZHH310M2cBOChdBORAFwE50EVAvRq67GvZsmVNGgOgfroIyIEuAnKgi4B6dNT7hosWLYqTTjqpmbMAHJQuAnKgi4Ac6CKgXnWf+aNUgBzoIiAHugjIgS4C6uW3fQEAAABUWN2XfUEjiqI9fWhn2sxa+VbSvLZiYtI8Dj1lWUuaVxTjcT9fJE2rlf1J8yIi4rAm9Bvj3pSJ0yJisNVjDKutmNDqEQ55zXiO8FLP6qR5A4ObkuZNvHRh0rzy3keS5jG8MgYiYleiNN2Rm+e2/lPyzLe+0Zs0748e/ouked+bc2PSvEPZePzOAgAAAGDcsPwBAAAAqDDLHwAAAIAKs/wBAAAAqDDLHwAAAIAKs/wBAAAAqLCGlz+7du2K008/vRmzANRNFwE50EVADnQRcDDO/AEAAACoMMsfAAAAgArrGM1fGhgYiLPOOmuv2y644IK45pprkgwFUA9dBORAFwE50EXAgYxq+TNhwoR48sknU88C0BBdBORAFwE50EXAgbjsCwAAAKDCLH8AAAAAKszyBwAAAKDCLH8AAAAAKqyuF3ze91XjJ06cuN9tEeEFxoCm0kVADnQRkANdBDSiruWPwgByoIuAHOgiIAe6CGiEy74AAAAAKszyBwAAAKDC6rrsC3JQrl2XNK+YOSNpXvnY+qR5HIrKVg9wyGsrJiROTJ0XETtrSeNq5VtJ89qKiUnzoFb2J89sKw5PnsnYHHvdtUnz+u+5OWlerRxImhfRjK85h7724oiIGGz1GDTJh6fMbPUIB/Xof/9J2sA5aeMOZc78AQAAAKgwyx8AAACACrP8AQAAAKgwyx8AAACACrP8AQAAAKgwyx8AAACACqvrV713d3fv9ectW7bE0UcfHW1te++ONmzYkG4ygH3oIiAHugjIgS4CGlHX8mffwpgzZ0584xvfiBNOOKEpQwEMRxcBOdBFQA50EdAIl30BAAAAVNiolz9PP/10yjkARkUXATnQRUAOdBEwklEvf+644464+uqr47XXXks5D0BDdBGQA10E5EAXASMZ9fLn7rvvjt/7vd+LuXPnxt/93d+lnAmgbroIyIEuAnKgi4CR1PWCz8Npb2+PhQsXxumnnx5r1qyJc845J+VcAHXRRUAOdBGQA10EjGTUy58h06dPj+nTp6eYBWDUdBGQA10E5EAXAfvy274AAAAAKszyBwAAAKDCRnXZ18MPP5x6DoCG6SIgB7oIyIEuAg7EmT8AAAAAFWb5AwAAAFBhRVmWZWvuenNEDLbmrqEJipkzkuaVj61PmpdWe0R0tXqIRHQRjRksdyTN+0nfD5PmXfv4E0nz/v5ztyfNS0sXwXhRluk/P9pmfSpZ1tT3HB+b/ufjyfJaJ10Xpf562RaHJ82LiPjhS48kzfvECXOT5u2qbUma19H2O0nzmuHV3/40ad4/bXs1ad5Z75mWNC8iYkLb+xKmjfzcyJk/AAAAABVm+QMAAABQYZY/AAAAABVm+QMAAABQYZY/AAAAABVm+QMAAABQYZY/AAAAABVm+QMAAABQYZY/AAAAABVm+QMAAABQYR31vFF3d/fBgzo64oknnhjzQAAj0UVADnQRkANdBDSiruXPhg0bmj0HwEHpIiAHugjIgS4CGuGyLwAAAIAKs/wBAAAAqDDLHwAAAIAKs/wBAAAAqDDLHwAAAIAKs/wBAAAAqDDLHwAAAIAKs/wBAAAAqLCiLMuyNXe9OSIGW3PX7Kcsa0nzisJecayKT81Inln7m8cSJbVHURyXKKvV8u2i1J+XzeBzfezKcmfSvLYrPps0762/vD1pXkTEhPb3J0pqj4iuRFmtlq6LBmtbk+QMaW87OmleRERZpu3domhPmrez9krSvM629F8zU78PI4rEeWm/xXjXNRckzYuIeP221QnT2qMoTkiY1yr5Pi9qBt8DVV/qx7jt/E8lzYuIqK39QcK0kb9P89EJAAAAUGGWPwAAAAAVZvkDAAAAUGGWPwAAAAAVZvkDAAAAUGGjWv7cddddsWPHjtSzADREFwE50EVADnQRcCANL382btwY3/3ud+OII45oxjwAddFFQA50EZADXQQcTEc9b/TQQw/FjTfeGEcdddTu284999zd/79jx45YuHBhXHzxxckHBBiii4Ac6CIgB7oIaERdy5+IiM985jOxZMmSYY+tWLEi2UAAB6KLgBzoIiAHugiolxd8BgAAAKiwus/8WbNmTaxbt27YY0OnFAI0my4CcqCLgBzoIqBeLvsCDim6CMiBLgJyoIuAernsCwAAAKDCLH8AAAAAKsxr/gCHFF0E5EAXATnQRUC96lr+dHR0xPz58+MrX/nKsMddTwq8E3QRkANdBORAFwGNKMqyLFtz15sjYrA1d81+yrKWNK8oXFE4VsWnZiTPrP3NY4mS2qMojkuU1Wr5dlHqz8tm8Lk+dmW5M2le2xWfTZr31l/enjQvImJC+/sTJbVHRFeirFZL10WDta1Jcoa0tx2dNC8ioizT9m5RtCfN21l7JWleZ1v6r5mp34cRReK8tN9ivOuaC5LmRUS8ftvqhGntURQnJMxrlXyfFzWD74GqL/Vj3Hb+p5LmRUTU1v4gYdrI36f56AQAAACosLpf8yc9e6e8pP5pj8d3rKZ2Hd+E1FQ/GU37E9bWyvljNfXnZTPk/P47VKT9idTUKanPMJiYOC8iXYdU6eMv5b+lM2FWxKHR+alnnJA471B4H+Z95s/vvvvfJc17W8rH5VB4jOtRpV6th++Bqi/tYzz1PTl/j3bgrBZe9gUAAABAs1lNAgAAAFSY5Q8AAABAhVn+AAAAAFSY5Q8AAABAhVn+AAAAAFSY5Q8AAABAhVn+AAAAAFSY5Q8AAABAhVn+AAAAAFRY+7Jly5a1eohmWrp0afT398fJJ5/c6lGG1cz5br755njttdfi1FNPHVNOM2dcsWJF/PKXv4zf//3fT57N6Nx1113xwQ9+MCZMmNDqUSpFF+kiGqOLmmM8d1FEmj7SReOLLmoOXaSLaEyKLupIOM+YrF69Ou66667df96+fXt0dnbGYYcdtvu266+/Pj75yU+2YryIiHjyySfjS1/60gHfZuXKlXHaaae9QxNVz86dO+PVV1+N9773va0eJXbt2hVnnHFG/PznPx91Rnd3915/3rJlSxx99NHR1rb3SXcbNmwY9X2ktHHjxvjud78bl19+eatHaanc+0gXNV/Vuiji0OojXfQ2XYQuai1d9DZdRE5dFDH+vk9L1UXZLH8WLFgQCxYs2P3nG264Ic4888w4//zzx5zd09MTy5cvH/bY5MmTY+3atXXlnHXWWU158FPN9+yzz8YxxxwTxxxzzLDHf/GLX8SECRPipJNOatmMw3nhhRd2///QBvuOO+4YdV5O9v14mTNnTnzjG9+IE044oUUT7e+hhx6KG2+8MY466qjdt5177rm7/3/Hjh2xcOHCuPjii1swXWs0q4900dt0UWvk3ke6aH+6aH+Nfp43q4900ejpokOPLtqfLjr0jccuymb5s6/BwcFob29PkrV06dKYPXt2kqxmSDXfqlWrYsaMGTFjxoxhjz/xxBNx1FFHjeobrlQzDm1YBwYGYvr06XH88cfHI488Ep///Ofj61//ejz//PPx7W9/e8z3Q2M+85nPxJIlS4Y9tmLFind4mvyk6iNd9DZdxEh00YHposY1q490UbXpogPTRY3TRYxG6i7KZvmz72lXr7/+eqxbty5uueWWvW5v5WlX+844khxODcvV0Pvm7rvvjvb29ujv749jjjkmZs+eHTNnzozVq1fHpEmTWjzlvxkYGIizzjprr9suuOCCuOaaa0ad+fTTT9KIIXcAAAUMSURBVGezUWZ4ufeRLho7XfQ2fZQ3XVR9uuhtuihvuqj6DrUuivB92mhks/zZ95PxvPPOi1NPPTV6e3vHnH2g0+EWL14cs2bNanjG3t7eePHFF+O2227LZr5mSj3junXrYuXKlfHAAw/EvHnz4jvf+U5cccUV2ZXKhAkT4sknn0yaeccdd8Tjjz8eS5YsGfHUzwOp5wtcR0dHPPHEE3VnrlmzJtatWzfssaFTCseTZvWRLho7XZTWWPpIFzWfLhrbfM2ki9LSRXnTRWObr5nGaxdF5Pd92qHQRdksf/b0wgsvxNatW+OZZ56Jp556akyvMt7T0xM9PT3DHrvppptGlbljx45YvXp1fPOb34yIiL6+vnjuuef2ugavlfOllnrG9evXx8knnxzHHntsRERMnDgxFi1aFNdee23cd999lf9tCnfffXc8/PDDMXfu3LjlllvinHPOaejvN+OnFk5vHlmqPtJFY6eL0htLH+mid5Yu0kVVposOHbpIF1VZ1buo7eBv8s6q1WqxbNmyWLBgQSxZsiQWLlwYfX19TbmvLVu2xOTJkxv+e6tWrYoPfehDMW3atIiIeOONN2Lx4sXxxhtvZDHfO6nRGbdt2xa33nprXHXVVXvdPmfOnOjq6orrrrsudu3alXrMrLS3t8fChQvjxhtvjDVr1rR6HA7gneojXTR2umh09NGhQReNbb53ki4aHV10aNBFY5vvnaSLRqfqXZTVmT9bt26NP//zP4+yLOPKK6+Mzs7OuPDCC2PevHnR29sbH/nIRxrKO9CpVw8++GD09fXFlClTGsp8+umn45577on7779/920f+MAHYtq0aXH//ffHZZdd1tL5Uks94xe+8IW45JJLoqura79jy5cvj4suuiiuuOKK+Na3vlV35hlnnBHvfve7GzqFLgfTp0+P6dOnt3oMRpCyj3TR2Omi5tJH+dJFY5svNV3UXLooX7pobPOlpouaq6pdlM3yp7e3N+6///6YNWtW3HrrrdHZ2RkREZdffnm85z3viUsvvTQ+/vGPx+2331535nCnXpVlGUVRRH9/f2zatCmmTp1ad962bdviqquuiosvvji6urriueeei02bNsWvfvWr2LJlS6xZs6ahYkk9X0TEkiVLdp/+t3379ujs7Nx9Pe5orgtMPWN3d3fMmzcvduzYEUceeWTs2rUriqKIiIhJkybFqlWr4stf/nJDM37sYx+Ll19+uaG/w95c27631H2ki3QR9dFFe9NFY++iiLR9pIvGB120N12ki3RRa1T2NX+2bt0a9913X5xyyin7HZs7d26cffbZce+99475fhYtWhQbN26MgYGBmD9/fkMvXtXf3x+vvvpqPPDAA7F27do4+uij433ve1+ceOKJcdFFF8Xtt98eP/vZz+LMM89syXz7vuL+vlJdozyWGYeKt7e3Nx5//PF4880395rryCOPjJUrVzY807Jlyxr+O8PZ9xXjJ06cuN9tEdGUFztslY6Ojpg/f3585StfGfb4eLy2/Z3oI100drpIF1WdLhrbfBHvTB/pIl1UdbpIF7W6iyLGXx81o4uKsizLsQ7G25599tk48cQTx8WLYeXk+eefj5NOOqnVY0A2dFFr6CLYmy5qDV0Ee9NFraGL8mP5AwAAAFBh2f22LwAAAADSsfwBAAAAqDDLHwAAAIAKs/wBAAAAqDDLHwAAAIAKs/wBAAAAqDDLHwAAAIAK+/+rFwIE93mkTgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib.font_manager import FontProperties\n",
    "myfont=FontProperties(fname=r'wqy-microhei.ttc')\n",
    "def draw(data, x, y, ax):\n",
    "    ax.imshow(data, cmap='YlGn', vmin=0, vmax=1)\n",
    "    ax.set_xticks(np.arange(len(x)))\n",
    "    ax.set_yticks(np.arange(len(y)))\n",
    "    ax.set_xticklabels(x, fontproperties=myfont)\n",
    "    ax.set_yticklabels(y, fontproperties=myfont)\n",
    "    ax.grid(False)\n",
    "    \n",
    "for layer in range(1, 6, 2):\n",
    "    fig, axs = plt.subplots(1,4, figsize=(20, 10))\n",
    "    print(\"Encoder Layer\", layer+1)\n",
    "    for h in range(4):\n",
    "        draw(model.encoder.layers[layer].self_attn.attn[0, h].cpu().data, \n",
    "            sent, sent, ax=axs[h])\n",
    "    plt.show()\n",
    "    \n",
    "for layer in range(1, 6, 2):\n",
    "    fig, axs = plt.subplots(1,4, figsize=(20, 10))\n",
    "    print(\"Decoder Self Layer\", layer+1)\n",
    "    for h in range(4):\n",
    "        draw(model.decoder.layers[layer].self_attn.attn[0, h].cpu().data[:len(tgt_sent), :len(tgt_sent)], \n",
    "            tgt_sent, tgt_sent, ax=axs[h])\n",
    "    plt.show()\n",
    "    print(\"Decoder Src Layer\", layer+1)\n",
    "    fig, axs = plt.subplots(1,4, figsize=(20, 10))\n",
    "    for h in range(4):\n",
    "        draw(model.decoder.layers[layer].self_attn.attn[0, h].cpu().data[:len(sent), :len(sent)], \n",
    "            sent, sent, ax=axs[h])\n",
    "    plt.show()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc-showcode": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}