Custom Transformer and Tokenizer with low loss.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f571414e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fa58a3fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.utils.data as data\n",
    "import torch.nn.functional as F\n",
    "import math\n",
    "import json\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c72274d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('separated_translations.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fd70c3b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import math\n",
    "\n",
    "# Define the Multi-Head Attention module\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self, d_model, num_heads):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        assert d_model % num_heads == 0, \"d_model must be divisible by num_heads\"\n",
    "        \n",
    "        self.dim_model = d_model  # Dimension of the input and output\n",
    "        self.num_heads = num_heads  # Number of attention heads\n",
    "        self.dim_head = d_model // num_heads  # Dimension of each attention head\n",
    "        \n",
    "        # Linear transformations for Q, K, V, and output\n",
    "        self.W_q = nn.Linear(d_model, d_model)\n",
    "        self.W_k = nn.Linear(d_model, d_model)\n",
    "        self.W_v = nn.Linear(d_model, d_model)\n",
    "        self.W_o = nn.Linear(d_model, d_model)\n",
    "        \n",
    "        self.dropout = nn.Dropout(0.1)  # Dropout layer for regularization\n",
    "        self.layer_norm = nn.LayerNorm(d_model)  # Layer normalization\n",
    "        \n",
    "    def scaled_dot_product_attention(self, Q, K, V, mask=None):\n",
    "        # Compute attention scores by performing matrix multiplication\n",
    "        attn_scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.dim_head)\n",
    "        \n",
    "        if mask is not None:\n",
    "            # Apply mask to attention scores\n",
    "            attn_scores = attn_scores.masked_fill(mask == 0, -1e9)\n",
    "        \n",
    "        # Apply softmax to obtain attention probabilities\n",
    "        attn_probs = torch.softmax(attn_scores, dim=-1)\n",
    "        \n",
    "        # Apply dropout and compute the output by multiplying attention probabilities with values\n",
    "        output = torch.matmul(self.dropout(attn_probs), V)\n",
    "        return output\n",
    "        \n",
    "    def split_heads(self, x):\n",
    "        # Reshape input tensor to split the last dimension into multiple heads\n",
    "        batch_size, seq_length, dim_model = x.size()\n",
    "        return x.view(batch_size, seq_length, self.num_heads, self.dim_head).transpose(1, 2)\n",
    "        \n",
    "    def combine_heads(self, x):\n",
    "        # Reshape tensor to concatenate heads and restore the original dimensions\n",
    "        batch_size, _, seq_length, dim_head = x.size()\n",
    "        return x.transpose(1, 2).contiguous().view(batch_size, seq_length, self.dim_model)\n",
    "        \n",
    "    def forward(self, Q, K, V, mask=None):\n",
    "        # Apply linear transformations to the input and split into heads\n",
    "        Q = self.split_heads(self.W_q(self.layer_norm(Q)))\n",
    "        K = self.split_heads(self.W_k(self.layer_norm(K)))\n",
    "        V = self.split_heads(self.W_v(self.layer_norm(V)))\n",
    "        \n",
    "        # Perform scaled dot-product attention\n",
    "        attn_output = self.scaled_dot_product_attention(Q, K, V, mask)\n",
    "        \n",
    "        # Combine heads and apply the final linear transformation\n",
    "        output = self.W_o(self.combine_heads(attn_output))\n",
    "        return output\n",
    "    \n",
    "# Define the Position-wise Feed Forward module\n",
    "class PositionWiseFeedForward(nn.Module):\n",
    "    def __init__(self, d_model, d_ff, dropout=0.1):\n",
    "        super(PositionWiseFeedForward, self).__init__()\n",
    "        self.linear1 = nn.Linear(d_model, d_ff)  # First linear transformation\n",
    "        self.dropout = nn.Dropout(dropout)  # Dropout layer for regularization\n",
    "        self.linear2 = nn.Linear(d_ff, d_model)  # Second linear transformation\n",
    "        self.layer_norm = nn.LayerNorm(d_model)  # Layer normalization\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.layer_norm(x)  # Apply layer normalization\n",
    "        x = self.linear1(x)  # Apply the first linear transformation\n",
    "        x = F.relu(x)  # Apply ReLU activation function\n",
    "        x = self.dropout(x)  # Apply dropout\n",
    "        x = self.linear2(x)  # Apply the second linear transformation\n",
    "        return x\n",
    "\n",
    "# Define the Positional Encoding module\n",
    "class PositionalEncoding(nn.Module):\n",
    "    def __init__(self, dim_model, max_seq_length):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        \n",
    "        pe = torch.zeros(max_seq_length, dim_model)  # Initialize the positional encoding matrix\n",
    "        \n",
    "        # Compute the position and div_term values for the positional encoding\n",
    "        position = torch.arange(0, max_seq_length, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, dim_model, 2).float() * -(math.log(10000.0) / dim_model))\n",
    "        \n",
    "        # Compute sine and cosine values for the positional encoding\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        \n",
    "        self.register_buffer('pe', pe.unsqueeze(0))  # Register the positional encoding as a buffer\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return x + self.pe[:, :x.size(1)]  # Add the positional encoding to the input\n",
    "\n",
    "# Define the Encoder Layer module\n",
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(self, dim_model, num_heads, dim_ff, dropout):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        self.self_attn = MultiHeadAttention(dim_model, num_heads)  # Self-attention mechanism\n",
    "        self.feed_forward = PositionWiseFeedForward(dim_model, dim_ff)  # Position-wise feed-forward network\n",
    "        self.norm1 = nn.LayerNorm(dim_model)  # Layer normalization\n",
    "        self.norm2 = nn.LayerNorm(dim_model)  # Layer normalization\n",
    "        self.dropout = nn.Dropout(dropout)  # Dropout layer for regularization\n",
    "        \n",
    "    def forward(self, x, mask):\n",
    "        attn_output = self.self_attn(x, x, x, mask)  # Self-attention operation\n",
    "        x = x + self.dropout(attn_output)  # Residual connection with dropout\n",
    "        x = self.norm1(x)  # Apply layer normalization\n",
    "        ff_output = self.feed_forward(x)  # Position-wise feed-forward operation\n",
    "        x = x + self.dropout(ff_output)  # Residual connection with dropout\n",
    "        x = self.norm2(x)  # Apply layer normalization\n",
    "        return x\n",
    "\n",
    "# Define the Decoder Layer module\n",
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self, dim_model, num_heads, dim_ff, dropout):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.self_attn = MultiHeadAttention(dim_model, num_heads)  # Self-attention mechanism\n",
    "        self.cross_attn = MultiHeadAttention(dim_model, num_heads)  # Cross-attention mechanism\n",
    "        self.feed_forward = PositionWiseFeedForward(dim_model, dim_ff)  # Position-wise feed-forward network\n",
    "        self.norm1 = nn.LayerNorm(dim_model)  # Layer normalization\n",
    "        self.norm2 = nn.LayerNorm(dim_model)  # Layer normalization\n",
    "        self.norm3 = nn.LayerNorm(dim_model)  # Layer normalization\n",
    "        self.dropout = nn.Dropout(dropout)  # Dropout layer for regularization\n",
    "        \n",
    "    def forward(self, x, enc_output, src_mask, tgt_mask):\n",
    "        attn_output = self.self_attn(x, x, x, tgt_mask)  # Self-attention operation\n",
    "        x = x + self.dropout(attn_output)  # Residual connection with dropout\n",
    "        x = self.norm1(x)  # Apply layer normalization\n",
    "        attn_output = self.cross_attn(x, enc_output, enc_output, src_mask)  # Cross-attention operation\n",
    "        x = x + self.dropout(attn_output)  # Residual connection with dropout\n",
    "        x = self.norm2(x)  # Apply layer normalization\n",
    "        ff_output = self.feed_forward(x)  # Position-wise feed-forward operation\n",
    "        x = x + self.dropout(ff_output)  # Residual connection with dropout\n",
    "        x = self.norm3(x)  # Apply layer normalization\n",
    "        return x\n",
    "\n",
    "# Define the Transformer model\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self, src_vocab_size, tgt_vocab_size, dim_model, num_heads, num_layers, dim_ff, max_seq_length, dropout):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.max_seq_length = max_seq_length \n",
    "        self.tokenizer = tokenizer\n",
    "        self.encoder_embedding = nn.Embedding(src_vocab_size, dim_model)  # Encoder embedding layer\n",
    "        self.decoder_embedding = nn.Embedding(tgt_vocab_size, dim_model)  # Decoder embedding layer\n",
    "        self.positional_encoding = PositionalEncoding(dim_model, max_seq_length)  # Positional encoding layer\n",
    "\n",
    "        self.encoder_layers = nn.ModuleDict(\n",
    "            {f\"encoder_layer_{i}\": EncoderLayer(dim_model, num_heads, dim_ff, dropout) for i in range(num_layers)}\n",
    "        )  # Stack of encoder layers\n",
    "        self.decoder_layers = nn.ModuleDict(\n",
    "            {f\"decoder_layer_{i}\": DecoderLayer(dim_model, num_heads, dim_ff, dropout) for i in range(num_layers)}\n",
    "        )  # Stack of decoder layers\n",
    "\n",
    "        self.fc = nn.Linear(dim_model, tgt_vocab_size)  # Output linear layer\n",
    "        self.dropout = nn.Dropout(dropout)  # Dropout layer for regularization\n",
    "\n",
    "    #def generate_mask(self, src, tgt):\n",
    "     #   src_mask = (src != 0).unsqueeze(1).unsqueeze(2)  # Mask for source sequence\n",
    "     #   tgt_mask = (tgt != 0).unsqueeze(1).unsqueeze(3)  # Mask for target sequence\n",
    "      #  seq_length = tgt.size(1)\n",
    "      #  nopeak_mask = (1 - torch.triu(torch.ones(1, seq_length, seq_length), diagonal=1)).bool()  # Mask for future tokens in target sequence\n",
    "      #  tgt_mask = tgt_mask & nopeak_mask  # Apply the future mask to the target mask\n",
    "      #  return src_mask, tgt_mask\n",
    "    \n",
    "    def generate_mask(self, src, tgt):\n",
    "        src_mask = (src != 0).unsqueeze(1).unsqueeze(2)  # Mask for source sequence\n",
    "    \n",
    "        if tgt is not None:\n",
    "            tgt_mask = (tgt != 0).unsqueeze(1).unsqueeze(3)  # Mask for target sequence\n",
    "            seq_length = tgt.size(1)\n",
    "            nopeak_mask = (1 - torch.triu(torch.ones(1, seq_length, seq_length), diagonal=1)).bool()  # Mask for future tokens in target sequence\n",
    "            tgt_mask = tgt_mask & nopeak_mask\n",
    "        else:\n",
    "            tgt_mask = None\n",
    "\n",
    "        return src_mask, tgt_mask\n",
    "    \n",
    "    # Define a translation function\n",
    "    def translate_sentence(self, sentence):\n",
    "        self.eval()  # Set the model to evaluation mode\n",
    "        tokens = self.tokenizer.tokenize([sentence])  # Tokenize the input sentence\n",
    "        src_data = torch.tensor(tokens)  # Convert tokens to tensor\n",
    "        src_mask = self.generate_mask(src_data, None)[0]  # Create a source mask\n",
    "        with torch.no_grad():\n",
    "            output = self.generate(src_data, src_mask)  # Generate the translation\n",
    "        translated_sentence = self.tokenizer.detokenize(output.tolist())  # Convert tokens to sentence\n",
    "        return translated_sentence\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    def forward(self, src, tgt):\n",
    "        src_mask, tgt_mask = self.generate_mask(src, tgt)  # Generate masks for source and target sequences\n",
    "        \n",
    "        src_embedded = self.positional_encoding(self.encoder_embedding(src[:, :self.max_seq_length]))\n",
    "\n",
    "        #src_embedded = self.positional_encoding(self.encoder_embedding(src))  # Apply positional encoding to the encoder input\n",
    "        tgt_embedded = self.positional_encoding(self.decoder_embedding(tgt))  # Apply positional encoding to the decoder input\n",
    "\n",
    "        enc_output = self.dropout(src_embedded)  # Apply dropout to the encoder input\n",
    "        for enc_layer in self.encoder_layers.values():\n",
    "            enc_output = enc_layer(enc_output, src_mask)  # Apply each encoder layer\n",
    "\n",
    "        dec_output = self.dropout(tgt_embedded)  # Apply dropout to the decoder input\n",
    "        for dec_layer in self.decoder_layers.values():\n",
    "            dec_output = dec_layer(dec_output, enc_output, src_mask, tgt_mask)  # Apply each decoder layer\n",
    "\n",
    "        output = self.fc(dec_output)  # Linear transformation of the decoder output\n",
    "        \n",
    "        print(\"src.shape:\", src.shape)\n",
    "        print(\"self.encoder_embedding.weight.shape:\", self.encoder_embedding.weight.shape)\n",
    "        return output\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cccc6e89",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tokenizer:\n",
    "    def __init__(self):\n",
    "        self.word2idx = {}\n",
    "        self.idx2word = {}\n",
    "\n",
    "    def tokenize(self, sentences):\n",
    "        tokens = [[] for _ in range(len(sentences))]\n",
    "        for i, sentence in enumerate(sentences):\n",
    "            sentence_tokens = sentence.split()  # Split the sentence into tokens\n",
    "            token_ids = [self.get_id(token) for token in sentence_tokens]\n",
    "            tokens[i] = token_ids\n",
    "        return tokens\n",
    "\n",
    "\n",
    "\n",
    "    def get_id(self, token):\n",
    "        if token not in self.word2idx:\n",
    "            idx = len(self.word2idx)\n",
    "            self.word2idx[token] = idx\n",
    "            self.idx2word[idx] = token\n",
    "        return self.word2idx[token]\n",
    "\n",
    "    def load_word2idx(self, word2idx_dict):\n",
    "        self.word2idx = word2idx_dict\n",
    "        self.idx2word = {idx: word for word, idx in word2idx_dict.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "703816f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 1, Loss: 5.673375129699707\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 2, Loss: 4.623656749725342\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 3, Loss: 4.074891090393066\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 4, Loss: 3.5908701419830322\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 5, Loss: 3.0492727756500244\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 6, Loss: 2.5157065391540527\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 7, Loss: 2.011024236679077\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 8, Loss: 1.5900921821594238\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 9, Loss: 1.2382868528366089\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 10, Loss: 1.0386254787445068\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 11, Loss: 0.8460837006568909\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 12, Loss: 0.7300639152526855\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 13, Loss: 0.6036867499351501\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 14, Loss: 0.4864348769187927\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 15, Loss: 0.44471389055252075\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 16, Loss: 0.3589198887348175\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 17, Loss: 0.3148011267185211\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 18, Loss: 0.2806098759174347\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 19, Loss: 0.2298925668001175\n",
      "src.shape: torch.Size([100, 4])\n",
      "self.encoder_embedding.weight.shape: torch.Size([226, 512])\n",
      "Epoch: 20, Loss: 0.20778177678585052\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "english_sentences = df['English'].tolist()\n",
    "arabic_sentences = df['Arabic'].tolist()\n",
    "english_sentences = english_sentences[:100]\n",
    "arabic_sentences = arabic_sentences[:100]\n",
    "\n",
    "\n",
    "# Initialize the tokenizer\n",
    "tokenizer = Tokenizer()\n",
    "\n",
    "\n",
    "# Initialize an empty vocabulary dictionary\n",
    "word2idx_dict = {}\n",
    "\n",
    "\n",
    "# Iterate over your training data and update the vocabulary dictionary\n",
    "for sentence in english_sentences + arabic_sentences:\n",
    "    tokens = sentence.split()  # Split the sentence into tokens\n",
    "    for token in tokens:\n",
    "        if token not in word2idx_dict:\n",
    "            # Assign a unique index to each token in the vocabulary\n",
    "            word2idx_dict[token] = len(word2idx_dict)\n",
    "\n",
    "# Save the vocabulary dictionary\n",
    "with open(\"word2idx_dict.json\", \"w\") as f:\n",
    "    json.dump(word2idx_dict, f)\n",
    "\n",
    "# Load the saved vocabulary dictionary\n",
    "with open(\"word2idx_dict.json\", \"r\") as f:\n",
    "    saved_word2idx_dict = json.load(f)\n",
    "\n",
    "# Load the vocabulary into the tokenizer\n",
    "tokenizer.load_word2idx(saved_word2idx_dict)\n",
    "\n",
    "\n",
    "# Tokenize the sentences\n",
    "english_tokens = tokenizer.tokenize(english_sentences)\n",
    "arabic_tokens = tokenizer.tokenize(arabic_sentences)\n",
    "\n",
    "\n",
    "\n",
    "# Get the maximum sequence length\n",
    "max_seq_length = max(len(tokens) for tokens in english_tokens + arabic_tokens)\n",
    "\n",
    "# Pad or truncate the sequences to match max_seq_length\n",
    "pad_token_id = tokenizer.get_id(\"<PAD>\")\n",
    "english_tokens = [tokens[:max_seq_length] + [pad_token_id] * (max_seq_length - len(tokens)) for tokens in english_tokens]\n",
    "arabic_tokens = [tokens[:max_seq_length] + [pad_token_id] * (max_seq_length - len(tokens)) for tokens in arabic_tokens]\n",
    "\n",
    "# Convert tokens to tensors\n",
    "src_data = torch.tensor(english_tokens)  # (num_sentences, seq_length)\n",
    "tgt_data = torch.tensor(arabic_tokens)   # (num_sentences, seq_length)\n",
    "\n",
    "# Initialize the Transformer model\n",
    "src_vocab_size = len(tokenizer.word2idx)\n",
    "tgt_vocab_size = len(tokenizer.word2idx)\n",
    "dim_model = 512\n",
    "num_heads = 8\n",
    "num_layers = 6\n",
    "dim_ff = 2048\n",
    "dropout = 0.1\n",
    "\n",
    "transformer = Transformer(src_vocab_size, tgt_vocab_size, dim_model, num_heads, num_layers, dim_ff, max_seq_length, dropout)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=pad_token_id)\n",
    "optimizer = optim.Adam(transformer.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)\n",
    "\n",
    "transformer.train()\n",
    "\n",
    "for epoch in range(20):\n",
    "    optimizer.zero_grad()\n",
    "    output = transformer(src_data, tgt_data[:, :-1])\n",
    "    loss = criterion(output.contiguous().view(-1, tgt_vocab_size), tgt_data[:, 1:].contiguous().view(-1))\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    print(f\"Epoch: {epoch+1}, Loss: {loss.item()}\")\n",
    "    \n",
    "# Save the trained model\n",
    "torch.save(transformer.state_dict(), 'transformer_model.pth')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dfba714",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "634aa951",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb1c4810",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}