updated_neural_translation.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TGEWVutRNLN3",
        "outputId": "abfca11f-672d-4edb-e4a0-d118b6aa0ea4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (3.8.1)\n",
            "Collecting pyarabic\n",
            "  Downloading PyArabic-0.6.15-py3-none-any.whl (126 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/126.4 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━\u001b[0m \u001b[32m122.9/126.4 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.4/126.4 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk) (1.3.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk) (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk) (4.66.1)\n",
            "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from pyarabic) (1.16.0)\n",
            "Installing collected packages: pyarabic\n",
            "Successfully installed pyarabic-0.6.15\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Original Text: مثال على نص عربي لاختبار التحليل اللغوي\n",
            "Tokenized Text: ['مثال', 'على', 'نص', 'عربي', 'لاختبار', 'التحليل', 'اللغوي']\n"
          ]
        }
      ],
      "source": [
        "# Install NLTK and PyArabic\n",
        "!pip install nltk pyarabic\n",
        "\n",
        "# Import libraries\n",
        "import nltk\n",
        "import pyarabic.araby as araby\n",
        "\n",
        "# Download NLTK data\n",
        "nltk.download('punkt')\n",
        "\n",
        "# Function for Arabic tokenization using PyArabic\n",
        "def tokenize_arabic(text):\n",
        "    tokens = araby.tokenize(text)\n",
        "    return tokens\n",
        "\n",
        "# Example usage\n",
        "example_text = \"مثال على نص عربي لاختبار التحليل اللغوي\"\n",
        "print(\"Original Text:\", example_text)\n",
        "print(\"Tokenized Text:\", tokenize_arabic(example_text))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "uyLUQb9cj0AW",
        "outputId": "88b5a8ac-c648-4406-f6e6-50a6c7e89db4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 1\n",
            "\tTrain Loss: 8.994\n",
            "\tVal. Loss: 8.607\n",
            "Epoch: 2\n",
            "\tTrain Loss: 7.750\n",
            "\tVal. Loss: 8.636\n",
            "Epoch: 3\n",
            "\tTrain Loss: 6.847\n",
            "\tVal. Loss: 8.976\n",
            "Epoch: 4\n",
            "\tTrain Loss: 6.165\n",
            "\tVal. Loss: 9.365\n",
            "Epoch: 5\n",
            "\tTrain Loss: 5.699\n",
            "\tVal. Loss: 9.446\n",
            "Epoch: 6\n",
            "\tTrain Loss: 5.444\n",
            "\tVal. Loss: 9.683\n",
            "Epoch: 7\n",
            "\tTrain Loss: 5.294\n",
            "\tVal. Loss: 9.718\n",
            "Epoch: 8\n",
            "\tTrain Loss: 5.164\n",
            "\tVal. Loss: 9.860\n",
            "Epoch: 9\n",
            "\tTrain Loss: 5.057\n",
            "\tVal. Loss: 9.965\n",
            "Epoch: 10\n",
            "\tTrain Loss: 4.973\n",
            "\tVal. Loss: 10.127\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "from sklearn.model_selection import train_test_split\n",
        "import pandas as pd\n",
        "from collections import Counter\n",
        "import nltk\n",
        "import pyarabic.araby as araby\n",
        "import random\n",
        "import math\n",
        "\n",
        "# Ensure NLTK is downloaded (for tokenizing English text)\n",
        "nltk.download('punkt')\n",
        "\n",
        "# Tokenization functions\n",
        "def tokenize_english(text):\n",
        "    return nltk.word_tokenize(text)\n",
        "\n",
        "def tokenize_arabic(text):\n",
        "    return araby.tokenize(text)\n",
        "\n",
        "def build_vocab(texts, min_freq=1):\n",
        "    # Flatten the list of lists into a single list of words\n",
        "    flat_texts = [word for sentence in texts for word in sentence]\n",
        "\n",
        "    word_freq = Counter(flat_texts)\n",
        "    word_freq = {word: freq for word, freq in word_freq.items() if freq >= min_freq}\n",
        "    word_to_idx = {word: idx + 2 for idx, word in enumerate(word_freq)}\n",
        "    word_to_idx['<unk>'] = 0\n",
        "    word_to_idx['<pad>'] = 1\n",
        "    idx_to_word = {idx: word for word, idx in word_to_idx.items()}\n",
        "    return word_to_idx, idx_to_word\n",
        "\n",
        "\n",
        "# Function to convert text to indices\n",
        "def text_to_indices(tokenized_text, vocab):\n",
        "    return [vocab.get(word, vocab['<unk>']) for word in tokenized_text]\n",
        "\n",
        "# Load dataset\n",
        "df = pd.read_csv('/content/separated_translations.csv')  # Update the path to your dataset\n",
        "\n",
        "# Tokenize and build vocab\n",
        "tokenized_en = [tokenize_english(sentence) for sentence in df['English']]\n",
        "tokenized_ar = [tokenize_arabic(sentence) for sentence in df['Arabic']]\n",
        "en_vocab, en_inv_vocab = build_vocab(tokenized_en)\n",
        "ar_vocab, ar_inv_vocab = build_vocab(tokenized_ar)\n",
        "\n",
        "# Convert text to indices\n",
        "indexed_en = [text_to_indices(sentence, en_vocab) for sentence in tokenized_en]\n",
        "indexed_ar = [text_to_indices(sentence, ar_vocab) for sentence in tokenized_ar]\n",
        "\n",
        "# Pad sequences\n",
        "def pad_sequences(sequences, padding_value=0):\n",
        "    max_len = max(len(seq) for seq in sequences)\n",
        "    return [seq + [padding_value] * (max_len - len(seq)) for seq in sequences]\n",
        "\n",
        "padded_en = pad_sequences(indexed_en, padding_value=en_vocab['<pad>'])\n",
        "padded_ar = pad_sequences(indexed_ar, padding_value=ar_vocab['<pad>'])\n",
        "\n",
        "# TranslationDataset class\n",
        "class TranslationDataset(Dataset):\n",
        "    def __init__(self, src_data, trg_data):\n",
        "        self.src_data = src_data\n",
        "        self.trg_data = trg_data\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.src_data)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        src = torch.tensor(self.src_data[idx], dtype=torch.long)\n",
        "        trg = torch.tensor(self.trg_data[idx], dtype=torch.long)\n",
        "        return src, trg\n",
        "\n",
        "# Split data and create datasets\n",
        "train_src, valid_src, train_trg, valid_trg = train_test_split(padded_en, padded_ar, test_size=0.2)\n",
        "train_dataset = TranslationDataset(train_src, train_trg)\n",
        "valid_dataset = TranslationDataset(valid_src, valid_trg)\n",
        "\n",
        "# DataLoader\n",
        "def collate_fn(batch):\n",
        "    src_batch, trg_batch = zip(*batch)\n",
        "    src_batch_padded = pad_sequence(src_batch, padding_value=en_vocab['<pad>'])\n",
        "    trg_batch_padded = pad_sequence(trg_batch, padding_value=ar_vocab['<pad>'])\n",
        "    return src_batch_padded, trg_batch_padded\n",
        "\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=16, collate_fn=collate_fn)  # Reduced from 32\n",
        "valid_dataloader = DataLoader(valid_dataset, batch_size=16, collate_fn=collate_fn)\n",
        "\n",
        "\n",
        "# Encoder\n",
        "class Encoder(nn.Module):\n",
        "    def __init__(self, input_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout):\n",
        "        super().__init__()\n",
        "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
        "        self.rnn = nn.GRU(emb_dim, enc_hid_dim, bidirectional=True)\n",
        "        self.fc = nn.Linear(enc_hid_dim * 2, dec_hid_dim)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, src):\n",
        "        embedded = self.dropout(self.embedding(src))\n",
        "        outputs, hidden = self.rnn(embedded)\n",
        "        hidden = torch.tanh(self.fc(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1)))\n",
        "        return outputs, hidden\n",
        "\n",
        "# Attention\n",
        "class Attention(nn.Module):\n",
        "    def __init__(self, enc_hid_dim, dec_hid_dim):\n",
        "        super().__init__()\n",
        "        self.attn = nn.Linear(enc_hid_dim * 2 + dec_hid_dim, dec_hid_dim)\n",
        "        self.v = nn.Linear(dec_hid_dim, 1, bias=False)\n",
        "\n",
        "    def forward(self, hidden, encoder_outputs):\n",
        "        batch_size = encoder_outputs.shape[1]\n",
        "        src_len = encoder_outputs.shape[0]\n",
        "        hidden = hidden.unsqueeze(1).repeat(1, src_len, 1)\n",
        "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
        "        energy = torch.tanh(self.attn(torch.cat((hidden, encoder_outputs), dim=2)))\n",
        "        attention = self.v(energy).squeeze(2)\n",
        "        return torch.softmax(attention, dim=1)\n",
        "\n",
        "# Decoder\n",
        "class Decoder(nn.Module):\n",
        "    def __init__(self, output_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout, attention):\n",
        "        super().__init__()\n",
        "        self.output_dim = output_dim\n",
        "        self.attention = attention\n",
        "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
        "        self.rnn = nn.GRU(enc_hid_dim * 2 + emb_dim, dec_hid_dim)\n",
        "        self.fc_out = nn.Linear(enc_hid_dim * 2 + dec_hid_dim + emb_dim, output_dim)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, input, hidden, encoder_outputs):\n",
        "        input = input.unsqueeze(0)\n",
        "        embedded = self.dropout(self.embedding(input))\n",
        "        a = self.attention(hidden, encoder_outputs).unsqueeze(1)\n",
        "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
        "        weighted = torch.bmm(a, encoder_outputs).permute(1, 0, 2)\n",
        "        rnn_input = torch.cat((embedded, weighted), dim=2)\n",
        "        output, hidden = self.rnn(rnn_input, hidden.unsqueeze(0))\n",
        "        embedded = embedded.squeeze(0)\n",
        "        output = output.squeeze(0)\n",
        "        weighted = weighted.squeeze(0)\n",
        "        prediction = self.fc_out(torch.cat((output, weighted, embedded), dim=1))\n",
        "        return prediction, hidden.squeeze(0)\n",
        "\n",
        "# Seq2Seq Model\n",
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self, encoder, decoder, device):\n",
        "        super().__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.device = device\n",
        "\n",
        "    def forward(self, src, trg, teacher_forcing_ratio=0.5):\n",
        "        trg_len = trg.shape[0]\n",
        "        batch_size = trg.shape[1]\n",
        "        trg_vocab_size = self.decoder.output_dim\n",
        "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n",
        "        encoder_outputs, hidden = self.encoder(src)\n",
        "        input = trg[0,:]\n",
        "        for t in range(1, trg_len):\n",
        "            output, hidden = self.decoder(input, hidden, encoder_outputs)\n",
        "            outputs[t] = output\n",
        "            teacher_force = random.random() < teacher_forcing_ratio\n",
        "            top1 = output.argmax(1)\n",
        "            input = trg[t] if teacher_force else top1\n",
        "        return outputs\n",
        "\n",
        "\n",
        "# Define hyperparameters\n",
        "INPUT_DIM = len(en_vocab)\n",
        "OUTPUT_DIM = len(ar_vocab)\n",
        "ENC_EMB_DIM = 128  # Reduce embedding dimension\n",
        "DEC_EMB_DIM = 128\n",
        "ENC_HID_DIM = 256  # Reduce hidden layer size\n",
        "DEC_HID_DIM = 256\n",
        "ENC_DROPOUT = 0.5  # Dropout rate for the encoder\n",
        "DEC_DROPOUT = 0.5  # Dropout rate for the decoder\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn)\n",
        "model = Seq2Seq(enc, dec, device).to(device)\n",
        "\n",
        "\n",
        "# Instantiate the model, optimizer, criterion\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = Seq2Seq(enc, dec, device).to(device)\n",
        "optimizer = optim.Adam(model.parameters())\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=ar_vocab['<pad>'])\n",
        "\n",
        "\n",
        "\n",
        "# Training function\n",
        "def train(model, iterator, optimizer, criterion, clip):\n",
        "    model.train()  # Set the model to training mode\n",
        "    epoch_loss = 0\n",
        "\n",
        "    for i, batch in enumerate(iterator):\n",
        "        src, trg = batch\n",
        "        src, trg = src.to(device), trg.to(device)\n",
        "\n",
        "        optimizer.zero_grad()  # Clear the gradients\n",
        "\n",
        "        output = model(src, trg[:-1, :])  # Forward pass, trg[:-1, :] omits the last token\n",
        "\n",
        "        # Reshape output and target to compute loss\n",
        "        output_dim = output.shape[-1]\n",
        "        output = output.contiguous().view(-1, output_dim)\n",
        "        trg = trg[1:].contiguous().view(-1)\n",
        "\n",
        "        loss = criterion(output, trg)  # Compute loss\n",
        "\n",
        "        loss.backward()  # Backpropagation\n",
        "\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)  # Gradient clipping\n",
        "\n",
        "        optimizer.step()  # Update model parameters\n",
        "\n",
        "        epoch_loss += loss.item()\n",
        "\n",
        "    return epoch_loss / len(iterator)\n",
        "\n",
        "def evaluate(model, iterator, criterion):\n",
        "    model.eval()\n",
        "    epoch_loss = 0\n",
        "    with torch.no_grad():\n",
        "        for src, trg in iterator:\n",
        "            src, trg = src.to(device), trg.to(device)\n",
        "\n",
        "            # Forward pass, without teacher forcing\n",
        "            output = model(src, trg, 0)  # turn off teacher forcing\n",
        "\n",
        "            # Calculate loss\n",
        "            output_dim = output.shape[-1]\n",
        "            output = output[1:].view(-1, output_dim)\n",
        "            trg = trg[1:].view(-1)\n",
        "\n",
        "            loss = criterion(output, trg)\n",
        "            epoch_loss += loss.item()\n",
        "    return epoch_loss / len(iterator)\n",
        "\n",
        "\n",
        "# Training loop\n",
        "N_EPOCHS = 10\n",
        "CLIP = 1\n",
        "for epoch in range(N_EPOCHS):\n",
        "    train_loss = train(model, train_dataloader, optimizer, criterion, CLIP)\n",
        "    valid_loss = evaluate(model, valid_dataloader, criterion)\n",
        "\n",
        "    print(f'Epoch: {epoch+1}')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f}')\n",
        "    print(f'\\tVal. Loss: {valid_loss:.3f}')\n",
        "\n",
        "# Save model\n",
        "torch.save(model.state_dict(), 'translation_model.pt')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "scBmeH3D7ajl"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}