3. dl_model_data_prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning Model Data Preparation\n",
    "This Jupyter notebook is dedicated to preparing the data for subsequent use in the deep learning model. It saves the data in pickle files to ensure efficient loading into the model in the next step, thereby avoiding memory load issues on the computer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/embeddings_new_3_1.pkl\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def load_and_process_embedding(file_path):\n",
    "    \"\"\"\n",
    "    Loads the embeddings pickle files.\n",
    "    \"\"\"\n",
    "    with open(file_path, 'rb') as file:\n",
    "        embeddings = pickle.load(file)\n",
    "    embeddings = pd.DataFrame(embeddings)\n",
    "    embeddings.set_index('Uniprot_ID', inplace=True)\n",
    "    embeddings = embeddings.drop(columns=['full_sequence'])\n",
    "    return embeddings\n",
    "\n",
    "embeddings = pd.DataFrame()\n",
    "\n",
    "embedding_files = [\n",
    "    'data/embeddings_new_3_1.pkl',\n",
    "    # 'data/embeddings_new_3_2.pkl',\n",
    "    # 'data/embeddings_new_3_3.pkl',\n",
    "]\n",
    "\n",
    "for file_path in embedding_files:\n",
    "    print(file_path)\n",
    "    current_embeddings = load_and_process_embedding(file_path)\n",
    "    embeddings = pd.concat([embeddings, current_embeddings])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_ = embeddings.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Creating trimmed embeddings: because of the first and last tokens\n",
    "# (https://www.science.org/doi/suppl/10.1126/science.ade2574/suppl_file/science.ade2574_sm.pdf)\n",
    "# We used BOS and EOS tokens to signal the beginning and end of a real protein, to allow the model to separate a full-sized protein from a cropped one.\n",
    "trimmed_embeddings = [np.array(embedding)[1:-1, :] for embedding in embeddings['full_embedding']]\n",
    "trimmed_embeddings_df = pd.DataFrame({'embeddings': trimmed_embeddings}, index=embeddings.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>embeddings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Uniprot_ID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Q04739</th>\n",
       "      <td>[[0.01952355168759823, 0.0036598031874746084, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P53172</th>\n",
       "      <td>[[0.06398601084947586, -0.04467635229229927, -...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P33298</th>\n",
       "      <td>[[0.04065754637122154, -0.021706463769078255, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q12045</th>\n",
       "      <td>[[0.0963035449385643, -0.10714740306138992, -0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q03264</th>\n",
       "      <td>[[0.013744623400270939, 0.028120659291744232, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   embeddings\n",
       "Uniprot_ID                                                   \n",
       "Q04739      [[0.01952355168759823, 0.0036598031874746084, ...\n",
       "P53172      [[0.06398601084947586, -0.04467635229229927, -...\n",
       "P33298      [[0.04065754637122154, -0.021706463769078255, ...\n",
       "Q12045      [[0.0963035449385643, -0.10714740306138992, -0...\n",
       "Q03264      [[0.013744623400270939, 0.028120659291744232, ..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can see that we have the trimmed embedings ready to be used\n",
    "trimmed_embeddings_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Binary Position Data\n",
    "Load the binary position data and corresponding target values from the CSV file created in notebook 1. This data is pre-processed, ensuring each binary position sequence is padded to a length of one thousand."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Log2FC(LiP_norm)</th>\n",
       "      <th>Qvalue(LiP)</th>\n",
       "      <th>Padded_Binary_Positions</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Uniprot_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>P15703</th>\n",
       "      <td>0</td>\n",
       "      <td>-2.176707</td>\n",
       "      <td>0.003686</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P06169</th>\n",
       "      <td>101</td>\n",
       "      <td>2.264383</td>\n",
       "      <td>0.003686</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P38174</th>\n",
       "      <td>18</td>\n",
       "      <td>1.216913</td>\n",
       "      <td>0.003686</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P00359</th>\n",
       "      <td>280</td>\n",
       "      <td>2.790874</td>\n",
       "      <td>0.003686</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P0CH08;P0CH09</th>\n",
       "      <td>331</td>\n",
       "      <td>-0.647026</td>\n",
       "      <td>0.004177</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               index  Log2FC(LiP_norm)  Qvalue(LiP)  \\\n",
       "Uniprot_ID                                            \n",
       "P15703             0         -2.176707     0.003686   \n",
       "P06169           101          2.264383     0.003686   \n",
       "P38174            18          1.216913     0.003686   \n",
       "P00359           280          2.790874     0.003686   \n",
       "P0CH08;P0CH09    331         -0.647026     0.004177   \n",
       "\n",
       "                                         Padded_Binary_Positions  \n",
       "Uniprot_ID                                                        \n",
       "P15703         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "P06169         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "P38174         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "P00359         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "P0CH08;P0CH09  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the dataset\n",
    "df = pd.read_csv('data/OsmoticStress_with_binary_positions_padded_5000.csv')\n",
    "df['Binary_Positions'] = df['Binary_Positions'].apply(json.loads)\n",
    "df['Padded_Binary_Positions'] = df['Padded_Binary_Positions'].apply(json.loads)\n",
    "df.drop(columns=['full_sequence', 'Peptide_sequence', \"Binary_Positions\"], inplace=True)\n",
    "df.set_index('Uniprot_ID', inplace=True)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P32324    116\n",
      "P02994    108\n",
      "P10081     40\n",
      "P06169     39\n",
      "P00549     36\n",
      "         ... \n",
      "P32466      1\n",
      "P38066      1\n",
      "Q04223      1\n",
      "Q12513      1\n",
      "P47137      1\n",
      "Name: Uniprot_ID, Length: 1490, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Count the number of peptides per protein\n",
    "df_ = df.reset_index()\n",
    "counts = df_['Uniprot_ID'].value_counts()\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare training dataset\n",
    "Merge both embeddings and padded binary position together."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(637, 5)\n"
     ]
    }
   ],
   "source": [
    "# Merge embeddings with the dataset with binary positions\n",
    "dataset = df.merge(trimmed_embeddings_df, left_index=True, right_index=True, how='inner')\n",
    "print(dataset.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P15624    9\n",
      "P22203    6\n",
      "P00927    6\n",
      "P32356    5\n",
      "Q12154    5\n",
      "         ..\n",
      "P40099    1\n",
      "P40091    1\n",
      "P40078    1\n",
      "P40056    1\n",
      "O13547    1\n",
      "Name: Uniprot_ID, Length: 455, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Checks for missing values\n",
    "dataset_ = dataset.reset_index()\n",
    "counts = dataset_['Uniprot_ID'].value_counts()\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Log2FC(LiP_norm)</th>\n",
       "      <th>Qvalue(LiP)</th>\n",
       "      <th>Padded_Binary_Positions</th>\n",
       "      <th>embeddings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Uniprot_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>O13547</th>\n",
       "      <td>28401</td>\n",
       "      <td>0.240404</td>\n",
       "      <td>0.049373</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[[0.020321033895015717, 0.057242896407842636, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P00572</th>\n",
       "      <td>28283</td>\n",
       "      <td>0.485887</td>\n",
       "      <td>0.049034</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[[0.06270206719636917, -0.10046922415494919, -...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P00812</th>\n",
       "      <td>24722</td>\n",
       "      <td>0.476236</td>\n",
       "      <td>0.039900</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[[0.1953696757555008, -0.1298731565475464, 0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P00812</th>\n",
       "      <td>24723</td>\n",
       "      <td>-0.530722</td>\n",
       "      <td>0.046914</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, ...</td>\n",
       "      <td>[[0.1953696757555008, -0.1298731565475464, 0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P00927</th>\n",
       "      <td>24626</td>\n",
       "      <td>0.541523</td>\n",
       "      <td>0.039875</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[[0.10202319175004959, 0.04706750065088272, -0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q99258</th>\n",
       "      <td>24759</td>\n",
       "      <td>0.157271</td>\n",
       "      <td>0.044713</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[[-0.018650177866220474, 0.04459123685956001, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q99258</th>\n",
       "      <td>24760</td>\n",
       "      <td>-0.516572</td>\n",
       "      <td>0.046971</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[[-0.018650177866220474, 0.04459123685956001, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q99258</th>\n",
       "      <td>24761</td>\n",
       "      <td>-0.535843</td>\n",
       "      <td>0.049788</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[[-0.018650177866220474, 0.04459123685956001, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q99321</th>\n",
       "      <td>25764</td>\n",
       "      <td>0.189942</td>\n",
       "      <td>0.042230</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[[0.04402398690581322, -0.057297028601169586, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q99321</th>\n",
       "      <td>25765</td>\n",
       "      <td>1.063220</td>\n",
       "      <td>0.048786</td>\n",
       "      <td>[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, ...</td>\n",
       "      <td>[[0.04402398690581322, -0.057297028601169586, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>637 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  Log2FC(LiP_norm)  Qvalue(LiP)  \\\n",
       "Uniprot_ID                                         \n",
       "O13547      28401          0.240404     0.049373   \n",
       "P00572      28283          0.485887     0.049034   \n",
       "P00812      24722          0.476236     0.039900   \n",
       "P00812      24723         -0.530722     0.046914   \n",
       "P00927      24626          0.541523     0.039875   \n",
       "...           ...               ...          ...   \n",
       "Q99258      24759          0.157271     0.044713   \n",
       "Q99258      24760         -0.516572     0.046971   \n",
       "Q99258      24761         -0.535843     0.049788   \n",
       "Q99321      25764          0.189942     0.042230   \n",
       "Q99321      25765          1.063220     0.048786   \n",
       "\n",
       "                                      Padded_Binary_Positions  \\\n",
       "Uniprot_ID                                                      \n",
       "O13547      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "P00572      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "P00812      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "P00812      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, ...   \n",
       "P00927      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "...                                                       ...   \n",
       "Q99258      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "Q99258      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "Q99258      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "Q99321      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "Q99321      [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, ...   \n",
       "\n",
       "                                                   embeddings  \n",
       "Uniprot_ID                                                     \n",
       "O13547      [[0.020321033895015717, 0.057242896407842636, ...  \n",
       "P00572      [[0.06270206719636917, -0.10046922415494919, -...  \n",
       "P00812      [[0.1953696757555008, -0.1298731565475464, 0.0...  \n",
       "P00812      [[0.1953696757555008, -0.1298731565475464, 0.0...  \n",
       "P00927      [[0.10202319175004959, 0.04706750065088272, -0...  \n",
       "...                                                       ...  \n",
       "Q99258      [[-0.018650177866220474, 0.04459123685956001, ...  \n",
       "Q99258      [[-0.018650177866220474, 0.04459123685956001, ...  \n",
       "Q99258      [[-0.018650177866220474, 0.04459123685956001, ...  \n",
       "Q99321      [[0.04402398690581322, -0.057297028601169586, ...  \n",
       "Q99321      [[0.04402398690581322, -0.057297028601169586, ...  \n",
       "\n",
       "[637 rows x 5 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saves the dataset to a pickle file\n",
    "dataset.to_pickle('data/dataset_3.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-l-schewinski-tp-2]",
   "language": "python",
   "name": "conda-env-.conda-l-schewinski-tp-2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}