notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b43acd43",
   "metadata": {},
   "source": [
    "An excercise, recreating the results reported in [this paper](https://academic.oup.com/bioinformatics/article/39/4/btad187/7114029) using [the codes proviced by the authors](https://github.com/biomed-AI/GraphBepi)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "975ab329",
   "metadata": {},
   "source": [
    "importing necessary packages:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7b008e8e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/askari/.local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchmetrics as tm\n",
    "import torch.nn.functional as F\n",
    "import os\n",
    "import pytorch_lightning as pl\n",
    "from torch.nn.utils.rnn import pad_sequence,pack_sequence,pack_padded_sequence,pad_packed_sequence\n",
    "import pandas as pd\n",
    "import pickle as pk\n",
    "from tqdm import tqdm,trange\n",
    "import esm\n",
    "import warnings\n",
    "from torch.utils.data import DataLoader,Dataset\n",
    "import numpy as np\n",
    "import requests as rq\n",
    "import time\n",
    "import random\n",
    "from collections import defaultdict\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "from pytorch_lightning.callbacks import Callback,EarlyStopping,ModelCheckpoint\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bb3a2cdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def force_cudnn_initialization():\n",
    "    s = 32\n",
    "    dev = torch.device('cuda')\n",
    "    torch.nn.functional.conv2d(torch.zeros(s, s, s, s, device=dev), torch.zeros(s, s, s, s, device=dev))\n",
    "force_cudnn_initialization()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "157718d9",
   "metadata": {},
   "source": [
    "contents of tool.py, used to evaluate the perfomance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1b82e8fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "class METRICS:\n",
    "    def __init__(self,device='cpu'):\n",
    "        self.device=device\n",
    "        self.auroc=tm.AUROC(task='binary').to(device)\n",
    "        self.auprc=tm.AveragePrecision(task='binary').to(device)\n",
    "        self.roc=tm.ROC(task='binary').to(device)\n",
    "        self.prc=tm.PrecisionRecallCurve(task='binary').to(device)\n",
    "        self.rec=tm.Recall(task='binary').to(device)\n",
    "        self.prec=tm.Precision(task='binary').to(device)\n",
    "        self.f1=tm.F1Score(task='binary').to(device)\n",
    "        self.mcc=tm.MatthewsCorrCoef(task='binary').to(device)\n",
    "        f=lambda a,b,c,d,e:(a/(a+d)+c/(b+c))/2\n",
    "        self.stat=tm.StatScores(task='binary').to(device)\n",
    "        self.bacc=lambda x,y:f(*self.stat(x,y))\n",
    "\n",
    "    def to(self,pred,y):\n",
    "        return pred.to(self.device),y.to(self.device)\n",
    "    def calc_thresh(self,pred,y):\n",
    "        pred,y=self.to(pred,y)\n",
    "        prec, rec, thresholds = self.prc(pred,y)\n",
    "        f1=(2*prec*rec/(prec+rec)).nan_to_num(0)[:-1]\n",
    "        threshold = thresholds[torch.argmax(f1)]\n",
    "        return threshold\n",
    "    def calc_prc(self,pred,y):\n",
    "        pred,y=self.to(pred,y)\n",
    "        auroc = self.auroc(pred,y)\n",
    "        prec, rec, th1 = self.prc(pred,y)\n",
    "        auprc = self.auprc(pred,y)\n",
    "        fpr, tpr, th2 = self.roc(pred,y)\n",
    "        return {\n",
    "            'AUROC':auroc.cpu().item(),'AUPRC':auprc.cpu().item(),'prc':[rec[:-1],prec[:-1],th1],'roc':[fpr,tpr,th2]\n",
    "        }\n",
    "    def __call__(self,pred,y,threshold=None):\n",
    "        pred,y=self.to(pred,y)\n",
    "        auroc = self.auroc(pred,y)\n",
    "        prec, rec, thresholds = self.prc(pred,y)\n",
    "        auprc = self.auprc(pred,y)\n",
    "        if threshold is None:\n",
    "            f1=(2*prec*rec/(prec+rec)).nan_to_num(0)[:-1]\n",
    "            threshold = thresholds[torch.argmax(f1)]\n",
    "        threshold=torch.tensor(threshold)\n",
    "        self.f1.threshold=threshold\n",
    "        self.rec.threshold=threshold\n",
    "        self.mcc.threshold=threshold\n",
    "        self.stat.threshold=threshold\n",
    "        self.prec.threshold=threshold\n",
    "        f1 = self.f1(pred,y)\n",
    "        rec = self.rec(pred,y)\n",
    "        mcc = self.mcc(pred,y)\n",
    "        bacc = self.bacc(pred,y)\n",
    "        prec = self.prec(pred,y)\n",
    "        return {\n",
    "            'AUROC':auroc.cpu().item(),'AUPRC':auprc.cpu().item(),\n",
    "            'RECALL':rec.cpu().item(),'PRECISION':prec.cpu().item(),\n",
    "            'F1':f1.cpu().item(),'MCC':mcc.cpu().item(),\n",
    "            'BACC':bacc.cpu().item(),'threshold':threshold.cpu().item(),\n",
    "        }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1f5ab63",
   "metadata": {},
   "source": [
    "contents of EGAT.py, the graph attention layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "891b7014",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AE(nn.Module):\n",
    "    def __init__(self, dim_in, dim_out, hidden, dropout = 0., bias=True):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(\n",
    "            nn.Linear(dim_in, hidden, bias=bias),\n",
    "            nn.LayerNorm(hidden),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(dropout),\n",
    "            nn.Linear(hidden, dim_out, bias=bias),\n",
    "            nn.LayerNorm(dim_out),\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        return self.net(x)\n",
    "class EGraphAttentionLayer(nn.Module):\n",
    "    def __init__(self, in_features, out_features, dropout, alpha, concat=True):\n",
    "        super().__init__()\n",
    "        self.dropout = dropout\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.alpha = alpha\n",
    "        self.concat = concat\n",
    "\n",
    "        self.W = nn.Parameter(torch.empty(size=(in_features, out_features)))\n",
    "        nn.init.xavier_uniform_(self.W.data, gain=1.414)\n",
    "        self.a = nn.Parameter(torch.empty(size=(2*out_features, 1)))\n",
    "        nn.init.xavier_uniform_(self.a.data, gain=1.414)\n",
    "        self.leakyrelu = nn.LeakyReLU(self.alpha)\n",
    "\n",
    "    def forward(self, h, edge_attr):\n",
    "        Wh = torch.mm(h, self.W) # h.shape: (N, in_features), Wh.shape: (N, out_features)\n",
    "        e = self._prepare_attentional_mechanism_input(Wh)\n",
    "        e = e*edge_attr\n",
    "        zero_vec = -9e15*torch.ones_like(e)\n",
    "        e = torch.where(edge_attr > 0, e, zero_vec)\n",
    "        e = F.softmax(e, dim=1)\n",
    "        e = F.dropout(e, self.dropout, training=self.training)\n",
    "\n",
    "        h_prime=[]\n",
    "        for i in range(edge_attr.shape[0]):\n",
    "            h_prime.append(torch.matmul(e[i],Wh))\n",
    "\n",
    "        if self.concat:\n",
    "            h_prime = torch.cat(h_prime,dim=1)\n",
    "        else:\n",
    "            h_prime = torch.stack(h_prime,dim=0).mean(0)\n",
    "        return F.elu(h_prime),e\n",
    "\n",
    "    #compute attention coefficient\n",
    "    def _prepare_attentional_mechanism_input(self, Wh):\n",
    "        # Wh.shape (N, out_feature)\n",
    "        # self.a.shape (2 * out_feature, 1)\n",
    "        # Wh1&2.shape (N, 1)\n",
    "        # e.shape (N, N)\n",
    "        Wh1 = torch.matmul(Wh, self.a[:self.out_features, :])\n",
    "        Wh2 = torch.matmul(Wh, self.a[self.out_features:, :])\n",
    "        # broadcast add\n",
    "        e = Wh1 + Wh2.T\n",
    "        return self.leakyrelu(e)\n",
    "\n",
    "    def __repr__(self):\n",
    "        return self.__class__.__name__ + ' (' + str(self.in_features) + ' -> ' + str(self.out_features) + ')'\n",
    "class EGAT(nn.Module):\n",
    "    def __init__(self, nfeat, nhid, efeat, dropout=0.2, alpha=0.2):\n",
    "        super().__init__()\n",
    "        self.dropout = dropout\n",
    "        self.in_att = EGraphAttentionLayer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True)\n",
    "        self.out_att = EGraphAttentionLayer(nhid*efeat, nfeat, dropout=dropout, alpha=alpha, concat=False)\n",
    "    def forward(self, x, edge_attr):\n",
    "        x_cut=x\n",
    "        x = F.dropout(x, self.dropout, training=self.training)\n",
    "        x, edge_attr=self.in_att(x, edge_attr)\n",
    "        x, edge_attr=self.out_att(x, edge_attr)\n",
    "        return x+x_cut, edge_attr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "623aca64",
   "metadata": {},
   "source": [
    "contents of model.py, full GraphBepi model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bc16161a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GraphBepi(pl.LightningModule):\n",
    "    def __init__(\n",
    "        self,\n",
    "        feat_dim=2560, hidden_dim=256,\n",
    "        exfeat_dim=13, edge_dim=51,\n",
    "        augment_eps=0.05, dropout=0.2,\n",
    "        lr=1e-6, metrics=None, result_path=None\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.metrics=metrics\n",
    "        self.path=result_path\n",
    "        # loss function\n",
    "        self.loss_fn=nn.BCELoss()\n",
    "        # Hyperparameters\n",
    "        self.exfeat_dim=exfeat_dim\n",
    "        self.augment_eps = augment_eps\n",
    "        self.lr = lr\n",
    "        self.cls = 1\n",
    "        bias=False\n",
    "        self.W_v = nn.Linear(feat_dim, hidden_dim, bias=bias)\n",
    "        self.W_u1 = AE(exfeat_dim,hidden_dim,hidden_dim, bias=bias)\n",
    "        self.edge_linear=nn.Sequential(\n",
    "            nn.Linear(edge_dim,hidden_dim//4, bias=True),\n",
    "            nn.ELU(),\n",
    "        )\n",
    "        self.gat=EGAT(2*hidden_dim,hidden_dim,hidden_dim//4,dropout)\n",
    "        self.lstm1 = nn.LSTM(hidden_dim,hidden_dim//2,3,batch_first=True,bidirectional=True,dropout=dropout)\n",
    "        self.lstm2 = nn.LSTM(hidden_dim,hidden_dim//2,3,batch_first=True,bidirectional=True,dropout=dropout)\n",
    "        # output\n",
    "        self.mlp=nn.Sequential(\n",
    "            nn.Linear(4*hidden_dim,hidden_dim,bias=True),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_dim,1,bias=True),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "        # Initialization\n",
    "        for p in self.parameters():\n",
    "            if p.dim() > 1:\n",
    "                nn.init.xavier_uniform_(p)\n",
    "\n",
    "    def forward(self, V, edge):\n",
    "        h=[]\n",
    "        V = pad_sequence(V, batch_first=True, padding_value=0).float()\n",
    "        mask=V.sum(-1)!=0\n",
    "        if self.training and self.augment_eps > 0:\n",
    "            aug=torch.randn_like(V)\n",
    "            aug[~mask]=0\n",
    "            V = V+self.augment_eps * aug\n",
    "        mask=mask.sum(1)\n",
    "        feats,exfeats=self.W_v(V[:,:,:-self.exfeat_dim]),self.W_u1(V[:,:,-self.exfeat_dim:])\n",
    "        x_gcns=[]\n",
    "        for i in range(len(V)):\n",
    "            E=self.edge_linear(edge[i]).permute(2,0,1)\n",
    "            x1,x2=feats[i,:mask[i]],exfeats[i,:mask[i]]\n",
    "            x_gcn=torch.cat([x1,x2],-1)\n",
    "            x_gcn,E=self.gat(x_gcn,E)\n",
    "            x_gcns.append(x_gcn)\n",
    "        feats=pack_padded_sequence(feats,mask.cpu(),True,False)\n",
    "        exfeats=pack_padded_sequence(exfeats,mask.cpu(),True,False)\n",
    "        feats=pad_packed_sequence(self.lstm1(feats)[0],True)[0]\n",
    "        exfeats=pad_packed_sequence(self.lstm2(exfeats)[0],True)[0]\n",
    "        x_attns=torch.cat([feats,exfeats],-1)\n",
    "\n",
    "        x_attns=[x_attns[i,:mask[i]] for i in range(len(x_attns))]\n",
    "        h=[torch.cat([x_attn,x_gcn],-1) for x_attn,x_gcn in zip(x_attns,x_gcns)]\n",
    "        h=torch.cat(h,0)\n",
    "        return self.mlp(h)\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        feat, edge, y = batch\n",
    "        pred = self(feat, edge).squeeze(-1)\n",
    "        loss=self.loss_fn(pred,y.float())\n",
    "        self.log('train_loss', loss.cpu().item(), on_step=False, on_epoch=True, prog_bar=True, logger=True)\n",
    "        if self.metrics is not None:\n",
    "            result=self.metrics.calc_prc(pred.detach().clone(),y.detach().clone())\n",
    "            self.log('train_auc', result['AUROC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('train_prc', result['AUPRC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "        return loss\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        feat, edge, y = batch\n",
    "        pred = self(feat, edge).squeeze(-1)\n",
    "        return pred,y\n",
    "    def validation_epoch_end(self,outputs):\n",
    "        pred,y=[],[]\n",
    "        for i,j in outputs:\n",
    "            pred.append(i)\n",
    "            y.append(j)\n",
    "        pred=torch.cat(pred,0)\n",
    "        y=torch.cat(y,0)\n",
    "        loss=self.loss_fn(pred,y.float())\n",
    "        self.log('val_loss', loss.cpu().item(), on_epoch=True, prog_bar=True, logger=True)\n",
    "        if self.metrics is not None:\n",
    "            result=self.metrics(pred.detach().clone(),y.detach().clone())\n",
    "            self.log('val_AUROC', result['AUROC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('val_AUPRC', result['AUPRC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('val_mcc', result['MCC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('val_f1', result['F1'], on_epoch=True, prog_bar=True, logger=True)\n",
    "\n",
    "    def test_step(self, batch, batch_idx):\n",
    "        feat, edge, y = batch\n",
    "        pred = self(feat, edge).squeeze(-1)\n",
    "        return pred,y\n",
    "    def test_epoch_end(self,outputs):\n",
    "        pred,y=[],[]\n",
    "        for i,j in outputs:\n",
    "            pred.append(i)\n",
    "            y.append(j)\n",
    "        pred=torch.cat(pred,0)\n",
    "        y=torch.cat(y,0)\n",
    "        loss=self.loss_fn(pred,y.float())\n",
    "        if self.path:\n",
    "            if not os.path.exists(self.path):\n",
    "                os.system(f'mkdir -p {self.path}')\n",
    "            torch.save({'pred':pred.cpu(),'gt':y.cpu()},f'{self.path}/result.pkl')\n",
    "        if self.metrics is not None:\n",
    "            result=self.metrics(pred.detach().clone(),y.detach().clone())\n",
    "            self.log('test_loss', loss.cpu().item(), on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('test_AUROC', result['AUROC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('test_AUPRC', result['AUPRC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('test_recall', result['RECALL'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('test_precision', result['PRECISION'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('test_f1', result['F1'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('test_mcc', result['MCC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('test_bacc', result['BACC'], on_epoch=True, prog_bar=True, logger=True)\n",
    "            self.log('test_threshold', result['threshold'], on_epoch=True, prog_bar=True, logger=True)\n",
    "    def configure_optimizers(self):\n",
    "        return torch.optim.Adam(self.parameters(), betas=(0.9, 0.99), lr=self.lr, weight_decay=1e-5, eps=1e-5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57ad1d23",
   "metadata": {},
   "source": [
    "contents of preprocess.py, used to preprocess the inputs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "36a3395b",
   "metadata": {},
   "outputs": [],
   "source": [
    "DICT={\n",
    "    'ALA': 'A', 'CYS': 'C', 'CCS': 'C', 'ASP': 'D', 'GLU': 'E', 'PHE': 'F',\n",
    "    'GLY': 'G', 'HIS': 'H', 'ILE': 'I', 'LYS': 'K', 'LEU': 'L',\n",
    "    'MET': 'M', 'MSE': 'M', 'ASN': 'N', 'PRO': 'P', 'GLN': 'Q', 'ARG': 'R',\n",
    "    'SER': 'S', 'THR': 'T', 'VAL': 'V', 'TRP': 'W', 'TYR': 'Y',\n",
    "}\n",
    "def pdb_split(line):\n",
    "    order=int(line[6:11].strip())\n",
    "    atom=line[11:16].strip()\n",
    "    amino=line[16:21].strip()\n",
    "    chain=line[21]\n",
    "    site=line[22:28].strip()\n",
    "    x=line[28:38].strip()\n",
    "    y=line[38:46].strip()\n",
    "    z=line[46:54].strip()\n",
    "    return order,atom,amino,chain,site,x,y,z\n",
    "def judge(line,filt_atom='CA'):\n",
    "    kind=line[:6].strip()\n",
    "    if kind not in ['HETATM','ATOM']:\n",
    "        return None\n",
    "    order,atom,amino,chain,site,x,y,z=pdb_split(line)\n",
    "    if filt_atom is not None and atom!=filt_atom:\n",
    "        return None\n",
    "    prefix=''\n",
    "    if len(amino)>3:\n",
    "        prefix=amino[0]\n",
    "        amino=amino[-3:]\n",
    "    if amino=='MSE':\n",
    "        amino='MET'\n",
    "    elif amino=='CCS' or amino[:-1]=='CS':\n",
    "        amino='CYS'\n",
    "    elif amino not in DICT.keys():\n",
    "        return None\n",
    "    return prefix+amino,chain,site,float(x),float(y),float(z)\n",
    "def process_dssp(dssp_file):\n",
    "    aa_type = \"ACDEFGHIKLMNPQRSTVWY\"\n",
    "    SS_type = \"HBEGITSC\"\n",
    "    rASA_std = [115, 135, 150, 190, 210, 75, 195, 175, 200, 170,\n",
    "                185, 160, 145, 180, 225, 115, 140, 155, 255, 230]\n",
    "    with open(dssp_file, \"r\") as f:\n",
    "        lines = f.readlines()\n",
    "    seq = \"\"\n",
    "    dssp_feature = []\n",
    "    position = []\n",
    "    p = 0\n",
    "    while lines[p].strip()[0] != \"#\":\n",
    "        p += 1\n",
    "    for i in range(p + 1, len(lines)):\n",
    "        aa = lines[i][13]\n",
    "        if aa == \"!\" or aa == \"*\":\n",
    "            continue\n",
    "        seq += aa\n",
    "        POS = lines[i][5:11].strip()\n",
    "        position.append(POS)\n",
    "        SS = lines[i][16]\n",
    "        if SS == \" \":\n",
    "            SS = \"C\"\n",
    "        SS_vec = np.zeros(8)\n",
    "        SS_vec[SS_type.find(SS)] = 1\n",
    "        PHI = float(lines[i][103:109].strip())\n",
    "        PSI = float(lines[i][109:115].strip())\n",
    "        ACC = float(lines[i][34:38].strip())\n",
    "        ASA = min(100, round(ACC / rASA_std[aa_type.find(aa)] * 100)) / 100\n",
    "        dssp_feature.append(np.concatenate((np.array([PHI, PSI, ASA]), SS_vec)))\n",
    "\n",
    "    return seq, dssp_feature,position\n",
    "def transform_dssp(dssp_feature):\n",
    "    dssp_feature = np.array(dssp_feature)\n",
    "    angle = dssp_feature[:,0:2]\n",
    "    ASA_SS = dssp_feature[:,2:]\n",
    "    radian = angle * (np.pi / 180)\n",
    "    dssp_feature = np.concatenate([np.sin(radian), np.cos(radian), ASA_SS], axis = 1)\n",
    "    return dssp_feature\n",
    "def get_dssp(ID,root):\n",
    "    if not os.path.exists(f\"{root}/dssp/\"):\n",
    "        os.mkdir(f\"{root}/dssp/\")\n",
    "    os.system(f\"./mkdssp/mkdssp -i {root}/purePDB/{ID}.pdb -o {root}/dssp/{ID}.dssp\")\n",
    "    if not os.path.exists(f\"{root}/dssp/\" + ID + \".dssp\"):\n",
    "        return None\n",
    "    dssp_seq, dssp_matrix,position = process_dssp(f\"{root}/dssp/\" + ID + \".dssp\")\n",
    "    np.save(f\"{root}/dssp/\" + ID, transform_dssp(dssp_matrix))\n",
    "    np.save(f\"{root}/dssp/\"+ID+\"_pos\",position)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d4df520",
   "metadata": {},
   "source": [
    "contents of graph_construction.py, used to create a representation of the structural information:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cd0325eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "ID={\n",
    "    'A': 0, 'C': 1, 'D': 2, 'E': 3, 'F': 4,\n",
    "    'G': 5, 'H': 6, 'I': 7, 'K': 8, 'L': 9,\n",
    "    'M': 10, 'N': 11, 'P': 12, 'Q': 13, 'R': 14,\n",
    "    'S': 15, 'T': 16, 'V': 17, 'W': 18, 'Y': 19\n",
    "}\n",
    "\n",
    "def calcPROgraph(seq,coord,dseq=3,dr=10,dlong=5,k=10):\n",
    "    nodes=coord.shape[0]\n",
    "    adj=torch.zeros((nodes,nodes))\n",
    "    E=torch.zeros((nodes,nodes,21*2+2*dseq+3))\n",
    "    # C=coord.to('cuda:1')\n",
    "    dist=torch.cdist(coord,coord,2)\n",
    "    knn=dist.argsort(1)[:,1:k+1]\n",
    "    for i in range(nodes):\n",
    "        # knn=dist[i].argsort()[1:k+1]\n",
    "        for j in range(nodes):\n",
    "            not_edge=True\n",
    "            dij_seq=abs(i-j)\n",
    "            if dij_seq<dseq:\n",
    "                E[i][j][41+i-j+dseq]=1\n",
    "                not_edge=False\n",
    "            if dist[i][j]<dr and dij_seq>=dlong:\n",
    "                E[i][j][41+2*dseq]=1\n",
    "                not_edge=False\n",
    "            if j in knn[i] and dij_seq>=dlong:\n",
    "                E[i][j][42+2*dseq]=1\n",
    "                not_edge=False\n",
    "            if not_edge:\n",
    "                continue\n",
    "            adj[i][j]=1\n",
    "            E[i][j][ID.get(seq[i],20)]=1\n",
    "            E[i][j][21+ID.get(seq[j],20)]=1\n",
    "            E[i][j][43+2*dseq]=dij_seq\n",
    "            E[i][j][44+2*dseq]=dist[i][j]\n",
    "    idx=adj.nonzero().T\n",
    "    data=adj[idx[0],idx[1]]\n",
    "    adj=torch.sparse.FloatTensor(idx,data,adj.shape)\n",
    "    idx=E.nonzero().T\n",
    "    data=E[idx[0],idx[1],idx[2]]\n",
    "    E=torch.sparse.FloatTensor(idx,data,E.shape)\n",
    "    return {'adj':adj,'edge':E}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "823bb00f",
   "metadata": {},
   "source": [
    "contents of utils.py, methods for loading and working with the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ae61a248",
   "metadata": {},
   "outputs": [],
   "source": [
    "amino2id={\n",
    "    '<null_0>': 0, '<pad>': 1, '<eos>': 2, '<unk>': 3,\n",
    "    'L': 4, 'A': 5, 'G': 6, 'V': 7, 'S': 8, 'E': 9, 'R': 10,\n",
    "    'T': 11, 'I': 12, 'D': 13, 'P': 14, 'K': 15, 'Q': 16,\n",
    "    'N': 17, 'F': 18, 'Y': 19, 'M': 20, 'H': 21, 'W': 22,\n",
    "    'C': 23, 'X': 24, 'B': 25, 'U': 26, 'Z': 27, 'O': 28,\n",
    "    '.': 29, '-': 30, '<null_1>': 31, '<mask>': 32, '<cath>': 33, '<af2>': 34\n",
    "}\n",
    "class chain:\n",
    "    def __init__(self):\n",
    "        self.sequence=[]\n",
    "        self.amino=[]\n",
    "        self.coord=[]\n",
    "        self.site={}\n",
    "        self.date=''\n",
    "        self.length=0\n",
    "        self.adj=None\n",
    "        self.edge=None\n",
    "        self.feat=None\n",
    "        self.dssp=None\n",
    "        self.name=''\n",
    "        self.chain_name=''\n",
    "        self.protein_name=''\n",
    "    def add(self,amino,pos,coord):\n",
    "        self.sequence.append(DICT[amino])\n",
    "        self.amino.append(amino2id[DICT[amino]])\n",
    "        self.coord.append(coord)\n",
    "        self.site[pos]=self.length\n",
    "        self.length+=1\n",
    "    def process(self):\n",
    "        self.amino=torch.LongTensor(self.amino)\n",
    "        self.coord=torch.FloatTensor(self.coord)\n",
    "        self.label=torch.zeros_like(self.amino)\n",
    "        self.sequence=''.join(self.sequence)\n",
    "    def extract(self,model,device,path):\n",
    "        if len(self)>1024 or model is None:\n",
    "            return\n",
    "        f=lambda x:model(x.to(device).unsqueeze(0),[36])['representations'][36].squeeze(0).cpu()\n",
    "        with torch.no_grad():\n",
    "            feat=f(self.amino)\n",
    "        torch.save(feat,f'{path}/feat/{self.name}_esm2.ts')\n",
    "    def load_dssp(self,path):\n",
    "        dssp=torch.Tensor(np.load(f'{path}/dssp/{self.name}.npy'))\n",
    "        pos=np.load(f'{path}/dssp/{self.name}_pos.npy')\n",
    "        self.dssp=torch.Tensor([\n",
    "            -2.4492936e-16, -2.4492936e-16,\n",
    "            1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0\n",
    "        ]).repeat(self.length,1)\n",
    "        self.rsa=torch.zeros(self.length)\n",
    "        for i in range(len(dssp)):\n",
    "            self.dssp[self.site[pos[i]]]=dssp[i]\n",
    "            if dssp[i][4]>0.15:\n",
    "                self.rsa[i]=1\n",
    "        self.rsa=self.rsa.bool()\n",
    "    def load_feat(self,path):\n",
    "        self.feat=torch.load(f'{path}/feat/{self.name}_esm2.ts')\n",
    "    def load_adj(self,path,self_cycle=False):\n",
    "        graph=torch.load(f'{path}/graph/{self.name}.graph')\n",
    "        self.adj=graph['adj'].to_dense()\n",
    "        self.edge=graph['edge'].to_dense()\n",
    "        if not self_cycle:\n",
    "            self.adj[range(len(self)),range(len(self))]=0\n",
    "            self.edge[range(len(self)),range(len(self))]=0\n",
    "    def get_adj(self,path,dseq=3,dr=10,dlong=5,k=10):\n",
    "        graph=calcPROgraph(self.sequence,self.coord,dseq,dr,dlong,k)\n",
    "        torch.save(graph,f'{path}/graph/{self.name}.graph')\n",
    "    def update(self,pos,amino):\n",
    "        if amino not in DICT.keys():\n",
    "            return\n",
    "        amino_id=amino2id[DICT[amino]]\n",
    "        idx=self.site.get(pos,None)\n",
    "        if idx is None:\n",
    "            for i in self.site.keys():\n",
    "                # print(i,pos)\n",
    "                if i[:len(pos)]==pos:\n",
    "                    idx=self.site.get(i)\n",
    "                    if amino_id==self.amino[idx]:\n",
    "                        self.label[idx]=1\n",
    "                        return\n",
    "        elif amino_id!=self.amino[idx]:\n",
    "            for i in self.site.keys():\n",
    "                if i[:len(pos)]==pos:\n",
    "                    idx=self.site.get(i)\n",
    "                    if amino_id==self.amino[idx]:\n",
    "                        self.label[idx]=1\n",
    "                        return\n",
    "        else:\n",
    "            self.label[idx]=1\n",
    "    def __len__(self):\n",
    "        return self.length\n",
    "    def __getitem__(self,idx):\n",
    "        return self.amino[idx],self.coord[idx],self.label[idx]\n",
    "def collate_fn(batch):\n",
    "    edges = [item['edge'] for item in batch]\n",
    "    feats = [item['feat'] for item in batch]\n",
    "    labels = torch.cat([item['label'] for item in batch],0)\n",
    "    return feats,edges,labels\n",
    "\n",
    "def extract_chain(root,pid,chain,force=False):\n",
    "    if not force and os.path.exists(f'{root}/purePDB/{pid}_{chain}.pdb'):\n",
    "        return True\n",
    "    if not os.path.exists(f'{root}/PDB/{pid}.pdb'):\n",
    "        retry=5\n",
    "        pdb=None\n",
    "        with rq.get(f'https://files.rcsb.org/download/{pid}.pdb') as f:\n",
    "            if f.status_code==200:\n",
    "                pdb=f.content\n",
    "        while retry>0:\n",
    "            try:\n",
    "                with rq.get(f'https://files.rcsb.org/download/{pid}.pdb') as f:\n",
    "                    if f.status_code==200:\n",
    "                        pdb=f.content\n",
    "                        break\n",
    "            except:\n",
    "                retry-=1\n",
    "                continue\n",
    "        if pdb is None:\n",
    "            print(f'PDB file {pid} failed to download')\n",
    "            return False\n",
    "        with open(f'{root}/PDB/{pid}.pdb','wb') as f:\n",
    "            f.write(pdb)\n",
    "    lines=[]\n",
    "    with open(f'{root}/PDB/{pid}.pdb','r') as f:\n",
    "        for line in f:\n",
    "            if line[:6]=='HEADER':\n",
    "                lines.append(line)\n",
    "            if line[:6].strip()=='TER' and line[21]==chain:\n",
    "                lines.append(line)\n",
    "                break\n",
    "            feats=judge(line,None)\n",
    "            if feats is not None and feats[1]==chain:\n",
    "                lines.append(line)\n",
    "    with open(f'{root}/purePDB/{pid}_{chain}.pdb','w') as f:\n",
    "        for i in lines:\n",
    "            f.write(i)\n",
    "    return True\n",
    "def process_chain(data,root,pid,model,device):\n",
    "    get_dssp(pid,root)\n",
    "    same={}\n",
    "    with open(f'{root}/purePDB/{pid}.pdb','r') as f:\n",
    "        for line in f:\n",
    "            if line[:6]=='HEADER':\n",
    "                date=line[50:59].strip()\n",
    "                data.date=date\n",
    "                continue\n",
    "            feats=judge(line,'CA')\n",
    "            if feats is None:\n",
    "                continue\n",
    "            amino,_,site,x,y,z=feats\n",
    "            if len(amino)>3:\n",
    "                if same.get(site) is None:\n",
    "                    same[site]=amino[0]\n",
    "                if same[site]!=amino[0]:\n",
    "                    continue\n",
    "                amino=amino[-3:]\n",
    "            data.add(amino,site,[x,y,z])\n",
    "    data.process()\n",
    "    data.get_adj(root)\n",
    "    data.extract(model,device,root)\n",
    "    return data\n",
    "def initial(file,root,model=None,device='cpu',from_native_pdb=True):\n",
    "    df=pd.read_csv(f'{root}/{file}',header=0,index_col=0)\n",
    "    prefix=df.index\n",
    "    labels=df['Epitopes (resi_resn)']\n",
    "    samples=[]\n",
    "    with tqdm(prefix) as tbar:\n",
    "        for i in tbar:\n",
    "            tbar.set_postfix(protein=i)\n",
    "            if from_native_pdb:\n",
    "                state=extract_chain(root,i[:4],i[-1])\n",
    "                if not state:\n",
    "                    continue\n",
    "            data=chain()\n",
    "            p,c=i.split('_')\n",
    "            data.protein_name=p\n",
    "            data.chain_name=c\n",
    "            data.name=f\"{p}_{c}\"\n",
    "            process_chain(data,root,i,model,device)\n",
    "            label=labels.loc[i].split(', ')\n",
    "            for j in label:\n",
    "                site,amino=j.split('_')\n",
    "                data.update(site,amino)\n",
    "            samples.append(data)\n",
    "    with open(f'{root}/total.pkl','wb') as f:\n",
    "        pk.dump(samples,f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ba8ef7f",
   "metadata": {},
   "source": [
    "first part of the contents of dataset.py, a class for handling the protein data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ac9e0cf1",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "class PDB(Dataset):\n",
    "    def __init__(\n",
    "        self,mode='train',fold=-1,root='.',self_cycle=False\n",
    "    ):\n",
    "        self.root=root\n",
    "        assert mode in ['train','val','test']\n",
    "        if mode in ['train','val']:\n",
    "            with open(f'{self.root}/train.pkl','rb') as f:\n",
    "                self.samples=pk.load(f)\n",
    "        else:\n",
    "            with open(f'{self.root}/test.pkl','rb') as f:\n",
    "                self.samples=pk.load(f)\n",
    "        self.data=[]\n",
    "        idx=np.load(f'{self.root}/cross-validation.npy')\n",
    "        cv=10\n",
    "        inter=len(idx)//cv\n",
    "        ex=len(idx)%cv\n",
    "        if mode=='train':\n",
    "            order=[]\n",
    "            for i in range(cv):\n",
    "                if i==fold:\n",
    "                    continue\n",
    "                order+=list(idx[i*inter:(i+1)*inter+ex*(i==cv-1)])\n",
    "        elif mode=='val':\n",
    "            order=list(idx[fold*inter:(fold+1)*inter+ex*(fold==cv-1)])\n",
    "        else:\n",
    "            order=list(range(len(self.samples)))\n",
    "        order.sort()\n",
    "        tbar=tqdm(order)\n",
    "        for i in tbar:\n",
    "            tbar.set_postfix(chain=f'{self.samples[i].name}')\n",
    "            self.samples[i].load_feat(self.root)\n",
    "            self.samples[i].load_dssp(self.root)\n",
    "            self.samples[i].load_adj(self.root,self_cycle)\n",
    "            self.data.append(self.samples[i])\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    def __getitem__(self,idx):\n",
    "        seq=self.data[idx]\n",
    "        feat=torch.cat([seq.feat,seq.dssp],1)\n",
    "        return {\n",
    "            'feat':feat,\n",
    "            'label':seq.label,\n",
    "            'adj':seq.adj,\n",
    "            'edge':seq.edge,\n",
    "        }\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68416757",
   "metadata": {},
   "source": [
    "second part of dataset.py, downloads and creates the necessary datasets (needs to be ran just once to create the needed files):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "00b4ad52",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_exists = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4a776b23",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument('--root', type=str, default='./data/BCE_633', help='dataset path')\n",
    "parser.add_argument('--gpu', type=int, default=0, help='gpu.')\n",
    "args = parser.parse_args()\n",
    "root = args.root\n",
    "device='cpu' if args.gpu==-1 else f'cuda:{args.gpu}'\n",
    "'''\n",
    "if not dataset_exists:\n",
    "    root = '.'\n",
    "    device = 'cuda'\n",
    "\n",
    "    os.system(f'cd {root} && mkdir PDB purePDB feat dssp graph')\n",
    "    model,_=esm.pretrained.esm2_t36_3B_UR50D()\n",
    "    model=model.to(device)\n",
    "    model.eval()\n",
    "    train='DATA/total.csv'\n",
    "    initial(train,root,model,device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04580901",
   "metadata": {},
   "source": [
    "third part of dataset.py:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c46a1d40",
   "metadata": {},
   "outputs": [],
   "source": [
    "root = '.'\n",
    "device = 'cuda'\n",
    "with open(f'{root}/total.pkl','rb') as f:\n",
    "    dataset=pk.load(f)\n",
    "    dates={i.name:i.date for i in dataset}\n",
    "    filt_data=[]\n",
    "    for i in dataset:\n",
    "        if len(i)<1024 and i.label.sum()>0:\n",
    "            filt_data.append(i)\n",
    "    month={'JAN':1,'FEB':2,'MAR':3,'APR':4,'MAY':5,'JUN':6,'JUL':7,'AUG':8,'SEP':9,'OCT':10,'NOV':11,'DEC':12}\n",
    "    trainset,valset,testset=[],[],[]\n",
    "    D,M,Y=[],[],[]\n",
    "    test=20210401\n",
    "    dates_=[]\n",
    "    for i in filt_data:\n",
    "        d,m,y=dates[i.name].split('-')\n",
    "        d,m,y=int(d),month[m],int(y)\n",
    "        if y<23:\n",
    "            y+=2000\n",
    "        else:\n",
    "            y+=1900\n",
    "        date=y*10000+m*100+d\n",
    "        if date<test:\n",
    "            dates_.append(date)\n",
    "            trainset.append(i)\n",
    "        else:\n",
    "            testset.append(i)\n",
    "    with open(f'{root}/train.pkl','wb') as f:\n",
    "        pk.dump(trainset,f)\n",
    "    with open(f'{root}/test.pkl','wb') as f:\n",
    "        pk.dump(testset,f)\n",
    "    idx=np.array(dates_).argsort()\n",
    "    np.save(f'{root}/cross-validation.npy',idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cddf6e75",
   "metadata": {},
   "source": [
    "seed everything:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "168e3970",
   "metadata": {},
   "outputs": [],
   "source": [
    "def seed_everything(seed=2022):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "seed_everything(2022)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04429c82",
   "metadata": {},
   "source": [
    "set parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7dd8691f",
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu=0\n",
    "fold=-1\n",
    "lr=1e-6\n",
    "batch=4\n",
    "epochs=30\n",
    "root='.'\n",
    "log_name=f'BCE_633_GraphBepi'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a32117cb",
   "metadata": {},
   "source": [
    "load data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "013e6513",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████| 577/577 [00:15<00:00, 37.79it/s, chain=3lh2_V]\n",
      "0it [00:00, ?it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████| 56/56 [00:01<00:00, 33.55it/s, chain=7ue9_C]\n"
     ]
    }
   ],
   "source": [
    "trainset=PDB(mode='train',fold=fold,root=root)\n",
    "valset=PDB(mode='val',fold=fold,root=root)\n",
    "testset=PDB(mode='test',root=root)\n",
    "train_loader=DataLoader(trainset, batch, shuffle=True, collate_fn=collate_fn, drop_last=True)\n",
    "val_loader=DataLoader(valset, batch, shuffle=False, collate_fn=collate_fn)\n",
    "test_loader=DataLoader(testset, batch, shuffle=False, collate_fn=collate_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2160ddfb",
   "metadata": {},
   "source": [
    "create models:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "776c8c69",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "device='cpu' if gpu==-1 else f'cuda:{gpu}'\n",
    "metrics=METRICS(device)\n",
    "es=EarlyStopping('val_AUPRC',patience=40,mode='max')\n",
    "mc=ModelCheckpoint(\n",
    "    f'./model/{log_name}/',f'model_{fold}',\n",
    "    'val_AUPRC',\n",
    "    mode='max',\n",
    "    save_weights_only=True, \n",
    ")\n",
    "logger = TensorBoardLogger(\n",
    "    './log', \n",
    "    name=f'{log_name}_{fold}'\n",
    ")\n",
    "cb=[mc,es]\n",
    "trainer = pl.Trainer( \n",
    "    max_epochs=epochs, callbacks=cb,\n",
    "    logger=logger,check_val_every_n_epoch=1,\n",
    ")\n",
    "model=GraphBepi(\n",
    "    feat_dim=2560,                     # esm2 representation dim\n",
    "    hidden_dim=256,                    # hidden representation dim\n",
    "    exfeat_dim=13,                     # dssp feature dim\n",
    "    edge_dim=51,                       # edge feature dim\n",
    "    augment_eps=0.05,                  # random noise rate\n",
    "    dropout=0.2,\n",
    "    lr=lr,                             # learning rate\n",
    "    metrics=metrics,                   # an implement to compute performance\n",
    "    result_path=f'./model/{log_name}', # path to save temporary result file of testset\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "300c2869",
   "metadata": {},
   "source": [
    "train:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "710c0058",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  | Name        | Type       | Params\n",
      "-------------------------------------------\n",
      "0 | loss_fn     | BCELoss    | 0     \n",
      "1 | W_v         | Linear     | 655 K \n",
      "2 | W_u1        | AE         | 69.9 K\n",
      "3 | edge_linear | Sequential | 3.3 K \n",
      "4 | gat         | EGAT       | 8.5 M \n",
      "5 | lstm1       | LSTM       | 1.2 M \n",
      "6 | lstm2       | LSTM       | 1.2 M \n",
      "7 | mlp         | Sequential | 262 K \n",
      "-------------------------------------------\n",
      "11.9 M    Trainable params\n",
      "0         Non-trainable params\n",
      "11.9 M    Total params\n",
      "47.536    Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0:  91%|██████▍| 144/158 [48:29<04:42, 20.20s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:13,  1.03s/it]\u001b[A\n",
      "Epoch 0:  92%|██████▍| 145/158 [48:30<04:20, 20.07s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:22,  1.87s/it]\u001b[A\n",
      "Epoch 0:  92%|██████▍| 146/158 [48:32<03:59, 19.95s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:05<00:23,  2.11s/it]\u001b[A\n",
      "Epoch 0:  93%|██████▌| 147/158 [48:34<03:38, 19.83s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:08<00:23,  2.37s/it]\u001b[A\n",
      "Epoch 0:  94%|██████▌| 148/158 [48:37<03:17, 19.71s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:09<00:16,  1.88s/it]\u001b[A\n",
      "Epoch 0:  94%|██████▌| 149/158 [48:38<02:56, 19.59s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:11<00:15,  1.92s/it]\u001b[A\n",
      "Epoch 0:  95%|██████▋| 150/158 [48:40<02:35, 19.47s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:14<00:14,  2.14s/it]\u001b[A\n",
      "Epoch 0:  96%|██████▋| 151/158 [48:43<02:15, 19.36s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:19<00:18,  3.02s/it]\u001b[A\n",
      "Epoch 0:  96%|██████▋| 152/158 [48:48<01:55, 19.26s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:21<00:13,  2.66s/it]\u001b[A\n",
      "Epoch 0:  97%|██████▊| 153/158 [48:50<01:35, 19.15s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:26<00:14,  3.63s/it]\u001b[A\n",
      "Epoch 0:  97%|██████▊| 154/158 [48:55<01:16, 19.06s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:28<00:08,  2.91s/it]\u001b[A\n",
      "Epoch 0:  98%|██████▊| 155/158 [48:57<00:56, 18.95s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:29<00:04,  2.45s/it]\u001b[A\n",
      "Epoch 0:  99%|██████▉| 156/158 [48:58<00:37, 18.84s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:32<00:02,  2.76s/it]\u001b[A\n",
      "Epoch 0:  99%|██████▉| 157/158 [49:02<00:18, 18.74s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132]\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:35<00:00,  2.55s/it]\u001b[A\n",
      "Epoch 0: 100%|█| 158/158 [49:04<00:00, 18.63s/it, loss=0.384, v_num=5, train_auc_step=0.563, train_prc_step=0.132, val_l\u001b[A\n",
      "Epoch 1:  91%|▉| 144/158 [49:13<04:47, 20.51s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:19,  1.53s/it]\u001b[A\n",
      "Epoch 1:  92%|▉| 145/158 [49:15<04:24, 20.38s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:04<00:31,  2.64s/it]\u001b[A\n",
      "Epoch 1:  92%|▉| 146/158 [49:18<04:03, 20.27s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:08<00:34,  3.14s/it]\u001b[A\n",
      "Epoch 1:  93%|▉| 147/158 [49:22<03:41, 20.15s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:12<00:32,  3.29s/it]\u001b[A\n",
      "Epoch 1:  94%|▉| 148/158 [49:26<03:20, 20.04s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:13<00:23,  2.58s/it]\u001b[A\n",
      "Epoch 1:  94%|▉| 149/158 [49:27<02:59, 19.92s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:16<00:20,  2.61s/it]\u001b[A\n",
      "Epoch 1:  95%|▉| 150/158 [49:30<02:38, 19.80s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:19<00:20,  2.94s/it]\u001b[A\n",
      "Epoch 1:  96%|▉| 151/158 [49:33<02:17, 19.69s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:26<00:25,  4.26s/it]\u001b[A\n",
      "Epoch 1:  96%|▉| 152/158 [49:40<01:57, 19.61s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:28<00:16,  3.34s/it]\u001b[A\n",
      "Epoch 1:  97%|▉| 153/158 [49:42<01:37, 19.49s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:34<00:16,  4.16s/it]\u001b[A\n",
      "Epoch 1:  97%|▉| 154/158 [49:48<01:17, 19.40s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:35<00:09,  3.25s/it]\u001b[A\n",
      "Epoch 1:  98%|▉| 155/158 [49:49<00:57, 19.29s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:37<00:05,  2.80s/it]\u001b[A\n",
      "Epoch 1:  99%|▉| 156/158 [49:51<00:38, 19.17s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:41<00:03,  3.14s/it]\u001b[A\n",
      "Epoch 1:  99%|▉| 157/158 [49:55<00:19, 19.08s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:43<00:00,  2.97s/it]\u001b[A\n",
      "Epoch 1: 100%|█| 158/158 [49:57<00:00, 18.97s/it, loss=0.388, v_num=5, train_auc_step=0.639, train_prc_step=0.288, val_l\u001b[A\n",
      "Epoch 2:  91%|▉| 144/158 [49:05<04:46, 20.46s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:11,  1.12it/s]\u001b[A\n",
      "Epoch 2:  92%|▉| 145/158 [49:06<04:24, 20.32s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:25,  2.11s/it]\u001b[A\n",
      "Epoch 2:  92%|▉| 146/158 [49:09<04:02, 20.20s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:25,  2.33s/it]\u001b[A\n",
      "Epoch 2:  93%|▉| 147/158 [49:12<03:40, 20.08s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:26,  2.62s/it]\u001b[A\n",
      "Epoch 2:  94%|▉| 148/158 [49:15<03:19, 19.97s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:17,  1.98s/it]\u001b[A\n",
      "Epoch 2:  94%|▉| 149/158 [49:16<02:58, 19.84s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:12<00:16,  2.10s/it]\u001b[A\n",
      "Epoch 2:  95%|▉| 150/158 [49:18<02:37, 19.72s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:15,  2.21s/it]\u001b[A\n",
      "Epoch 2:  96%|▉| 151/158 [49:21<02:17, 19.61s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:20<00:18,  3.10s/it]\u001b[A\n",
      "Epoch 2:  96%|▉| 152/158 [49:26<01:57, 19.51s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:21<00:13,  2.63s/it]\u001b[A\n",
      "Epoch 2:  97%|▉| 153/158 [49:27<01:36, 19.40s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:27<00:14,  3.66s/it]\u001b[A\n",
      "Epoch 2:  97%|▉| 154/158 [49:33<01:17, 19.31s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:28<00:08,  2.84s/it]\u001b[A\n",
      "Epoch 2:  98%|▉| 155/158 [49:34<00:57, 19.19s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:30<00:04,  2.41s/it]\u001b[A\n",
      "Epoch 2:  99%|▉| 156/158 [49:36<00:38, 19.08s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:33<00:02,  2.80s/it]\u001b[A\n",
      "Epoch 2:  99%|▉| 157/158 [49:39<00:18, 18.98s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:36<00:00,  2.70s/it]\u001b[A\n",
      "Epoch 2: 100%|█| 158/158 [49:42<00:00, 18.88s/it, loss=0.363, v_num=5, train_auc_step=0.674, train_prc_step=0.233, val_l\u001b[A\n",
      "Epoch 3:  91%|▉| 144/158 [49:47<04:50, 20.75s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:14,  1.15s/it]\u001b[A\n",
      "Epoch 3:  92%|▉| 145/158 [49:48<04:27, 20.61s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:04<00:27,  2.30s/it]\u001b[A\n",
      "Epoch 3:  92%|▉| 146/158 [49:51<04:05, 20.49s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:07<00:28,  2.62s/it]\u001b[A\n",
      "Epoch 3:  93%|▉| 147/158 [49:54<03:44, 20.37s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:10<00:28,  2.81s/it]\u001b[A\n",
      "Epoch 3:  94%|▉| 148/158 [49:58<03:22, 20.26s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:11<00:19,  2.13s/it]\u001b[A\n",
      "Epoch 3:  94%|▉| 149/158 [49:58<03:01, 20.13s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:13<00:18,  2.27s/it]\u001b[A\n",
      "Epoch 3:  95%|▉| 150/158 [50:01<02:40, 20.01s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:16<00:17,  2.51s/it]\u001b[A\n",
      "Epoch 3:  96%|▉| 151/158 [50:04<02:19, 19.90s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:22<00:20,  3.40s/it]\u001b[A\n",
      "Epoch 3:  96%|▉| 152/158 [50:09<01:58, 19.80s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:23<00:14,  2.80s/it]\u001b[A\n",
      "Epoch 3:  97%|▉| 153/158 [50:11<01:38, 19.68s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:30<00:15,  3.95s/it]\u001b[A\n",
      "Epoch 3:  97%|▉| 154/158 [50:17<01:18, 19.60s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:31<00:09,  3.23s/it]\u001b[A\n",
      "Epoch 3:  98%|▉| 155/158 [50:19<00:58, 19.48s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:33<00:05,  2.73s/it]\u001b[A\n",
      "Epoch 3:  99%|▉| 156/158 [50:21<00:38, 19.37s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:35<00:02,  2.70s/it]\u001b[A\n",
      "Epoch 3:  99%|▉| 157/158 [50:23<00:19, 19.26s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:38<00:00,  2.52s/it]\u001b[A\n",
      "Epoch 3: 100%|█| 158/158 [50:25<00:00, 19.15s/it, loss=0.358, v_num=5, train_auc_step=0.687, train_prc_step=0.309, val_l\u001b[A\n",
      "Epoch 4:  91%|▉| 144/158 [50:07<04:52, 20.89s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:12,  1.07it/s]\u001b[A\n",
      "Epoch 4:  92%|▉| 145/158 [50:08<04:29, 20.75s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:23,  1.93s/it]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4:  92%|▉| 146/158 [50:11<04:07, 20.63s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:25,  2.31s/it]\u001b[A\n",
      "Epoch 4:  93%|▉| 147/158 [50:14<03:45, 20.51s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:26,  2.62s/it]\u001b[A\n",
      "Epoch 4:  94%|▉| 148/158 [50:17<03:23, 20.39s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.08s/it]\u001b[A\n",
      "Epoch 4:  94%|▉| 149/158 [50:18<03:02, 20.26s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:12<00:17,  2.19s/it]\u001b[A\n",
      "Epoch 4:  95%|▉| 150/158 [50:20<02:41, 20.14s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:16,  2.42s/it]\u001b[A\n",
      "Epoch 4:  96%|▉| 151/158 [50:23<02:20, 20.02s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:21<00:20,  3.38s/it]\u001b[A\n",
      "Epoch 4:  96%|▉| 152/158 [50:29<01:59, 19.93s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:22<00:14,  2.84s/it]\u001b[A\n",
      "Epoch 4:  97%|▉| 153/158 [50:30<01:39, 19.81s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:28<00:15,  3.80s/it]\u001b[A\n",
      "Epoch 4:  97%|▉| 154/158 [50:36<01:18, 19.72s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:29<00:08,  2.97s/it]\u001b[A\n",
      "Epoch 4:  98%|▉| 155/158 [50:37<00:58, 19.60s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:31<00:05,  2.52s/it]\u001b[A\n",
      "Epoch 4:  99%|▉| 156/158 [50:39<00:38, 19.48s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:35<00:02,  2.87s/it]\u001b[A\n",
      "Epoch 4:  99%|▉| 157/158 [50:43<00:19, 19.38s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:38<00:00,  2.88s/it]\u001b[A\n",
      "Epoch 4: 100%|█| 158/158 [50:45<00:00, 19.28s/it, loss=0.374, v_num=5, train_auc_step=0.737, train_prc_step=0.154, val_l\u001b[A\n",
      "Epoch 5:  91%|▉| 144/158 [49:25<04:48, 20.59s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:13,  1.02s/it]\u001b[A\n",
      "Epoch 5:  92%|▉| 145/158 [49:26<04:25, 20.46s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:23,  1.99s/it]\u001b[A\n",
      "Epoch 5:  92%|▉| 146/158 [49:28<04:04, 20.34s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:27,  2.48s/it]\u001b[A\n",
      "Epoch 5:  93%|▉| 147/158 [49:32<03:42, 20.22s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:25,  2.59s/it]\u001b[A\n",
      "Epoch 5:  94%|▉| 148/158 [49:34<03:20, 20.10s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:17,  1.99s/it]\u001b[A\n",
      "Epoch 5:  94%|▉| 149/158 [49:35<02:59, 19.97s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:12<00:16,  2.03s/it]\u001b[A\n",
      "Epoch 5:  95%|▉| 150/158 [49:37<02:38, 19.85s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:15,  2.25s/it]\u001b[A\n",
      "Epoch 5:  96%|▉| 151/158 [49:40<02:18, 19.74s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:20<00:18,  3.06s/it]\u001b[A\n",
      "Epoch 5:  96%|▉| 152/158 [49:45<01:57, 19.64s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:21<00:12,  2.59s/it]\u001b[A\n",
      "Epoch 5:  97%|▉| 153/158 [49:46<01:37, 19.52s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:27<00:13,  3.48s/it]\u001b[A\n",
      "Epoch 5:  97%|▉| 154/158 [49:52<01:17, 19.43s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:27<00:08,  2.69s/it]\u001b[A\n",
      "Epoch 5:  98%|▉| 155/158 [49:53<00:57, 19.31s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:29<00:04,  2.43s/it]\u001b[A\n",
      "Epoch 5:  99%|▉| 156/158 [49:55<00:38, 19.20s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:33<00:02,  2.74s/it]\u001b[A\n",
      "Epoch 5:  99%|▉| 157/158 [49:58<00:19, 19.10s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:35<00:00,  2.72s/it]\u001b[A\n",
      "Epoch 5: 100%|█| 158/158 [50:01<00:00, 19.00s/it, loss=0.365, v_num=5, train_auc_step=0.620, train_prc_step=0.199, val_l\u001b[A\n",
      "Epoch 6:  91%|▉| 144/158 [50:20<04:53, 20.98s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:14,  1.09s/it]\u001b[A\n",
      "Epoch 6:  92%|▉| 145/158 [50:21<04:30, 20.84s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:23,  1.96s/it]\u001b[A\n",
      "Epoch 6:  92%|▉| 146/158 [50:24<04:08, 20.72s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:27,  2.50s/it]\u001b[A\n",
      "Epoch 6:  93%|▉| 147/158 [50:27<03:46, 20.60s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:27,  2.71s/it]\u001b[A\n",
      "Epoch 6:  94%|▉| 148/158 [50:30<03:24, 20.48s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.05s/it]\u001b[A\n",
      "Epoch 6:  94%|▉| 149/158 [50:31<03:03, 20.35s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:13<00:17,  2.15s/it]\u001b[A\n",
      "Epoch 6:  95%|▉| 150/158 [50:33<02:41, 20.23s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:15,  2.23s/it]\u001b[A\n",
      "Epoch 6:  96%|▉| 151/158 [50:36<02:20, 20.11s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:20<00:18,  3.15s/it]\u001b[A\n",
      "Epoch 6:  96%|▉| 152/158 [50:41<02:00, 20.01s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:22<00:13,  2.71s/it]\u001b[A\n",
      "Epoch 6:  97%|▉| 153/158 [50:43<01:39, 19.89s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:28<00:14,  3.63s/it]\u001b[A\n",
      "Epoch 6:  97%|▉| 154/158 [50:48<01:19, 19.80s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:28<00:08,  2.78s/it]\u001b[A\n",
      "Epoch 6:  98%|▉| 155/158 [50:49<00:59, 19.67s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:30<00:04,  2.50s/it]\u001b[A\n",
      "Epoch 6:  99%|▉| 156/158 [50:51<00:39, 19.56s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:34<00:02,  2.82s/it]\u001b[A\n",
      "Epoch 6:  99%|▉| 157/158 [50:55<00:19, 19.46s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:36<00:00,  2.69s/it]\u001b[A\n",
      "Epoch 6: 100%|█| 158/158 [50:57<00:00, 19.35s/it, loss=0.355, v_num=5, train_auc_step=0.689, train_prc_step=0.204, val_l\u001b[A\n",
      "Epoch 7:  91%|▉| 144/158 [50:39<04:55, 21.11s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:15,  1.16s/it]\u001b[A\n",
      "Epoch 7:  92%|▉| 145/158 [50:40<04:32, 20.97s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:25,  2.10s/it]\u001b[A\n",
      "Epoch 7:  92%|▉| 146/158 [50:43<04:10, 20.84s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:25,  2.36s/it]\u001b[A\n",
      "Epoch 7:  93%|▉| 147/158 [50:45<03:47, 20.72s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:26,  2.64s/it]\u001b[A\n",
      "Epoch 7:  94%|▉| 148/158 [50:48<03:26, 20.60s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.03s/it]\u001b[A\n",
      "Epoch 7:  94%|▉| 149/158 [50:49<03:04, 20.47s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:12<00:17,  2.14s/it]\u001b[A\n",
      "Epoch 7:  95%|▉| 150/158 [50:52<02:42, 20.35s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:16,  2.39s/it]\u001b[A\n",
      "Epoch 7:  96%|▉| 151/158 [50:55<02:21, 20.23s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:21<00:20,  3.45s/it]\u001b[A\n",
      "Epoch 7:  96%|▉| 152/158 [51:00<02:00, 20.14s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:23<00:14,  2.88s/it]\u001b[A\n",
      "Epoch 7:  97%|▉| 153/158 [51:02<01:40, 20.02s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:29<00:16,  4.07s/it]\u001b[A\n",
      "Epoch 7:  97%|▉| 154/158 [51:09<01:19, 19.93s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:31<00:09,  3.20s/it]\u001b[A\n",
      "Epoch 7:  98%|▉| 155/158 [51:10<00:59, 19.81s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:33<00:05,  2.85s/it]\u001b[A\n",
      "Epoch 7:  99%|▉| 156/158 [51:12<00:39, 19.70s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:37<00:03,  3.17s/it]\u001b[A\n",
      "Epoch 7:  99%|▉| 157/158 [51:16<00:19, 19.60s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:39<00:00,  3.00s/it]\u001b[A\n",
      "Epoch 7: 100%|█| 158/158 [51:19<00:00, 19.49s/it, loss=0.373, v_num=5, train_auc_step=0.654, train_prc_step=0.227, val_l\u001b[A\n",
      "Epoch 8:  91%|▉| 144/158 [50:35<04:55, 21.08s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:16,  1.29s/it]\u001b[A\n",
      "Epoch 8:  92%|▉| 145/158 [50:37<04:32, 20.95s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:23,  1.99s/it]\u001b[A\n",
      "Epoch 8:  92%|▉| 146/158 [50:39<04:09, 20.82s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:26,  2.43s/it]\u001b[A\n",
      "Epoch 8:  93%|▉| 147/158 [50:42<03:47, 20.70s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:26,  2.70s/it]\u001b[A\n",
      "Epoch 8:  94%|▉| 148/158 [50:45<03:25, 20.58s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.06s/it]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8:  94%|▉| 149/158 [50:46<03:04, 20.45s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:13<00:17,  2.16s/it]\u001b[A\n",
      "Epoch 8:  95%|▉| 150/158 [50:48<02:42, 20.33s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:16,  2.36s/it]\u001b[A\n",
      "Epoch 8:  96%|▉| 151/158 [50:51<02:21, 20.21s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:20<00:18,  3.14s/it]\u001b[A\n",
      "Epoch 8:  96%|▉| 152/158 [50:56<02:00, 20.11s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:22<00:13,  2.64s/it]\u001b[A\n",
      "Epoch 8:  97%|▉| 153/158 [50:57<01:39, 19.99s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:28<00:14,  3.62s/it]\u001b[A\n",
      "Epoch 8:  97%|▉| 154/158 [51:03<01:19, 19.89s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:29<00:08,  2.82s/it]\u001b[A\n",
      "Epoch 8:  98%|▉| 155/158 [51:04<00:59, 19.77s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:30<00:04,  2.35s/it]\u001b[A\n",
      "Epoch 8:  99%|▉| 156/158 [51:06<00:39, 19.65s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:33<00:02,  2.66s/it]\u001b[A\n",
      "Epoch 8:  99%|▉| 157/158 [51:09<00:19, 19.55s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:35<00:00,  2.55s/it]\u001b[A\n",
      "Epoch 8: 100%|█| 158/158 [51:11<00:00, 19.44s/it, loss=0.345, v_num=5, train_auc_step=0.622, train_prc_step=0.224, val_l\u001b[A\n",
      "Epoch 9:  91%|▉| 144/158 [50:52<04:56, 21.20s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:17,  1.33s/it]\u001b[A\n",
      "Epoch 9:  92%|▉| 145/158 [50:53<04:33, 21.06s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:24,  2.05s/it]\u001b[A\n",
      "Epoch 9:  92%|▉| 146/158 [50:56<04:11, 20.93s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:26,  2.42s/it]\u001b[A\n",
      "Epoch 9:  93%|▉| 147/158 [50:59<03:48, 20.81s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:27,  2.74s/it]\u001b[A\n",
      "Epoch 9:  94%|▉| 148/158 [51:02<03:26, 20.69s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.07s/it]\u001b[A\n",
      "Epoch 9:  94%|▉| 149/158 [51:03<03:05, 20.56s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:13<00:17,  2.20s/it]\u001b[A\n",
      "Epoch 9:  95%|▉| 150/158 [51:05<02:43, 20.44s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:16<00:17,  2.48s/it]\u001b[A\n",
      "Epoch 9:  96%|▉| 151/158 [51:08<02:22, 20.32s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:21<00:20,  3.37s/it]\u001b[A\n",
      "Epoch 9:  96%|▉| 152/158 [51:14<02:01, 20.23s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:23<00:13,  2.75s/it]\u001b[A\n",
      "Epoch 9:  97%|▉| 153/158 [51:15<01:40, 20.10s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:29<00:15,  3.88s/it]\u001b[A\n",
      "Epoch 9:  97%|▉| 154/158 [51:22<01:20, 20.01s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:30<00:08,  2.94s/it]\u001b[A\n",
      "Epoch 9:  98%|▉| 155/158 [51:22<00:59, 19.89s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:32<00:05,  2.63s/it]\u001b[A\n",
      "Epoch 9:  99%|▉| 156/158 [51:24<00:39, 19.77s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:36<00:03,  3.06s/it]\u001b[A\n",
      "Epoch 9:  99%|▉| 157/158 [51:28<00:19, 19.67s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:38<00:00,  2.79s/it]\u001b[A\n",
      "Epoch 9: 100%|█| 158/158 [51:31<00:00, 19.56s/it, loss=0.311, v_num=5, train_auc_step=0.702, train_prc_step=0.238, val_l\u001b[A\n",
      "Epoch 10:  56%|▌| 88/158 [30:55<24:35, 21.08s/it, loss=0.342, v_num=5, train_auc_step=0.618, train_prc_step=0.232, val_l\u001b[A"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15:  91%|▉| 144/158 [50:33<04:54, 21.07s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:12,  1.07it/s]\u001b[A\n",
      "Epoch 15:  92%|▉| 145/158 [50:34<04:32, 20.93s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:25,  2.11s/it]\u001b[A\n",
      "Epoch 15:  92%|▉| 146/158 [50:37<04:09, 20.81s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:07<00:29,  2.72s/it]\u001b[A\n",
      "Epoch 15:  93%|▉| 147/158 [50:41<03:47, 20.69s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:10<00:27,  2.79s/it]\u001b[A\n",
      "Epoch 15:  94%|▉| 148/158 [50:44<03:25, 20.57s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:11<00:19,  2.19s/it]\u001b[A\n",
      "Epoch 15:  94%|▉| 149/158 [50:45<03:03, 20.44s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:14<00:18,  2.37s/it]\u001b[A\n",
      "Epoch 15:  95%|▉| 150/158 [50:47<02:42, 20.32s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:17<00:18,  2.59s/it]\u001b[A\n",
      "Epoch 15:  96%|▉| 151/158 [50:50<02:21, 20.20s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:22<00:20,  3.46s/it]\u001b[A\n",
      "Epoch 15:  96%|▉| 152/158 [50:56<02:00, 20.11s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:23<00:14,  2.85s/it]\u001b[A\n",
      "Epoch 15:  97%|▉| 153/158 [50:57<01:39, 19.99s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:30<00:16,  4.01s/it]\u001b[A\n",
      "Epoch 15:  97%|▉| 154/158 [51:04<01:19, 19.90s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:31<00:09,  3.17s/it]\u001b[A\n",
      "Epoch 15:  98%|▉| 155/158 [51:05<00:59, 19.78s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:33<00:05,  2.74s/it]\u001b[A\n",
      "Epoch 15:  99%|▉| 156/158 [51:07<00:39, 19.66s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:37<00:03,  3.16s/it]\u001b[A\n",
      "Epoch 15:  99%|▉| 157/158 [51:11<00:19, 19.56s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:40<00:00,  3.00s/it]\u001b[A\n",
      "Epoch 15: 100%|█| 158/158 [51:14<00:00, 19.46s/it, loss=0.371, v_num=5, train_auc_step=0.832, train_prc_step=0.335, val_\u001b[A\n",
      "Epoch 16:  91%|▉| 144/158 [51:39<05:01, 21.53s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:12,  1.03it/s]\u001b[A\n",
      "Epoch 16:  92%|▉| 145/158 [51:40<04:37, 21.38s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:04<00:27,  2.33s/it]\u001b[A\n",
      "Epoch 16:  92%|▉| 146/158 [51:44<04:15, 21.26s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:07<00:30,  2.76s/it]\u001b[A\n",
      "Epoch 16:  93%|▉| 147/158 [51:47<03:52, 21.14s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:10<00:29,  2.92s/it]\u001b[A\n",
      "Epoch 16:  94%|▉| 148/158 [51:50<03:30, 21.02s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:11<00:20,  2.24s/it]\u001b[A\n",
      "Epoch 16:  94%|▉| 149/158 [51:51<03:07, 20.88s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:13<00:17,  2.24s/it]\u001b[A\n",
      "Epoch 16:  95%|▉| 150/158 [51:53<02:46, 20.76s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:17<00:17,  2.54s/it]\u001b[A\n",
      "Epoch 16:  96%|▉| 151/158 [51:56<02:24, 20.64s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:22<00:20,  3.49s/it]\u001b[A\n",
      "Epoch 16:  96%|▉| 152/158 [52:02<02:03, 20.54s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:24<00:14,  2.95s/it]\u001b[A\n",
      "Epoch 16:  97%|▉| 153/158 [52:04<01:42, 20.42s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:30<00:15,  3.86s/it]\u001b[A\n",
      "Epoch 16:  97%|▉| 154/158 [52:10<01:21, 20.33s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:31<00:08,  2.94s/it]\u001b[A\n",
      "Epoch 16:  98%|▉| 155/158 [52:10<01:00, 20.20s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:33<00:05,  2.63s/it]\u001b[A\n",
      "Epoch 16:  99%|▉| 156/158 [52:12<00:40, 20.08s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:36<00:03,  3.01s/it]\u001b[A\n",
      "Epoch 16:  99%|▉| 157/158 [52:16<00:19, 19.98s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:39<00:00,  2.90s/it]\u001b[A\n",
      "Epoch 16: 100%|█| 158/158 [52:19<00:00, 19.87s/it, loss=0.375, v_num=5, train_auc_step=0.615, train_prc_step=0.225, val_\u001b[A\n",
      "Epoch 17:  91%|▉| 144/158 [49:38<04:49, 20.69s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:14,  1.08s/it]\u001b[A\n",
      "Epoch 17:  92%|▉| 145/158 [49:40<04:27, 20.55s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:04<00:27,  2.26s/it]\u001b[A\n",
      "Epoch 17:  92%|▉| 146/158 [49:43<04:05, 20.43s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:07<00:29,  2.70s/it]\u001b[A\n",
      "Epoch 17:  93%|▉| 147/158 [49:46<03:43, 20.32s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:10<00:27,  2.78s/it]\u001b[A\n",
      "Epoch 17:  94%|▉| 148/158 [49:49<03:21, 20.20s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:11<00:19,  2.20s/it]\u001b[A\n",
      "Epoch 17:  94%|▉| 149/158 [49:50<03:00, 20.07s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:14<00:19,  2.46s/it]\u001b[A\n",
      "Epoch 17:  95%|▉| 150/158 [49:53<02:39, 19.96s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:17<00:17,  2.55s/it]\u001b[A\n",
      "Epoch 17:  96%|▉| 151/158 [49:56<02:18, 19.84s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:22<00:20,  3.39s/it]\u001b[A\n",
      "Epoch 17:  96%|▉| 152/158 [50:01<01:58, 19.75s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:24<00:14,  2.91s/it]\u001b[A\n",
      "Epoch 17:  97%|▉| 153/158 [50:03<01:38, 19.63s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:30<00:15,  3.97s/it]\u001b[A\n",
      "Epoch 17:  97%|▉| 154/158 [50:09<01:18, 19.54s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:31<00:09,  3.05s/it]\u001b[A\n",
      "Epoch 17:  98%|▉| 155/158 [50:10<00:58, 19.42s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:33<00:05,  2.64s/it]\u001b[A\n",
      "Epoch 17:  99%|▉| 156/158 [50:12<00:38, 19.31s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:37<00:03,  3.13s/it]\u001b[A\n",
      "Epoch 17:  99%|▉| 157/158 [50:16<00:19, 19.21s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:40<00:00,  3.01s/it]\u001b[A\n",
      "Epoch 17: 100%|█| 158/158 [50:19<00:00, 19.11s/it, loss=0.347, v_num=5, train_auc_step=0.702, train_prc_step=0.375, val_\u001b[A\n",
      "Epoch 18:  91%|▉| 144/158 [48:01<04:40, 20.01s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:15,  1.21s/it]\u001b[A\n",
      "Epoch 18:  92%|▉| 145/158 [48:02<04:18, 19.88s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:04<00:28,  2.35s/it]\u001b[A\n",
      "Epoch 18:  92%|▉| 146/158 [48:05<03:57, 19.77s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:08<00:32,  2.97s/it]\u001b[A\n",
      "Epoch 18:  93%|▉| 147/158 [48:09<03:36, 19.66s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:11<00:29,  2.98s/it]\u001b[A\n",
      "Epoch 18:  94%|▉| 148/158 [48:12<03:15, 19.54s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:11<00:19,  2.18s/it]\u001b[A\n",
      "Epoch 18:  94%|▉| 149/158 [48:13<02:54, 19.42s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:14<00:18,  2.27s/it]\u001b[A\n",
      "Epoch 18:  95%|▉| 150/158 [48:15<02:34, 19.31s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:17<00:17,  2.48s/it]\u001b[A\n",
      "Epoch 18:  96%|▉| 151/158 [48:18<02:14, 19.20s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:22<00:20,  3.40s/it]\u001b[A\n",
      "Epoch 18:  96%|▉| 152/158 [48:24<01:54, 19.11s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:24<00:14,  2.89s/it]\u001b[A\n",
      "Epoch 18:  97%|▉| 153/158 [48:25<01:34, 18.99s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:30<00:15,  3.95s/it]\u001b[A\n",
      "Epoch 18:  97%|▉| 154/158 [48:32<01:15, 18.91s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:31<00:09,  3.05s/it]\u001b[A\n",
      "Epoch 18:  98%|▉| 155/158 [48:33<00:56, 18.79s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:33<00:05,  2.69s/it]\u001b[A\n",
      "Epoch 18:  99%|▉| 156/158 [48:35<00:37, 18.69s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:37<00:03,  3.05s/it]\u001b[A\n",
      "Epoch 18:  99%|▉| 157/158 [48:38<00:18, 18.59s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:39<00:00,  2.84s/it]\u001b[A\n",
      "Epoch 18: 100%|█| 158/158 [48:41<00:00, 18.49s/it, loss=0.33, v_num=5, train_auc_step=0.690, train_prc_step=0.176, val_l\u001b[A\n",
      "Epoch 19:  91%|▉| 144/158 [46:37<04:31, 19.42s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:12,  1.08it/s]\u001b[A\n",
      "Epoch 19:  92%|▉| 145/158 [46:38<04:10, 19.30s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:24,  2.00s/it]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19:  92%|▉| 146/158 [46:40<03:50, 19.18s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:27,  2.52s/it]\u001b[A\n",
      "Epoch 19:  93%|▉| 147/158 [46:43<03:29, 19.07s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:27,  2.70s/it]\u001b[A\n",
      "Epoch 19:  94%|▉| 148/158 [46:46<03:09, 18.97s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.02s/it]\u001b[A\n",
      "Epoch 19:  94%|▉| 149/158 [46:47<02:49, 18.84s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:12<00:17,  2.13s/it]\u001b[A\n",
      "Epoch 19:  95%|▉| 150/158 [46:50<02:29, 18.73s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:16,  2.33s/it]\u001b[A\n",
      "Epoch 19:  96%|▉| 151/158 [46:52<02:10, 18.63s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:20<00:19,  3.19s/it]\u001b[A\n",
      "Epoch 19:  96%|▉| 152/158 [46:57<01:51, 18.54s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:22<00:13,  2.69s/it]\u001b[A\n",
      "Epoch 19:  97%|▉| 153/158 [46:59<01:32, 18.43s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:27<00:14,  3.55s/it]\u001b[A\n",
      "Epoch 19:  97%|▉| 154/158 [47:04<01:13, 18.34s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:28<00:08,  2.70s/it]\u001b[A\n",
      "Epoch 19:  98%|▉| 155/158 [47:05<00:54, 18.23s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:30<00:04,  2.48s/it]\u001b[A\n",
      "Epoch 19:  99%|▉| 156/158 [47:07<00:36, 18.13s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:34<00:02,  2.84s/it]\u001b[A\n",
      "Epoch 19:  99%|▉| 157/158 [47:11<00:18, 18.03s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:37<00:00,  2.84s/it]\u001b[A\n",
      "Epoch 19: 100%|█| 158/158 [47:14<00:00, 17.94s/it, loss=0.352, v_num=5, train_auc_step=0.723, train_prc_step=0.174, val_\u001b[A\n",
      "Epoch 20:  91%|▉| 144/158 [45:23<04:24, 18.92s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:10,  1.19it/s]\u001b[A\n",
      "Epoch 20:  92%|▉| 145/158 [45:24<04:04, 18.79s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:19,  1.64s/it]\u001b[A\n",
      "Epoch 20:  92%|▉| 146/158 [45:26<03:44, 18.68s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:05<00:22,  2.05s/it]\u001b[A\n",
      "Epoch 20:  93%|▉| 147/158 [45:29<03:24, 18.57s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:07<00:21,  2.19s/it]\u001b[A\n",
      "Epoch 20:  94%|▉| 148/158 [45:31<03:04, 18.46s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:08<00:14,  1.65s/it]\u001b[A\n",
      "Epoch 20:  94%|▉| 149/158 [45:32<02:45, 18.34s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:10<00:13,  1.64s/it]\u001b[A\n",
      "Epoch 20:  95%|▉| 150/158 [45:34<02:25, 18.23s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:12<00:13,  1.89s/it]\u001b[A\n",
      "Epoch 20:  96%|▉| 151/158 [45:36<02:06, 18.12s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:16<00:15,  2.56s/it]\u001b[A\n",
      "Epoch 20:  96%|▉| 152/158 [45:40<01:48, 18.03s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:17<00:10,  2.16s/it]\u001b[A\n",
      "Epoch 20:  97%|▉| 153/158 [45:41<01:29, 17.92s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:22<00:12,  3.01s/it]\u001b[A\n",
      "Epoch 20:  97%|▉| 154/158 [45:46<01:11, 17.84s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:23<00:07,  2.34s/it]\u001b[A\n",
      "Epoch 20:  98%|▉| 155/158 [45:47<00:53, 17.73s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:25<00:04,  2.03s/it]\u001b[A\n",
      "Epoch 20:  99%|▉| 156/158 [45:48<00:35, 17.62s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:28<00:02,  2.45s/it]\u001b[A\n",
      "Epoch 20:  99%|▉| 157/158 [45:52<00:17, 17.53s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:30<00:00,  2.28s/it]\u001b[A\n",
      "Epoch 20: 100%|█| 158/158 [45:54<00:00, 17.43s/it, loss=0.364, v_num=5, train_auc_step=0.681, train_prc_step=0.263, val_\u001b[A\n",
      "Epoch 21:  91%|▉| 144/158 [47:24<04:36, 19.75s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:13,  1.02s/it]\u001b[A\n",
      "Epoch 21:  92%|▉| 145/158 [47:25<04:15, 19.62s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:22,  1.92s/it]\u001b[A\n",
      "Epoch 21:  92%|▉| 146/158 [47:27<03:54, 19.51s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:28,  2.60s/it]\u001b[A\n",
      "Epoch 21:  93%|▉| 147/158 [47:31<03:33, 19.40s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:27,  2.74s/it]\u001b[A\n",
      "Epoch 21:  94%|▉| 148/158 [47:34<03:12, 19.28s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.11s/it]\u001b[A\n",
      "Epoch 21:  94%|▉| 149/158 [47:35<02:52, 19.16s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:13<00:17,  2.17s/it]\u001b[A\n",
      "Epoch 21:  95%|▉| 150/158 [47:37<02:32, 19.05s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:16<00:16,  2.38s/it]\u001b[A\n",
      "Epoch 21:  96%|▉| 151/158 [47:40<02:12, 18.94s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:21<00:19,  3.24s/it]\u001b[A\n",
      "Epoch 21:  96%|▉| 152/158 [47:45<01:53, 18.85s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:22<00:14,  2.81s/it]\u001b[A\n",
      "Epoch 21:  97%|▉| 153/158 [47:47<01:33, 18.74s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:28<00:14,  3.56s/it]\u001b[A\n",
      "Epoch 21:  97%|▉| 154/158 [47:52<01:14, 18.65s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:29<00:08,  2.86s/it]\u001b[A\n",
      "Epoch 21:  98%|▉| 155/158 [47:53<00:55, 18.54s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:30<00:04,  2.43s/it]\u001b[A\n",
      "Epoch 21:  99%|▉| 156/158 [47:55<00:36, 18.43s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:34<00:02,  2.69s/it]\u001b[A\n",
      "Epoch 21:  99%|▉| 157/158 [47:58<00:18, 18.33s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:36<00:00,  2.52s/it]\u001b[A\n",
      "Epoch 21: 100%|█| 158/158 [48:00<00:00, 18.23s/it, loss=0.389, v_num=5, train_auc_step=0.583, train_prc_step=0.088, val_\u001b[A\n",
      "Epoch 22:  91%|▉| 144/158 [46:47<04:32, 19.50s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:14,  1.14s/it]\u001b[A\n",
      "Epoch 22:  92%|▉| 145/158 [46:48<04:11, 19.37s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:05<00:38,  3.25s/it]\u001b[A\n",
      "Epoch 22:  92%|▉| 146/158 [46:53<03:51, 19.27s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:08<00:32,  2.99s/it]\u001b[A\n",
      "Epoch 22:  93%|▉| 147/158 [46:56<03:30, 19.16s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:11<00:28,  2.83s/it]\u001b[A\n",
      "Epoch 22:  94%|▉| 148/158 [46:58<03:10, 19.05s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:11<00:18,  2.06s/it]\u001b[A\n",
      "Epoch 22:  94%|▉| 149/158 [46:59<02:50, 18.92s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:13<00:15,  1.95s/it]\u001b[A\n",
      "Epoch 22:  95%|▉| 150/158 [47:01<02:30, 18.81s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:16<00:15,  2.21s/it]\u001b[A\n",
      "Epoch 22:  96%|▉| 151/158 [47:04<02:10, 18.70s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:20<00:17,  2.86s/it]\u001b[A\n",
      "Epoch 22:  96%|▉| 152/158 [47:08<01:51, 18.61s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:21<00:11,  2.38s/it]\u001b[A\n",
      "Epoch 22:  97%|▉| 153/158 [47:09<01:32, 18.49s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:26<00:12,  3.11s/it]\u001b[A\n",
      "Epoch 22:  97%|▉| 154/158 [47:14<01:13, 18.40s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:27<00:07,  2.46s/it]\u001b[A\n",
      "Epoch 22:  98%|▉| 155/158 [47:15<00:54, 18.29s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:29<00:04,  2.16s/it]\u001b[A\n",
      "Epoch 22:  99%|▉| 156/158 [47:16<00:36, 18.18s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:32<00:02,  2.45s/it]\u001b[A\n",
      "Epoch 22:  99%|▉| 157/158 [47:19<00:18, 18.09s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:34<00:00,  2.37s/it]\u001b[A\n",
      "Epoch 22: 100%|█| 158/158 [47:22<00:00, 17.99s/it, loss=0.366, v_num=5, train_auc_step=0.697, train_prc_step=0.199, val_\u001b[A\n",
      "Epoch 23:  91%|▉| 144/158 [44:40<04:20, 18.61s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:13,  1.00s/it]\u001b[A\n",
      "Epoch 23:  92%|▉| 145/158 [44:41<04:00, 18.49s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:02<00:18,  1.58s/it]\u001b[A\n",
      "Epoch 23:  92%|▉| 146/158 [44:43<03:40, 18.38s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:04<00:19,  1.75s/it]\u001b[A\n",
      "Epoch 23:  93%|▉| 147/158 [44:44<03:20, 18.27s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:07<00:19,  1.93s/it]\u001b[A\n",
      "Epoch 23:  94%|▉| 148/158 [44:47<03:01, 18.16s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:07<00:13,  1.51s/it]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23:  94%|▉| 149/158 [44:47<02:42, 18.04s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:10<00:14,  1.77s/it]\u001b[A\n",
      "Epoch 23:  95%|▉| 150/158 [44:50<02:23, 17.93s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:11<00:12,  1.77s/it]\u001b[A\n",
      "Epoch 23:  96%|▉| 151/158 [44:52<02:04, 17.83s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:15<00:14,  2.40s/it]\u001b[A\n",
      "Epoch 23:  96%|▉| 152/158 [44:55<01:46, 17.74s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:16<00:10,  2.04s/it]\u001b[A\n",
      "Epoch 23:  97%|▉| 153/158 [44:57<01:28, 17.63s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:22<00:12,  3.07s/it]\u001b[A\n",
      "Epoch 23:  97%|▉| 154/158 [45:02<01:10, 17.55s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:23<00:07,  2.39s/it]\u001b[A\n",
      "Epoch 23:  98%|▉| 155/158 [45:03<00:52, 17.44s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:24<00:04,  2.08s/it]\u001b[A\n",
      "Epoch 23:  99%|▉| 156/158 [45:04<00:34, 17.34s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:27<00:02,  2.19s/it]\u001b[A\n",
      "Epoch 23:  99%|▉| 157/158 [45:07<00:17, 17.24s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:28<00:00,  2.12s/it]\u001b[A\n",
      "Epoch 23: 100%|█| 158/158 [45:09<00:00, 17.15s/it, loss=0.393, v_num=5, train_auc_step=0.696, train_prc_step=0.199, val_\u001b[A\n",
      "Epoch 24:  91%|▉| 144/158 [43:33<04:14, 18.15s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:13,  1.04s/it]\u001b[A\n",
      "Epoch 24:  92%|▉| 145/158 [43:35<03:54, 18.03s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:24,  2.05s/it]\u001b[A\n",
      "Epoch 24:  92%|▉| 146/158 [43:37<03:35, 17.93s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:26,  2.43s/it]\u001b[A\n",
      "Epoch 24:  93%|▉| 147/158 [43:40<03:16, 17.83s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:26,  2.69s/it]\u001b[A\n",
      "Epoch 24:  94%|▉| 148/158 [43:43<02:57, 17.73s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.01s/it]\u001b[A\n",
      "Epoch 24:  94%|▉| 149/158 [43:44<02:38, 17.61s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:12<00:16,  2.02s/it]\u001b[A\n",
      "Epoch 24:  95%|▉| 150/158 [43:46<02:20, 17.51s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:15,  2.28s/it]\u001b[A\n",
      "Epoch 24:  96%|▉| 151/158 [43:49<02:01, 17.41s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:20<00:18,  3.08s/it]\u001b[A\n",
      "Epoch 24:  96%|▉| 152/158 [43:54<01:43, 17.33s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:21<00:13,  2.67s/it]\u001b[A\n",
      "Epoch 24:  97%|▉| 153/158 [43:55<01:26, 17.23s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:28<00:15,  3.76s/it]\u001b[A\n",
      "Epoch 24:  97%|▉| 154/158 [44:02<01:08, 17.16s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:29<00:08,  2.92s/it]\u001b[A\n",
      "Epoch 24:  98%|▉| 155/158 [44:03<00:51, 17.05s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:30<00:04,  2.47s/it]\u001b[A\n",
      "Epoch 24:  99%|▉| 156/158 [44:04<00:33, 16.95s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:34<00:02,  2.98s/it]\u001b[A\n",
      "Epoch 24:  99%|▉| 157/158 [44:08<00:16, 16.87s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:37<00:00,  2.77s/it]\u001b[A\n",
      "Epoch 24: 100%|█| 158/158 [44:11<00:00, 16.78s/it, loss=0.355, v_num=5, train_auc_step=0.770, train_prc_step=0.321, val_\u001b[A\n",
      "Epoch 25:  91%|▉| 144/158 [58:28<05:41, 24.36s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:01<00:15,  1.17s/it]\u001b[A\n",
      "Epoch 25:  92%|▉| 145/158 [58:29<05:14, 24.20s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:20,  1.68s/it]\u001b[A\n",
      "Epoch 25:  92%|▉| 146/158 [58:31<04:48, 24.05s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:05<00:22,  2.01s/it]\u001b[A\n",
      "Epoch 25:  93%|▉| 147/158 [58:34<04:22, 23.91s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:08<00:21,  2.19s/it]\u001b[A\n",
      "Epoch 25:  94%|▉| 148/158 [58:36<03:57, 23.76s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:09<00:15,  1.73s/it]\u001b[A\n",
      "Epoch 25:  94%|▉| 149/158 [58:37<03:32, 23.61s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:10<00:14,  1.80s/it]\u001b[A\n",
      "Epoch 25:  95%|▉| 150/158 [58:39<03:07, 23.46s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:13<00:13,  1.91s/it]\u001b[A\n",
      "Epoch 25:  96%|▉| 151/158 [58:41<02:43, 23.32s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:17<00:15,  2.66s/it]\u001b[A\n",
      "Epoch 25:  96%|▉| 152/158 [58:45<02:19, 23.20s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:18<00:11,  2.23s/it]\u001b[A\n",
      "Epoch 25:  97%|▉| 153/158 [58:47<01:55, 23.05s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:23<00:12,  3.16s/it]\u001b[A\n",
      "Epoch 25:  97%|▉| 154/158 [58:52<01:31, 22.94s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:24<00:07,  2.45s/it]\u001b[A\n",
      "Epoch 25:  98%|▉| 155/158 [58:53<01:08, 22.79s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:26<00:04,  2.12s/it]\u001b[A\n",
      "Epoch 25:  99%|▉| 156/158 [58:54<00:45, 22.66s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:29<00:02,  2.40s/it]\u001b[A\n",
      "Epoch 25:  99%|▉| 157/158 [58:57<00:22, 22.53s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:31<00:00,  2.26s/it]\u001b[A\n",
      "Epoch 25: 100%|█| 158/158 [58:59<00:00, 22.40s/it, loss=0.347, v_num=5, train_auc_step=0.793, train_prc_step=0.279, val_\u001b[A\n",
      "Epoch 26:  91%|▉| 144/158 [46:59<04:34, 19.58s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:12,  1.05it/s]\u001b[A\n",
      "Epoch 26:  92%|▉| 145/158 [47:00<04:12, 19.45s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:22,  1.91s/it]\u001b[A\n",
      "Epoch 26:  92%|▉| 146/158 [47:03<03:52, 19.34s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:26,  2.39s/it]\u001b[A\n",
      "Epoch 26:  93%|▉| 147/158 [47:06<03:31, 19.23s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:09<00:27,  2.76s/it]\u001b[A\n",
      "Epoch 26:  94%|▉| 148/158 [47:09<03:11, 19.12s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:10<00:18,  2.03s/it]\u001b[A\n",
      "Epoch 26:  94%|▉| 149/158 [47:10<02:50, 18.99s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:12<00:17,  2.15s/it]\u001b[A\n",
      "Epoch 26:  95%|▉| 150/158 [47:12<02:31, 18.88s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:15<00:16,  2.32s/it]\u001b[A\n",
      "Epoch 26:  96%|▉| 151/158 [47:15<02:11, 18.78s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:21<00:20,  3.36s/it]\u001b[A\n",
      "Epoch 26:  96%|▉| 152/158 [47:20<01:52, 18.69s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:22<00:13,  2.75s/it]\u001b[A\n",
      "Epoch 26:  97%|▉| 153/158 [47:22<01:32, 18.58s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:29<00:15,  3.88s/it]\u001b[A\n",
      "Epoch 26:  97%|▉| 154/158 [47:28<01:13, 18.50s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:29<00:08,  2.93s/it]\u001b[A\n",
      "Epoch 26:  98%|▉| 155/158 [47:29<00:55, 18.38s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:31<00:04,  2.49s/it]\u001b[A\n",
      "Epoch 26:  99%|▉| 156/158 [47:30<00:36, 18.28s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:35<00:02,  2.88s/it]\u001b[A\n",
      "Epoch 26:  99%|▉| 157/158 [47:34<00:18, 18.18s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:37<00:00,  2.80s/it]\u001b[A\n",
      "Epoch 26: 100%|█| 158/158 [47:37<00:00, 18.08s/it, loss=0.364, v_num=5, train_auc_step=0.746, train_prc_step=0.290, val_\u001b[A\n",
      "Epoch 27:  91%|▉| 144/158 [44:48<04:21, 18.67s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:12,  1.03it/s]\u001b[A\n",
      "Epoch 27:  92%|▉| 145/158 [44:49<04:01, 18.55s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:22,  1.90s/it]\u001b[A\n",
      "Epoch 27:  92%|▉| 146/158 [44:52<03:41, 18.44s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:06<00:25,  2.28s/it]\u001b[A\n",
      "Epoch 27:  93%|▉| 147/158 [44:55<03:21, 18.33s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:08<00:22,  2.27s/it]\u001b[A\n",
      "Epoch 27:  94%|▉| 148/158 [44:57<03:02, 18.23s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:09<00:16,  1.88s/it]\u001b[A\n",
      "Epoch 27:  94%|▉| 149/158 [44:58<02:42, 18.11s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:11<00:15,  1.88s/it]\u001b[A\n",
      "Epoch 27:  95%|▉| 150/158 [45:00<02:24, 18.00s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:14<00:14,  2.07s/it]\u001b[A\n",
      "Epoch 27:  96%|▉| 151/158 [45:02<02:05, 17.90s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:18<00:16,  2.68s/it]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27:  96%|▉| 152/158 [45:06<01:46, 17.81s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:19<00:11,  2.21s/it]\u001b[A\n",
      "Epoch 27:  97%|▉| 153/158 [45:08<01:28, 17.70s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:24<00:12,  3.12s/it]\u001b[A\n",
      "Epoch 27:  97%|▉| 154/158 [45:13<01:10, 17.62s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:25<00:07,  2.43s/it]\u001b[A\n",
      "Epoch 27:  98%|▉| 155/158 [45:14<00:52, 17.51s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:27<00:04,  2.27s/it]\u001b[A\n",
      "Epoch 27:  99%|▉| 156/158 [45:15<00:34, 17.41s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:29<00:02,  2.44s/it]\u001b[A\n",
      "Epoch 27:  99%|▉| 157/158 [45:18<00:17, 17.32s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:31<00:00,  2.25s/it]\u001b[A\n",
      "Epoch 27: 100%|█| 158/158 [45:20<00:00, 17.22s/it, loss=0.346, v_num=5, train_auc_step=0.847, train_prc_step=0.224, val_\u001b[A\n",
      "Epoch 28:  91%|▉| 144/158 [44:13<04:18, 18.43s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:12,  1.05it/s]\u001b[A\n",
      "Epoch 28:  92%|▉| 145/158 [44:14<03:58, 18.31s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:22,  1.84s/it]\u001b[A\n",
      "Epoch 28:  92%|▉| 146/158 [44:17<03:38, 18.20s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:05<00:21,  1.98s/it]\u001b[A\n",
      "Epoch 28:  93%|▉| 147/158 [44:19<03:19, 18.09s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:08<00:22,  2.22s/it]\u001b[A\n",
      "Epoch 28:  94%|▉| 148/158 [44:22<02:59, 17.99s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:09<00:15,  1.77s/it]\u001b[A\n",
      "Epoch 28:  94%|▉| 149/158 [44:23<02:40, 17.87s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:10<00:14,  1.76s/it]\u001b[A\n",
      "Epoch 28:  95%|▉| 150/158 [44:24<02:22, 17.77s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:13<00:13,  1.94s/it]\u001b[A\n",
      "Epoch 28:  96%|▉| 151/158 [44:27<02:03, 17.66s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:17<00:16,  2.67s/it]\u001b[A\n",
      "Epoch 28:  96%|▉| 152/158 [44:31<01:45, 17.57s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:18<00:11,  2.28s/it]\u001b[A\n",
      "Epoch 28:  97%|▉| 153/158 [44:32<01:27, 17.47s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:24<00:12,  3.21s/it]\u001b[A\n",
      "Epoch 28:  97%|▉| 154/158 [44:38<01:09, 17.39s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:25<00:07,  2.59s/it]\u001b[A\n",
      "Epoch 28:  98%|▉| 155/158 [44:39<00:51, 17.29s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:26<00:04,  2.22s/it]\u001b[A\n",
      "Epoch 28:  99%|▉| 156/158 [44:40<00:34, 17.18s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:29<00:02,  2.41s/it]\u001b[A\n",
      "Epoch 28:  99%|▉| 157/158 [44:43<00:17, 17.09s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:31<00:00,  2.23s/it]\u001b[A\n",
      "Epoch 28: 100%|█| 158/158 [44:45<00:00, 17.00s/it, loss=0.334, v_num=5, train_auc_step=0.783, train_prc_step=0.254, val_\u001b[A\n",
      "Epoch 29:  91%|▉| 144/158 [46:55<04:33, 19.56s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation: 0it [00:00, ?it/s]\u001b[A\n",
      "Validation:   0%|                                                                                | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   0%|                                                                   | 0/14 [00:00<?, ?it/s]\u001b[A\n",
      "Validation DataLoader 0:   7%|████▏                                                      | 1/14 [00:00<00:12,  1.03it/s]\u001b[A\n",
      "Epoch 29:  92%|▉| 145/158 [46:56<04:12, 19.43s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  14%|████████▍                                                  | 2/14 [00:03<00:23,  2.00s/it]\u001b[A\n",
      "Epoch 29:  92%|▉| 146/158 [46:59<03:51, 19.31s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  21%|████████████▋                                              | 3/14 [00:05<00:23,  2.11s/it]\u001b[A\n",
      "Epoch 29:  93%|▉| 147/158 [47:01<03:31, 19.20s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  29%|████████████████▊                                          | 4/14 [00:08<00:22,  2.22s/it]\u001b[A\n",
      "Epoch 29:  94%|▉| 148/158 [47:04<03:10, 19.08s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  36%|█████████████████████                                      | 5/14 [00:09<00:16,  1.80s/it]\u001b[A\n",
      "Epoch 29:  94%|▉| 149/158 [47:05<02:50, 18.96s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  43%|█████████████████████████▎                                 | 6/14 [00:11<00:14,  1.80s/it]\u001b[A\n",
      "Epoch 29:  95%|▉| 150/158 [47:07<02:30, 18.85s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  50%|█████████████████████████████▌                             | 7/14 [00:13<00:13,  1.92s/it]\u001b[A\n",
      "Epoch 29:  96%|▉| 151/158 [47:09<02:11, 18.74s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  57%|█████████████████████████████████▋                         | 8/14 [00:17<00:15,  2.61s/it]\u001b[A\n",
      "Epoch 29:  96%|▉| 152/158 [47:13<01:51, 18.64s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  64%|█████████████████████████████████████▉                     | 9/14 [00:18<00:11,  2.26s/it]\u001b[A\n",
      "Epoch 29:  97%|▉| 153/158 [47:14<01:32, 18.53s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation DataLoader 0:  71%|█████████████████████████████████████████▍                | 10/14 [00:24<00:12,  3.17s/it]\u001b[A\n",
      "Epoch 29:  97%|▉| 154/158 [47:20<01:13, 18.44s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  79%|█████████████████████████████████████████████▌            | 11/14 [00:25<00:07,  2.49s/it]\u001b[A\n",
      "Epoch 29:  98%|▉| 155/158 [47:21<00:54, 18.33s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  86%|█████████████████████████████████████████████████▋        | 12/14 [00:26<00:04,  2.12s/it]\u001b[A\n",
      "Epoch 29:  99%|▉| 156/158 [47:22<00:36, 18.22s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0:  93%|█████████████████████████████████████████████████████▊    | 13/14 [00:29<00:02,  2.37s/it]\u001b[A\n",
      "Epoch 29:  99%|▉| 157/158 [47:25<00:18, 18.12s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Validation DataLoader 0: 100%|██████████████████████████████████████████████████████████| 14/14 [00:31<00:00,  2.25s/it]\u001b[A\n",
      "Epoch 29: 100%|█| 158/158 [47:27<00:00, 18.02s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n",
      "Epoch 29: 100%|█| 158/158 [47:27<00:00, 18.02s/it, loss=0.337, v_num=5, train_auc_step=0.671, train_prc_step=0.230, val_\u001b[A\n"
     ]
    }
   ],
   "source": [
    "trainer.fit(model, train_loader, test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2f0d504",
   "metadata": {},
   "source": [
    "Test:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2ac4ffa8",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(\n",
    "    torch.load(f'./model/{log_name}/model_{fold}.ckpt')['state_dict'],\n",
    ")\n",
    "trainer = pl.Trainer(gpus=[gpu],logger=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4c252003",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing DataLoader 0: 100%|█████████████████████████████████████████████████████████████| 14/14 [00:02<00:00,  5.56it/s]\n",
      "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
      "       Test metric             DataLoader 0\n",
      "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
      "       test_AUPRC           0.18235257267951965\n",
      "       test_AUROC           0.6983789801597595\n",
      "        test_bacc           0.6147208213806152\n",
      "         test_f1            0.2584644854068756\n",
      "        test_loss           0.28255847096443176\n",
      "        test_mcc            0.17051027715206146\n",
      "     test_precision         0.19350232183933258\n",
      "       test_recall          0.38908830285072327\n",
      "     test_threshold         0.1518811732530594\n",
      "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n"
     ]
    }
   ],
   "source": [
    "result = trainer.test(model,test_loader)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}