neural_network_forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "\n",
    "DATA_DIR = 'data/train-scene classification'\n",
    "MODEL_DIR = \"saved_model/\"\n",
    "\n",
    "IMG_DIR = os.path.join(DATA_DIR, 'train')\n",
    "\n",
    "TRAIN_CSV = os.path.join(DATA_DIR, 'train.csv')\n",
    "TEST_CSV = \"data/test_WyRytb0.csv\"\n",
    "\n",
    "RESULTS = \"results/\"\n",
    "\n",
    "image = 'image_name'\n",
    "label = 'label'\n",
    "\n",
    "H = W = 150\n",
    "ORIG_H = ORIG_W = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>resnet34train_transforms3_7</th>\n",
       "      <th>resnet34train_transforms3_5</th>\n",
       "      <th>resnet34train_transforms2_6</th>\n",
       "      <th>resnet34no_transforms_7</th>\n",
       "      <th>resnet34train_transforms_8</th>\n",
       "      <th>resnet34train_transforms2_1</th>\n",
       "      <th>resnet34train_transforms_9</th>\n",
       "      <th>resnet34train_transforms3_9</th>\n",
       "      <th>resnet34train_transforms_6</th>\n",
       "      <th>...</th>\n",
       "      <th>resnet34no_transforms_2</th>\n",
       "      <th>resnet34train_transforms_7</th>\n",
       "      <th>resnet34no_transforms_8</th>\n",
       "      <th>resnet34train_transforms2_5</th>\n",
       "      <th>resnet34train_transforms_3</th>\n",
       "      <th>resnet34train_transforms_5</th>\n",
       "      <th>resnet34train_transforms_2</th>\n",
       "      <th>resnet34no_transforms_6</th>\n",
       "      <th>resnet34train_transforms2_8</th>\n",
       "      <th>resnet34no_transforms_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.jpg</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.jpg</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.jpg</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14.jpg</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  image_name  resnet34train_transforms3_7  resnet34train_transforms3_5  \\\n",
       "0      3.jpg                            5                            5   \n",
       "1      5.jpg                            0                            0   \n",
       "2      6.jpg                            4                            4   \n",
       "3     11.jpg                            2                            2   \n",
       "4     14.jpg                            5                            5   \n",
       "\n",
       "   resnet34train_transforms2_6  resnet34no_transforms_7  \\\n",
       "0                            5                        5   \n",
       "1                            0                        0   \n",
       "2                            4                        4   \n",
       "3                            3                        2   \n",
       "4                            5                        5   \n",
       "\n",
       "   resnet34train_transforms_8  resnet34train_transforms2_1  \\\n",
       "0                           5                            5   \n",
       "1                           0                            0   \n",
       "2                           4                            4   \n",
       "3                           2                            2   \n",
       "4                           5                            5   \n",
       "\n",
       "   resnet34train_transforms_9  resnet34train_transforms3_9  \\\n",
       "0                           5                            5   \n",
       "1                           0                            0   \n",
       "2                           4                            4   \n",
       "3                           2                            4   \n",
       "4                           5                            5   \n",
       "\n",
       "   resnet34train_transforms_6           ...             \\\n",
       "0                           5           ...              \n",
       "1                           0           ...              \n",
       "2                           4           ...              \n",
       "3                           2           ...              \n",
       "4                           5           ...              \n",
       "\n",
       "   resnet34no_transforms_2  resnet34train_transforms_7  \\\n",
       "0                        5                           5   \n",
       "1                        0                           0   \n",
       "2                        4                           4   \n",
       "3                        2                           2   \n",
       "4                        5                           5   \n",
       "\n",
       "   resnet34no_transforms_8  resnet34train_transforms2_5  \\\n",
       "0                        5                            5   \n",
       "1                        4                            0   \n",
       "2                        4                            4   \n",
       "3                        2                            2   \n",
       "4                        5                            5   \n",
       "\n",
       "   resnet34train_transforms_3  resnet34train_transforms_5  \\\n",
       "0                           5                           5   \n",
       "1                           0                           0   \n",
       "2                           4                           4   \n",
       "3                           2                           2   \n",
       "4                           5                           5   \n",
       "\n",
       "   resnet34train_transforms_2  resnet34no_transforms_6  \\\n",
       "0                           5                        5   \n",
       "1                           0                        0   \n",
       "2                           4                        4   \n",
       "3                           2                        3   \n",
       "4                           5                        5   \n",
       "\n",
       "   resnet34train_transforms2_8  resnet34no_transforms_9  \n",
       "0                            5                        5  \n",
       "1                            0                        0  \n",
       "2                            4                        4  \n",
       "3                            2                        2  \n",
       "4                            5                        5  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data = pd.read_csv(TEST_CSV)\n",
    "for folder in [\"resnet34/\"]:#, \"resnet50/\", \"se_resnet50/\"]:\n",
    "    for res in os.listdir(RESULTS+folder):\n",
    "        df = pd.read_csv(RESULTS+\"resnet34/\"+res)\n",
    "        test_data[folder[:-1]+res.split(\".\")[0]]=df[\"label\"]\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data.corr().to_csv(\"correlation_120.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mukesh/miniconda3/envs/tgs_torch/lib/python3.6/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.jpg</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.jpg</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.jpg</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.jpg</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14.jpg</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  image_name  label\n",
       "0      3.jpg    5.0\n",
       "1      5.jpg    0.0\n",
       "2      6.jpg    4.0\n",
       "3     11.jpg    2.0\n",
       "4     14.jpg    5.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = test_data[[\"image_name\"]]\n",
    "x= list(test_data.mode(axis=1,numeric_only=True)[0])\n",
    "sub[\"label\"] = x\n",
    "sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv(RESULTS+\"resnet34/ensemble_rnet34.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "pickle_dump = []\n",
    "for res in os.listdir(\"numpy_outs/resnet34/\"):\n",
    "    pickle_dump.append(pickle.load(open(\"numpy_outs/resnet34/{}\".format(res), \"rb\")))\n",
    "len(pickle_dump)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7301"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pickle_dump[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7301"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for pickles in pickle_dump:\n",
    "x = [sum(pik) for pik in zip(*pickle_dump)]  \n",
    "len(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7301"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "gth = [np.argmax(y) for y in x]\n",
    "len(gth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(TEST_CSV)\n",
    "test_data[\"label\"]=gth\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data.to_csv(\"numpy_pickle_rnet34.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}