02_18-07-2018_srm_iwildcam-split-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwd = Path.cwd()\n",
    "cwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = cwd/'data'\n",
    "data_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/val.csv'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/test_information.json'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/train_val'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/train_annotations.json'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/val_annotations.json'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/valid'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/train'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/train.csv')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(data_dir.glob('*'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_animal = data_dir/'train'/'animal'\n",
    "train_no_animal = data_dir/'train'/'no_animal'\n",
    "val_animal = data_dir/'valid'/'animal'\n",
    "val_no_animal = data_dir/'valid'/'no_animal'\n",
    "\n",
    "train_animal.mkdir(parents=True, exist_ok=True)\n",
    "train_no_animal.mkdir(parents=True, exist_ok=True)\n",
    "val_animal.mkdir(parents=True, exist_ok=True)\n",
    "val_no_animal.mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/val.csv'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/test_information.json'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/train_val'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/train_annotations.json'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/val_annotations.json'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/valid'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/train'),\n",
       " PosixPath('/home/srm/Desktop/projects/microsoft/iwildcam-torch/data/train.csv')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(data_dir.glob('*'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(data_dir/'train.csv')\n",
    "val_df = pd.read_csv(data_dir/'val.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>location</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_val/5968c0f9-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>26</td>\n",
       "      <td>no_animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_val/5a0b016f-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>26</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_val/59b93afb-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>38</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_val/59641f56-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>38</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_val/5a1e5306-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>33</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           file_name  location category_id\n",
       "0  train_val/5968c0f9-23d2-11e8-a6a3-ec086b02610b...        26   no_animal\n",
       "1  train_val/5a0b016f-23d2-11e8-a6a3-ec086b02610b...        26      animal\n",
       "2  train_val/59b93afb-23d2-11e8-a6a3-ec086b02610b...        38      animal\n",
       "3  train_val/59641f56-23d2-11e8-a6a3-ec086b02610b...        38      animal\n",
       "4  train_val/5a1e5306-23d2-11e8-a6a3-ec086b02610b...        33      animal"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "animal       55902\n",
       "no_animal    50526\n",
       "Name: category_id, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['category_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>location</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_val/5961064f-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>70</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_val/594b602b-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>70</td>\n",
       "      <td>no_animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_val/59531a77-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>70</td>\n",
       "      <td>no_animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_val/5949d1bf-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>70</td>\n",
       "      <td>no_animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_val/595636dc-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>70</td>\n",
       "      <td>no_animal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           file_name  location category_id\n",
       "0  train_val/5961064f-23d2-11e8-a6a3-ec086b02610b...        70      animal\n",
       "1  train_val/594b602b-23d2-11e8-a6a3-ec086b02610b...        70   no_animal\n",
       "2  train_val/59531a77-23d2-11e8-a6a3-ec086b02610b...        70   no_animal\n",
       "3  train_val/5949d1bf-23d2-11e8-a6a3-ec086b02610b...        70   no_animal\n",
       "4  train_val/595636dc-23d2-11e8-a6a3-ec086b02610b...        70   no_animal"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "no_animal    8275\n",
       "animal       4444\n",
       "Name: category_id, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df['category_id'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "def transfer_data(data_dir, folder, df):\n",
    "    for _, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "        src = data_dir / row.file_name\n",
    "        dest = data_dir / folder / row.category_id / src.name        \n",
    "        src.rename(dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 106428/106428 [00:14<00:00, 7269.00it/s]\n"
     ]
    }
   ],
   "source": [
    "transfer_data(data_dir, 'train', train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 12719/12719 [00:01<00:00, 7749.86it/s]\n"
     ]
    }
   ],
   "source": [
    "transfer_data(data_dir, 'valid', val_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Balance the loading set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import shutil\n",
    "count = 0\n",
    "for file in val_no_animal.iterdir():\n",
    "    if count < 3781:\n",
    "        src = file\n",
    "        dest = train_no_animal / file.name\n",
    "        shutil.move(src, dest)\n",
    "    else:\n",
    "        break\n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}