{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Transfer learning with densenet\n1. Number of doodle classes are 340. This is quite big, so deep CNN networks are required.\n2. But we don't have much resouces and are not able to train such a huge network for a long time.\n3. Transfer learning comes in! DenseNet is a useful backbone structure for image recognition and pretrained weights are provided.\n\nI refered to this kernel[https://www.kaggle.com/leighplt/pytorch-starter-kit]"},{"metadata":{"trusted":true},"cell_type":"code","source":"#module import\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport cv2\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\nfrom torchvision import transforms, utils\nimport copy\nimport tqdm\nimport sys\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# variables and hyperparameters\n\ntrain_dir = '../input/train_simplified'\ntest_dir = '../input'\ntest_file = 'test_simplified.csv'\n\ntrain_nrows = 2000\nval_nrows = 100\n\ntrain_files = os.listdir(train_dir)\ntrain_files = sorted(train_files)\n\nw2i_dict = {fn[:-4].replace(' ','_'): i for i, fn in enumerate(train_files)}\ni2w_dict = {v: k for k, v in w2i_dict.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make training / validation Datasets\n\nclass DoodleDataset(Dataset):\n    def __init__(self, file, dir, mode='train', nrows = None, skiprows=None, transform = None):\n        self.file = file\n        self.dir = dir\n        self.mode = mode\n        self.nrows = nrows\n        self.transform = transform\n        # only use 'nrows' to reduce memory usage / skiprows is used when we make validation set\n        self.pd_frame = pd.read_csv(os.path.join(self.dir, self.file), usecols=['drawing'], nrows=nrows, skiprows=skiprows)\n        if self.mode == 'train':\n            self.label = w2i_dict[file[:-4].replace(' ','_')]\n        \n    @staticmethod\n    # used in __getitem__ function / converts points into line\n    def _point2line(strokes):\n        # one channel as a grayscale\n        img = np.zeros((256,256), dtype = np.uint8)\n        for stroke in strokes:\n            x_coords = stroke[0]\n            y_coords = stroke[1]\n            for i in range(len(stroke[0])-1):\n                # draw line in color 255\n                cv2.line(img, (x_coords[i], y_coords[i]), (x_coords[i+1], y_coords[i+1]), color = 255, thickness = 2)\n        return img\n    \n    def __len__(self):\n        return len(self.pd_frame)\n    \n    def __getitem__(self, idx):\n        # string to type variable\n        strokes = eval(self.pd_frame.drawing[idx])\n        sample = self._point2line(strokes)\n        if self.transform:\n            sample = self.transform(sample)\n        if self.mode=='train':\n            # after transform, return images as a numpy array\n            sample = sample.numpy()\n            return sample.astype('float32'), self.label\n        else:\n            # add batch size\n            sample = sample.numpy()\n            return sample.astype('float32')\n\n        \ndata_transform = transforms.Compose([\n    # numpy ndarray to PIL image\n    transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(),\n    transforms.Resize((128,128)),\n    transforms.ToTensor()\n])\n\nval_data_transform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.Resize((128,128)),\n    transforms.ToTensor()\n])\n\n\ndoodles = ConcatDataset([DoodleDataset(fn, train_dir, nrows=train_nrows, transform = data_transform) for fn in train_files])\ndoodles_val = ConcatDataset([DoodleDataset(fn, train_dir, nrows=val_nrows, transform = val_data_transform, skiprows=range(1, train_nrows+1)) for fn in train_files])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make dataloader\n\ndataloader = DataLoader(doodles, batch_size = 16, shuffle = True, num_workers = 4)\nvalloader = DataLoader(doodles_val, batch_size = 16, shuffle = True, num_workers = 4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check if dataset and dataloader are formed well\n\nsamples, labels = iter(dataloader).next()\nplt.figure(figsize=(16,24))\ngrid_imgs = torchvision.utils.make_grid(samples[:24])\nnp_grid_imgs = grid_imgs.numpy()\n# in tensor, image is (batch, height, width), so you have to transpose it to (height, width, batch) in numpy to show it.\nplt.imshow(np.transpose(np_grid_imgs, (1,2,0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#score function\n\ndef mapk(output, target, k=3):\n    with torch.no_grad():\n        batch_size = target.size(0)\n        _, pred = output.topk(k, 1, True, True)\n        pred = pred.t()\n        correct = pred.eq(target.view(1, -1).expand_as(pred))\n        for i in range(k):\n            correct[i] = correct[i]*(k-i)\n        score = correct[:k].view(-1).float().sum(0, keepdim=True)\n        score.mul_(1.0 / (k * batch_size))\n        return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\nclass RAdam(Optimizer):\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        super(RAdam, self).__init__(params, defaults)\n\n        # step_buffer stores [step, N_sma, step_size]\n        self.step_buffer = [None, None, None]\n        self.N_sma_max = 2 / (1-betas[1]) -1\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n        loss = None if closure is None else closure()\n\n        for group in self.param_groups:\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n\n                # basic variables\n                p_data_fp32 = p.data.float()\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 1\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['step'] += 1\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n                \n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                beta1_t, beta2_t = beta1 ** state['step'], beta2 ** state['step']\n                bias_correction1, bias_correction2 = 1 - beta1_t, 1 - beta2_t\n\n                # get gradients\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')    \n\n                # update exponential average and exponential average squared\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n\n                # get N_sma and store in step_buffer\n                if state['step'] == self.step_buffer[0]:\n                    N_sma, step_size = self.step_buffer[1], self.step_buffer[2]\n                else:\n                    N_sma = self.N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    # step size reflects rectification and bias_correction term in advance\n                    if N_sma >= 5:\n                        rectification = math.sqrt( ((N_sma-4)*(N_sma-2)*self.N_sma_max) / ((self.N_sma_max-4)*(self.N_sma_max-2)*N_sma) )\n                        step_size = math.sqrt(bias_correction2) * rectification / bias_correction1\n                    else:\n                        step_size = 1.0 / bias_correction1\n                    \n                    self.step_buffer[0] = state['step']\n                    self.step_buffer[1] = N_sma\n                    self.step_buffer[2] = step_size\n\n                \n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # update parameter\n                if N_sma >= 5:            \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size * group['lr'], exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# transfer learning with densenet121 structure\n\ndevice = 'cuda'\n\n# densenet input image is RGB channel color image, so you have to reduce channel\ndef squeeze_weights(m):\n    m.weight.data = m.weight.data.sum(dim=1)[:,None]\n    m.in_channels = 1\nmodel = torchvision.models.densenet121(pretrained=True)    \n\n'''\nif you have small portion of data, only training classifier is a good way.\nfor param in model.parameters():\n    param.requires_grad = False\n'''\nnum_ftrs = model.classifier.in_features\nmodel.classifier = nn.Linear(num_ftrs, 340)\nmodel.features.conv0.apply(squeeze_weights)\n\n\nmodel = model.to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = RAdam(model.parameters(), lr=0.005)\nscheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30000,60000,90000], gamma=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training procedure\n\nepochs = 3\nitr = 1\np_itr = 5000\ntotal_loss, score = 0,0\nval_score = 0.0\nbest_model_wts = copy.deepcopy(model.state_dict())\nbest_score = 0.0\nloss_list = []\nscore_list = []\n\n\nfor epoch in range(epochs):\n    model.train()\n    for samples, labels in dataloader:\n        samples, labels = samples.to(device), labels.to(device)    \n        optimizer.zero_grad()\n        output = model(samples)\n        loss = criterion(output, labels)\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n        score += mapk(output, labels)[0].item()\n        scheduler.step()\n        \n        if itr%p_itr==0:\n            print('[Epoch {}/{}] Iteration {} -> Train Loss: {:.4f}, MAP@3: {:.3f}'.format(epoch+1, epochs, itr, total_loss/p_itr, score/p_itr))\n            loss_list.append(total_loss/p_itr)\n            score_list.append(score/p_itr)\n            total_loss, score = 0,0\n        itr += 1\n        \n    # use validation set for finding well updated weights\n    model.eval()\n    for samples, labels in valloader:\n        with torch.no_grad():\n            samples = samples.to(device)\n            labels = labels.to(device)\n            output = model(samples)\n            val_score += mapk(output, labels)[0].item()\n    print('[Epoch {}/{}] Validation Score: {:.4f}'.format(epoch+1, epochs, val_score/len(valloader)))\n    if val_score > best_score:\n        best_score = val_score\n        best_model_wts = copy.deepcopy(model.state_dict())\n    val_score = 0\n    \n    \n# best performed weights are chosen\nmodel.load_state_dict(best_model_wts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax1 = plt.subplots()\nax2 = ax1.twinx()\nax1.plot(loss_list, label='loss', color = 'g')\nax2.plot(score_list, label='score', color = 'b')\n\nax1.set_ylabel('loss')\nax2.set_ylabel('score')\n\nplt.title('training loss and score')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filename_pth = 'ckpt_densenet121.pth'\ntorch.save(model.state_dict(), filename_pth)\n\ntest_transform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.Resize((128,128)),\n    transforms.ToTensor()\n])\n\ntestset = DoodleDataset(test_file, test_dir, mode='test', transform = test_transform)\ntestloader = DataLoader(testset, batch_size=16, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\nlabels = np.empty((0,3))\nfor x in tqdm.tqdm_notebook(testloader):\n    with torch.no_grad():\n        x = x.to(device)\n        output = model(x)\n        _, pred = output.topk(3,1,True,True)\n        labels = np.concatenate([labels, pred.cpu()], axis=0)\n    \nsubmission = pd.read_csv('../input/test_simplified.csv', index_col='key_id')\nsubmission.drop(['countrycode','drawing'], axis=1, inplace=True)\nsubmission['word']=''\nfor i, label in enumerate(labels):\n    submission.word.iloc[i] = \" \".join([i2w_dict[l] for l in label])\n    \nsubmission.to_csv('preds_densenet121.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom IPython.display import FileLinks\nFileLinks('.') # input argument is specified folder\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}