{"cells":[{"metadata":{"_uuid":"ccb30bbb3598a1d4fb3790d978bc0c0e85389339"},"cell_type":"markdown","source":"### This version without validation and fixed MAP@3 function"},{"metadata":{"trusted":true,"_uuid":"9198fb677be3ac9abf16e8689a04edef08ca2c2a"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore') # to suppress some matplotlib deprecation warnings\n\nimport numpy as np # linear algebra\nnp.random.seed(60794)\nimport random\nrandom.seed(60794)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport ast\nimport cv2\n\nimport matplotlib.pyplot as plt\nimport matplotlib.style as style\n\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport os\nimport glob\nimport time\nimport tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"632b7bda026ef88df3ed781287076cf3cba00ded"},"cell_type":"code","source":"import torch\ntorch.manual_seed(60794)\ntorch.backends.cudnn.deterministic = True\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\n\nimport torchvision\nfrom torchvision import transforms, utils","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e09952898da920238ff4cb225343f7223d8bca8"},"cell_type":"code","source":"import torch.nn as nn\nimport torch.utils.model_zoo as model_zoo\n\n\n__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',\n           'resnet152']\n\n\nmodel_urls = {\n    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n}\n\ndef swish(x):\n    return torch.sigmoid(x) * x\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=1, bias=False)\n\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None):\n        super(BasicBlock, self).__init__()\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None):\n        super(Bottleneck, self).__init__()\n        self.conv1 = conv1x1(inplanes, planes)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = conv3x3(planes, planes, stride)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = conv1x1(planes, planes * self.expansion)\n        self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = swish(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = swish(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = swish(out)\n\n        return out\n\n\nclass ResNet(nn.Module):\n\n    def __init__(self, block, layers, num_classes=1000):\n        super(ResNet, self).__init__()\n        self.inplanes = 64\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                conv1x1(self.inplanes, planes * block.expansion, stride),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = swish(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n        return x\n\n\ndef resnet18(pretrained=False, **kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)\n    if pretrained:\n        model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d03279525b6ed7739361eb31d17fc6c25809d78"},"cell_type":"code","source":"# https://github.com/leigh-plt/cs231n_hw2018/blob/master/assignment2/pytorch_tutorial.ipynb\n\"\"\"\nSave and Load Checkpoints\nBased on Model Training\nSource: https://github.com/leigh-plt/cs231n_hw2018/blob/master/assignment2/pytorch_tutorial.ipynb\nWill be the basis for checkpoint ensembling\n\"\"\"\ndef save_checkpoint(checkpoint_path, model, optimizer):\n    state = {'state_dict': model.state_dict(),\n             'optimizer' : optimizer.state_dict()}\n    torch.save(state, checkpoint_path)\n    print('model saved to %s' % checkpoint_path)\n    \ndef load_checkpoint(checkpoint_path, model, optimizer):\n    state = torch.load(checkpoint_path)\n    model.load_state_dict(state['state_dict'])\n    optimizer.load_state_dict(state['optimizer'])\n    print('model loaded from %s' % checkpoint_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dbef81eb0e43eddca9272b4eb06aaed7dc731e8e"},"cell_type":"code","source":"en_dict = {}\npath = '../input/quickdraw-doodle-recognition/train_simplified/'\n\nfilenames = glob.glob(os.path.join(path, '*.csv'))\nfilenames = sorted(filenames)\n\ndef encode_files():\n    \"\"\" Encode all label by name of csv_files \"\"\"\n    counter = 0\n    for fn in filenames:\n        en_dict[fn[:-4].split('/')[-1].replace(' ', '_')] = counter\n        counter += 1\n        \n# collect file names and encode label\nencode_files()\n\ndec_dict = {v: k for k, v in en_dict.items()}\ndef decode_labels(label):\n    return dec_dict[label]\n\ndef get_label(nfile):\n    \"\"\" Return encoded label for class by name of csv_files \"\"\"\n    return en_dict[nfile.replace(' ', '_')[:-4]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ee3b325e9705009c9d4a1f0f9ec009691c4c97c"},"cell_type":"code","source":"class DoodlesDataset(Dataset):\n    \"\"\"Doodles csv dataset.\"\"\"\n\n    def __init__(self, csv_file, root_dir, mode='train', nrows=1000, skiprows=None, size=256, transform=None):\n        \"\"\"\n        Args:\n            csv_file (string): Path to the csv file with annotations.\n            root_dir (string): Directory with all the images.\n            mode (string): Train or test mode.\n            nrows (int): Number of rows of file to read. Useful for reading pieces of large files.\n            skiprows (list-like or integer or callable): \n                    Line numbers to skip (0-indexed) or number of lines to skip (int) at the start of the file.\n            size (int): Size of output image.\n            transform (callable, optional): Optional transform to be applied\n                on a sample.\n        \"\"\"\n        self.root_dir = root_dir\n        file = os.path.join(self.root_dir, csv_file)\n        self.size = size\n        self.mode = mode\n        self.doodle = pd.read_csv(file, usecols=['drawing'], nrows=nrows, skiprows=skiprows)\n        self.transform = transform\n        if self.mode == 'train':\n            self.label = get_label(csv_file)\n\n    @staticmethod\n    def _draw(raw_strokes, size=256, lw=6, time_color=True):\n        BASE_SIZE = 256\n        img = np.zeros((BASE_SIZE, BASE_SIZE), np.uint8)\n        for t, stroke in enumerate(raw_strokes):\n            for i in range(len(stroke[0]) - 1):\n                color = 255 - min(t, 10) * 13 if time_color else 255\n                _ = cv2.line(img, (stroke[0][i], stroke[1][i]),\n                             (stroke[0][i + 1], stroke[1][i + 1]), color, lw)\n        if size != BASE_SIZE:\n            return cv2.resize(img, (size, size))\n        else:\n            return img\n    \n    def __len__(self):\n        return len(self.doodle)\n\n    def __getitem__(self, idx):\n        raw_strokes = ast.literal_eval(self.doodle.drawing[idx])\n        sample = self._draw(raw_strokes, size=self.size, lw=6, time_color=True)\n        if self.transform:\n            sample = self.transform(sample)\n        if self.mode == 'train':\n            return (sample[None]/255).astype('float32'), self.label\n        else:\n            return (sample[None]/255).astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ac104312411ce6a9d461eeab7787f11c92378c5"},"cell_type":"code","source":"SIZE = 224 # for matching to imagenet\n# collect all single csvset in one\nselect_nrows = 8000 # origingal 2000 for 5 folds\ndoodle_all = ConcatDataset([DoodlesDataset(fn.split('/')[-1], path,\n                                           skiprows=list(range(1,78000)),\n                                           nrows=select_nrows, size=SIZE) for fn in filenames])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64634d84532ae3f7a8ccc54cc8b9b60cd2a04f2d"},"cell_type":"code","source":"# select some rows for validation\n# valid_rows = 100\n# validationset = ConcatDataset([DoodlesDataset(fn.split('/')[-1], path, nrows=valid_rows, size=SIZE,\n#                                            skiprows=range(1, select_nrows+1)) for fn in filenames])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f1372d9e14013e607980ce390b09f2599d673a6"},"cell_type":"code","source":"# total images in set\nprint('Train set:', len(doodle_all))\n# print('Validation set:', len(validationset))\n# Use the torch dataloader to iterate through the dataset\nloader = DataLoader(doodle_all, batch_size=128, shuffle=True, num_workers=0)\n# valid_loader = DataLoader(validationset, batch_size=128, shuffle=False, num_workers=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"d6a92e3cf66fc593a980c5abf01473146a92755d"},"cell_type":"code","source":"# functions to show an image\ndef imshow(img):\n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n\n# get some images\ndataiter = iter(loader)\nimages, label = dataiter.next()\n\n# show images\nplt.figure(figsize=(16,24))\nimshow(torchvision.utils.make_grid(images[:24]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"541587df1a149bc48ff54503cb70d41d411e61db"},"cell_type":"code","source":"# validation function \ndef validation(lossf, scoref):\n    model.eval()\n    loss, score = 0, 0\n    vlen = len(valid_loader)\n    for x, y in valid_loader:\n        x, y = x.to(device), y.to(device)\n        output = model(x)\n        loss += lossf(output, y).item()\n        score += scoref(output, y)[0].item()\n    model.train()\n    return loss/vlen, score/vlen","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bb3cff215025bfd4867b8ef44785cafdda085aa"},"cell_type":"code","source":"def accuracy(output, target, topk=(3,)):\n    \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n    with torch.no_grad():\n        maxk = max(topk)\n        batch_size = target.size(0)\n\n        _, pred = output.topk(maxk, 1, True, True)\n        pred = pred.t()\n        correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n        res = []\n        for k in topk:\n            correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n            res.append(correct_k.mul_(100.0 / batch_size))\n        return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcfe289cea5ab2caac9d6bce085dae515784f3e9"},"cell_type":"code","source":"def mapk(output, target, k=3):\n    \"\"\"\n    Computes the mean average precision at k.\n    \n    Parameters\n    ----------\n    output (torch.Tensor): A Tensor of predicted elements.\n                           Shape: (N,C)  where C = number of classes, N = batch size\n    target (torch.int): A Tensor of elements that are to be predicted. \n                        Shape: (N) where each value is  0≤targets[i]≤C−1\n    k (int, optional): The maximum number of predicted elements\n    \n    Returns\n    -------\n    score (torch.float):  The mean average precision at k over the output\n    \"\"\"\n    with torch.no_grad():\n        batch_size = target.size(0)\n\n        _, pred = output.topk(k, 1, True, True)\n        pred = pred.t()\n        correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n        for i in range(k):\n            correct[i] = correct[i]*(k-i)\n            \n        score = correct[:k].view(-1).float().sum(0, keepdim=True)\n        score.mul_(1.0 / (k * batch_size))\n        return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c4c981e8162b32a5700d8462208302cfb9c47b3"},"cell_type":"code","source":"model = resnet18(pretrained=True)\n#model = torchvision.models.resnet18(pretrained=False,num_classes=340)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d13f4cbbf05a8d7228771ce1414e30a621a81ae"},"cell_type":"code","source":"# Chenge number of inputs channels and number of classes \n# Its first and last layers in model\ndef squeeze_weights(m):\n        m.weight.data = m.weight.data.sum(dim=1)[:,None]\n        m.in_channels = 1\n        \nmodel.conv1.apply(squeeze_weights);\n\nnum_classes = 340\nmodel.fc = nn.Linear(in_features=512, out_features=num_classes, bias=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e742abf34ddbd5c1b1218a8b8e506f7b41d5ceb1"},"cell_type":"code","source":"%%time\n# test with random data\nmodel(torch.randn(12,1,224,224)).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4535539aeea5fe64996cd69a4d0555024952a47"},"cell_type":"code","source":"device = 'cuda'\nmodel.to(device);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f9756c44222d039a7cff9886c253c5d3cdc1cb6"},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001, amsgrad=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"451e68b4c50c6a705aa82beddebf2bfccf599772"},"cell_type":"code","source":"load_checkpoint(\"../input/fold-two-doodler5/doodle-fold-two-2.pth\", model, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df12311a7e01e25c2f969041e050b775acccbe43"},"cell_type":"code","source":"# PyTorch scheduler:\n# https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\nscheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5000,12000,18000], gamma=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce361e3f6370b69b2b74a1f692665ef525c2346f"},"cell_type":"code","source":"%%time\nepochs = 1\nlsize = len(loader)\nitr = 1\np_itr = 1000 # print every N iteration\nmodel.train()\ntloss, score = 0, 0\nfor epoch in range(epochs):\n    for x, y in tqdm.tqdm(loader):\n        x, y = x.to(device), y.to(device)\n        optimizer.zero_grad()\n        output = model(x)\n        loss = criterion(output, y)\n        loss.backward()\n        optimizer.step()\n        tloss += loss.item()\n        score += mapk(output, y)[0].item()\n        scheduler.step()\n        if itr%p_itr==0:\n#             vloss, vscore = validation(criterion, mapk)\n            print('Iteration {} -> Train Loss: {:.4f}, MAP@3: {:.3f}'.format(itr, tloss/p_itr, score/p_itr))\n#             print('Iteration {} -> Validation Loss: {:.4f}, MAP@3: {:.3f}'.format(itr, vloss, vscore))\n#             scheduler.step(vscore)\n#             time.sleep(0.1)\n            tloss, score = 0, 0\n        itr +=1\n# vloss, vscore = validation(criterion, mapk) old comment\nitr = itr%p_itr\nprint('Final -> Loss: {:.4f}, MAP@3: {:.3f}'.format(loss/itr, score/itr))\nsave_checkpoint('doodle-fold-two-%i.pth' % epochs, model, optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c10f196d429b3ca77c8612242885a8128988e763"},"cell_type":"code","source":"testset = DoodlesDataset('test_simplified.csv', '../input/quickdraw-doodle-recognition', mode='test', nrows=None, size=SIZE)\ntestloader = DataLoader(testset, batch_size=128, shuffle=False, num_workers=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b595d5b9bd8d0c6ec5043717d7f9678a147f063","scrolled":true},"cell_type":"code","source":"model.eval()\nlabels = np.empty((0,3))\nfor x in tqdm.tqdm(testloader):\n    x = x.to(device)\n    output = model(x)\n    _, pred = output.topk(3, 1, True, True)\n    labels = np.concatenate([labels, pred], axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3177c36bc834f5910cc9e2fc743b2673ce637406"},"cell_type":"code","source":"%%time\nsubmission = pd.read_csv('../input/quickdraw-doodle-recognition/test_simplified.csv', index_col='key_id')\nsubmission.drop(['countrycode', 'drawing'], axis=1, inplace=True)\nsubmission['word'] = ''\nfor i, label in enumerate(labels):\n    submission.word.iloc[i] = \" \".join([decode_labels(l) for l in label])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b980509c4869d040547074b077710a9d533f2797"},"cell_type":"code","source":"submission.to_csv('preds_resnet18_foldone.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fce5761dcef0721906f446ad8a6b22e71538d45"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}