{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:52:58.067376Z","iopub.execute_input":"2023-10-29T05:52:58.067648Z","iopub.status.idle":"2023-10-29T05:52:58.07289Z","shell.execute_reply.started":"2023-10-29T05:52:58.067606Z","shell.execute_reply":"2023-10-29T05:52:58.072123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n\nimport ast\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\n\nimport torchvision\nfrom torchvision import transforms, utils\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport cv2  # to generate image from vectors (strokes)\n\nimport tqdm\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else \"cpu\")","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2023-10-29T05:52:59.114284Z","iopub.execute_input":"2023-10-29T05:52:59.114555Z","iopub.status.idle":"2023-10-29T05:52:59.120939Z","shell.execute_reply.started":"2023-10-29T05:52:59.114513Z","shell.execute_reply":"2023-10-29T05:52:59.120151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NROWS = 10000\nVAL_NROWS = 500\n\npath = '/kaggle/input/googledoodles'\nfilenames = glob.glob(os.path.join(path, '*.csv'))\nNUM_CLASSES = len(filenames)","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:52:59.742999Z","iopub.execute_input":"2023-10-29T05:52:59.743277Z","iopub.status.idle":"2023-10-29T05:52:59.749991Z","shell.execute_reply.started":"2023-10-29T05:52:59.743232Z","shell.execute_reply":"2023-10-29T05:52:59.748626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:53:00.418573Z","iopub.execute_input":"2023-10-29T05:53:00.418876Z","iopub.status.idle":"2023-10-29T05:53:00.423576Z","shell.execute_reply.started":"2023-10-29T05:53:00.418827Z","shell.execute_reply":"2023-10-29T05:53:00.422791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this drawing function was adopted from https://github.com/ebouteillon/kaggle-quickdraw-doodle-recognition-challenge/blob/master/2-training-resnet18-from-scratch-with-128px-images.ipynb\n\nshift_colors = (\n    (255, 0, 0),\n    (255, 128, 0),\n    (255, 255, 0),\n    (128, 255, 0),\n    (0, 255, 0),\n    (0, 255, 128),\n    (0, 255, 255),\n    (0, 128, 255),\n    (0, 0, 255),\n    (128, 0, 255),\n    (255, 0, 255),\n    (255, 0, 128)\n)\n\n\ndef draw_cv2(raw_strokes, size=128, lw=1):\n    # draw function inspired from https://towardsdatascience.com/10-lessons-learned-from-participating-to-google-ai-challenge-268b4aa87efa\n    BASE_SIZE = 256\n    border = 2  # keep some margin with image border\n\n    img = np.zeros((size, size, 3), np.uint8)\n    coef = (size - 2 * lw - 2 * border) / (BASE_SIZE - 1)\n    num_stokes = len(raw_strokes)\n    for t, stroke in enumerate(raw_strokes[::-1]):  # iterate in reverse order, so that earlier strokes, which are more important, are drawn later so that they are not overlapped\n        rgb = shift_colors[(num_stokes-t-1)%12]\n\n        for i in range(len(stroke[0]) - 1):\n            p1 = (int(coef * stroke[0][i] + lw + border), int(coef * stroke[1][i] + lw+ border))\n            p2 = (int(coef * stroke[0][i + 1] + lw + border), int(coef * stroke[1][i + 1] + lw + border))\n            _ = cv2.line(img, p1, p2, rgb, lw, cv2.LINE_AA)\n    return img","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:53:36.708256Z","iopub.execute_input":"2023-10-29T05:53:36.708562Z","iopub.status.idle":"2023-10-29T05:53:36.721235Z","shell.execute_reply.started":"2023-10-29T05:53:36.708502Z","shell.execute_reply":"2023-10-29T05:53:36.720458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encode_dict = {}\npath = '/kaggle/input/googledoodles'\n\nfilenames = glob.glob(os.path.join(path, '*.csv'))\nfilenames = sorted(filenames)\nprint(len(filenames))\n\nfor ix, filename in enumerate(filenames):\n    class_name = filename.split('/')[-1].split('.')[0].replace(' ', '_')\n    encode_dict[class_name] = ix\n    \ndecode_dict = {value:key for key, value in encode_dict.items()}","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:53:37.023125Z","iopub.execute_input":"2023-10-29T05:53:37.023399Z","iopub.status.idle":"2023-10-29T05:53:37.030844Z","shell.execute_reply.started":"2023-10-29T05:53:37.023356Z","shell.execute_reply":"2023-10-29T05:53:37.030153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ClassDataset(Dataset):\n    def __init__(self, csv_file_path, mode='train', nrows=15000, skiprows=0, size=128):\n        # try nrows=20000\n        super().__init__()\n        \n        self.df = pd.read_csv(csv_file_path, usecols=['strokes'], nrows=nrows, skiprows=0)\n        self.mode = mode\n        self.size = size\n        if self.mode == 'train':\n            self.class_name = csv_file_path.split('/')[-1].split('.')[0].replace(' ', '_')\n            self.label = encode_dict[self.class_name]\n            \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        raw_strokes = ast.literal_eval(self.df.strokes[index])\n        image = draw_cv2(raw_strokes, size=self.size)  # (size, size, 3)\n        image = image.transpose(2, 0, 1)\n        \n        if self.mode == 'train':\n            return (image/255).astype('float32'), self.label\n        else:\n            return (image/255).astype('float32')\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:00.660125Z","iopub.execute_input":"2023-10-29T05:54:00.660468Z","iopub.status.idle":"2023-10-29T05:54:00.66847Z","shell.execute_reply.started":"2023-10-29T05:54:00.66041Z","shell.execute_reply":"2023-10-29T05:54:00.667492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dset = ConcatDataset([ClassDataset(filename, nrows=NROWS) for filename in filenames])\nval_dset = ConcatDataset([ClassDataset(filename, nrows=VAL_NROWS, skiprows=NROWS) for filename in filenames])","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:00.669603Z","iopub.execute_input":"2023-10-29T05:54:00.66988Z","iopub.status.idle":"2023-10-29T05:54:01.936519Z","shell.execute_reply.started":"2023-10-29T05:54:00.669817Z","shell.execute_reply":"2023-10-29T05:54:01.935913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(dset))  # 340 * NROWS\nprint(len(val_dset))   # 340 * VAL_NROWS","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:04.830008Z","iopub.execute_input":"2023-10-29T05:54:04.830329Z","iopub.status.idle":"2023-10-29T05:54:04.834874Z","shell.execute_reply.started":"2023-10-29T05:54:04.830271Z","shell.execute_reply":"2023-10-29T05:54:04.833823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dloader = DataLoader(dset, batch_size=128, shuffle=True, num_workers=2)\nval_dloader = DataLoader(val_dset, batch_size=128, num_workers=2, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:07.23365Z","iopub.execute_input":"2023-10-29T05:54:07.233972Z","iopub.status.idle":"2023-10-29T05:54:07.283264Z","shell.execute_reply.started":"2023-10-29T05:54:07.233922Z","shell.execute_reply":"2023-10-29T05:54:07.282326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = next(iter(dloader))\nprint(batch)  # it works well, shuffled.","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:09.298704Z","iopub.execute_input":"2023-10-29T05:54:09.299055Z","iopub.status.idle":"2023-10-29T05:54:10.240056Z","shell.execute_reply.started":"2023-10-29T05:54:09.299001Z","shell.execute_reply":"2023-10-29T05:54:10.239244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(np.transpose(batch[0][0], (1, 2, 0)))","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:10.375047Z","iopub.execute_input":"2023-10-29T05:54:10.375329Z","iopub.status.idle":"2023-10-29T05:54:10.681633Z","shell.execute_reply.started":"2023-10-29T05:54:10.375285Z","shell.execute_reply":"2023-10-29T05:54:10.68059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# our dataloader is ready.\n# let's make model\n\n# building blocks for mobilenet.\n# name Conv and Conv_dw are following terms used in mobilenet paper (https://arxiv.org/pdf/1704.04861.pdf)\n\nclass Conv(nn.Module):\n    def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1):\n        super().__init__()\n        self.layers = nn.Sequential(\n                nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, stride=stride, padding=padding, bias=False),\n                nn.BatchNorm2d(out_channel),\n                nn.ReLU(inplace=True)\n            )\n        \n    def forward(self, input):\n        return self.layers(input)\n    \nclass Conv_dw_Conv(nn.Module):\n    # Conv dw layer followed by Conv layer.\n    # implemented this way since every conv dw layer is followed by conv layer with kernel size 1, stride 1 with some out_channel\n    def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1):\n        super().__init__()\n        self.layers = nn.Sequential(\n                nn.Conv2d(in_channel, in_channel, kernel_size, stride, padding, bias=False, groups=in_channel),\n                nn.BatchNorm2d(in_channel),\n                nn.ReLU(inplace=True),\n                Conv(in_channel, out_channel, kernel_size=1, stride=1, padding=0)\n            )\n        \n    def forward(self, input):\n        return self.layers(input)\n    \n    \nclass MobileNet(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        \n        self.num_classes = num_classes\n        \n        self.model = nn.Sequential(\n                Conv(3, 32, stride=2),\n            \n                Conv_dw_Conv(32, 64, kernel_size=3, stride=1),\n                Conv_dw_Conv(64, 128, kernel_size=3, stride=2),\n                Conv_dw_Conv(128, 128, kernel_size=3, stride=1),\n                Conv_dw_Conv(128, 256, kernel_size=3, stride=2),\n                Conv_dw_Conv(256, 256, kernel_size=3, stride=1),\n                Conv_dw_Conv(256, 512, kernel_size=3, stride=2),\n            \n                Conv_dw_Conv(512, 512, kernel_size=3, stride=1),\n                Conv_dw_Conv(512, 512, kernel_size=3, stride=1),\n                Conv_dw_Conv(512, 512, kernel_size=3, stride=1),\n                Conv_dw_Conv(512, 512, kernel_size=3, stride=1),\n                Conv_dw_Conv(512, 512, kernel_size=3, stride=1),\n            \n                Conv_dw_Conv(512, 1024, kernel_size=3, stride=2),\n                Conv_dw_Conv(1024, 1024, kernel_size=3, stride=1)\n        )\n        \n        \n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(1024, num_classes)\n        \n    def forward(self, input):\n        x = self.model(input)\n        x = self.avg_pool(x)\n        x = x.view(-1, 1024)\n        out = self.fc(x)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:25.467248Z","iopub.execute_input":"2023-10-29T05:54:25.467527Z","iopub.status.idle":"2023-10-29T05:54:25.494397Z","shell.execute_reply.started":"2023-10-29T05:54:25.467486Z","shell.execute_reply":"2023-10-29T05:54:25.493596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# before training, let's set up our metric MAP@3\n# I slightly modified implementation of https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n\ndef apk(actual, predicted, k=3):\n    \"\"\"\n    Computes the average precision at k.\n    This function computes the average prescision at k between two lists of\n    items.\n    Parameters\n    ----------\n    actual : int\n             element that are to be predicted\n    predicted : list\n                A list of predicted elements (order does matter)\n    k : int, optional\n        The maximum number of predicted elements\n    Returns\n    -------\n    score : double\n            The average precision at k over the input lists\n    \"\"\"\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n\n    for i,p in enumerate(predicted):\n        if p == actual:\n            score = 1 / (i+1.0)\n    \n    return score\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Computes the mean average precision at k.\n    This function computes the mean average prescision at k between two lists\n    of lists of items.\n    Parameters\n    ----------\n    actual : list\n             A list of elements that are to be predicted \n    predicted : list\n                A list of lists of predicted elements\n                (order matters in the lists)\n    k : int, optional\n        The maximum number of predicted elements\n    Returns\n    -------\n    score : double\n            The mean average precision at k over the input lists\n    \"\"\"\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:28.372146Z","iopub.execute_input":"2023-10-29T05:54:28.37242Z","iopub.status.idle":"2023-10-29T05:54:28.381087Z","shell.execute_reply.started":"2023-10-29T05:54:28.372379Z","shell.execute_reply":"2023-10-29T05:54:28.380207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# one last thing before start training, we need a function that returns validation map@3.\n\ndef validation_score(model, val_data_loader):\n    model.eval()\n    sum_score = 0\n    count = 0\n    for images, labels in val_data_loader:\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = images.size(0)\n        output = model(images)\n        topk = output.detach().topk(3, dim=1)[1]\n        sum_score += mapk(labels.cpu().numpy(), topk.cpu().numpy()) * batch_size\n        count += batch_size\n        \n    model.train()\n    return sum_score / count\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:54:30.638144Z","iopub.execute_input":"2023-10-29T05:54:30.638416Z","iopub.status.idle":"2023-10-29T05:54:30.644421Z","shell.execute_reply.started":"2023-10-29T05:54:30.638374Z","shell.execute_reply":"2023-10-29T05:54:30.643486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MobileNet(340).to(device)\ncriterion = nn.CrossEntropyLoss().to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.002)\nmodel.load_state_dict(torch.load('/kaggle/input/models/model_checkpoint_best_val.ckpt'))","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:59:27.411247Z","iopub.execute_input":"2023-10-29T05:59:27.411543Z","iopub.status.idle":"2023-10-29T05:59:27.509237Z","shell.execute_reply.started":"2023-10-29T05:59:27.411502Z","shell.execute_reply":"2023-10-29T05:59:27.508429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_ftrs = model.fc.in_features\nmodel.fc = nn.Linear(num_ftrs, NUM_CLASSES)\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:03:55.15021Z","iopub.execute_input":"2023-10-29T06:03:55.150504Z","iopub.status.idle":"2023-10-29T06:03:55.156879Z","shell.execute_reply.started":"2023-10-29T06:03:55.150462Z","shell.execute_reply":"2023-10-29T06:03:55.156058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # train time!\n\n# model = MobileNet(NUM_CLASSES).to(device)\n\n# criterion = nn.CrossEntropyLoss().to(device)\n# optimizer = torch.optim.Adam(model.parameters(), lr=0.002)\n\n# scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[60000, 130000, 160000, 190000], gamma=0.5)","metadata":{"execution":{"iopub.status.busy":"2023-10-29T03:50:44.756597Z","iopub.status.idle":"2023-10-29T03:50:44.756971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprint(\"Start training...\")\n\nepochs = 40\n\nprint_every = 100 # print every N iterations\nvalidate_every = 100  # do validation every N iterations\nmodel.train()\n\nbest_val_score = 0\n\ncurr_iter = 0\navg_loss = 0\navg_score = 0\n\nfor epoch in range(epochs):\n\n\n    for ix, (images, labels) in enumerate(dloader):\n        \n        images = images.to(device)\n        labels = labels.to(device)\n        \n        model.zero_grad()\n        output = model(images)\n        loss = criterion(output, labels)\n        loss.backward()\n        optimizer.step()\n        \n        avg_loss += loss.item()\n        \n        topk = output.detach().topk(3, dim=1)[1]\n        avg_score += mapk(labels.cpu().numpy(), topk.cpu().numpy())\n        # scheduler.step()\n            \n        curr_iter += 1\n        if (curr_iter) % print_every == 0:\n            print('Epoch {}, Iteration {} - Train Loss: {:.4f}, MAP@3: {:.3f}'.format(epoch + 1, curr_iter, avg_loss/print_every, avg_score/print_every))\n            avg_loss = 0\n            avg_score = 0\n            \n        if curr_iter % validate_every == 0:\n            val_score = validation_score(model, val_dloader)\n            print('Epoch {}, Iteration {}: validation map@3: {}'.format(epoch + 1, curr_iter, val_score))\n            if val_score > best_val_score:\n                print('New best validation score: {}, saving model...'.format(val_score))\n                best_val_score = val_score\n                torch.save(model.state_dict(), 'model_checkpoint_best_val.ckpt')\n","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:04:08.748556Z","iopub.execute_input":"2023-10-29T06:04:08.748964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save model\n\ntorch.save(model.state_dict(), 'model_checkpoint.ckpt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make submission using best val checkpoint\n\nmodel.load_state_dict(torch.load('model_checkpoint_best_val.ckpt'))\nmodel.eval()\n\nlabels = []\nfor images in tqdm.tqdm(test_dloader):\n    images = images.to(device)\n    output = model(images)\n    _, pred = output.topk(3, 1)\n    for i in range(len(images)):\n        labels.append(' '.join([decode_dict[pred[i][j].item()]for j in range(3)]))\n        \nprint(len(labels))\n        \nsubmission = pd.read_csv('../input/test_simplified.csv', index_col='key_id' ,usecols=['key_id'])\nprint(len(submission))\nsubmission['word'] = labels\n\nsubmission.to_csv('submission_best_val.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-29T03:50:44.762555Z","iopub.status.idle":"2023-10-29T03:50:44.762956Z"},"trusted":true},"execution_count":null,"outputs":[]}]}