{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd\nimport codecs\nimport glob\nimport cv2\nimport timm\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\n#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nfrom PIL import Image, ImageDraw\nimport torch\ntorch.backends.cudnn.benchmark = False\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.autograd import Variable\nfrom torch.utils.data.dataset import Dataset\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:18.080736Z","iopub.execute_input":"2023-04-15T11:44:18.081076Z","iopub.status.idle":"2023-04-15T11:44:22.759761Z","shell.execute_reply.started":"2023-04-15T11:44:18.081044Z","shell.execute_reply":"2023-04-15T11:44:22.758613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#below are paramaters for data reading\nsampleNumber = 1000 # defining how many drawings to take from each class\nTEST_SPLIT_SIZE=0.05 # the train-test size\n#below are paramaters for image drawing\nBASE_SIZE = 299\nIMAGE_SIZE = 64 # the input size (both width and height) of the net\nMAX_STROKE_IDX = 10 # to encode the order of strokes with brightness, this is the max number of stroke that will affect the brightness\nBRIGHTNESS_CONSTANT = 13 # every new stroke's brightness will -13\nLINE_WIDTH = 6\nOFFSET = 22\nDROPOUT_RATE = 0.95\n#below are parameters for training\nepochs = 25\nBATCH_SIZE = 160","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:22.762007Z","iopub.execute_input":"2023-04-15T11:44:22.762763Z","iopub.status.idle":"2023-04-15T11:44:22.770059Z","shell.execute_reply.started":"2023-04-15T11:44:22.762717Z","shell.execute_reply":"2023-04-15T11:44:22.769201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getSample(number, use_simplified = True):\n    if use_simplified:\n        all_csv_path = glob.glob('/kaggle/input/quickdraw-doodle-recognition/train_simplified/*.csv')\n    else:\n        all_csv_path = glob.glob('/kaggle/input/quickdraw-doodle-recognition/train_raw/*.csv')\n    all_df = []\n    for path in all_csv_path:\n        temp_df = pd.read_csv(path, nrows=number, parse_dates=['timestamp'])\n        all_df.append(temp_df)\n    df = pd.concat(all_df, axis=0, ignore_index=True)  \n    #shuffle the result and encode the class label\n    df = df.reindex(np.random.permutation(df.index))\n    encoder = LabelEncoder().fit(df['word'])\n    df['word'] = encoder.transform(df['word'])\n    df_train, df_val = train_test_split(df, test_size=TEST_SPLIT_SIZE)\n    print('Train size:', len(df_train), 'Val size:', len(df_val))\n    print('Saving...')\n    df_train.to_pickle(os.path.join('/kaggle/working', 'train_' + str(number) + '.pkl'))\n    df_val.to_pickle(os.path.join('/kaggle/working', 'val_' + str(number) + '.pkl'))\n    print('Saving completed.')\ngetSample(sampleNumber, True)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:22.774997Z","iopub.execute_input":"2023-04-15T11:44:22.775308Z","iopub.status.idle":"2023-04-15T11:44:31.178517Z","shell.execute_reply.started":"2023-04-15T11:44:22.775277Z","shell.execute_reply":"2023-04-15T11:44:31.177453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#below is the code to conver the stroke into img matrix\n#here we encode the order of drawing into different colors\n#stroke consists of several lines, below is the function to draw a line\ndef draw_line_segment(image, start_point, end_point, color, width):\n        cv2.line(image, start_point, end_point, color, width)\n\n# below is the function to convert raw strokes into image\ndef draw_image(raw_strokes, size):\n    # create a basesize image first\n    canvas = np.zeros((BASE_SIZE, BASE_SIZE), np.uint8)\n    strokes = eval(raw_strokes)\n    for stroke_idx, single_stroke in enumerate(strokes):  \n        #get all the Xs and Ys for all points of the stroke\n        x, y = single_stroke\n        points = list(zip(x, y))\n        for point_idx, (start, end) in enumerate(zip(points[:-1], points[1:])):\n            #randomly dropout points\n            if np.random.uniform() > DROPOUT_RATE:\n                continue\n            #adjust the brightness of current stroke\n            brightness = 255 - min(stroke_idx, MAX_STROKE_IDX) * BRIGHTNESS_CONSTANT\n            start_point = (start[0] + OFFSET, start[1] + OFFSET)\n            end_point = (end[0] + OFFSET, end[1] + OFFSET)\n            draw_line_segment(canvas, start_point, end_point, brightness, LINE_WIDTH)\n    if size != BASE_SIZE:\n        return cv2.resize(canvas, (size, size))\n    else:\n        return canvas","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:31.181623Z","iopub.execute_input":"2023-04-15T11:44:31.182023Z","iopub.status.idle":"2023-04-15T11:44:31.193355Z","shell.execute_reply.started":"2023-04-15T11:44:31.181981Z","shell.execute_reply":"2023-04-15T11:44:31.190297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    def __init__(self, drawings, labels, image_size, transform=None):\n        self.drawings = drawings\n        self.labels = labels\n        self.image_size = image_size\n        self.transform = transform\n    def __getitem__(self, idx):\n        single_channel_img = draw_image(self.drawings[idx], self.image_size)\n        three_channel_img = np.repeat(single_channel_img[..., np.newaxis], 3, axis=2)\n        img_as_pil = Image.fromarray(np.uint8(three_channel_img))\n        if self.transform:\n            img_transformed = self.transform(img_as_pil)\n        else:\n            img_transformed = img_as_pil\n            \n        label_tensor = torch.from_numpy(np.array([self.labels[idx]]))\n\n        return img_transformed, label_tensor\n\n    def __len__(self):\n        return len(self.drawings)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:31.194814Z","iopub.execute_input":"2023-04-15T11:44:31.195256Z","iopub.status.idle":"2023-04-15T11:44:31.217506Z","shell.execute_reply.started":"2023-04-15T11:44:31.195218Z","shell.execute_reply":"2023-04-15T11:44:31.216497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getAccuraryTop1_3(prediction, label):\n    with torch.no_grad():\n        batch_size = label.size(0)\n#         print(\"batch_size =\",batch_size)\n        _, pred = prediction.topk(3, 1, True, True)\n        pred = pred.t()\n        result_list = pred.eq(label.view(1, -1).expand_as(pred))\n        final_result = []\n        correct_top1 = result_list[:1].float().sum()\n        correct_top3 = result_list[:3].float().sum()\n        final_result.append(correct_top1.mul_(100.0 / batch_size))\n        final_result.append(correct_top3.mul_(100.0 / batch_size))\n        return final_result","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:31.218943Z","iopub.execute_input":"2023-04-15T11:44:31.219344Z","iopub.status.idle":"2023-04-15T11:44:31.231434Z","shell.execute_reply.started":"2023-04-15T11:44:31.219286Z","shell.execute_reply":"2023-04-15T11:44:31.230375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup_data(x, y, alpha=1.0, use_cuda=True):\n    '''Returns mixed inputs, pairs of targets, and lambda'''\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n\n    batch_size = x.size()[0]\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n\n    # x 是一个batch 一批的输入\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:31.233008Z","iopub.execute_input":"2023-04-15T11:44:31.234087Z","iopub.status.idle":"2023-04-15T11:44:31.242382Z","shell.execute_reply.started":"2023-04-15T11:44:31.234049Z","shell.execute_reply":"2023-04-15T11:44:31.241278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:31.243952Z","iopub.execute_input":"2023-04-15T11:44:31.244407Z","iopub.status.idle":"2023-04-15T11:44:31.252241Z","shell.execute_reply.started":"2023-04-15T11:44:31.24435Z","shell.execute_reply":"2023-04-15T11:44:31.251166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Available Vision Transformer Models: \")\ntimm.list_models(\"vit*\")","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:31.25519Z","iopub.execute_input":"2023-04-15T11:44:31.256336Z","iopub.status.idle":"2023-04-15T11:44:31.267893Z","shell.execute_reply.started":"2023-04-15T11:44:31.256281Z","shell.execute_reply":"2023-04-15T11:44:31.26644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgsize=64\nmodelname=\"vit_base_patch16_224\"\nmodel = timm.create_model(modelname, num_classes=340,in_chans=3,pretrained=True,img_size=imgsize)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:31.271835Z","iopub.execute_input":"2023-04-15T11:44:31.272543Z","iopub.status.idle":"2023-04-15T11:44:47.636469Z","shell.execute_reply.started":"2023-04-15T11:44:31.272495Z","shell.execute_reply":"2023-04-15T11:44:47.635441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_pickle('/kaggle/working/train_{num}.pkl'.format(num = sampleNumber))\ndf_val = pd.read_pickle('/kaggle/working/val_{num}.pkl'.format(num = sampleNumber))\n\ntrain_loader = torch.utils.data.DataLoader(\n    ImageDataset(df_train['drawing'].values, df_train['word'].values, imgsize,\n                     transforms.Compose([\n                        transforms.RandomHorizontalFlip(),\n                        transforms.RandomVerticalFlip(),\n                        transforms.ToTensor(),\n                        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n        ])\n    ),\n    batch_size=BATCH_SIZE, shuffle=True, num_workers=2,\n)\n\nval_loader = torch.utils.data.DataLoader(\n    ImageDataset(df_val['drawing'].values, df_val['word'].values, imgsize,\n                     transforms.Compose([\n                        transforms.RandomHorizontalFlip(),\n                        transforms.RandomVerticalFlip(),\n                        transforms.ToTensor(),\n                        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n        ])\n    ),\n    batch_size=BATCH_SIZE, shuffle=False, num_workers=2,\n)\nmodel = model.cuda()\nloss_fn = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.0001)\n# scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[2, 3, 5, 7, 8], gamma=0.1)\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=len(train_loader) / 10, gamma=0.95)\n\nprint('Train:', df_train.shape[0], 'Val', df_val.shape[0])\nprint('Epoch\\t\\tTrain: loss/Top1/Top3\\t\\tTest: loss/Top1/Top3')\ntotal_train_loss=[]\ntotal_train_acc1=[]\ntotal_train_acc3=[]\ntotal_test_loss=[]\ntotal_test_acc1=[]\ntotal_test_acc3=[]\nfor epoch in range(epochs):\n    train_losss, train_acc1s, train_acc5s = [], [], []\n    for i, data in enumerate(train_loader):\n        # scheduler.step()\n        model = model.train()\n        train_img, train_label = data\n        optimizer.zero_grad()\n\n        # TODO: data paraell\n        # train_img = Variable(train_img).cuda(async=True)\n        # train_label = Variable(train_label.view(-1)).cuda()\n\n        train_img = Variable(train_img).cuda()\n        train_label = Variable(train_label.view(-1)).cuda()\n\n        # 加入mixup\n        if np.random.randint(1, 10) >= 5:\n            mixed_x, y_a, y_b, lam = mixup_data(train_img, train_label)\n            output = model(mixed_x)\n            train_loss = mixup_criterion(loss_fn, output, y_a, y_b, lam)\n        else:\n            output = model(train_img)\n            train_loss = loss_fn(output, train_label)\n\n        # output = model(train_img)\n        # train_loss = loss_fn(output, train_label)\n\n        train_loss.backward()\n        optimizer.step()\n    \n        train_losss.append(train_loss.item())\n        if i % int(100) == 0:\n            with torch.no_grad():\n                train_acc1, train_acc3 = getAccuraryTop1_3(output, train_label)\n                train_acc1s.append(train_acc1.data.item())\n                train_acc5s.append(train_acc3.item())\n    scheduler.step() \n    val_losss, val_acc1s, val_acc5s = [], [], []\n    with torch.no_grad():\n        for data in val_loader:\n            val_images, val_labels = data\n\n            # val_images = Variable(val_images).cuda(async=True)\n            # val_labels = Variable(val_labels.view(-1)).cuda()\n\n            val_images = Variable(val_images).cuda()\n            val_labels = Variable(val_labels.view(-1)).cuda() \n\n            output = model(val_images)\n            val_loss = loss_fn(output, val_labels)\n            val_acc1, val_acc3 = getAccuraryTop1_3(output, val_labels)\n\n            val_losss.append(val_loss.item())\n            val_acc1s.append(val_acc1.item())\n            val_acc5s.append(val_acc3.item())\n\n\n    logstr = '{0:2s}\\t\\t{1:.4f}/{2:.4f}/{3:.4f}\\t\\t{4:.4f}/{5:.4f}/{6:.4f}'.format(\n        str(epoch), \n        np.mean(train_losss, 0), np.mean(train_acc1s, 0), np.mean(train_acc5s, 0),\n        np.mean(val_losss, 0), np.mean(val_acc1s, 0), np.mean(val_acc5s, 0),\n    )\n        # torch.save(model.state_dict(), '/content/drive/MyDrive/ama564 project/data/output/model//{0}_{1}_{2}_{3}.pt'.format(modelname, imgsize, epoch, i))\n    print(logstr)\n    total_train_loss.append(np.mean(train_losss, 0))\n    total_train_acc1.append(np.mean(train_acc1s, 0))\n    total_train_acc3.append(np.mean(train_acc5s, 0))\n    total_test_loss.append(np.mean(val_losss, 0))\n    total_test_acc1.append(np.mean(val_acc1s, 0))\n    total_test_acc3.append(np.mean(val_acc5s, 0)) ","metadata":{"execution":{"iopub.status.busy":"2023-04-15T11:44:47.638215Z","iopub.execute_input":"2023-04-15T11:44:47.638667Z","iopub.status.idle":"2023-04-15T15:40:22.481869Z","shell.execute_reply.started":"2023-04-15T11:44:47.638618Z","shell.execute_reply":"2023-04-15T15:40:22.479695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-04-15T15:40:22.485368Z","iopub.execute_input":"2023-04-15T15:40:22.486128Z","iopub.status.idle":"2023-04-15T15:40:22.493362Z","shell.execute_reply.started":"2023-04-15T15:40:22.486087Z","shell.execute_reply":"2023-04-15T15:40:22.492256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(range(1,25+1),  total_test_loss, \"-r\", label = \"Test Loss\")\nplt.plot(range(1,25+1), total_train_loss, \"-b\", label = \"Train Loss\")\nplt.legend(loc=\"best\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.title(\"Loss\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T15:40:22.494838Z","iopub.execute_input":"2023-04-15T15:40:22.495297Z","iopub.status.idle":"2023-04-15T15:40:22.73382Z","shell.execute_reply.started":"2023-04-15T15:40:22.495235Z","shell.execute_reply":"2023-04-15T15:40:22.732864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(range(1,25+1),  total_test_acc1, \"-r\", label = \"Test Top1 Accuracy\")\nplt.plot(range(1,25+1), total_train_acc1, \"-b\", label = \"Train Top1 Accuracy\")\nplt.legend(loc=\"best\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.title(\"Top 1 Accuracy\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T15:40:22.736421Z","iopub.execute_input":"2023-04-15T15:40:22.737125Z","iopub.status.idle":"2023-04-15T15:40:22.950299Z","shell.execute_reply.started":"2023-04-15T15:40:22.737084Z","shell.execute_reply":"2023-04-15T15:40:22.949275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(range(1,25+1),  total_test_acc3, \"-r\", label = \"Test Top3 Accuracy\")\nplt.plot(range(1,25+1), total_train_acc3, \"-b\", label = \"Train Top3 Accuracy\")\nplt.legend(loc=\"best\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.title(\"Top 3 Accuracy\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T15:40:22.951866Z","iopub.execute_input":"2023-04-15T15:40:22.952512Z","iopub.status.idle":"2023-04-15T15:40:23.187152Z","shell.execute_reply.started":"2023-04-15T15:40:22.952472Z","shell.execute_reply":"2023-04-15T15:40:23.185976Z"},"trusted":true},"execution_count":null,"outputs":[]}]}