{"cells":[{"metadata":{},"cell_type":"markdown","source":"1.데이터 처리\n- 데이터는 340개의 class가 각 csvfile로 나뉘어 저장돼있음. 모두 합하여 약 5000만개의 이미지가, csv file의 각 행마다 [x,y] vector조합으로 표기돼있음. 데이터 로딩시간을 줄이기 위해 전체 데이터를 이미지화해 raw데이터로 저장하고 싶었지만, 시간과 메모리 문제로 인해 [x,y] vector와 class만을 pickle파일로 저장한 후 dataloader 부분에서 이미지화 해 불러오기로 함. 그를 위해 34개 class씩 10개의 pickle파일로 데이터를 저장함."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from set1.util.path import path_dropbox_data\nfrom set1.util.iomanager import join, readpickle, writepickle\nimport numpy as np\nimport json\nimport csv\nfrom tqdm import tqdm\n\nclass_len = readpickle(join(path_dropbox_data(), r'QuickDraw\\class_length'))\ndef load_data(filename):\n\tprint('loading class:{}'.format(filename))\n\tdata = []\n\twith open(join(path_dropbox_data(), 'QuickDraw\\\\train_simplified\\\\', filename + '.csv'))as file:\n\t\tread = csv.reader(file)\n\t\tread.__next__()\n\t\tfor row in read:\n\t\t\timg_str = row[1]\n\t\t\timg_array = np.array(json.loads(img_str))\n\t\t\tdata.append([img_array, sorted(list(class_len.keys())).index(filename)])\n\treturn data\n\nif __name__ == \"__main__\":\n\tmerged = dict()\n\tfor name in tqdm(list(class_len.keys())):\n\t\tmerged[name] = load_data(name)\n\twritepickle(join(path_dropbox_data(), 'QuickDraw'), merged, 'merged')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2.네트워크\n- 시간과 머신의 성능을 고려해 5000만개 데이터에 대해 학습을 돌리기 위해서는 간단한 네트워크를 사용하기로 함. Imagenet classification에 사용된 적 있는 resnet을 사용해 구현하기로 결정. 간단한 resnet을 pytorch를 사용해 구현함"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\ndef res_block(in_channel, out_channel):\n\tblock = nn.Sequential(\n\t\tnn.InstanceNorm2d(in_channel),\n\t\tnn.ReLU(),\n\t\tnn.Conv2d(in_channel, out_channel, kernel_size=3, stride=1, padding=1),\n\t\tnn.InstanceNorm2d(out_channel),\n\t\tnn.ReLU(),\n\t\tnn.Conv2d(out_channel, out_channel, kernel_size=3, stride=1, padding=1)\n\t)\n\treturn block\n\ndef big_block(in_channel, out_channel, n_block):\n\tlayers = []\n\tlayers.append(res_block(in_channel, out_channel))\n\tfor i in range(n_block-1):\n\t\tlayers.append(res_block(out_channel, out_channel))\n\treturn nn.Sequential(*layers)\n\nclass Resnet(nn.Module):\n\tdef __init__(self):\n\t\tsuper().__init__()\n\t\tself.block1 = big_block(64, 64, 3)\n\t\tself.block2 = big_block(128, 128, 4)\n\t\tself.block3 = big_block(256, 256, 6)\n\t\tself.block4 = big_block(512, 512, 3)\n\t\tself.downscale = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=8, stride=4, padding=2)\n\t\tself.downscale1 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=4, stride=2, padding=1)\n\t\tself.downscale2 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=4, stride=2, padding=1)\n\t\tself.downscale3 = nn.Conv2d(in_channels=512, out_channels=512, kernel_size=4, stride=2, padding=1)\n\t\tself.FCLayer = nn.Linear(512, 340)\n\n\tdef forward(self, input):\n\t\tskip = [None]*3\n\t\tout = self.downscale(input)\t#64,128,128\n\t\tskip[0] = out\t#64,128,128\n\n\t\tout = self.block1(out)\t#64,128,128\n\t\tout = self.downscale1(torch.cat([out, skip[0]], 1))\t#128,128,128 to 128,64,64\n\t\tskip[1] = out\t\t#128,64,64\n\n\t\tout = self.block2(out)\t#128,64,64\n\t\tout = self.downscale2(torch.cat([out, skip[1]], 1)) #256,64,64 to 256,32,32\n\t\tskip[2] = out\t#256,32,32\n\n\t\tout = self.block3(out)\t#256,32,32\n\t\tout = self.downscale3(torch.cat([out, skip[2]], 1))\t#512,32,32 to 512,16,16\n\n\t\tout = self.block4(out)\t#512,16,16\n\t\tout = F.avg_pool2d(out, 512)\n\n\t\tout = out.reshape([-1,512])\n\t\tout = self.FCLayer(out)\n\t\treturn out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"3.data loader와 loss\n- 학습 진행을 위해 pickle형태로 저장된 [x,y] vector형태를 이미지로 만들어줘 [image, class]형태로 반환하는 data loader와 loss function을 구현"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport cv2\nfrom set1.util.iomanager import join, readpickle\nfrom set1.util.path import path_dropbox_data\nimport torch.utils.data\nfrom torchvision.transforms import transforms\nimport torch.nn as nn\nimport torch\n\n\ndef draw_image(img_array):\n\timg = np.zeros((256, 256, 3), np.uint8)\n\tfor i in range(img_array.shape[0]):\n\t\tfor j in range(img_array[i][0].__len__() - 1):\n\t\t\timg = cv2.line(img, (img_array[i][0][j], img_array[i][1][j]), (img_array[i][0][j + 1], img_array[i][1][j + 1]), (256, 256, 256), 3)\n\treturn np.transpose(img[:, :, 0:1], (2, 0, 1))\n\n\nclass DoodleDataset(torch.utils.data.Dataset):\n\tdef __init__(self, filenumber):\n\t\tself.path = join(path_dropbox_data(), 'QuickDraw')\n\t\tself.data = readpickle(join(self.path, 'merged{}'.format(filenumber)))\n\t\tself.length = readpickle(join(self.path, 'file_length'))[filenumber]\n\n\tdef __getitem__(self, index):\n\t\timage = draw_image(self.data[index][0])\n\t\timage = torch.from_numpy(image).float()\n\t\tlabel = self.data[index][1]\n\t\treturn image, label\n\n\tdef __len__(self):\n\t\treturn self.length\n\n\nclass Doodle_loss(nn.Module):\n\tdef __init__(self):\n\t\tsuper().__init__()\n\t\tself.loss = nn.CrossEntropyLoss()\n\n\tdef forward(self, out, label):\n\t\tloss = self.loss(out, label)\n\t\treturn loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"3.학습\n- 위에서 만든 data loader와 network를 이용해 학습시키는 코드"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport argparse\nfrom set1.util.path import path_dropbox_data\nfrom set1.doodle.util import DoodleDataset, Doodle_loss\nfrom set1.doodle.network2 import Resnet\nfrom set1.YOLO.saver import Saver\nfrom set1.util.iomanager import join\n\nparser = argparse.ArgumentParser()\n\nparser.add_argument('--learning_rate', '-lr', type=float, default=1e-7)\nparser.add_argument('--eval_period', '-ep', type=int, default=100)\nparser.add_argument('--save_period', '-sp', type=int, default=10000)\nparser.add_argument('--num_iter', '-ne', type=int, default=100000)\nparser.add_argument('--network_name', '-nn', type=str, default='SR_GAN_24')\nparser.add_argument('--batch_size', '-bs', type=int, default=256)\nparser.add_argument('--path_root', '-pr', type=str, default=path_dropbox_data())\n\nargs = parser.parse_args([])\n\nlearning_rate = args.learning_rate\neval_period = args.eval_period\nsave_period = args.save_period\nbatch_size = args.batch_size\nnum_iter = args.num_iter\nnetwork_name = args.network_name\n\ndevice = torch.device(\"cuda:0\")\nmodel = Resnet()\nif torch.cuda.device_count() > 1:\n\tmodel = nn.DataParallel(model)\nmodel.to(device)\n\ncriterion = Doodle_loss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\ndoodle_saver = Saver(model, join(path_dropbox_data(), r'doodle', r'save'), \"doodle\", max_to_keep=100)\ndoodle_saver.load()\n\nfor epoch in range(100):\n\tdata_loader = torch.utils.data.DataLoader(dataset=DoodleDataset,\n\t\t\t\t\t\t\t\t\t\t\t  batch_size=batch_size,\n\t\t\t\t\t\t\t\t\t\t\t  shuffle=True)\n\tprint(\"loading {}th dataset\".format(epoch % 10))\n\tfor param in optimizer.param_groups:\n\t\tparam['lr'] *= 0.8\n\tfor i, (images, labels) in enumerate(data_loader):\n\t\timages = images.to(device)\n\t\tlabels = labels.to(device)\n\n\t\t# Forward pass\n\t\toutputs = model(images)\n\t\tloss = criterion(outputs, labels)\n\n\t\toptimizer.zero_grad()\n\t\tloss.backward()\n\t\toptimizer.step()\n\n\t\tif (i) % eval_period == 0:\n\t\t\tacc = torch.sum(labels == torch.argmax(outputs, 1)).float() / batch_size\n\t\t\tprint(\"Epoch [{}/{}], Step [{}] Loss: {:.4f} Accuracy: {}\"\n\t\t\t\t  .format(epoch + 1, 100, i, loss.item(), acc))\n\n\t\tif (i) % save_period == 0:\n\t\t\tdoodle_saver.save(i + epoch * data_loader.__len__())\n\t\t\tprint(\"saved at iter_{}\".format(i + epoch * data_loader.__len__()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"4.결과\n- 시간과 머신 성능으로 인해 learning rate를 높이고 각 34개 class씩 학습시키자 문제가 발생함. 새로운 classes를 학습시키자 이전에 학습시킨 다른 class의 정보가 잊혀짐. 그 결과 최근에 학습시킨 class는 90%가량의 accuracy가 나오지만, 다른 class들이 accuracy가 0에 가까운 결과가 나옴. 결과적으로 약 13%정도의 public score를 받게 됨"},{"metadata":{"trusted":true},"cell_type":"code","source":"filenumber [3], Step [0] accuracy : [9.0234375%]\nfilenumber [3], Step [10] accuracy : [89.9609375%]\nfilenumber [3], Step [20] accuracy : [90.46875%]\nfilenumber [3], Step [30] accuracy : [90.4296875%]\nfilenumber [3], Step [40] accuracy : [88.9453125%]\nfilenumber [3], Step [50] accuracy : [90.1953125%]\nfilenumber [3], Step [60] accuracy : [89.21875%]\nfilenumber [3], Step [70] accuracy : [90.625%]\nfilenumber [3], Step [80] accuracy : [90.3515625%]\nfilenumber [3], Step [90] accuracy : [89.453125%]\nfilenumber [3], Step [100] accuracy : [90.078125%]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filenumber [8], Step [0] accuracy : [0.0%]\nfilenumber [8], Step [10] accuracy : [0.0%]\nfilenumber [8], Step [20] accuracy : [0.0%]\nfilenumber [8], Step [30] accuracy : [0.0%]\nfilenumber [8], Step [40] accuracy : [0.0%]\nfilenumber [8], Step [50] accuracy : [0.0%]\nfilenumber [8], Step [60] accuracy : [0.0%]\nfilenumber [8], Step [70] accuracy : [0.0%]\nfilenumber [8], Step [80] accuracy : [0.0%]\nfilenumber [8], Step [90] accuracy : [0.0%]\nfilenumber [8], Step [100] accuracy : [0.0%]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"5.Discussion\n- 각 34개씩 10개로 나눈 class들이 너무 데이터가 많고 learning rate가 높고, network의 complexity가 부족해 이전 class들의 학습결과가 저장되지 않는것으로 추정됨.\n- 전체 데이터를 무작위로 고르게 섞은 데이터셋을 이용해 무작위하게 학습시키면 보다 accuracy의 분포가 일정해질 것으로 기대할 수 있음\n- resnet 자체의 depth를 높여도 complexity가 증가해 보다 많은 class의 \n- 머신의 성능이 좋다면 전체 5000만개의 데이터를 메모리에 한번에 올려서 학습시킬 수 있음. 그 경우 고르게 학습할 것으로 기대할 수 있음\n- 또, 시간이 더 주어진다면 learning rate를 줄여서 epoch을 진행하면 보다 고르게 학습해 고른 결과가 나올 수 있음."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}