{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Quick, Draw! Doodle Recognition\n\n#### _ 2019 Winter Coding _ 이기인\n\n1. 프레임워크\n   - Keras를 사용했습니다.\n   - Tensorflow 역시 사용할 줄 알지만, Keras가 모델 구축에 훨씬 편리하기 때문에 짧은 시간 내에 모델을 테스트하는데 적합하다고 생각했습니다.\n2. 데이터\n   - 학습 데이터셋은 총 340개의 클래스를 가지고 있고, 각 클래스별로 10,000개의 데이터를 가지고 있습니다.\n   - Overfitting을 방지하고 학습데이터를 최대한 활용하기 위해서 클래스별 CSV 파일들을 Shuffle해서 사용했고, 이를 Training에 사용할 때는 Cross-Validation 방식을 적용했습니다.\n   - 하드웨어의 제한으로 학습에 simplified dataset을 사용했습니다.\n   - 데이터에는 stroke가 기록되어있는데, 학습에 사용하기 위해서 3가지 encoding을 바탕으로 전처리를 했습니다.\n     1. stroke 여부 기준 : stroke가 된 부분은 255, 아니면 0\n     2. stroke 시간 기준 : 보통 윤곽을 먼저 그리고 디테일한 부분을 나중에 그립니다. 그래서 첫번째 stroke에 255를 주고, 125가 될 때 까지 다음 stroke는 13씩 값을 감소시켰습니다.\n     3. 각 stroke에서의 시간 기준 : 각 stroke에서 point가 찍힌 시간을 기준으로 가중치를 준다면, stroke의 방향을 알 수 있습니다. 첫번째 point는 255를, 그리고 그 다음부터 20이 될 때까지 감소시켰습니다.\n3. 모델\n   - keras.application 패키지의 MobileNet을 사용했습니다.\n   - 캐글 커널에서만 작업해야하는 조건이었고, 최대한 가벼우면서 성능이 좋은 모델을 선택해야했고, MobileNet과 ResNet18이 후보였습니다. 일주일간 튜닝을 하면서 학습해본 결과 ResNet18보다는 MobileNet의 성능이 높게 나와서 최종적으로 MobileNet을 사용했습니다."},{"metadata":{},"cell_type":"markdown","source":"## 📑 Import Libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\n\nimport json\nimport datetime as dt\nfrom tqdm import tqdm\n\nimport ast\nimport math\nfrom glob import glob\nimport glob\nfrom sklearn.preprocessing import LabelEncoder\nfrom keras.utils.np_utils import to_categorical\nfrom multiprocessing.dummy import Pool\nfrom keras.models import load_model\nimport time\nimport keras\nimport random\n\nfrom skimage.draw import draw\nimport matplotlib.pyplot as plt\nimport matplotlib.style as style\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport os\nprint(os.listdir(\"../input/mobilenetfile\"))\nprint(os.listdir(\"./\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 📥 Shuffle CSVs and Load Data\n### 1. Shuffle CSVs\n<a href=\"https://kaggle.com/gaborfodor/shuffle-csvs\">beluga</a> 님의 코드를 참고했습니다."},{"metadata":{"trusted":true},"cell_type":"code","source":"def f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\nclass Simplified():\n    def __init__(self, input_path='./input'):\n        self.input_path = input_path\n\n    def list_all_categories(self):\n        files = os.listdir(os.path.join(self.input_path, 'train_simplified'))\n        return sorted([f2cat(f) for f in files], key=str.lower)\n\n    def read_training_csv(self, category, nrows=None, usecols=None, drawing_transform=False):\n        df = pd.read_csv(os.path.join(self.input_path, 'train_simplified', category + '.csv'),\n                         nrows=nrows, parse_dates=['timestamp'], usecols=usecols)\n        if drawing_transform:\n            df['drawing'] = df['drawing'].apply(json.loads)\n        return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# shuffle csv 만든적이 없다면 주석 풀고 실행 #\n\n# PATH = '../input/quickdraw-doodle-recognition'\n\n# start = dt.datetime.now()\n# s = Simplified(PATH)\n# NCSVS = 100\n# categories = s.list_all_categories()\n# print(len(categories))\n\n# for y, cat in tqdm(enumerate(categories)):\n#     df = s.read_training_csv(cat, nrows=30000)\n#     df['y'] = y\n#     df['cv'] = (df.key_id // 10 ** 7) % NCSVS\n#     for k in range(NCSVS):\n#         filename = 'train_k{}.csv'.format(k)\n#         chunk = df[df.cv == k]\n#         chunk = chunk.drop(['key_id'], axis=1)\n#         if y == 0:\n#             chunk.to_csv(filename, index=False)\n#         else:\n#             chunk.to_csv(filename, mode='a', header=False, index=False)\n\n# for k in tqdm(range(NCSVS)):\n#     filename = 'train_k{}.csv'.format(k)\n#     if os.path.exists(filename):\n#         df = pd.read_csv(filename)\n#         df['rnd'] = np.random.rand(len(df))\n#         df = df.sort_values(by='rnd').drop('rnd', axis=1)\n#         df.to_csv(filename + '.gz', compression='gzip', index=False)\n#         os.remove(filename)\n# print(df.shape)\n\n# end = dt.datetime.now()\n# print('Latest run {}.\\nTotal time {}s'.format(end, (end - start).seconds))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Load Data\n<a href=\"https://www.kaggle.com/echomil/mobilenet-126x126x3-100k-per-class\">Pawel Mieloch</a> 님의 코드를 참고했습니다."},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_DIR = '../input/quickdraw-doodle-recognition/'\nBASE_SIZE = 256\n\n# Cross Validation을 위해 추가\ndef split_train_val(): \n    ALL_FILES = glob.glob('../input/shuffle-csvs/*.csv.gz')\n    VALIDATION_FILE = '../input/shuffle-csvs/train_k'+str(int(random.random()*93))+'.csv.gz'\n    ALL_FILES.remove(VALIDATION_FILE)\n    np.random.seed(seed=1987)\n    return ALL_FILES, VALIDATION_FILE\n\n\ndef apk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    if len(predicted) > k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])\n\n\ndef preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])\n\ndef f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return sorted([f2cat(f) for f in files], key=str.lower)\n\n\ndef plot_batch(x):    \n    cols = 4\n    rows = 6\n    fig, axs = plt.subplots(nrows=rows, ncols=cols, sharex=True, sharey=True, figsize=(18, 18))\n    for i in range(rows):\n        for k in range(0,3):\n            ax = axs[i, k]\n            ax.imshow(x[i, :, :, k], cmap=plt.cm.gray)\n            ax.axis('off')\n        ax = axs[i, 3]\n        ax.imshow(x[i, :, :], )\n        ax.axis('off')\n    fig.tight_layout()\n    plt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 💻 Predictive Modeling\n### 1. Learning and data Hyper parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUGMENTATION = True\nSTEPS = 200\nBATCH_SIZE = 400\nEPOCHS = 10\nNCATS = 340\nLEARNING_RATE = 0.002\n\nIMG_SHAPE = (128,128,3)\nIMG_SIZE = IMG_SHAPE[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Image Encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_cv2(raw_strokes, size=256, lw=6, augmentation = False):\n    img = np.zeros((BASE_SIZE, BASE_SIZE, 3), np.uint8)\n    for t, stroke in enumerate(raw_strokes):\n        points_count = len(stroke[0]) - 1\n        grad = 255//points_count\n        for i in range(len(stroke[0]) - 1):\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]), (stroke[0][i + 1], stroke[1][i + 1]), (255, 255 - min(t,10)*13, max(255 - grad*i, 20)), lw)\n    if size != BASE_SIZE:\n        img = cv2.resize(img, (size, size))\n    if augmentation:\n        if random.random() > 0.5:\n            img = np.fliplr(img)\n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3. Data generators\nShuffle한 데이터 파일들을 하나로 통합해서 generator에 사용"},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_generator(size, batchsize, lw=6, augmentation = False):\n    while True:\n        for filename in ALL_FILES:\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                df['drawing'] = df['drawing'].apply(eval)\n                x = np.zeros((len(df), size, size,3))\n                for i, raw_strokes in enumerate(df.drawing.values):\n                    x[i] = draw_cv2(raw_strokes, size=size, lw=lw, augmentation = augmentation)\n                x = x / 255.\n                x = x.reshape((len(df), size, size, 3)).astype(np.float32)\n                y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n                yield x, y\n\ndef valid_generator(valid_df, size, batchsize, lw=6):\n    while(True):\n        for i in range(0,len(valid_df),batchsize):\n            chunk = valid_df[i:i+batchsize]\n            x = np.zeros((len(chunk), size, size,3))\n            for i, raw_strokes in enumerate(chunk.drawing.values):\n                x[i] = draw_cv2(raw_strokes, size=size, lw=lw)\n            x = x / 255.\n            x = x.reshape((len(chunk), size, size,3)).astype(np.float32)\n            y = keras.utils.to_categorical(chunk.y, num_classes=NCATS)\n            yield x,y\n        \ndef test_generator(test_df, size, batchsize, lw=6):\n    for i in range(0,len(test_df),batchsize):\n        chunk = test_df[i:i+batchsize]\n        x = np.zeros((len(chunk), size, size,3))\n        for i, raw_strokes in enumerate(chunk.drawing.values):\n            x[i] = draw_cv2(raw_strokes, size=size, lw=lw)\n        x = x / 255.\n        x = x.reshape((len(chunk), size, size, 3)).astype(np.float32)\n        yield x\n        \n\nALL_FILES, VALIDATION_FILE = split_train_val()\ntrain_datagen = image_generator(size=IMG_SIZE, batchsize=BATCH_SIZE, augmentation = AUGMENTATION)\n\nvalid_df = pd.read_csv(VALIDATION_FILE)\nvalid_df['drawing'] = valid_df['drawing'].apply(eval)\nvalidation_steps = len(valid_df)//BATCH_SIZE\nvalid_datagen = valid_generator(valid_df, size=IMG_SIZE, batchsize=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4.Visualization of image encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"single_class_df = valid_df[valid_df['y'] == 2]\nsingle_class_gen = valid_generator(single_class_df, size=IMG_SIZE, batchsize=BATCH_SIZE)\nx, y = next(single_class_gen)\nplot_batch(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5. Model definition"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Dense, Dropout, Flatten, Activation\nfrom keras.metrics import categorical_accuracy, top_k_categorical_accuracy, categorical_crossentropy\nfrom keras.models import Sequential\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom keras.optimizers import Adam\nfrom keras.applications.mobilenet import MobileNet\nfrom keras.applications.mobilenet import preprocess_input\nfrom keras.models import load_model\n\ndef top_3_accuracy(y_true, y_pred):\n    return keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)\n\nreducer = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5)\ncheckpointer = ModelCheckpoint(filepath='mobileNet_ckpt.hdf5', verbose=2, save_best_only=True)\nmodel = load_model('../input/mobilenetfile/mobileNet.hdf5', custom_objects = {'top_3_accuracy':top_3_accuracy})\nopt = Adam(lr = LEARNING_RATE)\nmodel.compile(optimizer = opt, loss = 'categorical_crossentropy', metrics = ['accuracy', top_3_accuracy])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 6. Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(train_datagen,\n                              steps_per_epoch=STEPS,\n                              epochs=EPOCHS,\n                              verbose=2,\n                              validation_data=valid_datagen,\n                              validation_steps=validation_steps,\n                              callbacks=[checkpointer,reducer])\nmodel.save('mobileNet.hdf5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 💻 Result"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))\nsubmission_df['drawing'] = submission_df['drawing'].apply(eval)\nsubmission_datagen = test_generator(submission_df, size=IMG_SIZE, batchsize=BATCH_SIZE)\nsubmission_predictions = model.predict_generator(submission_datagen, math.ceil(len(submission_df)/BATCH_SIZE))\ncats = list_all_categories()\nid2cat = {k: cat.replace(' ', '_') for k, cat in enumerate(cats)}\ntop3 = preds2catids(submission_predictions)\ntop3cats = top3.replace(id2cat)\nsubmission_df['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']\nsubmission = submission_df[['key_id', 'word']]\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}