{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 프로그래머스 DS 과제 [이보현]\n\n## 참조 오픈소스\n1. https://www.kaggle.com/gaborfodor/greyscale-mobilenet-lb-0-892\n2. https://www.kaggle.com/jpmiller/image-based-cnn/data\n\n## 개요\n\n졸업작품 때 CNN을 사용해본 경험과, 이미지 처리 그리고 제한된 디바이스 안에서 구현해야 했기 때문에 MobileNet을 사용했습니다. 오픈소스로 제출된 20위권 안에서 MobileNet을 Greyscale로만 제출이 되어있어서 RGB채널을 이용하여 전처리를 구현해 보았습니다.\n\n오픈소스 1을 기반으로 MobileNet 학습을 진행하였고, RGB채널을 모두 사용하게 전처리 한 경우 test 실행 시 메모리 초과가 있어서 오픈소스 2를 참고하여 chunk로 분할하여 데이터를 처리하였습니다."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"This kernel has three main components:\n\n* MobileNet\n* Fast and memory efficient Image Generator with temporal colored strokes\n* Full training & submission with Kaggle Kernel"},{"metadata":{"_uuid":"f7f2a9516140a84124bf7bbf538ee4c30860b778"},"cell_type":"markdown","source":"## Setup\nImport the necessary libraries and a few helper functions."},{"metadata":{"trusted":true,"_uuid":"ce6d2aa7de1fa341144def7d3a5b1ffdea26bc91","_kg_hide-input":true},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\nimport os\nimport json\nimport datetime as dt\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = [16, 10]\nplt.rcParams['font.size'] = 14\nimport seaborn as sns\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Activation\nfrom tensorflow.keras.metrics import categorical_accuracy, top_k_categorical_accuracy, categorical_crossentropy\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.applications import MobileNet\nfrom tensorflow.keras.applications.mobilenet import preprocess_input\nstart = dt.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"978b1e827e598c53df3ef09838a6d85591d83052"},"cell_type":"code","source":"DP_DIR = '../input/shuffle-csvs/'\nINPUT_DIR = '../input/quickdraw-doodle-recognition/'\n\nBASE_SIZE = 256\nNCSVS = 100\nNCATS = 340\nnp.random.seed(seed=1987)\ntf.set_random_seed(seed=1987)\n\ndef f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return files","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"b2fcd1a08ae1ae0619be38a113a244eb6515b63b"},"cell_type":"code","source":"def apk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    if len(predicted) > k:\n        predicted = predicted[:k]\n    score = 0.0\n    num_hits = 0.0\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n    if not actual:\n        return 0.0\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])\n\ndef preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])\n\ndef top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"264156422a95e4b350886d558d516ae8bd2e25c0"},"cell_type":"markdown","source":"## MobileNet\n\nMobileNets are based on a streamlined architecture that uses depthwise separable convolutions to build light weight deep neural networks.\n\n[MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/pdf/1704.04861.pdf)"},{"metadata":{"trusted":true,"_uuid":"54e5f0c637195b6624e2f3e6db5e7f8990e14eb7"},"cell_type":"code","source":"STEPS = 800\nEPOCHS = 9\nsize = 64\nbatchsize = 680","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"0860ec35bee03f0c5cd21202dc7471c2d201cf5f","_kg_hide-output":true},"cell_type":"code","source":"model = MobileNet(input_shape=(size, size, 3), alpha=1., weights=None, classes=NCATS)\nmodel.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',\n              metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab1834ea2757a53d602a3508efffcc34bc190dc7"},"cell_type":"markdown","source":"## Training with Image Generator"},{"metadata":{"trusted":true,"_uuid":"f6455bf9555b8381b6a4292098a64a0eb7ff54dc"},"cell_type":"code","source":"def draw_cv2(raw_strokes, size=256, lw=6, time_color=True):\n    img = np.zeros((BASE_SIZE, BASE_SIZE,3), np.uint8)\n    \n    colors = [ (255,0,0),\n               (255,127,0),\n               (255,255,0),\n               (0,255,0),\n               (0,0,255),\n               (75,0,130),\n               (139,0,255)]\n    \n    for t, stroke in enumerate(raw_strokes):\n        color = ( 255 - colors[t % 7][0], 255 - colors[t % 7][1], 255 - colors[t % 7][2])\n        for i in range(len(stroke[0]) - 1):\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]),\n                         (stroke[0][i + 1], stroke[1][i + 1]), color, lw)\n#     print(np.unique(img))\n    if size != BASE_SIZE:\n        return cv2.resize(img, (size, size))\n    else:\n        return img\n\ndef image_generator_xd(size, batchsize, ks, lw=6, time_color=True):\n    while True:\n        for k in np.random.permutation(ks):\n            filename = os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(k))\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                df['drawing'] = df['drawing'].apply(json.loads)\n                x = np.zeros((len(df), size, size, 3))\n                for i, raw_strokes in enumerate(df.drawing.values):\n                    x[i, :, :, :] = draw_cv2(raw_strokes, size=size, lw=lw,\n                                             time_color=time_color)\n                x = preprocess_input(x).astype(np.float32)\n                y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n                yield x, y\n\ndef df_to_image_array_xd(df, size, lw=6, time_color=True):\n    df['drawing'] = df['drawing'].apply(json.loads)\n    x = np.zeros((len(df), size, size, 3))\n    for i, raw_strokes in enumerate(df.drawing.values):\n#         print(raw_strokes)\n        x[i, :, :, :] = draw_cv2(raw_strokes, size=size, lw=lw, time_color=time_color)\n    x = preprocess_input(x).astype(np.float32)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98ff512e1a1b5e86e86d9eef4127525bedf3b9e1"},"cell_type":"code","source":"valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)), nrows=34000)\n# valid_df = pd.read_csv(os.path.join(INPUT_DIR, 'train_simplified'))\nx_valid = df_to_image_array_xd(valid_df, size)\ny_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)\nprint(x_valid.shape, y_valid.shape)\nprint('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3 ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d80ad7f4d378ea7f30479221d604eeeed559cae4"},"cell_type":"code","source":"train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ce5fb89fbb77777316d6fca7689b6636c0e6021"},"cell_type":"code","source":"x, y = next(train_datagen)\nn = 8\nfig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(12, 12))\nfor i in range(n**2):\n    ax = axs[i // n, i % n]\n    ax.imshow( (255*(-x[i, :, :, :] + 1)/2).astype(np.uint8) )\n    ax.axis('off')\nplt.tight_layout()\nfig.savefig('gs.png', dpi=300)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e7853cd5bcbea1e68b4b93298e7c1a548b7b538"},"cell_type":"code","source":"%%timeit\nx, y = next(train_datagen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da72d70fc1781e80427d45a80c07b3571dda0b36"},"cell_type":"code","source":"callbacks = [\n    ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,\n                          mode='max', min_lr=1e-5, verbose=1),\n    ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,\n                    save_weights_only=True),\n]\nhists = []\nhist = model.fit_generator(\n    train_datagen, steps_per_epoch=STEPS, epochs=1, verbose=1,\n    validation_data=(x_valid, y_valid),\n    callbacks = callbacks\n)\nhists.append(hist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c1927f22d3c45cba0bdee7d6f4b6c858d82d614"},"cell_type":"code","source":"valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)\nmap3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions).values)\nprint('Map3: {:.3f}'.format(map3))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be4577a9ba00611697eea8f241a42c504981e86f"},"cell_type":"markdown","source":"## Create Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nfrom dask import bag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image, ImageDraw \nfrom pprint import pprint\nimport ast\nimheight, imwidth = 64, 64 \n# # faster conversion function\n\n# for i, raw_strokes in enumerate(df.drawing.values):\n#     x[i, :, :, :] = draw_cv2(raw_strokes, size=size, lw=lw, time_color=time_color)\n    \ndef df_to_image_array_chunk(chunk_values, size=64, lw=6, time_color=True):\n#     df['drawing'] = df['drawing'].apply(json.loads)\n    x = np.zeros((len(chunk_values),size, size, 3), np.uint8)\n    for i, raw_strokes in enumerate(chunk_values):\n        x[i, :, :, :] = draw_cv2(raw_strokes, size=size, lw=lw, time_color=time_color)\n    x = preprocess_input(x).astype(np.float32)\n    return cv2.resize(x, (size, size))\n\ndef draw_it(raw_strokes, size=64, lw=6, time_color=True):\n    img = np.zeros((BASE_SIZE, BASE_SIZE,3), np.uint8)\n    colors = [ (255,0,0),\n               (255,127,0),\n               (255,255,0),\n               (0,255,0),\n               (0,0,255),\n               (75,0,130),\n               (139,0,255)]\n    print(len(raw_strokes), len(raw_strokes[0]))\n    for t, stroke in enumerate(raw_strokes):\n#         if t == 1 or t == 2:\n#             print(stroke)\n        color = ( 255 - colors[t % 7][0], 255 - colors[t % 7][1], 255 - colors[t % 7][2])\n        for i in range(len(stroke[0]) - 1):\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]),\n                         (stroke[0][i + 1], stroke[1][i + 1]), color, lw)\n#     pprint(np.unique(img))\n    img = preprocess_input(img).astype(np.float32)\n    return cv2.resize(img, (size, size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ttvlist = []\nreader = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'), index_col=['key_id'],\n    chunksize=2048)\nfor df in tqdm(reader, total=55):\n    df['drawing'] = df['drawing'].apply(json.loads)\n    \n    x = np.zeros((len(df), size, size, 3))\n    for i, raw_strokes in enumerate(df.drawing.values):\n        x[i, :, :, :] = draw_cv2(raw_strokes, size=size)\n    x = preprocess_input(x).astype(np.float32)\n    \n    testpreds = model.predict(x, verbose=0)\n    ttvs = np.argsort(-testpreds)[:, 0:3]  # top 3\n    ttvlist.append(ttvs)\n    \nttvarray = np.concatenate(ttvlist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #%% set label dictionary and params\n# classfiles = list_all_categories()\n# numstonames = {i: v[:-4].replace(\" \", \"_\") for i, v in enumerate(classfiles)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classfiles = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\nnumstonames = {i: v[:-4].replace(\" \", \"_\") for i, v in enumerate(classfiles)} #adds underscores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(numstonames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df = pd.DataFrame({'first': ttvarray[:,0], 'second': ttvarray[:,1], 'third': ttvarray[:,2]})\npreds_df = preds_df.replace(numstonames)\npreds_df['words'] = preds_df['first'] + \" \" + preds_df['second'] + \" \" + preds_df['third']\n\nsub = pd.read_csv('../input/quickdraw-doodle-recognition/sample_submission.csv', index_col=['key_id'])\nsub['word'] = preds_df.words.values\n# sub.to_csv('../input/quickdraw-doodle-recognition/submission_mobilenet.csv')\nsub.to_csv('submission_mobilenet.csv')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test = sub\n# submission = test[['key_id', 'word']]\n# submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)), index=False)\n# submission.head()\n# submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b418f4c06c4e4453aa1b5ab16dde344eb8b735c5"},"cell_type":"code","source":"end = dt.datetime.now()\nprint('Latest run {}.\\nTotal time {}s'.format(end, (end - start).seconds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}