{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\nimport os\nimport json\nimport datetime as dt\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = [16, 10]\nplt.rcParams['font.size'] = 14\nimport seaborn as sns\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation, BatchNormalization\nfrom tensorflow.keras.metrics import categorical_accuracy, top_k_categorical_accuracy, categorical_crossentropy\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras.applications.efficientnet import preprocess_input\n\nstart = dt.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_filter = sorted([\n 'apple',\n 'banana',\n 'bandage',\n 'beard',\n 'bed',\n 'bee',\n 'belt',\n 'bench',\n 'bicycle',\n 'bird',\n 'birthday cake',\n 'book',\n 'bucket',\n 'bus',\n 'butterfly',\n 'cactus',\n 'camel',\n 'car',\n 'carrot',\n 'castle',\n 'cat',\n 'ceiling fan',\n 'cell phone',\n 'dog',\n 'dolphin',\n 'donut',\n 'door',\n 'envelope',\n 'eraser',\n 'eye',\n 'eyeglasses',\n 'fish',\n 'guitar',\n 'ice cream',\n 'laptop',\n 'leaf',\n 'lollipop',\n 'nail',\n 'ocean',\n 'octopus',\n 'pants',\n 'paper clip',\n 'pencil',\n 'shark',\n 'shoe',\n 'skull',\n 'smiley face',\n 'snail',\n 'snake',\n 'stairs',\n 'star',\n 'sun',\n 'sword',\n 'toothbrush',\n 'toothpaste',\n 'train',\n 'tree',\n 'wheel',\n 'windmill',\n 'wristwatch',\n 'bush',\n])\n\n_target2id = {}\nfor _id, target in enumerate(_filter):\n    _target2id[target] = _id","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DP_DIR = './shufflecsvs60'\nINPUT_DIR = '../input/quickdraw-doodle-recognition/'\n\nBASE_SIZE = 256\nNCSVS = 100\nNCATS = len(_filter)\nnp.random.seed(seed=1987)\ntf.random.set_seed(seed=1987)\n\ndef f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return sorted([f2cat(f) for f in files], key=str.lower)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    if len(predicted) > k:\n        predicted = predicted[:k]\n    score = 0.0\n    num_hits = 0.0\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n    if not actual:\n        return 0.0\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])\n\ndef preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])\n\ndef top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_cv2(raw_strokes, size=256, lw=6, time_color=True):\n    img = np.zeros((BASE_SIZE, BASE_SIZE), np.uint8)\n    for t, stroke in enumerate(raw_strokes):\n        for i in range(len(stroke[0]) - 1):\n            color = 255 - min(t, 10) * 13 if time_color else 255\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]),\n                         (stroke[0][i + 1], stroke[1][i + 1]), color, lw)\n    if size != BASE_SIZE:\n        return cv2.resize(img, (size, size))\n    else:\n        return img\n\ndef image_generator_xd(size, batchsize, ks, lw=6, time_color=True):\n    while True:\n        for k in np.random.permutation(ks):\n            #filename = os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(k))\n            filename = os.path.join(DP_DIR, 'train_k{}.csv'.format(k))\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                df['drawing'] = df['drawing'].apply(json.loads)\n                x = np.zeros((len(df), size, size, 1))\n                for i, raw_strokes in enumerate(df.drawing.values):\n                    x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw,\n                                             time_color=time_color)\n                x = preprocess_input(x).astype(np.float32)\n                y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n                yield x, y\n\ndef df_to_image_array_xd(df, size, lw=6, time_color=True):\n    df['drawing'] = df['drawing'].apply(json.loads)\n    x = np.zeros((len(df), size, size, 1))\n    for i, raw_strokes in enumerate(df.drawing.values):\n        x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw, time_color=time_color)\n    x = preprocess_input(x).astype(np.float32)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from: https://www.kaggle.com/l0new0lf/efficientnet-finetune-quickdraw/output?select=model.h5\n!curl -O https://www.kaggleusercontent.com/kf/45195565/eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..-twkrbZnzh1BhlfNXqLOYw.RUXalrnkKZrWmiABfOqBjU9oeIhpfZ_ud_RQzqHgMDsN9eB--nK5XzEBXJFB7J2Wmgu8wfLV0sOOdeOaQpxea0LnSQrVpTahcoLtMGUW7QhoSye-mR87Z7enPdHw9_4htcUptalj7h9_7tzFhQq7bkzUeVKkqwkLFQemw6ntQLdvoHDuWDykbFyF4hreEd4RozQKKBnXNb-LUBIaPhF3UMALZV4ctP0UYSGrW1K5xRyv7A0T16VK1msPcdSJxqDlbjK4RtuJbHWqIlOrCNxbHyMTU-6RtpI6ZhkGrN3ap2HTwE-jYiPBVrNzjpMibg97FXx4IiV20ee3Yu8LVDsmlU11-Nr7nefx_viihiIJaymhwFVae9xmbsCPMjx-Sn3WqwpNHQrKimM4Bmo4gGKOUVNYBrcb6T2WvZUKzhFKlD83mdP5-yth-4bZOPIAvZgkTW30IIn0kSTjxrn5Kee7275foqUObYk61K4ESfGNf-WTC396siqLcC3TJeL6IX4w-RAnsNIJLeTWN58d4GlzL3_JmcA-_pQHYvZl4x2H9mrc2LwPWEvi6OUZKW1tl0Y7Rqxgu-tHFMzgrM8nb_Tauh8DtO_IoycBQkGl3AGJwZk6Ly-X2TLk7eGgbFhPAiDqcBP_zf4F6ZQRyLtjDDpwBw.Sxq4k0zgY6AT_K2JOhpAGw/model.h5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transfer Learning "},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS = 800\nEPOCHS = 16\nsize = 64\nbatchsize = 680","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = EfficientNetB0(input_shape=(size, size, 1), weights='model.h5', classes=340)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nNote: fine - tuning. Not using as extractor.\n\"\"\"\n\n# for layer in base_model.layers:\n#     layer.trainable = False\n#     #layer.name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rep_layer = 'avg_pool'\nlast_layer = base_model.get_layer(rep_layer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = Flatten()(last_layer.output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 1 fc w/ 256, bn, dropout and softmax\nx = Dense(256, activation='relu', name='lin')(x)\nx = BatchNormalization()(x)\nx = Dropout(0.5)(x)\nx = Dense(61, activation='softmax', name='softmax')(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_model = Model(\n    inputs = base_model.input,\n    outputs = x\n)\n\n#new_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_model.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy',\n              metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols_to_filter = ['word']\nsearch_values = _filter\nbase = '../input/quickdrawshufflecsvs/'\n\n# str.match avoid these\n# instead of str.contains\n# edge_cases = [\n#    'hot dog',\n#    'palm tree',\n#    'pineapple',\n#    'police car',\n#    'postcard',\n#    'school bus',\n#    'streetlight',\n# ]\n\nos.makedirs('shufflecsvs60featvecs', exist_ok = True)\nfor file in os.listdir('../input/quickdrawshufflecsvs/'):\n    path = base + file\n    df = pd.read_csv(path)\n    patt = '|'.join(search_values)\n    mask = df[cols_to_filter].apply(lambda x: x.str.match(patt)).any(1)\n    df_filtered = df[mask]\n    df_filtered['y'] = df_filtered['word'].apply(lambda x: _target2id[x])\n    df_filtered.to_csv('shufflecsvs60featvecs/'+file)\n    print(f'{len(df)}>>{len(df_filtered)} \\t: {len(np.unique(df_filtered[\"word\"]))} classes')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DP_DIR = 'shufflecsvs60featvecs'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)), nrows=34000)\nvalid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv'.format(NCSVS - 1)), nrows=34000)\nx_valid = df_to_image_array_xd(valid_df, size)\ny_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)\nprint(x_valid.shape, y_valid.shape)\nprint('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3 ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = next(train_datagen)\nn = 8\nfig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(12, 12))\nfor i in range(n**2):\n    ax = axs[i // n, i % n]\n    (-x[i]+1)/2\n    ax.imshow((-x[i, :, :, 0] + 1)/2, cmap=plt.cm.gray)\n    ax.axis('off')\nplt.tight_layout()\nfig.savefig('gs.png', dpi=300)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"batch shape:\", x.shape)\nprint(\"single image shape:\", x[0].shape)\nprint(\"range of pixel values - max:\", x[0].max(), \" min:\", x[0].min())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(\n    x[0].reshape(64,64),\n    cmap=plt.cm.gray\n)\nplt.colorbar()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def invert(x):\n    return (-x+1)/2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(\n    invert(x[0].reshape(64,64)),\n    cmap=plt.cm.gray\n)\nplt.colorbar()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\nx, y = next(train_datagen)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Note:** data is nothing like imagenet, so, training from scratch"},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [\n    ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,\n                          mode='max', min_lr=1e-5, verbose=1),\n    ModelCheckpoint('transfer_model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,\n                    save_weights_only=True),\n]\nhists = []\nhist = new_model.fit_generator(\n    train_datagen, steps_per_epoch=STEPS, epochs=100, verbose=1,\n    validation_data=(x_valid, y_valid),\n    callbacks = callbacks\n)\nhists.append(hist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_df = pd.concat([pd.DataFrame(hist.history) for hist in hists], sort=True)\nhist_df.index = np.arange(1, len(hist_df)+1)\nfig, axs = plt.subplots(nrows=2, sharex=True, figsize=(16, 10))\naxs[0].plot(hist_df.val_categorical_accuracy, lw=5, label='Validation Accuracy')\naxs[0].plot(hist_df.categorical_accuracy, lw=5, label='Training Accuracy')\naxs[0].set_ylabel('Accuracy')\naxs[0].set_xlabel('Epoch')\naxs[0].grid()\naxs[0].legend(loc=0)\naxs[1].plot(hist_df.val_categorical_crossentropy, lw=5, label='Validation MLogLoss')\naxs[1].plot(hist_df.categorical_crossentropy, lw=5, label='Training MLogLoss')\naxs[1].set_ylabel('MLogLoss')\naxs[1].set_xlabel('Epoch')\naxs[1].grid()\naxs[1].legend(loc=0)\nfig.savefig('hist.png', dpi=300)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_predictions = new_model.predict(x_valid, batch_size=128, verbose=1)\nmap3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions).values)\nprint('Map3: {:.3f}'.format(map3))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))\ntest.head()\nx_test = df_to_image_array_xd(test, size)\nprint(test.shape, x_test.shape)\nprint('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3 ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predictions = new_model.predict(x_test, batch_size=128, verbose=1)\n\ntop3 = preds2catids(test_predictions)\ntop3.head()\ntop3.shape\n\ncats = list_all_categories()\nid2cat = {k: cat.replace(' ', '_') for k, cat in enumerate(cats)}\ntop3cats = top3.replace(id2cat)\ntop3cats.head()\ntop3cats.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']\nsubmission = test[['key_id', 'word']]\nsubmission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)), index=False)\nsubmission.head()\nsubmission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"end = dt.datetime.now()\nprint('Latest run {}.\\nTotal time {}s'.format(end, (end - start).seconds))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}