{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Keras MobileNet Benchmark\n\nIn a previous benchmark we used a simple three layer ConvNet. This time we use a deeper MobileNet architecture on greyscale strokes. \n\nThis kernel has three main components:\n\n* MobileNet\n* Fast and memory efficient Image Generator with temporal colored strokes\n* Full training & submission with Kaggle Kernel\n\nI did some paramer search but it should not be hard to improve the current score.","execution_count":null},{"metadata":{"_uuid":"f7f2a9516140a84124bf7bbf538ee4c30860b778"},"cell_type":"markdown","source":"## Setup\nImport the necessary libraries and a few helper functions.","execution_count":null},{"metadata":{"trusted":true,"_uuid":"ce6d2aa7de1fa341144def7d3a5b1ffdea26bc91","_kg_hide-input":true},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\nimport os\nimport json\nimport datetime as dt\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = [16, 10]\nplt.rcParams['font.size'] = 14\nimport seaborn as sns\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Activation\nfrom tensorflow.keras.metrics import categorical_accuracy, top_k_categorical_accuracy, categorical_crossentropy\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.applications import MobileNet\nfrom tensorflow.keras.applications.mobilenet import preprocess_input\nstart = dt.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Using tensorflow version: ', tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"978b1e827e598c53df3ef09838a6d85591d83052"},"cell_type":"code","source":"DP_DIR = '../input/shuffle-csvs/'\nINPUT_DIR = '../input/quickdraw-doodle-recognition/'\n\nBASE_SIZE = 256\nNCSVS = 100\nNCATS = 340\nnp.random.seed(seed=1987)\ntf.random.set_seed(seed=1987)\n\ndef f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return sorted([f2cat(f) for f in files], key=str.lower)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"b2fcd1a08ae1ae0619be38a113a244eb6515b63b"},"cell_type":"code","source":"def apk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    if len(predicted) > k:\n        predicted = predicted[:k]\n    score = 0.0\n    num_hits = 0.0\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n    if not actual:\n        return 0.0\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])\n\ndef preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])\n\ndef top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"264156422a95e4b350886d558d516ae8bd2e25c0"},"cell_type":"markdown","source":"## MobileNet\n\nMobileNets are based on a streamlined architecture that uses depthwise separable convolutions to build light weight deep neural networks.\n\n[MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/pdf/1704.04861.pdf)","execution_count":null},{"metadata":{"trusted":true,"_uuid":"54e5f0c637195b6624e2f3e6db5e7f8990e14eb7"},"cell_type":"code","source":"STEPS = 800\nEPOCHS = 20\nsize = 128\nbatchsize = 200","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"0860ec35bee03f0c5cd21202dc7471c2d201cf5f","_kg_hide-output":true},"cell_type":"code","source":"from tensorflow.keras.models import load_model\ndependencies = { 'top_3_accuracy': top_3_accuracy }\nmodel = load_model('../input/doodlenetcheckpoint/model.h5', custom_objects=dependencies)\n\n#model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)\n#model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy', metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])\nprint(model.summary())\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab1834ea2757a53d602a3508efffcc34bc190dc7"},"cell_type":"markdown","source":"## Training with Image Generator","execution_count":null},{"metadata":{"trusted":true,"_uuid":"f6455bf9555b8381b6a4292098a64a0eb7ff54dc"},"cell_type":"code","source":"def draw_cv2(raw_strokes, size=256, lw=6, time_color=True):\n    img = np.zeros((BASE_SIZE, BASE_SIZE), np.uint8)\n    for t, stroke in enumerate(raw_strokes):\n        for i in range(len(stroke[0]) - 1):\n            color = 255 - min(t, 10) * 13 if time_color else 255\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]),\n                         (stroke[0][i + 1], stroke[1][i + 1]), color, lw)\n    if size != BASE_SIZE:\n        return cv2.resize(img, (size, size))\n    else:\n        return img\n\ndef image_generator_xd(size, batchsize, ks, lw=6, time_color=True):\n    while True:\n        for k in np.random.permutation(ks):\n            filename = os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(k))\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                df['drawing'] = df['drawing'].apply(json.loads)\n                x = np.zeros((len(df), size, size, 1))\n                for i, raw_strokes in enumerate(df.drawing.values):\n                    x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw, time_color=time_color)\n                x = preprocess_input(x).astype(np.float32)\n                y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n                yield x, y\n\ndef df_to_image_array_xd(df, size, lw=6, time_color=True):\n    df['drawing'] = df['drawing'].apply(json.loads)\n    x = np.zeros((len(df), size, size, 1))\n    for i, raw_strokes in enumerate(df.drawing.values):\n        x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw, time_color=time_color)\n    x = preprocess_input(x).astype(np.float32)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x, y = next(train_datagen)\n# cats = list_all_categories()\n\n# plt.title(cats[y[0].argmax()])\n# plt.imshow((-x[0, :, :, 0] + 1)/2, cmap=\"gray\")\n\n# preds = model.predict(x, verbose=1)\n# cats[preds[0].argmax()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98ff512e1a1b5e86e86d9eef4127525bedf3b9e1"},"cell_type":"code","source":"valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)), nrows=34000)\nx_valid = df_to_image_array_xd(valid_df, size)\ny_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)\nprint(x_valid.shape, y_valid.shape)\nprint('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3 ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d80ad7f4d378ea7f30479221d604eeeed559cae4"},"cell_type":"code","source":"train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = next(train_datagen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ce5fb89fbb77777316d6fca7689b6636c0e6021"},"cell_type":"code","source":"x, y = next(train_datagen)\nn = 8\nfig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(12, 12))\nfor i in range(n**2):\n    ax = axs[i // n, i % n]\n    (-x[i]+1)/2\n    ax.imshow((-x[i, :, :, 0] + 1)/2, cmap=plt.cm.gray)\n    ax.axis('off')\nplt.tight_layout()\nfig.savefig('gs.png', dpi=300)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e7853cd5bcbea1e68b4b93298e7c1a548b7b538"},"cell_type":"code","source":"%%timeit\nx, y = next(train_datagen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da72d70fc1781e80427d45a80c07b3571dda0b36"},"cell_type":"code","source":"callbacks = [\n    ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001, mode='max', min_lr=1e-5, verbose=1),\n    ModelCheckpoint('/kaggle/working/model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True, save_weights_only=False),\n]\nhists = []\nhist = model.fit_generator(\n    train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,\n    validation_data=(x_valid, y_valid),\n    callbacks = callbacks\n)\nhists.append(hist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05767778d356bc63b7cded355159fd4082eee1a5"},"cell_type":"code","source":"hist_df = pd.concat([pd.DataFrame(hist.history) for hist in hists], sort=True)\nhist_df.index = np.arange(1, len(hist_df)+1)\nfig, axs = plt.subplots(nrows=2, sharex=True, figsize=(16, 10))\naxs[0].plot(hist_df.val_categorical_accuracy, lw=5, label='Validation Accuracy')\naxs[0].plot(hist_df.categorical_accuracy, lw=5, label='Training Accuracy')\naxs[0].set_ylabel('Accuracy')\naxs[0].set_xlabel('Epoch')\naxs[0].grid()\naxs[0].legend(loc=0)\naxs[1].plot(hist_df.val_categorical_crossentropy, lw=5, label='Validation MLogLoss')\naxs[1].plot(hist_df.categorical_crossentropy, lw=5, label='Training MLogLoss')\naxs[1].set_ylabel('MLogLoss')\naxs[1].set_xlabel('Epoch')\naxs[1].grid()\naxs[1].legend(loc=0)\nfig.savefig('hist.png', dpi=300)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorflowjs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflowjs as tfjs\nimport os\nimport zipfile\n\ntfjs.converters.save_keras_model(model, '/kaggle/working')\n\n!mkdir -p /kaggle/working/model\n\ndef zipdir(path, ziph):\n    # ziph is zipfile handle\n    for root, dirs, files in os.walk(path):\n        files = [ fi for fi in files if not fi.endswith(\".zip\") ]\n        for file in files:\n            ziph.write(os.path.join(root, file))\n\nzipf = zipfile.ZipFile('/kaggle/working/model/model.zip', 'w', zipfile.ZIP_DEFLATED)\nzipdir('/kaggle/working', zipf)\nzipf.close()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}