{"cells":[{"metadata":{"_uuid":"1a923c2104562eadf00c5807ee9d5fcbd973f9ec"},"cell_type":"markdown","source":"# Keras Simple CNN Benchmark\n\nI assume you are already familiar with the competition dataset. \n\nThis kernel has two main components:\n\n* Simple Convolutional Network\n* Fast and memory efficient Image Generator\n\nSimplified versions could be trained without GPU."},{"metadata":{"_uuid":"c2529a98a02b39421f173f3267a13a113a8dc7a5"},"cell_type":"markdown","source":"## Setup\nImport the necessary libraries and a few helper functions."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\nimport os\nimport ast\nimport datetime as dt\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = [16, 10]\nplt.rcParams['font.size'] = 14\nimport seaborn as sns\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Activation\nfrom tensorflow.keras.metrics import categorical_accuracy, top_k_categorical_accuracy, categorical_crossentropy\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\nstart = dt.datetime.now()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"033879c8451c02295c28b3680ec9e51f023a059f","trusted":true},"cell_type":"code","source":"DP_DIR = '../input/shuffle-animal-csvs/'\nINPUT_DIR = '../input/quickdraw-doodle-recognition/'\nBASE_SIZE = 256\nNCSVS = 100\nnp.random.seed(seed=1987)\ntf.set_random_seed(seed=1987)\n\ndef f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return sorted([f2cat(f) for f in files], key=str.lower)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1927a294fb63e04463d72e4c89ec42ed4f63ccd8","trusted":true},"cell_type":"code","source":"animals = ['ant', 'bat', 'bear', 'bee', 'bird', 'butterfly', 'camel', 'cat', 'cow',\n           'crab', 'crocodile', 'dog', 'dolphin', 'dragon', 'duck', 'elephant', 'fish',\n           'flamingo', 'frog', 'giraffe', 'hedgehog', 'horse', 'kangaroo', 'lion',\n           'lobster', 'monkey', 'mosquito', 'mouse', 'octopus', 'owl', 'panda',\n           'parrot', 'penguin', 'pig', 'rabbit', 'raccoon', 'rhinoceros', 'scorpion',\n           'sea turtle', 'shark', 'sheep', 'snail', 'snake', 'spider', 'squirrel',\n           'swan', 'teddy-bear', 'tiger', 'whale', 'zebra']\nNCATS = len(animals)\nprint('We have {} animals'.format(NCATS))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"d1b7f42049a9bda3b2a797abddfe55b8d8e83742","trusted":true},"cell_type":"code","source":"def apk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    if len(predicted) > k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])\n\n\ndef preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6b6da3ac9bbf135258b03b04a3e7fd49f617fed"},"cell_type":"markdown","source":"## Simple ConvNet"},{"metadata":{"_uuid":"4eb6c530d54785696674d2b8240271c91b93f6f1","trusted":true},"cell_type":"code","source":"def top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08c624ee6f13f928ca915829e99f31d5c783921a","trusted":true},"cell_type":"code","source":"size = 32\nbatchsize = 512","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8cf540bbf09ab24bd34a207891eb9d1bd93e7f21","trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(128, kernel_size=(3, 3), padding='same', activation='relu',\n                 input_shape=(size, size, 1)))\nmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\nmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\nmodel.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(NCATS, activation='softmax'))\nmodel.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',\n              metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d378ee2118a50627c8fd0930249b82d3e9ee42b5"},"cell_type":"markdown","source":"## Training with Image Generator"},{"metadata":{"_uuid":"7c9a777674c6e20392037a3b5113a0e8c92e0a5e","trusted":true},"cell_type":"code","source":"def draw_cv2(raw_strokes, size=256, lw=6):\n    img = np.zeros((BASE_SIZE, BASE_SIZE), np.uint8)\n    for stroke in raw_strokes:\n        for i in range(len(stroke[0]) - 1):\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]), (stroke[0][i + 1], stroke[1][i + 1]), 255, lw)\n    if size != BASE_SIZE:\n        return cv2.resize(img, (size, size))\n    else:\n        return img\n\ndef image_generator(size, batchsize, ks, lw=6):\n    while True:\n        for k in np.random.permutation(ks):\n            filename = os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(k))\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                df['drawing'] = df['drawing'].apply(ast.literal_eval)\n                x = np.zeros((len(df), size, size))\n                for i, raw_strokes in enumerate(df.drawing.values):\n                    x[i] = draw_cv2(raw_strokes, size=size, lw=lw)\n                x = x / 255.\n                x = x.reshape((len(df), size, size, 1)).astype(np.float32)\n                y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n                yield x, y\n\ndef df_to_image_array(df, size, lw=6):\n    df['drawing'] = df['drawing'].apply(ast.literal_eval)\n    x = np.zeros((len(df), size, size))\n    for i, raw_strokes in enumerate(df.drawing.values):\n        x[i] = draw_cv2(raw_strokes, size=size, lw=lw)\n    x = x / 255.\n    x = x.reshape((len(df), size, size, 1)).astype(np.float32)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d20f0992f56f63efafb633b2e992f2a00746a22","trusted":true},"cell_type":"code","source":"valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)), nrows=10**5)\nx_valid = df_to_image_array(valid_df, size)\ny_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)\nprint(x_valid.shape, y_valid.shape)\nprint('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3 ))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b9413b82070175b63f940ddb2c29c53ffa2d666b","trusted":true},"cell_type":"code","source":"train_datagen = image_generator(size=size, batchsize=batchsize, ks=range(NCSVS - 1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"94084dbe4e224cb185d0107378497f6f63024069","trusted":true},"cell_type":"code","source":"x, y = next(train_datagen)\nn = 8\nfig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(12, 12))\nfor i in range(n**2):\n    ax = axs[i // n, i % n]\n    ax.imshow(x[i, :, :, 0], cmap=plt.cm.gray)\n    ax.axis('off')\nplt.tight_layout()\nfig.savefig('bw.png', dpi=300)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f37a0dc1bc4394daccf261d25bd2637c6719442c","trusted":true},"cell_type":"code","source":"%%timeit\nnext(train_datagen)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4e239911c2464297c9939d3487ccf747869d89c","trusted":true},"cell_type":"code","source":"callbacks = [\n    EarlyStopping(monitor='val_categorical_accuracy', patience=7, min_delta=0.0001, mode='max'),\n    ReduceLROnPlateau(monitor='val_categorical_accuracy', factor=0.5, patience=5, min_delta=0.005,\n                      mode='max', cooldown=3),\n    ModelCheckpoint('bw_animal_cnn.h5', monitor='val_top_3_accuracy', mode='max',\n                    save_best_only=True, save_weights_only=True),\n]\nhist = model.fit_generator(\n    train_datagen, steps_per_epoch=500, epochs=100, verbose=1,\n    validation_data=(x_valid, y_valid),\n    callbacks = callbacks\n)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"cbf298635892171c021d3a3171aa30918f0f182e","trusted":true},"cell_type":"code","source":"hist_df = pd.DataFrame(hist.history)\nhist_df.to_csv('bw_cnn_history.csv', index=False)\nfig, axs = plt.subplots(nrows=2, sharex=True, figsize=(16, 10))\naxs[0].plot(hist_df.val_categorical_accuracy, lw=5, label='Validation Accuracy')\naxs[0].plot(hist_df.categorical_accuracy, lw=5, label='Training Accuracy')\naxs[0].set_ylabel('Accuracy')\naxs[0].set_xlabel('Epoch')\naxs[0].grid()\naxs[0].legend(loc=0)\naxs[1].plot(hist_df.val_categorical_crossentropy, lw=5, label='Validation MLogLoss')\naxs[1].plot(hist_df.categorical_crossentropy, lw=5, label='Training MLogLoss')\naxs[1].set_ylabel('MLogLoss')\naxs[1].set_xlabel('Epoch')\naxs[1].grid()\naxs[1].legend(loc=0)\nfig.savefig('hist.png', dpi=300)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0540513f0592b6fb258a425ecb000b6b9f074726","trusted":true},"cell_type":"code","source":"valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)\nmap3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions).values)\nprint('Map3: {:.3f}'.format(map3))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c8c50504611ec3db916b0d6c0a8aec1ca1263bf","trusted":true},"cell_type":"code","source":"end = dt.datetime.now()\nprint('Latest run {}.\\nTotal time {}s'.format(end, (end - start).seconds))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}