{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Quick draw recognition with CNN\nIn this project I am going to use the Quick draw dataset. Quick draw is one of the Google's A.I. experiments - https://quickdraw.withgoogle.com\nI will first train the model. Then convert it to a tensorflow.js model and then I will create a web application with a canvas drawing space that will recognise the drawing using the generated model. "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np\nimport pandas as pd\nfrom tensorflow.keras import layers\nfrom tensorflow import keras \nimport tensorflow as tf\nimport glob\nimport json\nimport cv2\nfrom tensorflow.keras.applications import MobileNet\nfrom tensorflow.keras.applications.mobilenet import preprocess_input\nfrom tensorflow.keras.metrics import categorical_accuracy, top_k_categorical_accuracy, categorical_crossentropy\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport random\nfrom tensorflow.keras.optimizers import Adam\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_FILES = glob.glob('../input/shuffle-csvs*/*.csv.gz')\nTEST_FILE = pd.read_csv('../input/shuffle-csvs/train_k7.csv.gz', nrows=30000)\n# TEST_FILE = pd.read_csv('../input/quickdraw-doodle-recognition/test_simplified.csv', nrows=100)\nBASE_SIZE = 256\nBATCH_SIZE = 512\nNCATS = 340\nnp.random.seed(seed=1978)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def draw_image_from_strokes(raw_strokes, size=256, lw=6, augmentation = False, time_color=False):\n    img = np.zeros((BASE_SIZE, BASE_SIZE), np.uint8)\n    for t, stroke in enumerate(raw_strokes):\n        for i in range(len(stroke[0]) - 1):\n            color = 255 - min(t, 10) * 13 if time_color else 0\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]),\n                         (stroke[0][i + 1], stroke[1][i + 1]), color, lw)\n    if size != BASE_SIZE:\n        img = cv2.resize(img, (size, size))\n    if augmentation:\n        if random.random() > 0.5:\n            img = np.fliplr(img)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_generator(size, batchsize, lw=6, augmentation = True, time_color=True):\n    while True:\n        for filename in TRAIN_FILES:\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                df['drawing'] = df['drawing'].apply(json.loads)\n                x = np.zeros((len(df), size, size, 1))\n                for i, raw_strokes in enumerate(df.drawing.values):\n                    x[i, :, :, 0] = draw_image_from_strokes(raw_strokes, size=size, lw=lw, augmentation = augmentation,\n                                                           time_color=time_color)\n                x = x / 255.\n                x = preprocess_input(x).astype(np.float32)\n                y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n                yield x, y\n                \n    \ndef test_image_generator(df, size, lw=6, time_color=True):\n    df['drawing'] = df['drawing'].apply(json.loads)\n    x = np.zeros((len(df), size, size, 1))\n    for i, raw_strokes in enumerate(df.drawing.values):\n        x[i, :, :, 0] = draw_image_from_strokes(raw_strokes, size=size, lw=lw, time_color=time_color)\n    x = x / 255.\n    x = preprocess_input(x).astype(np.float32)\n    y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n    return x, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = image_generator(128, BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = next(train_datagen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images(x):\n    n = 8\n    fig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(12, 12))\n    for i in range(n**2):\n        ax = axs[i // n, i % n]\n        (-x[i]+1)/2\n        ax.imshow((-x[i, :, :, 0] + 1)/2, cmap=plt.cm.gray)\n        ax.axis('off')\n    plt.tight_layout()\n    fig.savefig('gs.png', dpi=300)\n    plt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"plot_images(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test, y_test = test_image_generator(TEST_FILE, 128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(x_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# THE MODELS\n\nI am creating two models. The first one is custom CNN with just 9 layers."},{"metadata":{"trusted":true},"cell_type":"code","source":"model = keras.Sequential()\nmodel.add(layers.Convolution2D(16, (3, 3), padding='same', input_shape=x.shape[1:], activation='relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Convolution2D(16, (3, 3), padding='same', activation='relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Convolution2D(16, (3, 3), padding='same', activation='relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(128, activation='relu'))\nmodel.add(layers.Dense(340, activation='softmax'))\n\ndef top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)\n\nmodel.compile(loss='categorical_crossentropy',\n             optimizer=Adam(learning_rate=0.002),\n             metrics=[categorical_accuracy, categorical_crossentropy, top_3_accuracy])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And the second is MobileNet which is based on a streamlined architecture that uses depthwise separable convolutions to build light weight deep neural networks."},{"metadata":{"trusted":true},"cell_type":"code","source":"model_mn = MobileNet(input_shape=(128, 128, 1), alpha=1., weights=None, classes=NCATS)\nmodel_mn.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',\n              metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])\nprint(model_mn.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [\n    ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,\n                          mode='max', min_lr=1e-5, verbose=1),\n    ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,\n                    save_weights_only=True),\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hists = []\nhist = model.fit_generator(\n    train_datagen, steps_per_epoch=BATCH_SIZE, epochs=70, verbose=1,\n    validation_data=(x_test, y_test),\n    callbacks = callbacks\n)\nhists.append(hist)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}