{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Quick, Draw! classification by Densenet"},{"metadata":{},"cell_type":"markdown","source":"This is a score which I got when an epoch was 10. The reason why I showed this socre image is that the score was not shown on top of the page after executing this kernel."},{"metadata":{},"cell_type":"markdown","source":"![Score](http://oi.wooks.xyz/static/images/quick_draw_densenet_score.png)"},{"metadata":{},"cell_type":"markdown","source":"## Table of contents"},{"metadata":{},"cell_type":"markdown","source":"* [Import libraries](#Import-libraries)\n* [Load dataset](#Load-dataset)\n* [Model construction](#Model-construction)\n* [Training](#Training)\n* [Test](#Test)\n* [Reference](#Reference)"},{"metadata":{},"cell_type":"markdown","source":"## Import libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorflow==1.15","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nfrom glob import glob\nimport re\nimport ast\nimport numpy as np \nimport pandas as pd\nfrom PIL import Image, ImageDraw \nfrom tqdm import tqdm\nfrom dask import bag\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D\nfrom tensorflow.keras.metrics import top_k_categorical_accuracy\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\n\nfrom tflearn.layers.conv import global_avg_pool\nfrom tensorflow.contrib.layers import batch_norm, flatten\nfrom tensorflow.contrib.framework import arg_scope","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT_DIR = '../input/quickdraw-doodle-recognition/'\nTRAIN_DIR = ROOT_DIR + 'train_simplified/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set label dictionary and params\nclassfiles = os.listdir(TRAIN_DIR)\nnumstonames = {i: v[:-4].replace(\" \", \"_\") for i, v in enumerate(classfiles)}\n\nnum_classes = 340\nimheight, imwidth = 32, 32  \nims_per_class = 2000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Image conversion function\ndef draw_it(strokes):\n    image = Image.new(\"P\", (256,256), color=255)\n    image_draw = ImageDraw.Draw(image)\n    for stroke in ast.literal_eval(strokes):\n        for i in range(len(stroke[0])-1):\n            image_draw.line([stroke[0][i], \n                             stroke[1][i],\n                             stroke[0][i+1], \n                             stroke[1][i+1]],\n                            fill=0, width=5)\n    image = image.resize((imheight, imwidth))\n    return np.array(image)/255.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load and preprocess train data\ntrain_grand = []\nclass_paths = glob(TRAIN_DIR + '*.csv')\nfor i,c in enumerate(tqdm(class_paths[0: num_classes])):\n    train = pd.read_csv(c, usecols=['drawing', 'recognized'], nrows=ims_per_class*5//4)\n    train = train[train.recognized == True].head(ims_per_class)\n    imagebag = bag.from_sequence(train.drawing.values).map(draw_it) \n    trainarray = np.array(imagebag.compute())  # PARALLELIZE\n    trainarray = np.reshape(trainarray, (ims_per_class, -1))    \n    labelarray = np.full((train.shape[0], 1), i)\n    trainarray = np.concatenate((labelarray, trainarray), axis=1)\n    train_grand.append(trainarray)\n\ntrain_grand = np.array([train_grand.pop() for i in np.arange(num_classes)]) #less memory than np.concatenate\ntrain_grand = train_grand.reshape((-1, (imheight*imwidth+1)))\n\ndel trainarray\ndel train\n\nnp.random.shuffle(train_grand)\ny_train, X_train = train_grand[:,0], train_grand[:,1:]\ny_train = keras.utils.to_categorical(y_train, num_classes)\nprint('X_train.shape:', X_train.shape)\nprint('y_train.shape:', y_train.shape)\n\ndel train_grand","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model construction"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hyperparameter\ngrowth_k = 12\nnb_block = 2 # how many (dense block + Transition Layer) ?\ninit_learning_rate = 1e-4\nepsilon = 1e-8 # AdamOptimizer epsilon\ndropout_rate = 0.2\n\n# Momentum Optimizer will use\nnesterov_momentum = 0.9\nweight_decay = 1e-4\n\n# Label & batch_size\nclass_num = num_classes\nbatch_size = 100\n\n# Epoch\ntotal_epochs = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_layer(input, filter, kernel, stride=1, layer_name=\"conv\"):\n    with tf.name_scope(layer_name):\n        network = tf.layers.conv2d(inputs=input, filters=filter, kernel_size=kernel, strides=stride, padding='SAME')\n        return network\n\ndef Global_Average_Pooling(x, stride=1):\n    return global_avg_pool(x, name='Global_avg_pooling')\n\ndef Batch_Normalization(x, training, scope):\n    with arg_scope([batch_norm],\n                   scope=scope,\n                   updates_collections=None,\n                   decay=0.9,\n                   center=True,\n                   scale=True,\n                   zero_debias_moving_mean=True) :\n        return tf.cond(training,\n                       lambda : batch_norm(inputs=x, is_training=training, reuse=None),\n                       lambda : batch_norm(inputs=x, is_training=training, reuse=True))\n\ndef Drop_out(x, rate, training) :\n    return tf.layers.dropout(inputs=x, rate=rate, training=training)\n\ndef Relu(x):\n    return tf.nn.relu(x)\n\ndef Average_pooling(x, pool_size=[2,2], stride=2, padding='VALID'):\n    return tf.layers.average_pooling2d(inputs=x, pool_size=pool_size, strides=stride, padding=padding)\n\ndef Max_Pooling(x, pool_size=[3,3], stride=2, padding='VALID'):\n    return tf.layers.max_pooling2d(inputs=x, pool_size=pool_size, strides=stride, padding=padding)\n\ndef Concatenation(layers) :\n    return tf.concat(layers, axis=3)\n\ndef Linear(x) :\n    return tf.layers.dense(inputs=x, units=class_num, name='linear')\n\n\nclass DenseNet():\n    def __init__(self, x, nb_blocks, filters, training):\n        self.nb_blocks = nb_blocks\n        self.filters = filters\n        self.training = training\n        self.model = self.Dense_net(x)\n\n\n    def bottleneck_layer(self, x, scope):\n        with tf.name_scope(scope):\n            x = Batch_Normalization(x, training=self.training, scope=scope+'_batch1')\n            x = Relu(x)\n            x = conv_layer(x, filter=4 * self.filters, kernel=[1,1], layer_name=scope+'_conv1')\n            x = Drop_out(x, rate=dropout_rate, training=self.training)\n\n            x = Batch_Normalization(x, training=self.training, scope=scope+'_batch2')\n            x = Relu(x)\n            x = conv_layer(x, filter=self.filters, kernel=[3,3], layer_name=scope+'_conv2')\n            x = Drop_out(x, rate=dropout_rate, training=self.training)\n\n            return x\n\n    def transition_layer(self, x, scope):\n        with tf.name_scope(scope):\n            x = Batch_Normalization(x, training=self.training, scope=scope+'_batch1')\n            x = Relu(x)\n            in_channel = int(x.shape[-1])\n            x = conv_layer(x, filter=in_channel*0.5, kernel=[1,1], layer_name=scope+'_conv1')\n            x = Drop_out(x, rate=dropout_rate, training=self.training)\n            x = Average_pooling(x, pool_size=[2,2], stride=2)\n\n            return x\n\n    def dense_block(self, input_x, nb_layers, layer_name):\n        with tf.name_scope(layer_name):\n            layers_concat = list()\n            layers_concat.append(input_x)\n\n            x = self.bottleneck_layer(input_x, scope=layer_name + '_bottleN_' + str(0))\n\n            layers_concat.append(x)\n\n            for i in range(nb_layers - 1):\n                x = Concatenation(layers_concat)\n                x = self.bottleneck_layer(x, scope=layer_name + '_bottleN_' + str(i + 1))\n                layers_concat.append(x)\n\n            x = Concatenation(layers_concat)\n\n            return x\n\n    def Dense_net(self, input_x):\n        x = conv_layer(input_x, filter=2 * self.filters, kernel=[7,7], stride=2, layer_name='conv0')\n        x = Max_Pooling(x, pool_size=[3,3], stride=2)\n\n        for i in range(self.nb_blocks) :\n            # 6 -> 12 -> 48\n            x = self.dense_block(input_x=x, nb_layers=4, layer_name='dense_'+str(i))\n            x = self.transition_layer(x, scope='trans_'+str(i))\n        x = self.dense_block(input_x=x, nb_layers=32, layer_name='dense_final')\n\n        # 100 Layer\n        x = Batch_Normalization(x, training=self.training, scope='linear_batch')\n        x = Relu(x)\n        x = Global_Average_Pooling(x)\n        x = flatten(x)\n        x = Linear(x)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.reset_default_graph()\nx = tf.placeholder(tf.float32, shape=[None, imheight * imwidth])\nbatch_images = tf.reshape(x, [-1, imheight, imwidth, 1])\nlabel = tf.placeholder(tf.float32, shape=[None, class_num])\n\ntraining_flag = tf.placeholder(tf.bool)\nlearning_rate = tf.placeholder(tf.float32, name='learning_rate')\nlogits = DenseNet(x=batch_images, nb_blocks=nb_block, filters=growth_k, training=training_flag).model\ncost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=label, logits=logits))\n\noptimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, epsilon=epsilon)\ntrain = optimizer.minimize(cost)\n\ncorrect_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(label, 1))\naccuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sess = tf.Session()\nsess.run(tf.global_variables_initializer())\n\nepoch_learning_rate = init_learning_rate\nfor epoch in range(total_epochs):\n    if epoch == (total_epochs * 0.5) or epoch == (total_epochs * 0.75):\n        epoch_learning_rate = epoch_learning_rate / 10\n\n    total_batch = int(len(X_train) / batch_size)\n    for step in range(total_batch):\n        start, end = batch_size * step, batch_size * step + batch_size\n        batch_x, batch_y = X_train[start:end], y_train[start:end]\n        train_feed_dict = {\n            x: batch_x,\n            label: batch_y,\n            learning_rate: epoch_learning_rate,\n            training_flag: True\n        }\n\n        _, loss = sess.run([train, cost], feed_dict=train_feed_dict)\n        if step % 1000 == 0:\n            train_accuracy = sess.run([accuracy], feed_dict=train_feed_dict)\n            print(\"Epoch:\", epoch, \"Step:\", step, \"Loss:\", loss, \"Training accuracy:\", train_accuracy)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"ttvlist = []\nreader = pd.read_csv(ROOT_DIR + 'test_simplified.csv', index_col=['key_id'], chunksize=2048)\nfor chunk in tqdm(reader, total=55):\n    imagebag = bag.from_sequence(chunk.drawing.values).map(draw_it)\n    testarray = np.array(imagebag.compute()).reshape([-1, 1024])\n    test_feed_dict = {x: testarray, training_flag : False}\n    testpreds = sess.run(logits, feed_dict=test_feed_dict)\n    ttvs = np.argsort(-testpreds)[:, 0:3]  # top 3\n    ttvlist.append(ttvs)\n\nttvarray = np.concatenate(ttvlist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df = pd.DataFrame({'first': ttvarray[:,0], 'second': ttvarray[:,1], 'third': ttvarray[:,2]})\npreds_df = preds_df.replace(numstonames)\npreds_df['words'] = preds_df['first'] + \" \" + preds_df['second'] + \" \" + preds_df['third']\n\nsub = pd.read_csv(ROOT_DIR + 'sample_submission.csv', index_col=['key_id'])\nsub['word'] = preds_df.words.values\nsub.to_csv('densenet_submission.csv')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reference"},{"metadata":{},"cell_type":"markdown","source":"* https://www.kaggle.com/jpmiller/image-based-cnn\n* https://github.com/taki0112/Densenet-Tensorflow"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}