{"cells":[{"metadata":{"_uuid":"97b50a690f5fb81efa6aa4b9fe4befebf237277a"},"cell_type":"markdown","source":"A Kernel implementing a model inspired from dilated, gated convolutions for the QuickDraw Challenge as an alternative to lstm. More info can be found at https://arxiv.org/abs/1612.08083 and https://arxiv.org/abs/1511.07122\nDue to kernel limitations the training is limited. "},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"9ffb8a9c9710b696415f0081ff86bfe7e97ae0a4"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom ast import literal_eval\nimport time\nimport os\nimport keras as k\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f53ab8ffe242eb3764649a38d067e5f14cfdaa9"},"cell_type":"markdown","source":"Defining a few helper functions to load and preprocess the data"},{"metadata":{"trusted":true,"_uuid":"29a25af08544896febd50b32ac111068a8fe61e5"},"cell_type":"code","source":"def space_replace(s):\n    return s.replace(' ', '_')\n\nfolder_name = '../input/train_simplified/'\nfiles= os.listdir(folder_name)\nfiles= [space_replace(os.path.splitext(item)[0]) for item in files]\n\nfiles_to_id = {j:i for i,j in enumerate(files)}\nget_file_name = {j:i for i,j in zip(files_to_id.keys(), files_to_id.values())}\n\n\ndef replace_name(item):\n    item= space_replace(item)\n    return files_to_id[item]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f9b443d4bd3faa84441a813503c3cfd3537a41f6"},"cell_type":"markdown","source":"Merge and suffle the data into a single csv file. This will simplify things with feeding data to the model, and prevent the bottleneck."},{"metadata":{"trusted":true,"_uuid":"498b05babe4fe0363ddf70753c7f02b86b491df0"},"cell_type":"code","source":"files= os.listdir(folder_name)\nnum_samp= 1000     #Number of lines to read in each csv every iteration\nnum_lines= 10000     #Number of lines from each csv to include in the final dataframe max on average 100k\nnum_blocks = int(num_lines / num_samp)\ndataframe_length= 0\nfor block in range(num_blocks):\n    dataframe = pd.DataFrame()\n    block_idx = list(np.arange(1,num_samp*block))\n    for item in files:\n        file_path = os.path.join(folder_name, item)\n        df= pd.read_csv(file_path, \n                        usecols=['drawing', 'recognized', 'word'],\n                        dtype={'drawing':str, 'recognized':str, 'word':str},\n                        nrows= num_samp, skiprows= block_idx, low_memory=False)\n        df= df.loc[df.recognized=='True']\n        dataframe = dataframe.append(df, ignore_index=True)\n    dataframe= dataframe.sample(frac=1).reset_index(drop=True)\n    dataframe_length += len(dataframe)\n    dataframe.to_csv('training_data_10k.csv', columns=['drawing', 'word'], index=False, header=False, mode='a')\n    print ('Done processing block: '+ str(block), end='\\r')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd34bb6f50352afe17fe1ec6e89a87cbd53afd5d"},"cell_type":"code","source":"def get_path_signatures(seq):\n    # An implementation of path signatures based \n    # on https://arxiv.org/pdf/1603.03788.pdf\n    # Here 1 dimensional paths were chosen\n\n    \n    seq = literal_eval(seq)\n    reshaped_seq = []\n    for i in range(len(seq)):\n        reshaped_seq.append(np.array(list(zip(*seq[i]))))\n    \n    all_arrays= np.concatenate(reshaped_seq)\n       \n    path01= all_arrays[:,0]\n    path02= all_arrays[:,1]\n    \n    path1, path2, = [],[]\n    path11, path12, path21, path22 = [],[],[],[]\n    path111, path222 = [], []\n    path1111, path2222 = [], []\n    \n    for i in range(len(all_arrays)-1):\n        path1.append(all_arrays[i+1,0] - all_arrays[i,0])\n        path2.append(all_arrays[i+1,1] - all_arrays[i,1])\n        path11.append(0.5*((all_arrays[i+1,0] - all_arrays[i,0])**2))\n        path111.append(((all_arrays[i+1,0] - all_arrays[i,0])**3)/6)\n        path1111.append(((all_arrays[i+1,0] - all_arrays[i,0])**4)/24)\n        path22.append(0.5*((all_arrays[i+1,1] - all_arrays[i,1])**2))\n        path222.append(((all_arrays[i+1,1] - all_arrays[i,1])**3)/6)\n        path2222.append(((all_arrays[i+1,1] - all_arrays[i,1])**4)/24)\n    \n    path1 = np.array(path1)\n    path1 = np.append(path1,[0])\n    path2 = np.array(path2)\n    path2 = np.append(path2,[0])\n    path11 = np.array(path11)\n    path11 = np.append(path11,[0])\n    path111 = np.array(path111)\n    path111 = np.append(path111,[0])\n    path1111 = np.array(path1111)\n    path1111 = np.append(path1111,[0])\n    path22 = np.array(path22)\n    path22 = np.append(path22,[0])\n    path222 = np.array(path222)\n    path222 = np.append(path222,[0])\n    path2222 = np.array(path2222)\n    path2222 = np.append(path2222,[0])\n    \n    stacked =  np.transpose(\n        np.stack((path01, path02, \n                  path1, path2, path11,\n                  path22, path111, path222,\n                  path1111, path2222\n                 )))\n    return stacked","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6dabf6802b4b23fe8beaee57544d0a5bf7e13dca"},"cell_type":"code","source":"dataframe_length = 31206300\n\nclass training_batch_generator(k.utils.Sequence):\n    # A generator to preprocess the input and \n    # feed the data in batches.\n    def __init__(self,\n                 tr_iterator,\n                 batch_size=50,  \n                 data_length=1000, \n                 n_classes=340):\n        \n        self.iterator = tr_iterator\n        self.batch_size = batch_size\n        self.data_length = data_length\n        self.n_classes = n_classes\n        self.chunk= next(self.iterator).sample(frac=1).reset_index(drop=True)\n        \n    def __len__(self):\n        return int(np.floor(self.data_length / self.batch_size))\n\n    def __getitem__(self,index):\n        \n        labels_in = self.chunk.loc[index*self.batch_size:\n                                  (index+1)*self.batch_size-1,1].apply(\n                                   replace_name).values\n    \n        xystrokes = self.chunk.loc[index*self.batch_size:\n                                  (index+1)*self.batch_size-1,0].apply(\n                                   get_path_signatures).values\n        xystrokes = pad_sequences(xystrokes, maxlen= 128, dtype=np.int32)\n        \n        labels_in = k.utils.to_categorical(labels_in, num_classes= self.n_classes)\n        \n        return xystrokes, labels_in\n    \n    def on_epoch_end(self):\n        self.chunk= next(self.iterator).sample(frac=1).reset_index(drop=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a367870c1edcccc523793c1cedb4f6f46360718"},"cell_type":"code","source":"dataframe_length = 31206300\n\nclass validation_batch_generator(k.utils.Sequence):\n    # A generator to preprocess the input and \n    # feed the data in batches.\n    def __init__(self,\n                 data_chunk,\n                 batch_size=50,  \n                 data_length=1000, \n                 n_classes=340):\n        \n        self.chunk = data_chunk\n        self.batch_size = batch_size\n        self.data_length = data_length\n        self.n_classes = n_classes\n        \n        \n    def __len__(self):\n        return int(np.floor(self.data_length / self.batch_size))\n\n    def __getitem__(self,index):\n        \n        labels_in = self.chunk.loc[index*self.batch_size:\n                                  (index+1)*self.batch_size-1,1].apply(\n                                   replace_name).values\n    \n        xystrokes = self.chunk.loc[index*self.batch_size:\n                                  (index+1)*self.batch_size-1,0].apply(\n                                   get_path_signatures).values\n        xystrokes = pad_sequences(xystrokes, maxlen= 128, dtype=np.int32)\n        \n        labels_in = k.utils.to_categorical(labels_in, num_classes= self.n_classes)\n        \n        return xystrokes, labels_in\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06314185c8470c4e2a51580c6c9b97df35bdb8c8"},"cell_type":"code","source":"def top3accuracy(y_ture, y_pred):\n    return k.metrics.top_k_categorical_accuracy(y_ture, y_pred, k=3)\n\nearlystop = k.callbacks.EarlyStopping(monitor='val_loss', mode='min', patience=4, verbose=1)\nreduce_lr = k.callbacks.ReduceLROnPlateau(monitor='val_loss', mode='min', factor=0.9, patience=1, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f68fbd758f77f6213dd93af8c6f855b66f56b43"},"cell_type":"markdown","source":"The backbone of the model. Here we define a pre-activated residual block where dilation rate increases expoentially, while the gating mechanism acts as activation."},{"metadata":{"trusted":true,"_uuid":"e8b07d593c17ec6c4d18d72eb6f1d764870b5dc2"},"cell_type":"code","source":"def gated_activation(layer_in):\n    x1,xg = tf.split(layer_in,2,2)\n    xg = Activation('sigmoid')(xg)\n    xg = Multiply()([x1 , xg])\n    return xg\n    \n\nbase = 680\ndef ConvBlock(x, \n              filters=base,\n              size=3,\n              d_rate=1\n              ):\n    \n    x1 = Lambda(gated_activation)(x)\n    x2 = Conv1D(filters, size, padding='same', dilation_rate=2**(d_rate-1))(x1)\n    x2a = Lambda(gated_activation)(x2)\n    x3 = Conv1D(filters, size, padding='same', dilation_rate=2**(d_rate))(x2a)\n    x3a = Add()([x3, x])\n    \n    return x3a\n","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"818d82242509e80d39ae30e3c1a00474f6ce360d"},"cell_type":"code","source":"input1 = Input(shape=(128,10))\nnorm = BatchNormalization()(input1)\nnorm_s = Conv1D(base,1,padding='same', use_bias=False)(norm)\nres = Conv1D(base,3,padding='same')(norm)\nres = Lambda(gated_activation)(res)\nres = Conv1D(base,3,padding='same')(res)\nres = Add()([norm_s, res])\n\nfor i in range(2):\n    rate = 2*i + 2\n#     rate = [2,4], so we get dilation rate = [2,4,8,16]\n    res = ConvBlock(res, base, d_rate=rate)\n\nres = Lambda(gated_activation)(res)\nres = Conv1D(base,3,padding='same',dilation_rate=32)(res)\nres = Lambda(gated_activation)(res)\n\npool = GlobalAveragePooling1D()(res)\noutput = Dense(340, activation='softmax')(pool)\n\nmodel = Model(inputs=input1, outputs=output)\nopt = k.optimizers.Adam(lr=0.001, decay=0.0)\n\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=opt,\n              metrics=['categorical_accuracy', top3accuracy])\n# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"55e0ba692de5f027738fa3a197d72eb11ac5f659"},"cell_type":"code","source":"data = pd.read_csv('training_data_10k.csv', \n                   usecols=[0,1], \n                   header=None,\n                   nrows=3000000,\n                   dtype={0:str, 1:str}, \n                   iterator=True,\n                   chunksize=100000)\n\nval_data = pd.read_csv('training_data_10k.csv', \n                       usecols=[0,1], \n                       header=None,\n                       skiprows=3000000,\n                       nrows= 10000,\n                       dtype={0:str, 1:str}, \n                       )\n\nval_generator = validation_batch_generator(data_chunk= val_data, \n                                batch_size=100, data_length=10000)\n\nt= time.time()\n\ntrain_generator= training_batch_generator(tr_iterator=data, \n                                batch_size=100, data_length=100000)\n\nr = model.fit_generator(train_generator,\n                        validation_data= val_generator,\n                        verbose=2,\n                        epochs=29,\n                        use_multiprocessing=False,\n                        callbacks=[earlystop, reduce_lr] \n                        )\ntt= time.time()\nprint('Training time:', (tt-t)/60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef7145427df8455b0088cc1689c0223c18e6b344"},"cell_type":"code","source":"plt.plot(r.history['loss'])\nplt.plot(r.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Iterations (x1000)')\nplt.legend(['Train', 'Test'], loc='upper right')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efc13cc953e5167b049bf99f8c3fa5faf3379c93"},"cell_type":"code","source":"plt.plot(r.history['categorical_accuracy'])\nplt.plot(r.history['val_categorical_accuracy'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Iterations (x1000)')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b909e44dea7dcae8062fbe5b7b70e98346ca261"},"cell_type":"code","source":"class predict_batch_generator(k.utils.Sequence):\n    # A generator to preprocess the input and \n    # feed the data in batches.\n    def __init__(self,\n                 data_chunk,\n                 batch_size=200,  \n                 data_length=1000\n                ):\n        \n        self.chunk = data_chunk\n        self.batch_size = batch_size\n        self.data_length = data_length\n        \n        \n    def __len__(self):\n        return int(np.floor(self.data_length / self.batch_size))\n\n    def __getitem__(self,index):\n    \n        xystrokes = self.chunk.loc[index*self.batch_size:\n                                  (index+1)*self.batch_size-1,0].apply(\n                                   get_path_signatures).values\n        xystrokes = pad_sequences(xystrokes, maxlen= 128, dtype=np.int32)\n        \n        return xystrokes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32a1e772706fe3cad36d920f98c8842d0282bb76"},"cell_type":"code","source":"test_df = pd.read_csv('training_data_10k.csv', \n                       usecols=[0,1], \n                       header=None,\n                       skiprows=3000000,\n                       nrows= 10000,\n                       dtype={0:str, 1:str}, \n                       )\n\npred_g = predict_batch_generator(data_chunk=test_df, batch_size=100, data_length=len(test_df))\npreds = model.predict_generator(pred_g,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96a239dd9fcd1f3d90a6fad6c63c379f1b5b378d"},"cell_type":"code","source":"def get_top_3(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=-1)[:, :3], columns=[1,2,3])\n\ntop3preds = get_top_3(preds)\nget_names = test_df.loc[:,1].apply(replace_name).values\nget_names = [[item] for item in get_names]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47c167b49a05de536218db4ea84eb7dec2204348"},"cell_type":"code","source":"def apk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    if len(predicted) > k:\n        predicted = predicted[:k]\n    score = 0.0\n    num_hits = 0.0\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n    if not actual:\n        return 0.0\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1343db35e10e29a4fe307106a65571453de318b"},"cell_type":"code","source":"mapk(get_names, top3preds.values, k=3)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":1}