{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n%matplotlib inline\n\nimport matplotlib.pyplot as plt\n\nfrom itertools import product\n\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"DP_DIR = '../input/shuffle-csvs/'\nINPUT_DIR = '../input/quickdraw-doodle-recognition/'\n\nBASE_SIZE = 256\nNCSVS = 100\nNCATS = 340\n\ndef f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return sorted([f2cat(f) for f in files], key=str.lower)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6cdbdf6b09c94839b02b23ae3eed7a5d82be582"},"cell_type":"code","source":"import ast\n\ndef to_triplets(drawing):\n    last_x, last_y = 0, 0\n    triplets = []\n    for stroke in drawing:\n        xs = stroke[0]\n        ys = stroke[1]\n        first_triplet = [[xs[0] - last_x, ys[0] - last_y, 1]]\n        stroke_triplets = list(zip(np.diff(xs), np.diff(ys), np.repeat([0], len(xs) - 1)))\n        new_triplets = first_triplet + stroke_triplets\n        triplets.extend(new_triplets)\n        last_x, last_y = xs[-1], ys[-1]\n    return np.array(triplets)\n\ndef to_strokes(triplets, th = 0.5):\n    strokes_triplets = np.array_split(triplets, np.where(triplets[:, 2] > th)[0])\n    if len(strokes_triplets[0]) == 0:\n        strokes_triplets = strokes_triplets[1:]\n    drawing = []\n    last_x, last_y = 0, 0\n    for stroke_triplets in strokes_triplets:\n        xs = (np.cumsum(stroke_triplets[:, 0]) + last_x).tolist()\n        ys = (np.cumsum(stroke_triplets[:, 1]) + last_y).tolist()\n        last_x, last_y = xs[-1], ys[-1]\n        drawing.append([xs, ys])\n    return drawing\n\nowls = pd.read_csv(INPUT_DIR + 'train_simplified/owl.csv', nrows=256)\nowls['drawing'] = owls['drawing'].apply(ast.literal_eval)\nowls.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fde01fc9d148bdd2d1366119edfb34193ad2aa6"},"cell_type":"code","source":"import os \nimport json\nfrom keras.preprocessing.sequence import pad_sequences\n\ndef image_generator_xd(batchsize, ks):\n    while True:\n        for k in np.random.permutation(ks):\n            filename = os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(k))\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                x = df_to_image_array_xd(df,  add_y=True)\n                yield x\n\ndef df_to_image_array_xd(df, add_y=False):\n    df['drawing_val'] = df['drawing'].apply(json.loads)\n    drawing = df['drawing_val'].apply(to_triplets)\n    inp = drawing.tolist()\n    inp = pad_sequences(inp, padding='post', truncating='post', dtype=np.float32, maxlen=150)\n    inp[:, :, :2] /= 256.\n    df.loc[df.countrycode.isnull(), 'countrycode'] = 'Nan'\n    if add_y:\n        y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n        return inp, y\n    return inp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92d6177538067c01cf780806e051bb425a46e761"},"cell_type":"code","source":"def apk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    if len(predicted) > k:\n        predicted = predicted[:k]\n    score = 0.0\n    num_hits = 0.0\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n    if not actual:\n        return 0.0\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])\n\ndef preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])\n\ndef top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c197303ef830d58a50212a9a18117ad6522d9b3a"},"cell_type":"code","source":"from keras.engine.topology import Layer\n\nfrom tensorflow.python.framework import function\n\n@function.Defun(tf.float32, tf.float32)\ndef norm_grad(x, dy):\n    return dy*(x/(tf.norm(x, axis=0) + 1e-6))\n\n@function.Defun(tf.float32, grad_func=norm_grad)\ndef norm(x):\n    return tf.norm(x, axis=0)\n\n@function.Defun(tf.float32, tf.float32)\ndef dist_neg_grad(x, dy):\n    d = tf.sqrt(x[0, :] ** 2 - x[1, :] ** 2) + 1e-6\n    return dy * tf.stack([x[0, :] / d, -x[1, :] / d])\n\n@function.Defun(tf.float32, grad_func=dist_neg_grad)\ndef dist_neg(x):\n    return tf.sqrt(x[0, :] ** 2 - x[1, :] ** 2)\n\ndef div_no_nan(x, y):\n    return tf.where(y == 0, tf.zeros_like(x), tf.div(x, y))\n\nsize = 32\nimage_size = 64\n\nclass NeuralRasterizationLayer(Layer):\n    def __init__(self, render_size=32, parallel_iterations=32, **kwargs):\n        self.size = render_size\n        self.parallel_iterations = parallel_iterations\n        self.support_mask = True\n        super().__init__(**kwargs)\n        \n    def compute_mask(self, x, mask=None):\n        return None\n    \n    def compute_mask(self, input_shape, input_mask=None):\n        return None\n    \n    def build(self, input_shape):\n        self.output_dim = (self.size, self.size, 1)\n        indices_np = np.array(list(product(np.arange(self.size),np.arange(self.size)))).reshape((self.size,self.size,2))\n        self.tf_indices_np = tf.constant(indices_np, dtype=tf.float32)\n        \n    def call(self, x, mask=None):\n        assert isinstance(x, list)\n        points, atts = x\n        inp = tf.concat([points, atts], axis=-1)\n        images = self.batch_draw_lines(inp)\n        return tf.reshape(images, (-1, self.size, self.size, 1))\n    \n    def tf_point_line_dist(self, points, p0, p1):\n        points0 = points[:, 0]\n        points1 = points[:, 1]\n        p00 = p0[:, 0]\n        p01 = p0[:, 1]\n        p10 = p1[:, 0]\n        p11 = p1[:, 1]\n        d1, d2 = p10-p00, p11-p01\n        enum = tf.abs(d1*points1-d2*points0+p11*p00-p10*p01)\n        denom = norm(tf.stack([d1, d2]))\n        denom = tf.reshape(denom, (-1,))\n        val = div_no_nan(enum, (denom + 1e-6))\n        val0 = tf.abs(points0 - p00)\n        pred0 = tf.cast(tf.equal(p00 - p10, 0), tf.float32)\n        val1 = tf.abs(points1 - p01)\n        pred1 = tf.cast(tf.equal(p01 - p11, 0), tf.float32)\n        return pred0 * val0 + pred1 * val1 + (1 - pred0 - pred1) * val\n\n    def tf_point_point_dist(self, points, p, width=1.0):\n        return norm(tf.stack([points[:, 0] - p[:, 0], points[:, 1] - p[:, 1]]))\n\n    def draw_lines2(self, points, width=0.5):\n        points_x = tf.cumsum(points[:, 1]) * self.size\n        points_y = tf.cumsum(points[:, 0]) * self.size\n        points0 = tf.stack([points_x[:-1], points_y[:-1], points[:-1, 3], points[:-1, 2]], axis=1)\n        points1 = tf.stack([points_x[1:], points_y[1:], points[1:, 3], points[1:, 2]], axis=1)\n\n        points0 = tf.expand_dims(points0, 1)\n        points1 = tf.expand_dims(points1, 1)\n        points0_tile = tf.tile(points0, [1, self.size * self.size, 1])\n        points1_tile = tf.tile(points1, [1, self.size * self.size, 1])\n        points0_tile = tf.reshape(points0_tile, ((-1, 4)))\n        points1_tile = tf.reshape(points1_tile, ((-1, 4)))\n        indices = tf.reshape(self.tf_indices_np, (-1,2))\n        indices = tf.tile(indices, [tf.shape(points0)[0], 1])\n\n        cond_mask1 = tf.not_equal(points1_tile[:, 0], 0.0)\n        cond_mask2 = tf.not_equal(points1_tile[:, 1], 0.0)\n        cond_mask3 = tf.not_equal(points0_tile[:, 0], 0.0)\n        cond_mask4 = tf.not_equal(points0_tile[:, 1], 0.0)\n        cond0 = tf.equal(points1_tile[:, 3], 0.0)\n        cond1 = tf.less_equal(indices[:, 0], points1_tile[:, 0] + width)\n        cond2 = tf.greater_equal(indices[:, 0], points0_tile[:, 0] -  width)\n        cond3 = tf.less_equal(indices[:, 1], points1_tile[:, 1] +  width)\n        cond4 = tf.greater_equal(indices[:, 1], points0_tile[:, 1] - width)\n        cond1b = tf.greater_equal(indices[:, 0], points1_tile[:, 0] - width)\n        cond2b = tf.less_equal(indices[:, 0], points0_tile[:, 0] +  width)\n        cond3b = tf.greater_equal(indices[:, 1], points1_tile[:, 1] -  width)\n        cond4b = tf.less_equal(indices[:, 1], points0_tile[:, 1] + width)\n        cond = cond0 & (((cond1 & cond2) | (cond1b & cond2b)) & ((cond3 & cond4) | (cond3b & cond4b)))\n        cond = cond & ((cond_mask1 & cond_mask2) | (cond_mask3 & cond_mask4))\n\n        ind = tf.where(cond)\n        indices = tf.gather_nd(indices, ind)\n        points0_tile = tf.gather_nd(points0_tile, ind)\n        points1_tile = tf.gather_nd(points1_tile, ind)\n\n        dist = self.tf_point_line_dist(indices, points0_tile, points1_tile)\n        dist_p0 = self.tf_point_point_dist(indices, points0_tile, width)\n        dist_p1 = self.tf_point_point_dist(indices, points1_tile, width)\n        l0 = dist_neg(tf.stack([dist_p0, dist]))\n        l1 = dist_neg(tf.stack([dist_p1, dist]))\n        l0 = tf.reshape(l0, (-1,))\n        l1 = tf.reshape(l1, (-1,))\n        l0 = tf.where(l0 > 0, l0, tf.zeros_like(l0))\n        l1 = tf.where(l1 > 0, l1, tf.zeros_like(l1))\n        lam0, lam1 = div_no_nan(l0, (l0 + l1 + 1e-6)),  div_no_nan(l1, (l0 + l1 + 1e-6))\n        points = tf.where(dist < width, tf.ones_like(dist), tf.zeros_like(dist))\n        points *= tf.cast(lam0 * points0_tile[:, 2] + lam1 * points1_tile[:, 2], tf.float32)\n\n        points_all = tf.scatter_nd(ind, points, (tf.cast(tf.shape(cond)[0], dtype=tf.int64),))\n\n        images = tf.reshape(points_all, ((-1, size, size)))\n        image = tf.reduce_max(images, axis=0)\n        image = tf.minimum(1.0, image)\n        image *= 2.0\n        image -= 1.0\n        return image\n\n    def batch_draw_lines(self, points_batch, width=1):\n        return tf.map_fn(self.draw_lines2, points_batch, parallel_iterations=32)\n    \n    def compute_output_shape(self, input_shape):\n        return (input_shape[0][0], self.size, self.size, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cea82a3f944e28d30eccf6692cbf83d64679d882"},"cell_type":"code","source":"from keras.layers import Lambda\n\nResize = Lambda(lambda x: tf.image.resize_images(x, (image_size, image_size)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfdad3ca6426fcbc5666b744d4d96f7f32fb6c40"},"cell_type":"code","source":"import keras\n\nfrom keras.layers import Input, Concatenate, Dense, BatchNormalization, Conv2D, Activation, Add, MaxPooling2D\nfrom keras.layers import GlobalAveragePooling2D, Dropout\nfrom keras.activations import relu\nfrom keras.models import Model\nfrom keras.optimizers import Adam\n\ndef ConvBnRelu(x, filters, kernel_size, strides, shortcut=False):\n\tinp = x\n\tx = Conv2D(filters, kernel_size, strides=strides, padding='same', use_bias=False)(x)\n\tx = BatchNormalization()(x)\n\tx = Activation('relu')(x)\n\tif shortcut:\n\t\tinp = Conv2D(filters, 1, strides=strides, use_bias=False)(inp)\n\t\tinp = BatchNormalization()(inp)\n\t\tinp = Activation('relu')(inp)\n\t\tx = Add()([x, inp])\n\treturn x\n\ndef SketchNetMini(input_shape):\n\tinp = Input(shape=input_shape) # 64\n\tx = ConvBnRelu(inp, 16, 5, 2) # 32\n\tx = MaxPooling2D(pool_size=3, strides=2, padding='same')(x) # 16\n\tx = ConvBnRelu(x, 32, 5, 1) # 16\n\tx = MaxPooling2D(pool_size=3, strides=2, padding='same')(x) # 8\n\tx = ConvBnRelu(x, 64, 3, 1, shortcut=True) # 8\n\tx = ConvBnRelu(x, 64, 3, 1, shortcut=True) # 8\n\tx = ConvBnRelu(x, 64, 3, 1, shortcut=True) # 8\n\tx = MaxPooling2D(pool_size=3, strides=2, padding='same')(x) # 4\n\tx = ConvBnRelu(x, 128, 3, 1)\n\tx = ConvBnRelu(x, 256, 1, 1)\n\tx = GlobalAveragePooling2D()(x)\n\tx = Dropout(rate=0.01)(x)\n\tmodel = Model(inp, x)\n\treturn model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4343fc4b050c72ae71eb7b93f98584fab1b67e27"},"cell_type":"code","source":"import keras\nfrom keras.layers import InputLayer, LSTM, CuDNNLSTM, Bidirectional, TimeDistributed, Input, Dense, Conv2D, GlobalAveragePooling2D\nfrom keras.layers import BatchNormalization, Dropout, Conv1D, Concatenate\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.metrics import categorical_crossentropy, categorical_accuracy, top_k_categorical_accuracy\n\nrnn_inp = Input(shape=(150, 3))\nnet = BatchNormalization()(rnn_inp)\nnet = Conv1D(32, (5,), activation = 'relu', padding='same')(net)\nnet = Conv1D(64, (5,), activation = 'relu', padding='same')(net)\nrnn = Bidirectional(CuDNNLSTM(64, return_sequences=True))(net)\natt = TimeDistributed(Dense(1, activation='sigmoid', bias_initializer=keras.initializers.zeros()))(rnn)\nimg = NeuralRasterizationLayer(render_size=size)([rnn_inp, att])\nimg = Resize(img)\ncnn_model = SketchNetMini((image_size, image_size, 1))\nimage_out = cnn_model(img)\npred_class = Dense(NCATS, activation='softmax', name='class')(image_out)\nmodel_img = Model(rnn_inp, img)\nmodel = Model(rnn_inp, pred_class)\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef63514067500cc643e1c2fc482ee72bfc96981e"},"cell_type":"code","source":"valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)), nrows=1024)\nx_valid = df_to_image_array_xd(valid_df)\nn = 10\nfig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(12, 12))\nfor i in range(n*n):\n    ax = axs[i % n, i // n]\n    images = model_img.predict(x_valid[i].reshape((1, -1, 3)))\n    images += 1.0\n    images /= 2.0\n    ax.imshow(images[0].reshape((image_size,image_size)), cmap='gray')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cb02632abd44028252e3ac57ef823ccef5fb3fa"},"cell_type":"code","source":"train_datagen = image_generator_xd(batchsize=32, ks=range(NCSVS - 1))\nvalid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)), nrows=3200)\nvalid_data = df_to_image_array_xd(valid_df, add_y=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b45f693c039887f9efd59860b123f9d07d84c992"},"cell_type":"code","source":"model.compile(optimizer=Adam(lr=5e-4), loss='categorical_crossentropy',\n              metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62cc71a625be878f0307ca2b3b68bbc357b6daa3"},"cell_type":"code","source":"hist = model.fit_generator(\n    train_datagen, steps_per_epoch=1000, epochs=10, verbose=1, \n    validation_data=valid_data\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"661fb5e1b3ecf8d1e26c19212fbbf5d2c9858f7f"},"cell_type":"code","source":"model.compile(optimizer=Adam(lr=1e-4), loss='categorical_crossentropy',\n              metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])\nhist = model.fit_generator(\n    train_datagen, steps_per_epoch=1000, epochs=10, verbose=1, \n    validation_data=valid_data\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c04543663a280d281587af54bfd739731e385e0"},"cell_type":"code","source":"valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)), nrows=1024)\nx_valid = df_to_image_array_xd(valid_df)\nn = 10\nfig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(12, 12))\nfor i in range(n*n):\n    ax = axs[i % n, i // n]\n    images = model_img.predict(x_valid[i].reshape((1, -1, 3)))\n    images += 1.0\n    images /= 2.0\n    ax.imshow(images[0].reshape((image_size,image_size)), cmap='gray')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}