{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from ast import literal_eval\nimport csv\nimport os\nimport shutil\nimport tarfile\n\nimport numpy as np\nfrom PIL import Image, ImageDraw\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7fa7c6c670b280592688baa74c518b8d7a536f5d"},"cell_type":"markdown","source":"## Sequential Experiments\n\n\n__v38__: Conv + BiCudnnLSTM + Dense, 50000 steps, 128 items / batch\n\n__accuracy: 0.86507815__ on first __5000__ items\n\n__loss: 1.2240986__ on random __5000__ items\n\n__Competition Score: 0.763__\n\n---\n\n__v40__: Conv + BiCudnnLSTM + Dense, 100000 steps, 128 items / batch\n\n__accuracy: 0.88125__ on first __5000__ items\n\n__loss: 1.0899909__ on random __5000__ items\n\n__Competition Score: 0.752__\n\n---\n\n__v48__: Conv + Recurrent CNN with BiCudnnLSTM, 100000 steps, 128 items / batch\n\n__accuracy: 0.89039063__ on first __5000__ items\n\n__loss: 1.0255966__ on first __5000__ items\n\n__Competition Score: 0.780__\n\n---\n\n__v49__: Conv + Recurrent CNN with BiCudnnLSTM, 100000 steps, 128 items / batch, exponentially decaying learning rate at 0.96/100000 steps\n\n__accuracy: 0.8897656__ on first __5000__ items\n\n__loss: 1.0400434__ on first __5000__ items\n\n__Competition Score: 0.769__\n\n---\n\n__v50__: Conv + BiCudnnLSTM + Dense, 100000 steps, 128 items / batch, exponentially decaying learning rate at 0.96/100000 steps\n\n__accuracy: 0.8934375__ on first __5000__ items\n\n__loss: 1.0174446__ on first __5000__ items\n\n__Competition Score: 0.782__\n\n---\n\n__v52__: Conv + Recurrent CNN with BiCudnnLSTM, 100000 steps, 128 items / batch, exponentially decaying learning rate at 0.8/10000 steps\n\n__accuracy: 0.8384375__ on first __5000__ items\n\n__loss: 1.3769166__ on first __5000__ items\n\n__Competition Score: N/A__\n\n---\n\n__v53__: Conv + BiCudnnLSTM + Dense, 100000 steps, 128 items / batch, exponentially decaying learning rate at 0.8/10000 steps\n\n__accuracy: 0.8714844__ on first __5000__ items\n\n__loss: 1.1495687__ on first __5000__ items\n\n__Competition Score: 0.718__\n\n---\n\n__v61__: Temporal CNN, 100000 steps, 128 items / batch, exponentially decaying learning rate at 0.8/10000 steps\n\n__accuracy: 0.635625__ on first __5000__ items\n\n__loss: 2.6789243__ on first __5000__ items\n\n__Competition Score: N/A__\n\n---\n\n~~__v30__: Conv + BiGRU, 25000 steps, 128 items / batch~~ TIMED OUT\n\n~~__v36__: Conv + BiGRU + Dense, 25000 steps, 128 items / batch~~ TIMED OUT\n\n\n## Image Experiments\n\n\n__v66__: Vanilla 4 layer CNN + 4 dense layers, 100000 steps, 128 items / batch, exponentially decaying learning rate at 0.8/100000 steps\n\n__accuracy: 0.8244531__ on first __5000__ items\n\n__loss: 1.4278741__ on first __5000__ items\n\n__Competition Score: N/A__\n\n## Image + Sequential Experiments\n\n__v70__: \n\n__Image Model__: Vanilla 4 layer CNN + 4 dense layers\n\n__Sequential Model__: Conv + BiCudnnLSTM + Dense\n\n100000 steps, 128 items / batch, exponentially decaying learning rate at 0.8/100000 steps\n\n__Replaced all dropout with batch norm, except for the RNN layer__\n\n__accuracy: 0.8975781__ on first __5000__ items\n\n__loss: 0.97098076__ on first __5000__ items\n\n__Competition Score: 0.825__\n\n__v82__: \n\n__Image Model__: Vanilla 4 layer CNN + 4 dense layers\n\n__Sequential Model__: Conv + BiCudnnLSTM + Dense\n\n__100000__ steps, 128 items / batch, exponentially decaying learning rate at __0.01__/100000 steps\n\n__Replaced all dropout with batch norm, except for the RNN layer__\n\n__Bigger RNN State, Bigger sequence final output, and considering the last sequence output instead of summing over all sequence outputs__\n\n__Remove a conv layer and a couple of dense layers from the image model__\n\n__Remove a layer from the convolutions at the beginning of the sequence model__\n\n__accuracy:__ on first __5000__ items\n\n__loss:__ on first __5000__ items\n\n__Competition Score:__"},{"metadata":{"trusted":true,"_uuid":"f94c1ccee53d92f645d889f3cb18ac1671e3b906"},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d670f3330d31683596a366149476be6241a2f531"},"cell_type":"code","source":"tf.logging.set_verbosity(tf.logging.INFO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bdef8eb451a3045df5e28d2e01718b1c36702be"},"cell_type":"code","source":"top_k = 3\n\nmax_steps = 100000\n\nbatch_size = 128\nsubmission_batch_size = 1\neval_dataset_size = 5000\ntrain_dataset_prefetch_size = 10000\ntrain_dataset_shuffle_buffer_size = 100000\n\nimage_conv_layers = [(64, (3, 3), 2), (192, (3, 3), 2), (512, (3, 3), 2)]\nimage_dense_layers = [1024, 64]\n\nsequence_conv_layers = [(128, 5), (256, 3)]\nsequence_rnn_num_layers = 2\nsequence_rnn_state_size = 256\nsequence_rnn_direction = \"bidirectional\"\nsequence_rnn_dropout_prob = 0.3\nsequence_dense_layers = [192,]\n\nlearning_rate = 0.0001\nlearning_rate_decay_steps = 100000\nlearning_rate_decay_rate = 0.01\ngradient_clipping_norm = 9.0\n\nmodel_dir = \"task\"\nsave_checkpoints_secs = 2400\nsave_summary_steps = 1000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"290ddc4bda0401532c762e47145fee4a57747aeb","scrolled":false},"cell_type":"code","source":"train_files = os.listdir(\"../input/train_simplified\")\nn_classes = len(train_files)\nindex = { x.split(\".\")[0]: i for i, x in enumerate(sorted(train_files)) }\ninv_index = { v: k.replace(\" \", \"_\") for k, v in index.items() }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d949c994928939dcd0d8afa786363ba31faa995"},"cell_type":"code","source":"# https://www.kaggle.com/huyenvyvy/bidirectional-lstm-using-data-generator-lb-0-825\ndef _parse_sequence(v):\n    a = literal_eval(v.decode(\"ascii\"))\n    strokes = [(xi, yi, i) for i, (x, y) in enumerate(a) for xi, yi in zip(x, y)]\n    strokes = np.stack(strokes)\n    strokes[:, 2] = [1] + np.diff(strokes[:, 2]).tolist()\n    # 2 for a new start and 1 for each stroke, as 0 is used for padding/masking\n    strokes[:, 2] += 1\n    return np.float32(strokes)\n\n# https://www.kaggle.com/marikekoch/quick-draw-mp\ndef _parse_image(v):\n    image = Image.new(\"P\", (256, 256), color=255)\n    image_draw = ImageDraw.Draw(image)\n    a = literal_eval(v.decode(\"ascii\"))\n    for stroke in a:\n        for i in range(len(stroke[0]) - 1):\n            image_draw.line([stroke[0][i], stroke[1][i], stroke[0][i + 1], stroke[1][i + 1]], fill=0, width=5)\n    image = image.resize((32, 32))\n    return np.float32(np.array(image) / 255)\n\ndef parse_train_row(_, v, k, __, ___, l):\n    f = tf.py_func(_parse_sequence, [v], tf.float32, stateful=False)\n    f.set_shape((None, 3))\n    \n    i = tf.py_func(_parse_image, [v], tf.float32, stateful=False)\n    i.set_shape((32, 32))\n    \n    l = tf.py_func(lambda i: np.int32(index[i.decode(\"ascii\")]), [l], tf.int32, stateful=False)\n    l.set_shape(())\n    \n    return { \"strokes\": f, \"images\": i, \"keys\": k, \"lengths\": tf.shape(f)[0] }, l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1460e5f513559e5931fe0375fa4e46176efb93f7"},"cell_type":"code","source":"dataset = tf.data.Dataset.list_files([\n    os.path.join(\"../input/train_simplified\", x) \n    for x in os.listdir(\"../input/train_simplified\")\n])\n\ndataset = dataset.repeat()\n\ndataset = dataset.interleave(lambda x: tf.contrib.data.CsvDataset(\n    x, \n    [tf.constant([\"\"], dtype=tf.string), tf.string, tf.string, tf.constant([\"\"], dtype=tf.string), tf.constant([\"\"], dtype=tf.string), tf.string],\n).skip(1), cycle_length=n_classes, block_length=1)\n\ndataset = dataset.map(parse_train_row)\neval_dataset = dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d839e7e062ae7c521f321ef6466c37c73ca30493"},"cell_type":"code","source":"dataset = dataset.prefetch(train_dataset_prefetch_size)\ndataset = dataset.shuffle(train_dataset_shuffle_buffer_size)\ndataset = dataset.padded_batch(batch_size, padded_shapes=dataset.output_shapes)\ndataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6e6f7134690a16318b1708279923c6e61f22f9d"},"cell_type":"code","source":"eval_dataset = eval_dataset.padded_batch(batch_size, padded_shapes=eval_dataset.output_shapes)\neval_dataset = eval_dataset.take(eval_dataset_size)\neval_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16a33f34fc6582ef2b1595ded44274627852c211"},"cell_type":"code","source":"def image_model_fn(images, training):\n    net = tf.expand_dims(images, -1)\n    net = tf.layers.batch_normalization(net, training=training)\n    \n    for l in image_conv_layers:\n        net = tf.layers.conv2d(net, filters=l[0], kernel_size=l[1], padding=\"same\", activation=None)\n        net = tf.layers.batch_normalization(net, training=training)\n        net = tf.nn.relu(net)\n        net = tf.layers.max_pooling2d(net, pool_size=l[2], strides=l[2])\n        \n    net = tf.layers.flatten(net)\n    \n    for l in image_dense_layers:\n        net = tf.layers.dense(net, l, activation=None)\n        net = tf.layers.batch_normalization(net, training=training)\n        net = tf.nn.relu(net)\n    \n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"129de2aa1c274475323a744fcb3ad84b45233826"},"cell_type":"code","source":"def sequence_model_fn(strokes, lengths, training):\n    net = tf.layers.batch_normalization(strokes, training=training)\n    \n    for l in sequence_conv_layers:\n        net = tf.layers.conv1d(net, filters=l[0], kernel_size=l[1], activation=None, padding=\"same\")\n        net = tf.layers.batch_normalization(net, training=training)\n        net = tf.nn.relu(net)\n        \n    # CudnnLSTM input is time major\n    net = tf.transpose(net, [1, 0, 2])\n    net, _ = tf.contrib.cudnn_rnn.CudnnLSTM(\n        num_layers=sequence_rnn_num_layers,\n        num_units=sequence_rnn_state_size,\n        direction=sequence_rnn_direction,\n        dropout=sequence_rnn_dropout_prob if training else 0.0,\n    )(net)\n    net = tf.transpose(net, [1, 0, 2])\n    \n    # mask out-of-length rnn outputs\n    mask = tf.tile(\n        tf.expand_dims(\n            tf.sequence_mask(\n                lengths,\n                tf.shape(net)[1],\n            ), \n            2,\n        ), \n        [1, 1, tf.shape(net)[2]],\n    )\n    \n    net = tf.where(mask, net, tf.zeros_like(net))\n    \n    net = net[:, -1, :]\n    \n    for l in sequence_dense_layers:\n        net = tf.layers.dense(net, l, activation=None)\n        net = tf.layers.batch_normalization(net, training=training)\n        net = tf.nn.relu(net)\n        \n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ea3bf426958df55b6bd50bf6b4f5e8011517688"},"cell_type":"code","source":"def model_fn(features, labels, mode):\n    \"\"\"\n    features: { \"strokes\": [?, ?, 3], \"keys\": [?,], \"images\": [?, 32, 32], lengths\": [?,] }\n    labels: [?,]\n    \"\"\"\n    training = mode == tf.estimator.ModeKeys.TRAIN\n    \n    image_output = image_model_fn(features[\"images\"], training=training) # [?, 128]\n    sequence_output = sequence_model_fn(features[\"strokes\"], features[\"lengths\"], training=training) # [?, 128]\n    \n    net = tf.concat([image_output, sequence_output], axis=1)\n    \n    logits = tf.layers.dense(net, n_classes)\n    predictions = tf.nn.softmax(logits, axis=1)\n    _, indices = tf.nn.top_k(predictions, k=top_k)\n    \n    if mode == tf.estimator.ModeKeys.PREDICT:\n        return tf.estimator.EstimatorSpec(\n            mode=mode, \n            predictions={\n                \"predictions\": indices,\n                \"keys\": features[\"keys\"],\n            },\n        )\n    \n    loss = tf.reduce_mean(\n        tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits),\n    )\n    \n    if mode == tf.estimator.ModeKeys.EVAL:\n        return tf.estimator.EstimatorSpec(\n            mode=mode, \n            loss=loss,\n            eval_metric_ops={\n                # https://stackoverflow.com/a/44801217\n                \"accuracy\": tf.metrics.mean(tf.nn.in_top_k(predictions=predictions, targets=labels, k=top_k)),\n            },\n        )\n    \n    # https://github.com/tensorflow/models/blob/master/tutorials/rnn/quickdraw/train_model.py#L233\n    train_op = tf.contrib.layers.optimize_loss(\n        loss=loss,\n        global_step=tf.train.get_global_step(),\n        learning_rate=learning_rate,\n        learning_rate_decay_fn=lambda l, s: tf.train.exponential_decay(l, s, learning_rate_decay_steps, learning_rate_decay_rate, staircase=True),\n        optimizer=\"Adam\",\n        # some gradient clipping stabilizes training in the beginning.\n        clip_gradients=gradient_clipping_norm,\n        summaries=[\"learning_rate\", \"loss\", \"gradients\", \"gradient_norm\"],\n    )\n    \n    return tf.estimator.EstimatorSpec(\n        mode=mode, \n        loss=loss,\n        train_op=train_op,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef7fc051bd59dc0e5a3ca2ffe0c9b2c1f87efd6a"},"cell_type":"code","source":"runConfig = tf.estimator.RunConfig(\n    model_dir=model_dir, \n    save_checkpoints_secs=save_checkpoints_secs,\n    save_summary_steps=save_summary_steps,\n)\n\nestimator = tf.estimator.Estimator(model_fn, config=runConfig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"557aa3f4ae185e6e44af9a03c0d5e55537ec25b1"},"cell_type":"code","source":"train_spec = tf.estimator.TrainSpec(lambda: dataset.make_one_shot_iterator().get_next(), max_steps=max_steps)\neval_spec = tf.estimator.EvalSpec(lambda: eval_dataset.make_one_shot_iterator().get_next())\n\ntf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cffd7dc734ff123b5c93209da0c9fcfe61c68dab"},"cell_type":"code","source":"def parse_submission_row(k, _, v):\n    f = tf.py_func(_parse_sequence, [v], tf.float32, stateful=False)\n    f.set_shape((None, 3))\n    \n    i = tf.py_func(_parse_image, [v], tf.float32, stateful=False)\n    i.set_shape((32, 32))\n    \n    return { \"strokes\": f, \"images\": i, \"keys\": k, \"lengths\": tf.shape(f)[0] }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea4bbc22175fbd12f447759c3e439920a6159597"},"cell_type":"code","source":"submission_dataset = tf.contrib.data.CsvDataset(\"../input/test_simplified.csv\", [tf.string, tf.constant([\"\"], dtype=tf.string), tf.string]).skip(1)\nsubmission_dataset = submission_dataset.map(parse_submission_row)\nsubmission_dataset = submission_dataset.padded_batch(submission_batch_size, padded_shapes=submission_dataset.output_shapes)\nsubmission_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"141bbc6cc7d45028477c482dc429b34d3ed232d6"},"cell_type":"code","source":"predictions = estimator.predict(lambda: submission_dataset.make_one_shot_iterator().get_next())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd322f8437e06dabfcb5bb4dfddc4b07bfa9026d"},"cell_type":"code","source":"rows = []\nrows.append([\"key_id\", \"word\"])\n\nfor p in predictions:\n    rows.append([p[\"keys\"].decode(\"ascii\"), \" \".join([inv_index[x] for x in p[\"predictions\"]])])\n\nlen(rows), rows[:25]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0131e5fa88316230dc3a1ec72ced4033b9b51cf"},"cell_type":"code","source":"with open(\"submission.csv\", \"w\") as f:\n    writer = csv.writer(f)\n    writer.writerows(rows)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bb37a7e7595faa3582a05a70d80c6232f6a4a29"},"cell_type":"code","source":"with tarfile.open(\"task.tar.gz\", \"w:gz\") as tar:\n    tar.add(\"task\", arcname=os.path.basename(\"task\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99b9be95649c5037571ded79daad21e64163f800"},"cell_type":"code","source":"shutil.rmtree(\"task\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}