{"cells":[{"metadata":{"_cell_guid":"522ce330-d4f9-4fbe-9684-4b65fd684cca","_uuid":"174484daa5f084ce4970f5048d3e05d2c4429787"},"cell_type":"markdown","source":"# Bidirectional LSTM"},{"metadata":{"_uuid":"d8ccded02a5f2c4a9d9ee2f7688114bcd2e1f11a"},"cell_type":"markdown","source":"\nThis notebook is a combination of the data generator from Beluga's notebook: https://www.kaggle.com/gaborfodor/greyscale-mobilenet-lb-0-892 and largely based on Kevin Mader's LSTM code, with modifications in the network architecture https://www.kaggle.com/kmader/quickdraw-baseline-lstm-reading-and-submission. \n\nI am grateful for their contributions and can take little credit for this notebook. Running this notebook should achieve 0.823 on the LB. "},{"metadata":{"trusted":true,"_uuid":"8b08fbab2000a563b388f126eac74362641e497c"},"cell_type":"code","source":"debug = False\nif debug: \n    STEPS = 200\n    val_steps = 10\nelse:\n    STEPS = 800\n    val_steps = 100\n    \nSTROKE_COUNT = 100\nEPOCHS = 45\nbatchsize = 1000\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport keras\nfrom keras.utils.np_utils import to_categorical\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.preprocessing import LabelEncoder\nimport pandas as pd\nfrom keras.metrics import top_k_categorical_accuracy\ndef top_3_accuracy(x,y): return top_k_categorical_accuracy(x,y, 3)\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nfrom glob import glob\nimport gc\ngc.enable()\ndef get_available_gpus():\n    from tensorflow.python.client import device_lib\n    local_device_protos = device_lib.list_local_devices()\n    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7a5e3b6503b24a34e60062db6248730a4e96adf"},"cell_type":"code","source":"def preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])\n\ndef top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7acacf8e960084782425ef1a1a3fd532a240ad48"},"cell_type":"code","source":"from ast import literal_eval\n\ndef _stack_it(raw_strokes):\n    \"\"\"preprocess the string and make \n    a standard Nx3 stroke vector\"\"\"\n    stroke_vec = literal_eval(raw_strokes) # string->list\n    # unwrap the list\n    in_strokes = [(xi,yi,i)  \n     for i,(x,y) in enumerate(stroke_vec) \n     for xi,yi in zip(x,y)]\n    c_strokes = np.stack(in_strokes)\n    # replace stroke id with 1 for continue, 2 for new\n    c_strokes[:,2] = [1]+np.diff(c_strokes[:,2]).tolist()\n    c_strokes[:,2] += 1 # since 0 is no stroke\n    # pad the strokes with zeros\n    return pad_sequences(c_strokes.swapaxes(0, 1), \n                         maxlen=STROKE_COUNT, \n                         padding='post').swapaxes(0, 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c38d52616f2fbec1c6cd6162b36627314c303b0a"},"cell_type":"code","source":"DP_DIR = '../input/shuffle-csv-50k'\nINPUT_DIR = '../input/quickdraw-doodle-recognition'\nBASE_SIZE = 256\nNCSVS = 100\nNCATS = 340\nnp.random.seed(seed=1987)\ntf.set_random_seed(seed=1987)\n\ndef f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return sorted([f2cat(f) for f in files], key=str.lower)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f5c302cba57d61320a39075d00ddc1aa08f76e5"},"cell_type":"code","source":"def image_generator_xd( batchsize, ks):\n    while True:\n        for k in np.random.permutation(ks):\n            filename = os.path.join(DP_DIR, 'train_k{}.csv.gz.gz'.format(k))\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                \n                df['drawing'] = df['drawing'].map(_stack_it)\n                x2 = np.stack(df['drawing'], 0)\n                y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n                yield x2, y\n\ndef df_to_image_array_xd(df):\n    df['drawing'] = df['drawing'].map(_stack_it)\n    x2 = np.stack(df['drawing'], 0)\n    return x2","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_datagen = image_generator_xd(batchsize=batchsize, ks=range(NCSVS - 2))\nval_datagen = image_generator_xd(batchsize=batchsize, ks=range(NCSVS - 2, NCSVS))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6d29237e-ece3-4dfd-9095-475296f4a608","_uuid":"8bae16a4973a215861fbb536a602c4f5abf3b4bf"},"cell_type":"markdown","source":"### Stroke-based Classification\nHere we use the stroke information to train a model and see if the strokes give us a better idea of what the shape could be. "},{"metadata":{"_cell_guid":"5e1d5bba-0fb4-432c-bd0b-ad69be0ef9ac","_uuid":"b4a087a17798c2ec8eb520bc916bcad38d4ebff2","collapsed":true},"cell_type":"markdown","source":"### LSTM to Parse Strokes\nThe model suggeted from the tutorial is\n\n![Suggested Model](https://www.tensorflow.org/versions/master/images/quickdraw_model.png)"},{"metadata":{"trusted":true,"_uuid":"f6d6f9011ea41c639a91c2b107575063f781574c"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import BatchNormalization, Conv1D, LSTM, Dense, Dropout, Bidirectional\n#if len(get_available_gpus())>0:\n    # https://twitter.com/fchollet/status/918170264608817152?lang=en\n#    from keras.layers import CuDNNLSTM as LSTM # this one is about 3x faster on GPU instances\nstroke_read_model = Sequential()\nstroke_read_model.add(BatchNormalization(input_shape = (None,)+(3,)))\n# filter count and length are taken from the script https://github.com/tensorflow/models/blob/master/tutorials/rnn/quickdraw/train_model.py\nstroke_read_model.add(Conv1D(256, (5,), activation = 'relu'))\nstroke_read_model.add(Dropout(0.2))\nstroke_read_model.add(Conv1D(256, (5,), activation = 'relu'))\nstroke_read_model.add(Dropout(0.2))\nstroke_read_model.add(Conv1D(256, (3,), activation = 'relu'))\nstroke_read_model.add(Dropout(0.2))\nstroke_read_model.add(Bidirectional(LSTM(128, dropout = 0.3, recurrent_dropout= 0.3,  return_sequences = True)))\nstroke_read_model.add(Bidirectional(LSTM(128,dropout = 0.3, recurrent_dropout= 0.3, return_sequences = True)))\nstroke_read_model.add(Bidirectional(LSTM(128,dropout = 0.3, recurrent_dropout= 0.3, return_sequences = False)))\nstroke_read_model.add(Dense(512, activation = 'relu'))\nstroke_read_model.add(Dropout(0.2))\nstroke_read_model.add(Dense(NCATS, activation = 'softmax'))\nstroke_read_model.compile(optimizer = 'adam', \n                          loss = 'categorical_crossentropy', \n                          metrics = ['categorical_accuracy', top_3_accuracy])\nstroke_read_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2a549512-a9d9-4afd-b748-3e1c3296e193","_uuid":"5fda10b30c47a8cf6ea822ed0a4a1d7cd2c81195","trusted":true},"cell_type":"code","source":"weight_path=\"{}_weights.best.hdf5\".format('stroke_lstm_bidirectional_relu')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=4, \n                                   verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=3) \ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"825b3af8-9451-487b-a1e1-538f2f1489e1","_uuid":"ed2fc26af74aed1a93bbc253d61b72db5a81f5cc","trusted":true},"cell_type":"code","source":"# Change the number of epochs to 20\n\n\nfrom IPython.display import clear_output\nhist = stroke_read_model.fit_generator(train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,\n                        validation_data=val_datagen, validation_steps = val_steps,\n                      callbacks = callbacks_list)\nclear_output()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"599c7c54e12353e6dcee3d6c6a29bc04064982d3"},"cell_type":"code","source":"hist_df = pd.DataFrame(hist.history) \nhist_df.to_csv('hist_training.csv')\nhist_df.index = np.arange(1, len(hist_df)+1)\nfig, axs = plt.subplots(nrows=2, sharex=True, figsize=(16, 10))\naxs[0].plot(hist_df.val_top_3_accuracy, lw=5, label='Validation Accuracy')\naxs[0].plot(hist_df.top_3_accuracy, lw=5, label='Training Accuracy')\naxs[0].set_ylabel('Accuracy')\naxs[0].set_xlabel('Epoch')\naxs[0].grid()\naxs[0].legend(loc=0)\naxs[1].plot(hist_df.val_loss, lw=5, label='Validation MLogLoss')\naxs[1].plot(hist_df.loss, lw=5, label='Training MLogLoss')\naxs[1].set_ylabel('MLogLoss')\naxs[1].set_xlabel('Epoch')\naxs[1].grid()\naxs[1].legend(loc=0)\nfig.savefig('hist.png', dpi=300)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a7eb5b62-cf57-4380-8786-9ddc05be658f","_uuid":"858059b6c16d81f86460bef8fcf595e0d68d12b2","trusted":true},"cell_type":"code","source":"valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz.gz'.format(NCSVS - 1)), nrows=34000)\nx_valid = df_to_image_array_xd(valid_df)\ny_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)\nlstm_results = stroke_read_model.evaluate(x_valid, y_valid, batch_size = 4096)\nprint('Accuracy: %2.1f%%, Top 3 Accuracy %2.1f%%' % (100*lstm_results[1], 100*lstm_results[2]))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e99b1ed154f26381d12918e2b4e12db807e6535f"},"cell_type":"markdown","source":"# Submission\n"},{"metadata":{"_cell_guid":"436a4fce-3843-4c84-8eeb-0161fe3c4e04","_uuid":"4f3a40e23f2e917b68171822944491ab348e15b3","trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))\nsub_df['drawing'] = sub_df['drawing'].map(_stack_it)\nsub_vec = np.stack(sub_df['drawing'].values, 0)\nsub_pred = stroke_read_model.predict(sub_vec, verbose=True, batch_size=4096)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72825ea87d35ad96b0254e3af5f5aaf64fb9c78f"},"cell_type":"code","source":"top3 = preds2catids(sub_pred)\ncats = list_all_categories()\nid2cat = {k: cat.replace(' ', '_') for k, cat in enumerate(cats)}\ntop3cats = top3.replace(id2cat)\nsub_df['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b5ece83cb6095e95ef5741e73508d9129be1e3d"},"cell_type":"code","source":"sub_df[['key_id', 'word']].to_csv('lstm_relu_datagen.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"366a5a7b8bbf29317bb182d46bf8d48c730c440c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}