{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"* * # Keras Using LSTM on full Data\n\n"},{"metadata":{"_uuid":"f7f2a9516140a84124bf7bbf538ee4c30860b778"},"cell_type":"markdown","source":"## Setup\nImport the necessary libraries and a few helper functions."},{"metadata":{"trusted":true,"_uuid":"eb9499fdbf8e80f9702e454563a0044f80c5a35d"},"cell_type":"code","source":"from IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\nimport os\nimport ast\nimport datetime as dt\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = [16, 10]\nplt.rcParams['font.size'] = 14\nimport seaborn as sns\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport keras\nfrom keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation\nfrom keras.metrics import categorical_accuracy, top_k_categorical_accuracy, categorical_crossentropy\nfrom keras.models import Sequential\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom keras.optimizers import Adam\nfrom keras.applications import MobileNet\nfrom keras.applications.mobilenet import preprocess_input\nfrom keras.preprocessing.sequence import pad_sequences\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce6d2aa7de1fa341144def7d3a5b1ffdea26bc91","_kg_hide-input":true},"cell_type":"code","source":"import time\ntic = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c305d1062c05bb0487b1a0197cf00469c10d3626"},"cell_type":"code","source":"np.random.seed(seed=1988)\ntf.set_random_seed(seed=1988)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"978b1e827e598c53df3ef09838a6d85591d83052"},"cell_type":"code","source":"DP_DIR = '../input/shuffle-csvs/'\nINPUT_DIR = '../input/quickdraw-doodle-recognition/'\n\n\nBASE_SIZE = 256\nNCSVS = 100\nNCATS = 340\n\ndef f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return sorted([f2cat(f) for f in files], key=str.lower)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"b2fcd1a08ae1ae0619be38a113a244eb6515b63b"},"cell_type":"code","source":"def avg_precision(actual, predicted, k=3):\n    if not actual:\n        return 0.\n    if len(predicted)>k:\n        predicted = predicted[:k]\n    score = 0.0\n    num_hits = 0.0\n    for i, n in enumerate(predicted):\n        if n in actual and n not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits/(i + 1.)\n    return score/min(len(actual),k)\n\ndef mapk(actual, predicted, k=3):\n    return np.mean([avg_precision(a,p,k) for a,p in zip(actual, predicted)])\n\ndef preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:,:3], columns=['a','b','c'])\n\ndef top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"264156422a95e4b350886d558d516ae8bd2e25c0"},"cell_type":"markdown","source":""},{"metadata":{"trusted":true,"_uuid":"54e5f0c637195b6624e2f3e6db5e7f8990e14eb7"},"cell_type":"code","source":"def _stack_it(stroke_vec):\n    \"\"\"preprocess the string and make \n    a standard Nx3 stroke vector\"\"\"\n    # unwrap the list\n    in_strokes = [(xi,yi,i)  \n     for i,(x,y) in enumerate(stroke_vec) \n     for xi,yi in zip(x,y)]\n    c_strokes = np.stack(in_strokes)\n    # replace stroke id with 1 for continue, 2 for new\n    c_strokes[:,2] = [1]+np.diff(c_strokes[:,2]).tolist()\n    c_strokes[:,2] += 1 # since 0 is no stroke\n    # pad the strokes with zeros\n    return pad_sequences(c_strokes.swapaxes(0, 1), \n                         maxlen=STROKE_COUNT, \n                         padding='post').swapaxes(0, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b07c06cb2d4c1c5ba3561f045eae8b1f00314ef"},"cell_type":"code","source":"def draw_cv2(raw_strokes, size=256, lw=6, time_color=True):\n    img = np.zeros((BASE_SIZE, BASE_SIZE), np.uint8)\n    for t, stroke in enumerate(raw_strokes):\n        for i in range(len(stroke[0])-1):\n            color = 255 - min(t, 10) * 13 if time_color else 255\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]),\n                        (stroke[0][i+1], stroke[1][i+1]), color, lw)\n    if size != BASE_SIZE:\n        return cv2.resize(img, (size,size))\n    else:\n        return img","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab1834ea2757a53d602a3508efffcc34bc190dc7"},"cell_type":"markdown","source":"## Training with Image Generator"},{"metadata":{"trusted":true,"_uuid":"612c334761584ee7554232c7231efc4aa2b68f0b"},"cell_type":"code","source":"batchsize = 256\nSTROKE_COUNT = 196\nSTEPS = 800\nEPOCHS = 20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6455bf9555b8381b6a4292098a64a0eb7ff54dc"},"cell_type":"code","source":"def image_generator_xd( batchsize, ks, lw=6, time_color=True):\n    while True:\n        for k in np.random.permutation(ks):\n            filename = os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(k))\n            if not os.path.exists(filename):\n                continue\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                # Generator of multiple batches. Each iter is on single file of batchsize\n                df['drawing'] = df['drawing'].apply(ast.literal_eval)\n                x = np.zeros((len(df), 196, 3))\n                for i, raw_strokes in enumerate(df.drawing.values):\n                    x[i, :, :] = _stack_it(raw_strokes)\n                y=keras.utils.to_categorical(df.y, num_classes=NCATS) # y should be equal to the word\n                yield x,y\n\ndef df_to_image_array_xd(df, lw=6, time_color=True):\n    df['drawing'] = df['drawing'].apply(ast.literal_eval)\n    x = np.zeros((len(df), 196, 3))\n    for i, raw_strokes in enumerate(df.drawing.values):\n        x[i,:,:] = _stack_it(raw_strokes)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98ff512e1a1b5e86e86d9eef4127525bedf3b9e1"},"cell_type":"code","source":"valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)), nrows=34000)\nx_valid = df_to_image_array_xd(valid_df)\ny_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)\nprint(x_valid.shape, y_valid.shape)\nprint('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3 ))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d80ad7f4d378ea7f30479221d604eeeed559cae4"},"cell_type":"code","source":"train_datagen = image_generator_xd( batchsize=batchsize, ks=range(NCSVS - 1))\nx, y = next(train_datagen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ce5fb89fbb77777316d6fca7689b6636c0e6021"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import BatchNormalization, Conv1D, LSTM, Dense, Dropout\nstroke_read_model = Sequential()\nstroke_read_model.add(BatchNormalization(input_shape = (None,)+x.shape[2:]))\n# filter count and length are taken from the script https://github.com/tensorflow/models/blob/master/tutorials/rnn/quickdraw/train_model.py\nstroke_read_model.add(Conv1D(48, (5,)))\nstroke_read_model.add(Dropout(0.3))\nstroke_read_model.add(Conv1D(64, (5,)))\nstroke_read_model.add(Dropout(0.3))\nstroke_read_model.add(Conv1D(96, (3,)))\nstroke_read_model.add(Dropout(0.3))\nstroke_read_model.add(LSTM(128, return_sequences = True))\nstroke_read_model.add(Dropout(0.3))\nstroke_read_model.add(LSTM(128, return_sequences = False))\nstroke_read_model.add(Dropout(0.3))\nstroke_read_model.add(Dense(512))\nstroke_read_model.add(Dropout(0.3))\nstroke_read_model.add(Dense(NCATS, activation = 'softmax'))\nstroke_read_model.compile(optimizer = 'adam', \n                          loss = 'categorical_crossentropy', \n                          metrics = ['categorical_accuracy', top_3_accuracy])\nstroke_read_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da72d70fc1781e80427d45a80c07b3571dda0b36"},"cell_type":"code","source":"weight_path=\"{}_weights.best.hdf5\".format('stroke_lstm_model_generator')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=10, \n                                   verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=5) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ada344bf3454765298e7b7ed7861c82bca2d2084","_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"stroke_read_model.fit_generator(train_datagen, \n                      validation_data = (x_valid, y_valid), \n                                verbose=1,\n                                steps_per_epoch=STEPS,\n                      epochs = EPOCHS,\n                      callbacks = callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c1927f22d3c45cba0bdee7d6f4b6c858d82d614"},"cell_type":"code","source":"valid_predictions = stroke_read_model.predict(x_valid, batch_size=128, verbose=1)\nmap3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions).values)\nprint('Map3: {:.3f}'.format(map3))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be4577a9ba00611697eea8f241a42c504981e86f"},"cell_type":"markdown","source":"## Create Submission"},{"metadata":{"trusted":true,"_uuid":"a7d14348150baf753e90cf2719b9f31dd564f6a2"},"cell_type":"code","source":"test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))\ntest.head()\nx_test = df_to_image_array_xd(test)\nprint(test.shape, x_test.shape)\nprint('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3 ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"608b02f5c7909ae62becbe5c931b7264171296e8"},"cell_type":"code","source":"test_predictions = stroke_read_model.predict(x_test, batch_size=128, verbose=1)\n\ntop3 = preds2catids(test_predictions)\ntop3.head()\ntop3.shape\n\ncats = list_all_categories()\nid2cat = {k: cat.replace(' ', '_') for k, cat in enumerate(cats)}\ntop3cats = top3.replace(id2cat)\ntop3cats.head()\ntop3cats.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52e0f9c44f2a9a38fd1550ffb9c07fb7ea22b17d"},"cell_type":"code","source":"test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']\nsubmission = test[['key_id', 'word']]\nsubmission.to_csv('lstm_generator_submission_{}.csv'.format(int(map3 * 10**4)), index=False)\nsubmission.head()\nsubmission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b418f4c06c4e4453aa1b5ab16dde344eb8b735c5"},"cell_type":"code","source":"toc=time.time()\nprint('Total time taken: %0.2f sec'%(toc-tic))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}