{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n%matplotlib inline\nimport os\nimport matplotlib.pyplot as plt\nimport json\nimport datetime as dt\nimport seaborn as sns\nimport cv2\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Keras Package Load\nimport tensorflow as tf\nimport keras\nfrom keras.layers import Conv2D, GlobalAveragePooling2D, MaxPooling2D, GlobalAveragePooling1D, Conv1D, MaxPooling1D\nfrom keras.layers import Dense, Dropout, Flatten, Activation, Input, BatchNormalization\nfrom keras.layers import CuDNNLSTM as LSTM\nfrom keras.layers.merge import concatenate\nfrom keras.metrics import categorical_crossentropy, top_k_categorical_accuracy, categorical_accuracy\nfrom keras.models import Sequential, Model\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint,TensorBoard, LearningRateScheduler\nfrom keras.optimizers import Adam,Adagrad,RMSprop,SGD\nfrom keras.applications import Xception, MobileNet, MobileNetV2, xception, mobilenet\n\n\nfrom keras.utils.np_utils import to_categorical\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.preprocessing import LabelEncoder\nfrom keras.metrics import top_k_categorical_accuracy\n\n#supplement package load\nfrom tqdm import tqdm_notebook as tqdm\nfrom ast import literal_eval\nimport glob\nfrom multiprocessing import Pool\nfrom functools import partial\nfrom itertools import repeat\nfrom itertools import product","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Outline\n\n## Concept\n* 단일 모델 기준으로 성능이 높게 나온 것은 Deptwise 기반의 모델 Xception을 적은 규모로 적용한 Mobilenet\n* But, Drawing pattern 같은 경우에는 Seqeuntial한 성질을 보임\n* Depth wise CNN + LSTM 기반의 모델을 통해 Dept wise CNN에 Sequence 성질을 혼합함\n\n## Processing Sequence\n1. Depth CNN, LSTM 개별 data generator - 병렬 처리 반영\n2. two generator concatenate\n3. Input을 Image matrix, Drawing Time Sequence matrix 두개를 취함\n4. Mobilenet, LSTM에 개별 학습을 통해 Output - 해당 Input의 Softmaxt 확률을 통한 Label\n\n## Metric\n1. MAP3 score를 기준으로 평가함 - precision을 통해 Label 3개 예측(TOP.3) 중에서 실제가 차지 하는 비중이 중요\n\n\n## Result Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"SH_DIR = '../input/shufflecsvs/shuffle-csvs/shuffle-csvs/'\nINPUT_DIR = '../input/quickdraw-doodle-recognition/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nnp.random.seed(seed=SEED)\ntf.set_random_seed(seed=SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metric\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def apk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    if len(predicted) > k:\n        predicted = predicted[:k]\n    score = 0.0\n    num_hits = 0.0\n    for i, p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i + 1.0)\n    if not actual:\n        return 0.0\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=3):\n    \"\"\"\n    Source: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py\n    \"\"\"\n    return np.mean([apk(a, p, k) for a, p in zip(actual, predicted)])\n\ndef preds2catids(predictions):\n    return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])\n\ndef top_3_accuracy(y_true, y_pred):\n    return top_k_categorical_accuracy(y_true, y_pred, k=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\ndef list_all_categories():\n    files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))\n    return sorted([f2cat(f) for f in files], key=str.lower)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## model spect argument"},{"metadata":{"trusted":true},"cell_type":"code","source":"mobile_BASE_SIZE = 256\nmobile_NCSVS = 100\nmobile_NCATS = 340\n\nmobile_STEPS = 800\nmobile_EPOCHS = 70\nmobile_SIZE = 64\nmobile_BATCHSIZE = 128\n# drop_rate = 0.5\n\n\nlstm_batch_size = 128\nlstm_STROKE_COUNT = 196\n\nBATCHSIZE = 128","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data generator"},{"metadata":{},"cell_type":"markdown","source":"## CNN generator\n* CV2 패키지를 활용해 stroke의 순서에 따라 GreyScale 조정\n* 256*256 사이즈로 변경"},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_cv2(raw_strokes, size=256, lw=6, time_color=True):\n    img = np.zeros((mobile_BASE_SIZE, mobile_BASE_SIZE), np.uint8)\n    for t, stroke in enumerate(raw_strokes):\n        for i in range(len(stroke[0]) - 1):\n            color = 255 - min(t, 10) * 13 if time_color else 255\n            _ = cv2.line(img, (stroke[0][i], stroke[1][i]),\n                         (stroke[0][i + 1], stroke[1][i + 1]), color, lw)\n    if size != mobile_BASE_SIZE:\n        return cv2.resize(img, (size, size))\n    else:\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_generator_xd(size, batchsize, ks, lw=6, time_color=True):\n    while True:\n        for k in np.random.permutation(ks):\n            filename = os.path.join(SH_DIR, 'train_k{}.csv.gz'.format(k))\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                df['drawing'] = df['drawing'].apply(json.loads)\n                x = np.zeros((len(df), size, size, 1))\n                for i, raw_strokes in enumerate(df.drawing.values):\n                    x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw,\n                                             time_color=time_color)\n                x = mobilenet.preprocess_input(x).astype(np.float32)\n                y = keras.utils.to_categorical(df.y, num_classes=NCATS)\n                yield x, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def df_to_image_array_xd(df, size, lw=6, time_color=True):\n    df['drawing'] = df['drawing'].apply(json.loads)\n    x = np.zeros((len(df), size, size, 1))\n    for i, raw_strokes in enumerate(df.drawing.values):\n        x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw, time_color=time_color)\n    x = mobilenet.preprocess_input(x).astype(np.float32)\n    return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LSTM generator\n* Stroke를 [x,y,binary(strat/end)] 표현\n* maximun 196 Sequence 데이터 형태로 변환"},{"metadata":{"trusted":true},"cell_type":"code","source":"def df_to_sequence_array(df, time_size):\n    df['drawing'] = df['drawing'].apply(json.loads)\n    x = np.zeros((len(df), time_size, 3))\n    for i, raw_strokes in enumerate(df.drawing.values):\n        x[i, : ,: ] = _stack_it(raw_strokes, time_size)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _stack_it(raw_strokes, time_size):\n    \"\"\"preprocess the string and make \n    a standard Nx3 stroke vector\"\"\"\n    \n#     stroke_vec = literal_eval(raw_strokes) # string->list\n    \n    # unwrap the list\n    in_strokes = [(xi,yi,i)  \n     for i,(x,y) in enumerate(raw_strokes) \n     for xi,yi in zip(x,y)]\n    c_strokes = np.stack(in_strokes)\n    # replace stroke id with 1 for continue, 2 for new\n    c_strokes[:,2] = [1]+np.diff(c_strokes[:,2]).tolist()\n    c_strokes[:,2] += 1 # since 0 is no stroke\n    # pad the strokes with zeros\n    return pad_sequences(c_strokes.swapaxes(0, 1), \n                         maxlen=time_size, \n                         padding='post').swapaxes(0, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_batch(samples=5, \n               start_row=0,\n               max_rows = 1000):\n    \"\"\"\n    load and process the csv files\n    this function is horribly inefficient but simple\n    \"\"\"\n    out_df_list = []\n    for c_path in ALL_TRAIN_PATHS:\n        c_df = pd.read_csv(c_path, nrows=max_rows, skiprows=start_row)\n        c_df.columns=COL_NAMES\n        out_df_list += [c_df.sample(samples)[['drawing', 'word']]]\n    full_df = pd.concat(out_df_list)\n    full_df['drawing'] = full_df['drawing'].\\\n        map(_stack_it)\n    \n    return full_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Total generator\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_generator_xd(size,time_size, batchsize, ks, lw=6, time_color=True):\n    while True:\n        for k in np.random.permutation(ks):\n            filename = os.path.join(SH_DIR, 'train_k{}.csv.gz'.format(k))\n            for df in pd.read_csv(filename, chunksize=batchsize):\n                df['drawing'] = df['drawing'].apply(json.loads)\n                x = np.zeros((len(df), size, size, 1))\n                x_lstm = np.zeros((len(df), time_size, 3))\n                for i, raw_strokes in enumerate(df.drawing.values):\n#                     print(raw_strokes)\n                    x[i, :, :, 0] = draw_cv2(raw_strokes, size=size, lw=lw,\n                                             time_color=time_color)\n#                     print(_stack_it(raw_strokes, time_size))\n                    x_lstm[i, :, :] = _stack_it(raw_strokes, time_size)\n                x = mobilenet.preprocess_input(x).astype(np.float32)\n                y = keras.utils.to_categorical(df.y, num_classes=mobile_NCATS)\n                yield [x, x_lstm],  y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = image_generator_xd(size = mobile_SIZE, time_size=lstm_STROKE_COUNT, batchsize=mobile_BASE_SIZE, ks= range(mobile_NCSVS-1))\nx, y =next(train_datagen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"CNN shape{} \\\n      \\nLSTM shape {}\".format(x[0].shape,x[1].shape))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation Dataset\n* Validation은 99번째 데이터 셋을 기준으로 행함\n* 단일 프로세서로 validation generating시 시간이 많이 소모되기에 multiprocessing 처리"},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df = pd.read_csv(os.path.join(SH_DIR, 'train_k{}.csv.gz'.format(mobile_NCSVS-1)), nrows = 34000)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Validation multiprocessing\n* Product를 통해 데이터 셋을 Chunk(100개)단위로 분화\n* 각 Chunk argument 할당 후 병렬 처리"},{"metadata":{"trusted":true},"cell_type":"code","source":"iterable_1 = product(np.array_split(valid_df, 100), [mobile_SIZE])\niterable_2 = product(np.array_split(valid_df,  100), [lstm_STROKE_COUNT])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Virtual Core count\nNJOBS = !grep -c processor /proc/cpuinfo\nNJOBS = int(NJOBS[0])\nprint(NJOBS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with Pool(processes=NJOBS) as p:\n    x_valid  = p.starmap(df_to_image_array_xd, iterable_1)\n#     x_lstm_valid = p.starmap(_stack_it, iterable_2)\n\n\nwith Pool(processes=NJOBS) as p:\n    x_lstm_valid = p.starmap(df_to_sequence_array, iterable_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_valid = np.vstack(np.array(x_valid))\nx_lstm_valid = np.vstack(np.array(x_lstm_valid))\ny_valid = keras.utils.to_categorical(valid_df.y, num_classes=mobile_NCATS)\n\nprint(\n    'mobile_valid size :{} \\\n    \\nRNN valid size : {} \\\n    \\ny_valid size : {}'.format(x_valid.shape, x_lstm_valid.shape,  y_valid.shape))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Construct"},{"metadata":{},"cell_type":"markdown","source":"## 1. Mobile net base model\n* Dense Layer 이전의 Layer를 동결 후 학습\n* Global AveragePooling2D를 통한 Channel 전체 평균 Layer 값 반환 -> 일반적인 Flatten에 비해 파라메터의 수 줄임"},{"metadata":{"trusted":true},"cell_type":"code","source":"def mobiel_net(size):\n    base_model = MobileNet(weights = None, input_shape=(size, size,1), include_top=False ,alpha=1.)\n    x= base_model.output\n    x= GlobalAveragePooling2D()(x)\n#     model = Model(inputs = base_model.input, outputs = base_model.outputs, name = 'mobile_net')\n    \n    return x, base_model.input","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2.LSTM base model\n* 실험적인 테스트 결과 중간에 MaxPooling을 넣어주는 것이 좋게 나왔음\n* LSTM return sequnce True -> return sequnce False를 통한  many to many -> many to one으로 전체 Time sequence 정보 반영\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lstm_model(time_size ,f_size ):\n    stroke_input = Input(shape=(time_size, f_size), name = 'lstm_input')\n#     batch_norm = BatchNormalization()(stroke_input)\n#     conv1 = Conv1D(48, (5,), activation='relu')(batch_norm)\n#     drop1 = Dropout(0.3)(conv1)\n#     conv2 = Conv1D(64, (5,), activation='relu')(drop1)\n#     drop2 = Dropout(0.3)(conv2)\n#     conv3 = Conv1D(96, (3,), activation='relu')(drop2)\n#     drop3 = Dropout(0.3)(conv3)\n#     lstm1 = LSTM(128, return_sequences=True)(drop3)\n#     drop4 = Dropout(0.3)(lstm1)\n#     lstm2 = LSTM(128, return_sequences=False)(drop4)\n#     return lstm2, stroke_input\n    \n    batch_norm = BatchNormalization()(stroke_input)\n    conv1 = Conv1D(48, (5,), activation='relu')(batch_norm)\n#     drop1 = Dropout(0.3)(conv1)\n    conv2 = Conv1D(64, (5,), activation='relu')(conv1)\n#     drop2 = Dropout(0.3)(conv2)\n    conv3 = Conv1D(96, (3,), activation='relu')(conv2)\n#     drop3 = Dropout(0.3)(conv3)\n    \n    \"\"\" BATCH norm test _4\"\"\"\n    max_pool = MaxPooling1D(pool_size  = 4)(conv3)\n    \"\"\" BATCH norm test _4\"\"\"\n    \n    lstm1 = LSTM(128, return_sequences=True)(max_pool)\n#     drop4 = Dropout(0.3)(lstm1)\n\n#     \"\"\" BATCH norm test _3\"\"\"\n#     batch_norm_2 = BatchNormalization()(lstm1)\n#     \"\"\"BATCH norm test _3\"\"\"\n\n    lstm2 = LSTM(128, return_sequences=False)(lstm1)\n    \"\"\" BATCH norm test _4\"\"\"\n    batch_norm_2 = BatchNormalization()(lstm2)\n    \"\"\"BATCH norm test _4\"\"\"\n    \n    return batch_norm_2, stroke_input","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. Single model concatnate\n* Batchnorm이 정규화 및 드랍아웃의 성질을 지녔기에 따로 드랍아웃 layer를 추가하지 않음"},{"metadata":{"trusted":true},"cell_type":"code","source":"mobile_model, mobile_input = mobiel_net(mobile_SIZE)\nsequence_model, sequence_input = lstm_model(lstm_STROKE_COUNT, 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(mobile_model.shape,\\\n      sequence_model.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merge = concatenate([mobile_model, sequence_model])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" BATCH norm test _4\"\"\"\nhidden_1 = Dense(512, activation='relu')(merge)\nmerge_batch_norm = BatchNormalization()(hidden_1)\n\"\"\"BATCH norm test _4\"\"\"\n# merge_drop1 = Dropout(0.3)(merge)\n# merge_hidden1 = Dense(512)(merge_drop1)\n# merge_drop2 = Dropout(0.3)(merge_hidden1)\noutput = Dense(mobile_NCATS, activation = 'softmax')(merge_batch_norm)\nmerge_model = Model(inputs = [mobile_input, sequence_input ], output = output, name = 'merge_model_fin_70')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merge_model.name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merge_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Train\n* 자동화된 컴파일 & 학습 함수 구성\n* callback을 통한 LR 감소 및 Metric에 따른 최고 Score 갱신마다 Weight 기록\n* Optimizer는 momentum 기반의 Adam을 적용하여 빠른 검증\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"! mkdir ../working/weights\n! mkdir ../working/logs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def directory_check(path):\n    if not os.path.isdir(path):\n        os.mkdir(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compile_and_train(model, num_epochs, BATCHSIZE, OPTIMIZER):\n    model.compile(optimizer=OPTIMIZER(lr=0.002), loss='categorical_crossentropy',\n              metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])\n    directory_path = '../working/weights/{}/'.format(model.name)\n    log_path = '../working/logs/{}/'.format(model.name)\n    directory_check(directory_path)\n    directory_check(log_path)\n    \n    file_name = model.name + '.{epoch:02d}-{loss:.2f}.h5'\n    filepath = os.path.join(directory_path, file_name)\n#     filepath = 'weights/' + model.name + '.{epoch:02d}-{loss:.2f}.h5'\n    callbacks = [\n        ReduceLROnPlateau(monitor='val_top_3_accuracy', factor = 0.75, patience=3, min_delta=0.001, mode='max', min_lr=1e-5, verbose=1),\n        ModelCheckpoint(filepath, monitor='val_top_3_accuracy', mode= 'max', save_best_only=True, save_weights_only=True),\n        TensorBoard(log_dir=log_path, histogram_freq=0, batch_size=BATCHSIZE)\n        \n]\n#     \n    \n    hist = model.fit_generator(\n        train_datagen, steps_per_epoch = mobile_STEPS, epochs = num_epochs, verbose = 1,\n        validation_data = ([x_valid,x_lstm_valid], y_valid), \n        callbacks = callbacks\n)\n#     weight_files = glob.glob(os.path.join(os.getcwd(), '{}*'.format(directory_path)))\n    weight_files = glob.glob('{}*'.format(directory_path))\n    weight_file = max(weight_files, key = os.path.getctime)\n#     for file in weight_files:\n#         if file == weight_file:\n#             pass\n#         else:\n            \n#             os.remove(file)\n#     hists.append(hist)\n    return hist, weight_file","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_his, model_weight = compile_and_train(merge_model, 60 , BATCHSIZE, Adam)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_df = pd.concat([pd.DataFrame(model_his.history)],sort = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(nrows=2, sharex=True, figsize=(16, 10))\naxs[0].plot(history_df.val_categorical_accuracy, lw = 3, label ='val_acc')\naxs[0].plot(history_df.categorical_accuracy, lw = 3, label ='train_acc')\naxs[0].set_ylabel('acc')\naxs[0].set_xlabel('epoch')\naxs[0].grid()\naxs[0].legend(loc=0)\n\naxs[1].plot(history_df.val_categorical_crossentropy, lw = 3, label ='val_loss')\naxs[1].plot(history_df.categorical_crossentropy, lw = 3, label ='train_loss')\naxs[1].set_ylabel('loss')\naxs[1].set_xlabel('epoch')\naxs[1].grid()\naxs[1].legend(loc=0)\nfig.savefig('merge_model_fin_70.png', dpi = 300)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_predictions = merge_model.predict([x_valid, x_lstm_valid], batch_size=BATCHSIZE, verbose=1)\nmap3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions).values)\nprint('MAP3 : {:.3f}'.format(map3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test  = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = df_to_image_array_xd(test.copy(), mobile_SIZE)\nx_lstm_test = df_to_sequence_array(test.copy() , lstm_STROKE_COUNT)\n\nprint(test.shape, x_test.shape)\nprint(test.shape, x_lstm_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predictions = merge_model.predict([x_test,x_lstm_test], batch_size=BATCHSIZE, verbose=1)\ntop3 = preds2catids(test_predictions)\ntop3.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cats = list_all_categories()\nid2cat = {k : cat.replace(' ', '_')for k, cat in enumerate(cats)}\ntop3cats = top3.replace(id2cat)\ntop3cats.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']\nsubmission = test[['key_id', 'word']]\nsubmission.to_csv('{}-submission-fin.csv'.format(merge_model.name), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}