{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom tqdm import *\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import GroupKFold\nimport math\n\nimport tensorflow as tf\nimport tensorflow.keras as keras\nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, Multiply, Lambda,\n                          Concatenate, GlobalAveragePooling2D, Softmax)\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n\nfrom lightgbm import LGBMClassifier\nimport gc\n\n\nNUM_CLASSES = 6\nNUM_MODELS = 9","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/rsna-oof-data-for-stacking/oof_{}models_post_with_meta.csv'.format(NUM_MODELS))\ntest_df = pd.read_csv('../input/rsna-oof-data-for-stacking/sub_{}models_post_with_meta.csv'.format(NUM_MODELS))\ntrain_meta = pd.read_csv('../input/rsna-oof-data-for-stacking/train_meta_with_label_stage2.csv')\n\ntrain_df = pd.merge(train_df, train_meta, on=\"sop_instance_uid\")\ntrain_df.rename(columns={\"patient_id_x\": \"patient_id\"}, inplace=True)\ntrain_df.drop(['patient_id_y'], axis=1, inplace=True)\n\nprint(train_df.shape, test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_lgbm = train_df.iloc[:, 1:(6*NUM_MODELS+1)].values\nX_train = X_train_lgbm.reshape((len(train_df), NUM_MODELS, NUM_CLASSES, 1))\nY_train = train_df.iloc[:, -6:].values.astype(float)\nX_test_lgbm = test_df.iloc[:, 1:(6*NUM_MODELS+1)].values\nX_test = X_test_lgbm.reshape((len(test_df), NUM_MODELS, NUM_CLASSES, 1))\nY_pred = np.zeros((X_test.shape[0], NUM_CLASSES)).astype(float)\nprint(X_train.shape, Y_train.shape, X_test.shape)\nprint(X_train_lgbm.shape, X_test_lgbm.shape, Y_pred.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create new features for lightgbm\nbase_train_pred = np.mean(X_train, axis=1).reshape(len(X_train), NUM_CLASSES)\nbase_test_pred = np.mean(X_test, axis=1).reshape(len(X_test), NUM_CLASSES)\n\nX_train_lgbm = np.concatenate((X_train_lgbm, base_train_pred), axis=1)\nX_test_lgbm = np.concatenate((X_test_lgbm, base_test_pred), axis=1)\nprint(X_train_lgbm.shape, X_test_lgbm.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Some necessary functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# parameters for LGBM model\nparams = {\n    'boosting_type': 'gbdt',\n    'objective': 'binary',\n    'metric': 'log_loss',\n    'n_estimators': 2000,\n    'learning_rate': 0.01,\n    'num_leaves': 31,\n    'max_depth': -1, \n    'n_jobs': -1,\n    'subsample': 0.5, \n    'subsample_freq': 2,\n    'colsample_bytree': 0.9,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_stacking_model():\n    \n    input_tensor = Input(shape=(NUM_MODELS, NUM_CLASSES, 1))\n    x = Conv2D(128, kernel_size=(NUM_MODELS, 1), activation='relu')(input_tensor)\n    x = Dropout(0.3)(x)\n    x = Conv2D(256, (1,NUM_CLASSES), activation='relu')(x)\n    x = Dropout(0.3)(x)\n    x = Flatten()(x)\n    x = Dense(512, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    output1 = Dense(NUM_CLASSES, activation='sigmoid',\n               name='output1')(x)\n    x = Dense(256, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    output2 = Dense(NUM_CLASSES, activation='sigmoid',\n                   name='output2')(x)\n    model = Model(input_tensor, [output1, output2])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# weighted log loss function for keras\ndef _weighted_log_loss(y_true, y_pred):\n    \n    class_weights = np.array([2, 1, 1, 1, 1, 1])\n\n    y_pred = tf.keras.backend.clip(y_pred, tf.keras.backend.epsilon(), 1.0-tf.keras.backend.epsilon())\n    out = -(         y_true  * tf.keras.backend.log(      y_pred) * class_weights\n            + (1.0 - y_true) * tf.keras.backend.log(1.0 - y_pred) * class_weights)\n    \n    return tf.keras.backend.mean(out, axis=-1)\n\n# weighted log loss function for evaluating\ndef multilabel_logloss(y_true, y_pred):\n    class_weights = np.array([2, 1, 1, 1, 1, 1])\n    eps = 1e-15\n    y_pred = np.clip(y_pred, eps, 1.0-eps)\n    out = -(         y_true  * np.log(      y_pred) * class_weights\n            + (1.0 - y_true) * np.log(1.0 - y_pred) * class_weights)\n    \n    return np.mean(out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mix-up generator for NN\n\ndef mixup_data(x, y, alpha=0.4):\n    \n    # 50% chance to keep original data\n    if(np.random.randint(2) == 1):\n        return x, y\n    \n    # 50% chance to apply mix-up augmentation\n    lam = np.random.beta(alpha, alpha)\n    sample_size = x.shape[0]\n    index_array = np.arange(sample_size)\n    np.random.shuffle(index_array)\n    \n    mixed_x = lam * x + (1 - lam) * x[index_array]\n    mixed_y = (lam * y) + ((1 - lam) * y[index_array])\n    \n    return mixed_x, mixed_y\n\ndef make_batches(size, batch_size):\n    nb_batch = int(np.ceil(size/float(batch_size)))\n    return [(i*batch_size, min(size, (i+1)*batch_size)) for i in range(0, nb_batch)]\n\ndef batch_generator(X,y,batch_size=128,shuffle=True,mixup=False):\n    sample_size = X.shape[0]\n    index_array = np.arange(sample_size)\n    \n    while True:\n        if shuffle:\n            np.random.shuffle(index_array)\n        batches = make_batches(sample_size, batch_size)\n        for batch_index, (batch_start, batch_end) in enumerate(batches):\n            batch_ids = index_array[batch_start:batch_end]\n            X_batch = X[batch_ids]\n            y_batch = y[batch_ids]\n            \n            if mixup:\n                X_batch, y_batch = mixup_data(X_batch, y_batch)\n            \n            yield X_batch, {'output1': y_batch, 'output2': y_batch} ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 512 * 4\nREPEAT = 1 # you can repeat many times to improve the stability\nNN_score = []\nbase_score = []\nLGBM_score = []\nSTACK_score = []\nEPOCH = 50\nNUM_FOLDS = 5\n\nfor num_repeat in range(REPEAT):\n    GKF = GroupKFold(n_splits=NUM_FOLDS)\n    for fold, (train_index, test_index) in enumerate(GKF.split(X_train, Y_train, train_df['patient_id'])):\n\n        print('***************  Fold %d  ***************'%(fold))\n\n        # dataset for NN\n        x_train_nn, x_valid_nn = X_train[train_index], X_train[test_index]\n        y_train_fold, y_valid_fold = Y_train[train_index], Y_train[test_index]\n        print(x_train_nn.shape, x_valid_nn.shape)\n        print(y_train_fold.shape, y_valid_fold.shape)\n        \n        # dataset for lgbm\n        x_train_lgbm, x_valid_lgbm = X_train_lgbm[train_index], X_train_lgbm[test_index]\n        print(x_train_lgbm.shape, x_valid_lgbm.shape)\n        \n        ####### average ################\n        base_fold_pred = np.mean(x_valid_nn, axis=1).reshape(len(x_valid_nn), NUM_CLASSES)\n        base_score.append(multilabel_logloss(y_valid_fold, base_fold_pred))\n        print('simple average score for this fold: ', multilabel_logloss(y_valid_fold, base_fold_pred))\n\n        ######## train NN ##################\n        early_stoping = EarlyStopping(monitor='val_loss', patience=7, verbose=0)\n        WEIGHTS_PATH = 'cnn_stacking_weights_repeat{}_fold{}.hdf5'.format(num_repeat, fold)\n        save_checkpoint = ModelCheckpoint(WEIGHTS_PATH, monitor = 'val_loss', verbose = 0,\n                                          save_best_only = True, save_weights_only = True, mode='min')\n        reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr = 1e-8, verbose=0)\n        callbacks = [save_checkpoint, early_stoping, reduce_lr]\n        \n        tr_gen = batch_generator(x_train_nn,y_train_fold,\n                                 batch_size=BATCH_SIZE,\n                                 shuffle=True, mixup=True)\n        val_gen = batch_generator(x_valid_nn,y_valid_fold,\n                                 batch_size=BATCH_SIZE,\n                                 shuffle=False)\n        \n        train_gen_dataset = tf.data.Dataset.from_generator(\n            lambda:tr_gen,\n            output_types=('float32', {'output1': 'float32', 'output2': 'float32'}),\n            output_shapes=(tf.TensorShape((None, NUM_MODELS, NUM_CLASSES, 1)),\n                           {'output1':tf.TensorShape((None, NUM_CLASSES)),\n                            'output2':tf.TensorShape((None, NUM_CLASSES))}))\n        \n        val_gen_dataset = tf.data.Dataset.from_generator(\n            lambda:val_gen,\n            output_types=('float32', {'output1': 'float32', 'output2': 'float32'}),\n            output_shapes=(tf.TensorShape((None, NUM_MODELS, NUM_CLASSES, 1)),\n                           {'output1':tf.TensorShape((None, NUM_CLASSES)),\n                            'output2':tf.TensorShape((None, NUM_CLASSES))}))\n        \n        strategy = tf.distribute.MirroredStrategy()\n        with strategy.scope():\n            model = create_stacking_model()\n            model.compile(loss=_weighted_log_loss, optimizer=Adam(lr=1e-3))\n            model.fit(train_gen_dataset,\n                    steps_per_epoch=math.ceil(float(len(y_train_fold)) / float(BATCH_SIZE)),\n                    validation_data=val_gen_dataset,\n                    validation_steps=math.ceil(float(len(y_valid_fold)) / float(BATCH_SIZE)),\n                    epochs=EPOCH, callbacks=callbacks,\n                    workers=2, max_queue_size=10,\n                    use_multiprocessing=True,\n                    verbose=0)\n        \n        model.load_weights(WEIGHTS_PATH)\n        valid_nn = model.predict(x_valid_nn, batch_size=BATCH_SIZE, verbose=0)\n        valid_nn = np.sum(valid_nn, axis=0)/2\n        nn_score = multilabel_logloss(y_valid_fold, valid_nn)\n        NN_score.append(nn_score)\n        print('NN score for this fold: ', nn_score)\n        tmp = model.predict(X_test, batch_size=BATCH_SIZE, verbose=0)\n        Y_pred += np.sum(tmp, axis=0)/(2*NUM_FOLDS)\n\n        ###### train lgbm #############################\n        valid_lgbm = np.zeros((y_valid_fold.shape))\n        for i in range(NUM_CLASSES):\n            lgbm_model = LGBMClassifier(**params)\n            lgbm_model.fit(x_train_lgbm, y_train_fold[:,i],\n                           eval_set=(x_valid_lgbm, y_valid_fold[:,i]),\n                           eval_metric='logloss',\n                           early_stopping_rounds=100,\n                           verbose=0)\n            valid_lgbm[:, i] += (lgbm_model.predict_proba(x_valid_lgbm,\n                                                num_iteration=lgbm_model.best_iteration_)[:,1])\n            Y_pred[:, i] += (lgbm_model.predict_proba(X_test_lgbm,\n                                   num_iteration=lgbm_model.best_iteration_)[:,1])/NUM_FOLDS\n        lgbm_score = multilabel_logloss(y_valid_fold, valid_lgbm)\n        LGBM_score.append(lgbm_score)\n        print('LGBM score for this fold: ', lgbm_score)\n        \n        stack_score = multilabel_logloss(y_valid_fold, (valid_lgbm+valid_nn)/2)\n        print('LGBM + NN score for this fold: ', stack_score)\n        STACK_score.append(stack_score)\n\n        del (x_train_nn, x_valid_nn, y_train_fold, y_valid_fold,\n             x_train_lgbm, valid_nn, valid_lgbm, tmp)\n        gc.collect()\n    \nY_pred = Y_pred/(2*REPEAT)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('mean simple average score: ', np.mean(base_score))\nprint('mean NN score: ', np.mean(NN_score))\nprint('mean LGBM score: ', np.mean(LGBM_score))\nprint('mean NN + LGBM score: ', np.mean(STACK_score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_testset(filename=\"stage_1_sample_submission.csv\"):\n    df = pd.read_csv(filename)\n    df[\"Image\"] = df[\"ID\"].str.slice(stop=12)\n    df[\"Diagnosis\"] = df[\"ID\"].str.slice(start=13)\n    \n    df = df.loc[:, [\"Label\", \"Diagnosis\", \"Image\"]]\n    df = df.set_index(['Image', 'Diagnosis']).unstack(level=-1)\n    \n    return df\n\nsubmit = read_testset(filename=\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_sample_submission.csv\")\n\nsubmit.iloc[:, :] = Y_pred\nsubmit = submit.stack().reset_index()\nsubmit.insert(loc=0, column='ID', value=submit['Image'].astype(str) + \"_\" + submit['Diagnosis'])\nsubmit = submit.drop([\"Image\", \"Diagnosis\"], axis=1)\nsubmit.to_csv('stacking_{}_models_repeat_{}_times_stage2.csv'.format(NUM_MODELS, REPEAT), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}