{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nimport matplotlib.pyplot as plt\nimport os\nimport tensorflow as tf\nfrom tensorflow.keras import Model\nfrom tqdm import notebook","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train  =  pd.read_csv(\"../input/vinbigdata-chest-xray-abnormalities-detection/train.csv\")\ntest = pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/sample_submission.csv')\n\ntrain_dir256 = \"../input/vinbigdata-chest-xray-resized-png-256x256/train\"\ntest_dir256 = \"../input/vinbigdata-chest-xray-resized-png-256x256/test\"\n\n\ntrain['image_png'] = train.image_id+'.png'\ntest['image_png'] = test.image_id+'.png'\n\nIMAGE_SIZE256 = [256, 256] \nBATCH_SIZE = 32  \nEPOCHS = 2\nOPTIMIZER = tf.keras.optimizers.Adam(learning_rate=0.001)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.info())\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"is_fold_train = train.groupby(\"image_png\")[\"class_id\"].agg(lambda s: \n(s == 14).sum()).reset_index().rename({\n    \"class_id\": \"num_normal_annotations\"}, axis=1)\nis_fold_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def change(x):\n    if (x==3):\n        x=1\n    return x\n\nis_fold_train['target'] = is_fold_train['num_normal_annotations'].apply(lambda x: change(x))\nis_fold_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skfolds = StratifiedKFold(n_splits=5, \n                          random_state=42, \n                          shuffle = True)\n    \nfor num_fold, (train_index, val_index) in enumerate(skfolds.split(is_fold_train, is_fold_train.target)):\n    is_fold_train.loc[val_index, 'fold'] = int(num_fold)\n    \nis_fold_train['target'] = is_fold_train.target.astype('str')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(is_fold_train.info())\nis_fold_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen_train = ImageDataGenerator(\n                        rotation_range=40,          \n                        width_shift_range=0.2,   \n                        height_shift_range=0.2,  \n                        zoom_range=0.2,           \n                        horizontal_flip=True,     \n                        vertical_flip=False      \n                                   )     \n\ndatagen_test =  ImageDataGenerator(validation_split = 0.2) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_augmentation = tf.keras.Sequential([\n  tf.keras.layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n  tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope(): \n\n    class MyModel(Model):\n        def __init__(self, appl, training=True):\n            super(MyModel, self).__init__()\n            self.d1_0 = tf.keras.applications.EfficientNetB0(\n                input_shape=(*IMAGE_SIZE256, 3),\n                weights='imagenet', pooling='avg', \n                include_top=False)\n            \n            self.d1_1 = tf.keras.applications.DenseNet121(\n                input_shape=(*IMAGE_SIZE256, 3),\n                weights=None, pooling='max') \n                                                           \n            self.d2 = tf.keras.layers.Dense(150, activation='relu')\n            self.d3 = tf.keras.layers.Dense(2, activation='sigmoid')\n            self.dropout = tf.keras.layers.Dropout(0.15)\n            \n            \n            #self.augmentation1 = tf.keras.layers.experimental.preprocessing.RandomFlip(\n                #\"horizontal_and_vertical\")\n            # self.augmentation2 =  tf.keras.layers.experimental.preprocessing.RandomRotation(0.2)\n            #self.augmentation = data_augmentation\n            \n            self.training = training\n            self.appl = appl\n\n        #def call(self, x, training=False, appl=True):\n        def call(self, x):\n    \n            if self.appl:\n                x = self.d1_0(x)\n            else:\n                #x = self.augmentation(x)\n                x = self.d1_1(x)\n            \n            if self.training:\n                x = self.dropout(x)\n            x = self.d2(x)\n            return self.d3(x)\n    \n    \nmodel1 = MyModel(appl=True)\nmodel2 = MyModel(appl=False)\n\nmodel1.compile(\n        optimizer = OPTIMIZER,\n        loss = \"binary_crossentropy\",\n        metrics = [tf.keras.metrics.BinaryAccuracy()]\n    )\n\nmodel2.compile(\n        optimizer = OPTIMIZER,\n        loss = \"binary_crossentropy\",\n        metrics = [tf.keras.metrics.BinaryAccuracy()]\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_history = [] \nval_loss_history = []\n\nbinary_accuracy_history = []\nval_binary_accuracy_history = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold_n in range(5): \n    print('Fold #{}'.format(fold_n+1))\n    \n    train_data = is_fold_train[is_fold_train.fold != fold_n]\n    val_data = is_fold_train[is_fold_train.fold == fold_n] \n    \n    train_dataset = datagen_train.flow_from_dataframe(\n        train_data,                                              \n        directory = train_dir256,\n        subset = \"training\",\n        x_col = \"image_png\",\n        y_col = \"target\",\n        shuffle=True,\n        batch_size=BATCH_SIZE)\n            \n            \n\n    valid_dataset = datagen_test.flow_from_dataframe(\n        val_data,\n        directory = train_dir256,\n        subset = \"validation\",\n        x_col = \"image_png\",\n        y_col = \"target\",\n        shuffle=True,\n        batch_size=BATCH_SIZE)\n    \n    model_fit = model1.fit(train_dataset, validation_data=valid_dataset, epochs=EPOCHS) \n    \n    loss_history.append(model_fit.history['loss'])\n    val_loss_history.append(model_fit.history['val_loss'])\n    binary_accuracy_history.append(model_fit.history['binary_accuracy'])\n    val_binary_accuracy_history.append(model_fit.history['val_binary_accuracy'])\n    \n    \nlh = [item for sublist in loss_history for item in sublist]\nvlh = [item for sublist in val_loss_history for item in sublist]\n\nbah = [item for sublist in binary_accuracy_history for item in sublist]\nvbah = [item for sublist in val_binary_accuracy_history for item in sublist]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(lh, label='train')\nplt.plot(vlh, label='test')\nplt.title('loss')\nplt.legend()\nplt.show()\nplt.plot(bah, label='train')\nplt.plot(vbah, label='test')\nplt.title('binary_accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train_loss_history = [] \ntest_loss_history = []\n\ntrain_loss = tf.keras.metrics.Mean(name='train_loss')\ntrain_accuracy = tf.keras.metrics.BinaryAccuracy(name='train_accuracy')\n\ntest_loss = tf.keras.metrics.Mean(name='test_loss')\ntest_accuracy = tf.keras.metrics.BinaryAccuracy(name='test_accuracy')\n\n#@tf.function\ndef train_step(images, labels):\n    with tf.GradientTape() as tape:\n        \n        predictions = model1(images, training=True)\n        loss_value = tf.keras.losses.binary_crossentropy(labels, predictions) \n    \n    train_loss_history.append(train_loss(loss_value.numpy())) \n    \n    grads = tape.gradient(loss_value, model1.trainable_variables)\n    OPTIMIZER.apply_gradients(zip(grads, model2.trainable_variables))\n\n    train_loss(loss_value)\n    train_accuracy(labels, predictions)\n    \n    \n#@tf.function\ndef test_step(images, labels):\n    \n    predictions = model2(images, training=False)\n    \n    t_loss = tf.keras.losses.binary_crossentropy(labels, predictions)\n    \n    test_loss_history.append(test_loss(t_loss.numpy()))\n    \n    test_loss(t_loss)\n    test_accuracy(labels, predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def train(epochs):\n    for fold_n in range(5):\n        \n        train_data = is_fold_train[is_fold_train.fold != fold_n]\n        val_data = is_fold_train[is_fold_train.fold == fold_n]\n        \n        for epoch in range(epochs):\n            print('Epoch: {}'.format(epoch+1))\n                  \n            train_dataset = datagen_train.flow_from_dataframe( \n                train_data,                                              \n                directory = train_dir256,\n                subset = \"training\",\n                x_col = \"image_png\",\n                y_col = \"target\",\n                shuffle=True,\n                batch_size=BATCH_SIZE\n                                                             )\n            \n            \n\n            valid_dataset = datagen_test.flow_from_dataframe( \n                val_data,\n                directory = train_dir256,\n                subset = \"validation\",\n                x_col = \"image_png\",\n                y_col = \"target\",\n                shuffle=True,\n                batch_size=BATCH_SIZE\n                                                             )\n            \n            for (batch, (images, labels)) in notebook.tqdm(enumerate(train_dataset)):\n                train_step(images, labels)\n                \n                \n            for (batch, (test_images, test_labels)) in enumerate(valid_dataset):            \n                test_step(test_images, test_labels)\n                \n            \n            template = 'Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n            \n            print(template.format(\n                             train_loss.result(),\n                             train_accuracy.result()*100,\n                             test_loss.result(),\n                             test_accuracy.result()*100\n            ))\n            ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#train(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_history = [] \nval_loss_history = []\n\nbinary_accuracy_history = []\nval_binary_accuracy_history = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold_n in range(5):\n    print('Fold #{}'.format(fold_n+1))\n    \n    train_data = is_fold_train[is_fold_train.fold != fold_n]\n    val_data = is_fold_train[is_fold_train.fold == fold_n]\n    \n                  \n    train_dataset = datagen_train.flow_from_dataframe(\n        train_data,                                              \n        directory = train_dir256,\n        subset = \"training\",\n        x_col = \"image_png\",\n        y_col = \"target\",\n        shuffle=True,\n        batch_size=BATCH_SIZE)\n            \n\n    valid_dataset = datagen_test.flow_from_dataframe(\n        val_data,\n        directory = train_dir256,\n        subset = \"validation\",\n        x_col = \"image_png\",\n        y_col = \"target\",\n        shuffle=True,\n        batch_size=BATCH_SIZE)\n            \n    model_fit = model2.fit(train_dataset, validation_data=valid_dataset, epochs=EPOCHS)\n    \n    loss_history.append(model_fit.history['loss'])\n    val_loss_history.append(model_fit.history['val_loss'])\n    binary_accuracy_history.append(model_fit.history['binary_accuracy'])\n    val_binary_accuracy_history.append(model_fit.history['val_binary_accuracy'])\n    \n            \nlh = [item for sublist in loss_history for item in sublist]\nvlh = [item for sublist in val_loss_history for item in sublist]\n\nbah = [item for sublist in binary_accuracy_history for item in sublist]\nvbah = [item for sublist in val_binary_accuracy_history for item in sublist]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(lh, label='train')\nplt.plot(vlh, label='test')\nplt.title('loss')\nplt.legend()\nplt.show()\nplt.plot(bah, label='train')\nplt.plot(vbah, label='test')\nplt.title('binary_accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = ImageDataGenerator().flow_from_dataframe(\n                                test,\n                                directory = test_dir256,\n                                x_col = \"image_png\",\n                                class_mode='raw',\n                                y_col = \"image_id\",\n                                batch_size=3000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = test_dataset.next()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(x.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1 = model1.predict(x)\npreds2 = model2.predict(x)\npredictions = np.argmax(preds1 + preds2, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'image_id': y, 'label': predictions})  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}