{"cells":[{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare to start <a class=\"anchor\" id=\"prepare\"></a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set()\nimport tensorflow as tf\nimport pydicom\n\nfrom os import listdir\n\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\n\nfrom sklearn.model_selection import train_test_split\n\nimport keras\nfrom keras.applications import ResNet50, VGG16, ResNet101\nfrom keras.applications.resnet50 import preprocess_input as preprocess_resnet_50\n#from keras.applications.resnet101 import preprocess_input as preprocess_resnet_101\nfrom keras.applications.vgg16 import preprocess_input as preprocess_vgg_16\nfrom keras.layers import GlobalAveragePooling2D, Dense, Activation, concatenate, Dropout\nfrom keras.initializers import glorot_normal\nfrom keras.regularizers import l2\nfrom keras.models import Model, load_model\nfrom keras.utils import Sequence\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.optimizers import Adam\n\n\nfrom tensorflow.nn import sigmoid_cross_entropy_with_logits\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import applications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"applications.ResNet101(include_top=False, weights=\"imagenet\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_brute_force = True\ntrain_anysubtype_network = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#MODELOUTPUT_PATH = \"\"\n#brute_force_model_input = \"../input/rsna-ih-detection-baseline-models/_bruteforce_best_model.hdf5\"\n#brute_force_losses_path = \"../input/rsna-ih-detection-baseline-models/brute_force_losses.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.listdir('../input/')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def rescale_pixelarray(dataset):\n    image = dataset.pixel_array\n    rescaled_image = image * dataset.RescaleSlope + dataset.RescaleIntercept\n    rescaled_image[rescaled_image < -1024] = -1024\n    return rescaled_image\n\ndef set_manual_window(hu_image, custom_center, custom_width):\n    min_value = custom_center - (custom_width/2)\n    max_value = custom_center + (custom_width/2)\n    hu_image[hu_image < min_value] = min_value\n    hu_image[hu_image > max_value] = max_value\n    return hu_image\n\nclass Preprocessor:    \n    \n    def __init__(self, path, backbone, ct_level, ct_width, augment=False):\n        self.path = path\n        self.backbone = backbone\n        self.nn_input_shape = backbone[\"nn_input_shape\"]\n        self.ct_level = ct_level\n        self.ct_width = ct_width\n        self.augment = augment\n        \n    # 1. We need to load the dicom dataset\n    def load_dicom_dataset(self, filename):\n        dataset = pydicom.dcmread(self.path + filename)\n        return dataset\n    \n    # 2. We need to rescale the pixelarray to Hounsfield units\n    #    and we need to focus on our custom window:\n    def get_hounsfield_window(self, dataset, level, width):\n        hu_image = rescale_pixelarray(dataset)\n        windowed_image = set_manual_window(hu_image, level, width)\n        return windowed_image\n        \n    \n    # 3. Resize the image to the input shape of our CNN\n    def resize(self, image):\n        image = resize(image, self.nn_input_shape)\n        return image\n    \n    # 4. If we like to augment our image, let's do it:\n    def augment_img(self, image):\n        augment_img = iaa.Sequential([\n            iaa.OneOf([\n                #iaa.Affine(rotate=(-4, 4)),\n                iaa.Fliplr(0.5),\n            ])], random_order=True)\n        \n        image_aug = augment_img.augment_image(image)\n        return image_aug\n    \n    def fill_channels(self, image):\n        filled_image = np.stack((image,)*3, axis=-1)\n        return filled_image\n    \n    def preprocess(self, identifier):\n        filename = identifier +  \".dcm\"\n        dataset = self.load_dicom_dataset(filename)\n        level,width = self.get_windowing(dataset)\n        windowed_image = self.get_hounsfield_window(dataset, level, width)\n        image = self.resize(windowed_image)\n        if self.augment:\n            image = self.augment_img(image)\n        image = self.fill_channels(image)\n        return image\n    \n    def _normalize(self, image):\n        x_max = image.max()\n        x_min = image.min()\n        if x_max != x_min:\n            image = (image - x_min) / (x_max - x_min)\n            return image\n        return np.zeros(image.shape)\n    \n    def get_windowing(self,data):\n        dicom_fields = [data[('0028','1050')].value,  #window center\n                        data[('0028','1051')].value] #window width\n#                         data[('0028','1052')].value, #intercept\n#                         data[('0028','1053')].value] #slope\n        return [self.get_first_of_dicom_field_as_int(x) for x in dicom_fields]\n    \n    def get_first_of_dicom_field_as_int(self,x):\n        #get x[0] as in int is x is a 'pydicom.multival.MultiValue', otherwise get int(x)\n        if type(x) == pydicom.multival.MultiValue:\n            return int(x[0])\n        else:\n            return int(x)\n    \n#     def normalize(self, image):\n#         image = 2*(image/255) - 1\n#         return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Custom dataloader <a class=\"anchor\" id=\"dataloader\"></a>\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataLoader(Sequence):\n    \n    def __init__(self, dataframe,\n                 preprocessor,\n                 batch_size,\n                 num_classes=6,\n                 shuffle=False,\n                 steps=None):\n        self.preprocessor = preprocessor\n        self.data_ids = dataframe.index.values\n        self.dataframe = dataframe\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.input_shape = self.preprocessor.backbone[\"nn_input_shape\"]\n        self.preprocess_fun = self.preprocessor.backbone[\"preprocess_fun\"]\n        self.num_classes = num_classes\n        self.current_epoch=0\n        self.steps=steps\n        \n    # defines the number of steps per epoch\n    def __len__(self):\n        if self.steps is None:\n            return np.int(np.floor(len(self.data_ids) / np.float(self.batch_size)))\n        else:\n            return np.int(self.steps)\n    \n    # at the end of an epoch we may like to shuffle the data_ids:\n    def on_epoch_end(self):\n        if self.shuffle == True:\n            np.random.RandomState(self.current_epoch).shuffle(self.data_ids)\n            self.current_epoch += 1\n    \n    # should return a batch of images\n    def __getitem__(self, step):\n        # select the ids of the current batch\n        current_ids = self.data_ids[step*self.batch_size:(step+1)*self.batch_size]\n        X, y = self.__generate_batch(current_ids)\n        return X, y\n    \n    # collect the preprocessed images and targets of one batch\n    def __generate_batch(self, current_ids):\n        X = np.empty((self.batch_size, *self.input_shape, 3))\n        y = np.empty((self.batch_size, self.num_classes))\n        for idx, ident in enumerate(current_ids):\n            # Store sample\n            image = self.preprocessor.preprocess(ident)\n            X[idx] = self.preprocessor._normalize(image)\n            # Store class\n            y[idx] = self.__get_target(ident)\n        return X, y\n    \n    # extract the targets of one image id:\n    def __get_target(self, ident):\n        targets = self.dataframe.loc[ident].values\n        return targets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preparing the data <a class=\"anchor\" id=\"dataframeprep\"></a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_PATH = \"../input/rsna-intracranial-hemorrhage-detection/\"\ntrain_dir = INPUT_PATH + \"stage_1_train_images/\"\ntest_dir = INPUT_PATH + \"stage_1_test_images/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(INPUT_PATH+\"stage_1_sample_submission.csv\")\nsubmission.head(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindf = pd.read_csv(INPUT_PATH + \"stage_1_train.csv\")\ntraindf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label = traindf.Label.values\ntraindf = traindf.ID.str.rsplit(\"_\", n=1, expand=True)\ntraindf.loc[:, \"label\"] = label\ntraindf = traindf.rename({0: \"id\", 1: \"subtype\"}, axis=1)\ntraindf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testdf = submission.ID.str.rsplit(\"_\", n=1, expand=True)\ntestdf = testdf.rename({0: \"id\", 1: \"subtype\"}, axis=1)\ntestdf.loc[:, \"label\"] = 0\ntestdf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To use our dataloader we need to turn our traindf into something more useful. For this purpose we need to have the image id in one column and the subtypes of hemorrhage in further individual columns. "},{"metadata":{"trusted":true},"cell_type":"code","source":"traindf = pd.pivot_table(traindf, index=\"id\", columns=\"subtype\", values=\"label\")\ntraindf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testdf = pd.pivot_table(testdf, index=\"id\", columns=\"subtype\", values=\"label\")\ntestdf.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pretrained_models_path = \"../input/keras-pretrained-models/\"\n#listdir(\"../input/keras-pretrained-models/\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Just add more if you like:"},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_models = {\n    \"resnet_50\": {\"weights\": \"imagenet\",\n                  \"nn_input_shape\": (224,224),\n                  \"preprocess_fun\": preprocess_resnet_50},\n    \"vgg_16\": {\"weights\": \"imagenet\",\n              \"nn_input_shape\": (224,224),\n              \"preprocess_fun\": preprocess_vgg_16},\n    \"resnet_101\": {\"weights\": \"imagenet\",\n          \"nn_input_shape\": (224,224),\n          \"preprocess_fun\": preprocess_resnet_50}\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I like to experiment with different models and for this purpose I will write separate model functions that we have to pass to the constructor of our network class. "},{"metadata":{"trusted":true},"cell_type":"code","source":"def resnet_50():\n    #weights_path = pretrained_models_path + pretrained_models[\"resnet_50\"][\"weights\"]\n    net = ResNet50(include_top=False, weights=\"imagenet\")\n    for layer in net.layers:\n        layer.trainable = False\n    return net\n\ndef vgg_16():\n    #weights_path = pretrained_models_path + pretrained_models[\"vgg_16\"][\"weights\"]\n    net = VGG16(include_top=False, weights=\"imagenet\")\n    for layer in net.layers:\n        layer.trainable = False\n    return net\n\ndef resnet_101():\n    net = ResNet101(include_top=False, weights = \"imagenet\")\n    for layer in net.layers:\n        layer.trainable = False\n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELOUTPUT_PATH = \"\"\nclass MyNetwork:\n    \n    def __init__(self,\n                 model_fun,\n                 loss_fun,\n                 metrics_list,\n                 train_generator,\n                 dev_generator,\n                 epochs,\n                 num_classes=6,\n                 checkpoint_path=MODELOUTPUT_PATH):\n        self.model_fun = model_fun\n        self.loss_fun = loss_fun\n        self.metrics_list = metrics_list\n        self.train_generator = train_generator\n        self.dev_generator = dev_generator\n        self.epochs = epochs\n        self.num_classes = num_classes\n        self.checkpoint_path = \"_bruteforce_best_model_2.hdf5\"\n        self.checkpoint = ModelCheckpoint(filepath=self.checkpoint_path,\n                                          mode=\"min\",\n                                          verbose=1,\n                                          save_best_only=True,\n                                          save_weights_only=True,\n                                          period=1)\n        \n    def build_model(self):\n        base_model = self.model_fun()\n        x = base_model.output\n        x = GlobalAveragePooling2D()(x)\n        x = Dropout(0.3)(x)\n        logits = Dense(self.num_classes,\n                     kernel_initializer=glorot_normal(seed=11))(x)\n                     #kernel_regularizer=l2(0.2),\n                     #bias_regularizer=l2(0.2)\n        self.model = Model(inputs=base_model.input, outputs=logits)\n    \n    def compile_model(self):\n        self.model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.005, beta_1=0.9, beta_2=0.999, amsgrad=False),\n                           loss=self.loss_fun, \n                           metrics=self.metrics_list)\n    \n    def learn(self):\n        return self.model.fit_generator(generator=self.train_generator,\n                    validation_data=self.dev_generator,\n                    epochs=self.epochs,\n                    callbacks=[self.checkpoint],\n                    #use_multiprocessing=False,\n                    workers=8)\n    \n    def load_weights(self, path):\n        self.model.load_weights(path)\n    \n    def predict(self, test_generator):\n        logits = self.model.predict_generator(test_generator, \n                                              workers=8)\n        predictions = 1./(1+np.exp(-logits))\n        return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"undersample_seed=0\n\nnum_ill_patients = traindf[traindf[\"any\"]==1].shape[0]\nnum_ill_patients","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"healthy_patients = traindf[traindf[\"any\"]==0].index.values\nhealthy_patients_selection = np.random.RandomState(undersample_seed).choice(\n    healthy_patients, size=num_ill_patients, replace=False\n)\nlen(healthy_patients_selection)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ok, next, we will fuse the selected healthy patient group and the sick ones:"},{"metadata":{"trusted":true},"cell_type":"code","source":"sick_patients = traindf[traindf[\"any\"]==1].index.values\nselected_patients = list(set(healthy_patients_selection).union(set(sick_patients)))\nlen(selected_patients)/2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"remove_list = ['ID_c6bbec638',\n 'ID_2fd4dda7c',\n 'ID_2ac7f01ed',\n 'ID_b055aafa9',\n 'ID_cec3997fa',\n 'ID_ff9674e53',\n 'ID_d6435f3bf',\n 'ID_ea0ddbaf9',\n 'ID_cbbb50e6d',\n 'ID_038f966b9',\n 'ID_7607dbd07',\n 'ID_6fbc30b5d',\n 'ID_445a92ac2',\n 'ID_def2a0e9f',\n 'ID_9b68c3f5f',\n 'ID_ab474037b',\n 'ID_f22730d7b',\n 'ID_9cdc7295b',\n 'ID_91b9ce430',\n 'ID_4e61fb0b2',\n 'ID_fd5c41761',\n 'ID_11c4f9f91',\n 'ID_b76de950b',\n 'ID_191369dca',\n 'ID_b19f52c76',\n 'ID_b1cea5abb',\n 'ID_767c42624',\n 'ID_80a2dbc4a',\n 'ID_b494c2115',\n 'ID_72dce7784',\n 'ID_567a36143',\n 'ID_ba7080372',\n 'ID_d1b2d9ad0',\n 'ID_12e3b6923',\n 'ID_75e3f7e5a',\n 'ID_b76b13444',\n 'ID_b12bb2b16',\n 'ID_61c646098',\n 'ID_b4adf8739',\n 'ID_21053fe7e',\n 'ID_fdbfb2c17',\n 'ID_d7777de78',\n 'ID_e4b636907',\n 'ID_28c4609b3',\n 'ID_d1afb9750',\n 'ID_55f7bbbf2',\n 'ID_97cd49666',\n 'ID_6a939bc17',\n 'ID_b8665a653',\n 'ID_3d5d23058',\n 'ID_9da128021',\n 'ID_403b4fc67',\n 'ID_c4575f13b',\n 'ID_b966185b8',\n 'ID_f698edc00',\n 'ID_3e60e696d',\n 'ID_ac47ba810',\n 'ID_49ecc6164',\n 'ID_94463e98f',\n 'ID_56ecdf5c1',\n 'ID_61d2718d2',\n 'ID_1e633cf27',\n 'ID_f145c3cf4',\n 'ID_d7229490a',\n 'ID_5ab140176',\n 'ID_155249efa',\n 'ID_75cbdae68',\n 'ID_0b0e59911',\n 'ID_dd3b5bf4e',\n 'ID_845f922f4',\n 'ID_9bc2b62cc',\n 'ID_73dee8958',\n 'ID_dabc2a818',\n 'ID_807b56a94',\n 'ID_18aac96c0',\n 'ID_c51cbe76b',\n 'ID_17103c79e',\n 'ID_950a06268',\n 'ID_8fde47d9f',\n 'ID_d3fd5220e',\n 'ID_a2f9ba4bf',\n 'ID_ff012ee5b',\n 'ID_2b3671dd9',\n 'ID_75d691728',\n 'ID_f03370d7c',\n 'ID_21d4bd6f3',\n 'ID_23d0b13b7',\n 'ID_7c08b7fb7',\n 'ID_ac1d14c29',\n 'ID_3f422852d',\n 'ID_10fe2031e',\n 'ID_7a02fdbea',\n 'ID_b77ba3355',\n 'ID_a40f9b2de',\n 'ID_a1bb9bc26',\n 'ID_bb2a4a01c',\n 'ID_155b9c546',\n 'ID_830f46cad',\n 'ID_d3b76ef6e',\n 'ID_af129aa8e',\n 'ID_27757c171',\n 'ID_15b3ba199',\n 'ID_3274f5977',\n 'ID_64b44f180',\n 'ID_451f60160',\n 'ID_0603b315e',\n 'ID_a880e377e',\n 'ID_ca4a832a1',\n 'ID_c1ff9eb46',\n 'ID_09aeb0bbd',\n 'ID_3ba8a116c',\n 'ID_04280250b',\n 'ID_081f4d071',\n 'ID_a9e98ab5e',\n 'ID_a7e689932',\n 'ID_85900eb84',\n 'ID_1690a6499',\n 'ID_676b0cb59',\n 'ID_3eb407dd8',\n 'ID_ae691dd29',\n 'ID_8a35660d5',\n 'ID_798d956d0',\n 'ID_25de55880',\n 'ID_942e2f95b',\n 'ID_9a3bba619',\n 'ID_66131f4c9',\n 'ID_8fd6d5047',\n 'ID_3cb1b59bc',\n 'ID_9b297fa83',\n 'ID_0f8aa5749',\n 'ID_c1a3f037f',\n 'ID_d4ea87a35',\n 'ID_079945c27',\n 'ID_6cb797177',\n 'ID_5dbe845c1',\n 'ID_9ece1bb21',\n 'ID_6dcedd2e1',\n 'ID_176e4f16d',\n 'ID_362423b57',\n 'ID_db48a633d',\n 'ID_ced5fabca',\n 'ID_c60e34466',\n 'ID_317330708',\n 'ID_291edd834',\n 'ID_4e0bdd2ba',\n 'ID_ca9462f49',\n 'ID_a432727fd',\n 'ID_898ff55b6',\n 'ID_5bf2ca43f',\n 'ID_19306ecc5',\n 'ID_3e31d57d0',\n 'ID_f242fed92',\n 'ID_c11582dc9',\n 'ID_6cc19ac41',\n 'ID_d1a1c9a6c',\n 'ID_6f92e4481',\n 'ID_de10fdac2',\n 'ID_2e690fe7c',\n 'ID_28d6a694f',\n 'ID_66accd2e4',\n 'ID_6b1a86148',\n 'ID_394ffb5fd',\n 'ID_a2e178cc7',\n 'ID_9dad2eb09',\n 'ID_c964e4096',\n 'ID_91c508c7a',\n 'ID_7e756c43b',\n 'ID_7940bb7d0',\n 'ID_53c71fb9d',\n 'ID_0e9ac1c5f',\n 'ID_97e5a203e',\n 'ID_ac39010dc',\n 'ID_69974dd3e',\n 'ID_cade293be',\n 'ID_36ab2e72a',\n 'ID_8756b0c04',\n 'ID_d9840380c',\n 'ID_c64131283',\n 'ID_d77fa1286',\n 'ID_82ec3b736',\n 'ID_046ba342c',\n 'ID_4e14d0fe8',\n 'ID_3bc141392',\n 'ID_abcd58e88',\n 'ID_1291d1943',\n 'ID_6431af929',\n 'ID_f4c2157d8',\n 'ID_c45659d3d',\n 'ID_8c5fc9e44',\n 'ID_c2738e8b1',\n 'ID_f0d55b727',\n 'ID_bc97a5f4f',\n 'ID_1bb3b44c7',\n 'ID_b9938c32c',\n 'ID_8caa68ebd',\n 'ID_cf4d76860',\n 'ID_7917d368d',\n 'ID_dfa4e344a',\n 'ID_3dcbd1b5e',\n 'ID_be3fb6c17',\n 'ID_c6463f07d',\n 'ID_cb970c6dc',\n 'ID_3d7a23dbb',\n 'ID_19f266244',\n 'ID_1bc5771a7',\n 'ID_8fc348d44',\n 'ID_9a36e4b0e',\n 'ID_57d6a6455',\n 'ID_53f460f86',\n 'ID_985fb5e49',\n 'ID_f1fe5334e',\n 'ID_ae1689e1b',\n 'ID_4f0317d23',\n 'ID_5ffae2e26',\n 'ID_142f85eb8',\n 'ID_8144c7120',\n 'ID_842e85173',\n 'ID_bd4f3f06f',\n 'ID_184c541fa',\n 'ID_fe7327fab',\n 'ID_88b0d8b4f',\n 'ID_cef2af72d',\n 'ID_8f5d4b696',\n 'ID_ea2861e9a',\n 'ID_631f0b556',\n 'ID_0b2ec2d3f',\n 'ID_37c495912',\n 'ID_a356248db',\n 'ID_4c9fb82af',\n 'ID_7714ead69',\n 'ID_c07d2cb73',\n 'ID_10f34fb10',\n 'ID_c6f2d84be',\n 'ID_dfaa49f5c',\n 'ID_a3feeadf4',\n 'ID_6c19c9f7b',\n 'ID_0de0ab1d8',\n 'ID_60a1f0e24',\n 'ID_0144e4030',\n 'ID_6508563e0',\n 'ID_f188940f9',\n 'ID_22069463a',\n 'ID_ae7020fd1',\n 'ID_a9ab8569f',\n 'ID_7e870621c',\n 'ID_dd083e12a',\n 'ID_d0c52575a',\n 'ID_c037d5727',\n 'ID_6d7a27643',\n 'ID_0e1861e6d',\n 'ID_a3128aa77',\n 'ID_680b2194c',\n 'ID_882cd57de',\n 'ID_44d57858e',\n 'ID_b194d2a23',\n 'ID_ae7b11865',\n 'ID_8dc299456',\n 'ID_76f88846f',\n 'ID_6b15a7649',\n 'ID_f23f8e617',\n 'ID_aef6c6df9',\n 'ID_c35d5c858',\n 'ID_8d0ca7742',\n 'ID_0c4987103',\n 'ID_3c8b72361',\n 'ID_a23a8193f',\n 'ID_c65ca5466',\n 'ID_12a0d6d34',\n 'ID_f4891876d',\n 'ID_68e45bca7']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindf.drop(remove_list,axis = 0,inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#new_traindf = traindf.loc[selected_patients].copy()\ntraindf[\"any\"].value_counts()\nnew_traindf = traindf.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Validation\n\nOk, this part is important but I like to start simple again by using a single train_test_split. "},{"metadata":{"trusted":true},"cell_type":"code","source":"split_seed = 1\ntrain_data, dev_data = train_test_split(new_traindf,\n                                        test_size=0.2,\n                                        stratify=new_traindf.values,\n                                        random_state=split_seed)\nprint(train_data.shape)\nprint(dev_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_perc_train = train_data.sum() / train_data.shape[0] * 100\npos_perc_dev = dev_data.sum() / dev_data.shape[0] * 100\n\nfig, ax = plt.subplots(2,1,figsize=(20,14))\nsns.barplot(x=pos_perc_train.index, y=pos_perc_train.values, palette=\"Set2\", ax=ax[0]);\nax[0].set_title(\"Target distribution used for training data\")\nsns.barplot(x=pos_perc_dev.index, y=pos_perc_dev.values, palette=\"Set2\", ax=ax[1]);\nax[1].set_title(\"Target distribution used for dev data\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# multilabel loss (optional weighted)\ndef multilabel_loss(class_weights=None):\n    def multilabel_loss_inner(y_true, logits):\n        logits = tf.cast(logits, tf.float32)\n        y_true = tf.cast(y_true, tf.float32)\n        \n        # compute single class cross entropies:\n        contributions = tf.maximum(logits, 0) - tf.multiply(logits, y_true) + tf.log(1.+tf.exp(-tf.abs(logits)))\n\n        # contributions have shape (n_samples, n_classes), we need to reduce with mean over samples to obtain single class xentropies:\n        single_class_cross_entropies = tf.reduce_mean(contributions, axis=0)\n\n        # if None, weight equally:\n        if class_weights is None:\n            loss = tf.reduce_mean(single_class_cross_entropies)\n        else:\n            weights = tf.constant(class_weights, dtype=tf.float32)\n            loss = tf.reduce_sum(tf.multiply(weights, single_class_cross_entropies))\n        return loss\n    return multilabel_loss_inner","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def multilabel_focal_loss(class_weights=None, alpha=1, gamma=2):\n    def mutlilabel_focal_loss_inner(y_true, logits):\n        logits = tf.cast(logits, tf.float32)\n        y_true = tf.cast(y_true, tf.float32)\n\n        # compute predictions for each class in logits: \n        y = tf.sigmoid(logits)\n        # compute pred_t:\n        y_t = tf.where(tf.equal(y_true,1), y, 1.-y)\n\n        # compute single class cross entropies:\n        contributions = tf.maximum(logits, 0) - tf.multiply(logits, y_true) + tf.log(1.+tf.exp(-tf.abs(logits)))\n\n        # compute focal loss contributions\n        focal_loss_contributions = alpha * tf.multiply(tf.pow(1-y_t, gamma), contributions)\n\n        # our focal loss contributions have shape (n_samples, s_classes), we need to reduce with mean over samples:\n        focal_loss_per_class = tf.reduce_mean(focal_loss_contributions, axis=0)\n\n        # compute the overall loss if class weights are None (equally weighted):\n        if class_weights is None:\n            focal_loss = tf.reduce_mean(focal_loss_per_class)\n        else:\n            # weight the single class losses and compute the overall loss\n            weights = tf.constant(class_weights, dtype=tf.float32)\n            focal_loss = tf.reduce_sum(tf.multiply(weights, focal_loss_per_class))\n\n        return focal_loss\n    return mutlilabel_focal_loss_inner","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Let it run! :-)"},{"metadata":{"trusted":true},"cell_type":"code","source":"BACKBONE = \"resnet_101\"\nBATCH_SIZE = 32\nTEST_BATCH_SIZE = 32\nCT_LEVEL = 40\nCT_WIDTH = 150\n\nLR = 0.005","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_preprocessor = Preprocessor(path=train_dir,\n                                  backbone=pretrained_models[BACKBONE],\n                                  ct_level=CT_LEVEL,\n                                  ct_width=CT_WIDTH,\n                                  augment=False)\n\ndev_preprocessor = Preprocessor(path=train_dir,\n                                backbone=pretrained_models[BACKBONE],\n                                ct_level=CT_LEVEL,\n                                ct_width=CT_WIDTH,\n                                augment=False)\n\ntest_preprocessor = Preprocessor(path=test_dir,\n                                backbone=pretrained_models[BACKBONE],\n                                ct_level=CT_LEVEL,\n                                ct_width=CT_WIDTH,\n                                augment=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,4,figsize=(20,20))\n\n\nfor m in range(4):\n    example = train_data.index.values[m]\n    title = [col for col in train_data.loc[example,:].index if train_data.loc[example, col]==1]\n    if len(title) == 0:\n        title=\"Healthy\"\n    preprocess_example = train_preprocessor.preprocess(example)\n    ax[m].imshow(preprocess_example[:,:,0], cmap=\"Spectral\")\n    ax[m].grid(False)\n    ax[m].set_title(title);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataloader = DataLoader(train_data,\n                              train_preprocessor,\n                              BATCH_SIZE,\n                              shuffle=True,\n                              steps = 8000)\n\ndev_dataloader = DataLoader(dev_data, \n                            dev_preprocessor,\n                            BATCH_SIZE,\n                            shuffle=False)\n\ntest_dataloader = DataLoader(testdf, \n                             test_preprocessor,\n                             TEST_BATCH_SIZE,\n                             shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dev_dataloader.__len__()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataloader.__len__()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataloader.__len__()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_class_weights = [0.2, 0.16, 0.16, 0.16, 0.16, 0.16]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if train_brute_force:\n    model = MyNetwork(model_fun=vgg_16,\n                      loss_fun=multilabel_focal_loss(class_weights=my_class_weights, alpha=0.25, gamma=2),\n                      metrics_list=[multilabel_loss(class_weights=my_class_weights),\n                                    multilabel_focal_loss(class_weights=my_class_weights, alpha=0.25, gamma=2)],\n                      train_generator=train_dataloader,\n                      dev_generator=dev_dataloader,\n                      epochs=5,\n                      num_classes=6)\n    model.build_model()\n    model.compile_model()\n    history = model.learn()\n    \n    print(history.history.keys())\n    losses_df = pd.DataFrame(history.history[\"loss\"], columns=[\"multi_wfocal_loss_train\"])\n    losses_df.loc[:, \"multi_wfocal_loss_val\"] = history.history[\"val_loss\"]\n    losses_df.loc[:, \"multi_w_loss_train\"] = history.history[\"multilabel_loss_inner\"]\n    losses_df.loc[:, \"multi_w_loss_val\"] = history.history[\"val_multilabel_loss_inner\"]\n    losses_df.to_csv(\"brute_force_losses.csv\", index=False)\n    \n    plt.figure(figsize=(20,5))\n    plt.plot(history.history[\"loss\"], 'o-')\n    plt.plot(history.history[\"val_loss\"], 'o-')\n    \n    test_pred = model.predict(test_dataloader)\nelse:\n    losses_df = pd.read_csv(brute_force_losses_path)\n    model = MyNetwork(model_fun=resnet_50,\n                      loss_fun=multilabel_focal_loss(class_weights=my_class_weights,\n                                                     alpha=0.25,keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)\n                                                     gamma=2),\n                      metrics_list=[keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)\n                          multilabel_loss(class_weights=my_class_weights),\n                          multilabel_focal_loss(class_weights=my_class_weights,\n                                                alpha=0.25,\n                                                gamma=2)],\n                      train_generator=train_dataloader,\n                      dev_generator=dev_dataloader,\n                      epochs=5,\n                      num_classes=6)\n    model.build_model()\n    test_pred = model.predict(test_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testdf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred = model.predict(test_dataloader)\ntest_pred.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To make a first test submission we need to fill the submission template:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def turn_to_submission(test_data, pred, submission):\n    df = pd.DataFrame(pred, columns=test_data.columns, index=test_data.index)\n    df = df.stack().reset_index()\n    df.loc[:, \"ID\"] = df.id.str.cat(df.subtype, sep=\"_\")\n    df = df.drop([\"id\", \"subtype\"], axis=1)\n    df = df.set_index(\"ID\")\n    df = df.rename({0: \"Label\"}, axis=1)\n    submission.Label = submission.ID.map(df.Label)\n    return submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bruteforce_submission = turn_to_submission(testdf, test_pred, submission)\nbruteforce_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bruteforce_submission.to_csv(\"bruteforce_submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dev_pred = model.predict(dev_dataloader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building up the any-subtype network"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AnySubtypeNetwork(MyNetwork):\n    \n    def __init__(self,\n                 model_fun,\n                 loss_fun,\n                 metrics_list,\n                 train_generator,\n                 dev_generator,\n                 epochs,\n                 num_subtype_classes=5):\n        MyNetwork.__init__(self, \n                           model_fun=model_fun,\n                           loss_fun=loss_fun,\n                           metrics_list=metrics_list,\n                           train_generator=train_generator,\n                           dev_generator=dev_generator,\n                           epochs=epochs,\n                           num_classes=num_subtype_classes)\n    \n    def build_model(self):\n        base_model = self.model_fun()\n        x = base_model.output\n        x = GlobalAveragePooling2D()(x)\n        x = Dropout(0.3)(x)\n        any_logits = Dense(1, kernel_initializer=glorot_normal(seed=11),\n                     kernel_regularizer=l2(0.2),\n                     bias_regularizer=l2(0.2))(x)\n        any_pred = Activation(\"sigmoid\", name=\"any_predictions\")(any_logits)\n        x = concatenate([any_pred, x])\n        sub_logits = Dense(self.num_classes,\n                           name=\"subtype_logits\",\n                           kernel_initializer=glorot_normal(seed=12),\n                           kernel_regularizer=l2(0.2),\n                           bias_regularizer=l2(0.2))(x) \n        self.model = Model(inputs=base_model.input, outputs=[any_pred, sub_logits])\n    \n    def compile_model(self):\n        self.model.compile(optimizer=Adam(LR),\n                           loss=['binary_crossentropy', multilabel_focal_loss(alpha=0.25, gamma=2)], \n                           metrics=self.metrics_list)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now you can see that the any loss and the subtype loss are weighted the same. Doing so the any-part has comparable higher weight than any of the single subtypes. This is still a simplification. We don't know it neither for the any nor for single subtypes. **But we have taken the first advice \"weight any higher\" into account**."},{"metadata":{},"cell_type":"markdown","source":"## Multioutput DataLoader <a class=\"anchor\" id=\"twooutputsdataloader\"></a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AnySubtypeDataLoader(DataLoader):\n    \n    def __init__(self, dataframe,\n                 preprocessor,\n                 batch_size,\n                 num_classes=5,\n                 shuffle=False,\n                 steps=None):\n        DataLoader.__init__(self, \n                            dataframe=dataframe,\n                            preprocessor=preprocessor,\n                            batch_size=batch_size,\n                            num_classes=num_classes,\n                            shuffle=shuffle,\n                            steps=steps)\n    \n    # should return a batch of images\n    def __getitem__(self, step):\n        # select the ids of the current batch\n        current_ids = self.data_ids[step*self.batch_size:(step+1)*self.batch_size]\n        X, y_any, y_subtype = self.__generate_batch(current_ids)\n        return X, [y_any, y_subtype]\n    \n    # collect the preprocessed images and targets of one batch\n    def __generate_batch(self, current_ids):\n        X = np.empty((self.batch_size, *self.input_shape, 3))\n        y_subtype = np.empty((self.batch_size, self.num_classes))\n        y_any = np.empty((self.batch_size, 1))\n        for idx, ident in enumerate(current_ids):\n            # Store sample\n            image = self.preprocessor.preprocess(ident)\n            X[idx] = self.preprocess_fun(image)\n            # Store class\n            y_any[idx], y_subtype[idx] = self.__get_target(ident)\n        return X, y_any, y_subtype\n    \n    # extract the targets of one image id:\n    def __get_target(self, ident):\n        y_any = self.dataframe.loc[ident, \"any\"]\n        y_subtype = self.dataframe.drop(\"any\", axis=1).loc[ident].values\n        return y_any, y_subtype","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataloader = AnySubtypeDataLoader(train_data,\n                                        train_preprocessor,\n                                        BATCH_SIZE,\n                                        shuffle=True,\n                                        steps=100)\ndev_dataloader = AnySubtypeDataLoader(dev_data, \n                                      dev_preprocessor,\n                                      BATCH_SIZE,\n                                      shuffle=False,\n                                      steps=100)\ntest_dataloader = AnySubtypeDataLoader(testdf, \n                                       test_preprocessor,\n                                       TEST_BATCH_SIZE,\n                                       shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if train_anysubtype_network:\n    model = AnySubtypeNetwork(model_fun=resnet_50,\n                              loss_fun=None,\n                              metrics_list=None,\n                              train_generator=train_dataloader,\n                              dev_generator=dev_dataloader,\n                              epochs=5) \n    model.build_model()\n    model.compile_model()\n    history = model.learn()\n    \n    print(history.history.keys())\n    \n    plt.figure(figsize=(20,5))\n    plt.plot(history.history[\"loss\"], 'o-')\n    plt.plot(history.history[\"val_loss\"], 'o-')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f = os.listdir(train_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fascinating! During the first time it seems that only the any-loss changes and then very slowly the subtype loss follows. "}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":1}