{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install nb_black\n%load_ext nb_black","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport shutil\nimport multiprocessing\nimport pickle\nimport cv2\n\n!pip install iterative-stratification\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\nfrom sklearn.model_selection import train_test_split\n\n#!pip install tensorflow-gpu\nimport tensorflow as tf\n\nprint(tf.__version__)\nprint(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices(\"GPU\")))\n\nfrom keras_preprocessing.image import ImageDataGenerator\nfrom keras.applications import Xception, vgg16\nfrom keras.layers import (\n    GlobalAveragePooling2D,\n    Dense,\n    BatchNormalization,\n    Dropout,\n    Concatenate,\n    Add,\n)\nfrom keras.models import Model\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, Callback\nfrom keras.losses import binary_crossentropy\nfrom keras import backend as K\nfrom keras.utils import Sequence\n\n#!pip install -U efficientnet==0.0.4 #Used from Script Utility https://www.kaggle.com/ratthachat/efficientnet\n!pip install -U efficientnet #Used from Script Utility https://www.kaggle.com/ratthachat/efficientnet\nfrom efficientnet import EfficientNetB3, preprocess_input\n\n!pip install keras-rectified-adam\n# os.environ['TF_KERAS'] = '1' # Required to work RAdam work properly (maybe in future versions is solved)\nfrom keras_radam import RAdam\n\nfrom tqdm import tqdm_notebook\n\nfrom matplotlib import pyplot as plt\nimport matplotlib.image as image\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Custom codes to make code clear\nfrom dnn_helper import (\n    KFolds_flow_from_dataframe,\n    fold_training,\n    save_train,\n    visualize_training,\n    config_model_trainable,\n    bce_dice_loss,\n    BatchHistoryEarlyStopping,\n    make_pred,\n)\nfrom misc_utils import extract_zips, rebalance_data, oversampling_data\nfrom image_utils import preprocess_image, check_range, transform_range\nfrom dicom_utils import (\n    preprocess_dicom,\n    sample_bins_mean_std,\n    get_dcm_img,\n    normalize_img,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_PATH = \"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/\"\nTRAIN_DIR = BASE_PATH + \"stage_2_train/\"\nSUB_DIR = BASE_PATH + \"stage_2_test/\"\n\nIMAGE_DATA_BASE_PATH = \"../input/rsna-dicom-normalization-by-metadata-groups/\"\nCSV_DIR = IMAGE_DATA_BASE_PATH + \"csv/\"\nTRAIN_ZIP = IMAGE_DATA_BASE_PATH + \"train.zip\"\nTEST_ZIP = IMAGE_DATA_BASE_PATH + \"test.zip\"\nSUB_ZIP = IMAGE_DATA_BASE_PATH + \"sub.zip\"\n\n# Temporal data\nTRAIN_PNG = \"/kaggle/tmp/train/\"\nTEST_PNG = \"/kaggle/tmp/test/\"\nSUB_PNG = \"/kaggle/tmp/sub/\"\nAUG_TRAIN_PNG = \"/kaggle/tmp/aug_png/\"\n\n# Outputs\nPREDICTIONS_DIR = \"predictions/\"\nMODELS_DIR = \"model/\"\n\nCLASSES = [\n    \"any\",\n    \"epidural\",\n    \"intraparenchymal\",\n    \"intraventricular\",\n    \"subarachnoid\",\n    \"subdural\",\n]\n\n# As the training dataset is huge we could only train on a part of the whole training directory\nFRACTION_TRAINING = 0.2\n\nSEED = 42\nnp.random.seed(seed=SEED)\nif int(tf.__version__.split(\".\")[0]) >= 2:\n    tf.random.set_seed(seed=SEED)  # For tf v2 or higher\nelse:\n    tf.set_random_seed(seed=SEED)  # For tf v1 or lower","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_csv(path):\n    df = pd.read_csv(path)\n    df[\"filename\"] = df[\"ID\"].apply(lambda st: \"ID_\" + st.split(\"_\")[1] + \".png\")\n    df[\"type\"] = df[\"ID\"].apply(lambda st: st.split(\"_\")[2])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_df = load_csv(BASE_PATH + \"stage_2_train.csv\")\ndata_df.drop_duplicates([\"filename\", \"type\"], inplace=True)\ndata_df = data_df.pivot(\"filename\", \"type\", \"Label\").reset_index()  # Extract Labels\nsub_df = load_csv(BASE_PATH + \"stage_2_sample_submission.csv\")\nsub_df = pd.DataFrame(sub_df.filename.unique(), columns=[\"filename\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df, _ = train_test_split(data_df, train_size = FRACTION_TRAINING, stratify = data_df[CLASSES], random_state = SEED)\ntrain_df, test_df = train_test_split(\n    data_df, test_size=0.1, stratify=data_df[CLASSES], random_state=SEED\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[CLASSES].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"USE_AUGMENTED_DATA = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# In case of not using augmented data, will only do a simple undersampling on 'any' == 0 and oversampling on 'epidural' == 1\nif not USE_AUGMENTED_DATA:\n    # Undersampling depending on 'any' class to have a ratio 2-1 (2 samples of any = 0 for each sample of any = 1)\n    ratio = 2\n    train_df_indexes = rebalance_data(\n        train_df,\n        1,\n        target=\"any\",\n        negative_downsampling=True,\n        use_partial_data=True,\n        ratio_partial_data=ratio,\n    )[0]\n    train_df = train_df.loc[train_df_indexes]\n\n    # Oversampling on 'epidural' class because it has very low presence\n    train_df = oversampling_data(train_df, target=\"epidural\", goal_percentage=0.01)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[CLASSES].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train_df.copy()\nsample_weight_cols = []\nfor target in CLASSES:\n    df[\"count_\" + target] = df.groupby(target)[\"filename\"].transform(\"count\")\n    df[\"sample_weight_\" + target] = df[\"count_\" + target].max() / df[\"count_\" + target]\n    sample_weight_cols.append(\"sample_weight_\" + target)\n\ntrain_df[\"sample_weights\"] = df[sample_weight_cols].max(axis=1).copy()\ntrain_df[sample_weight_cols] = df[sample_weight_cols].copy()\ndel df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_FULL_MODEL = False\nBATCH_FREQ = 100\nEPOCHS = 50\nBATCH_SIZE = 64\n\nDROPOUT_RATE = 0.4\nimg_size = (250, 250)\n\nMODEL_NAME = \"efficientnet\"\nMODELS_LAST_BLOCK = {\"vgg\": 126, \"efficientnet\": \"multiply_16\", \"inception\": 172}\n\nLEARNING_RATE_TOP = 0.005\nLEARNING_RATE = 0.0001\nWARMUP_PROP = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenerator(Sequence):\n    def __init__(\n        self,\n        dataset,\n        training,\n        targets=CLASSES,\n        batch_size=16,\n        img_size=img_size,\n        img_dir=TRAIN_PNG,\n        preprocess=True,\n        bins=None,\n        multi_output=False,\n        verbose=False,\n        *args,\n        **kwargs\n    ):\n        self.dataset = dataset\n        self.ids = dataset.index\n        self.training = training\n        self.targets = targets if training else None\n        self.targets_names = [\"pred_\" + target for target in self.targets]\n        self.labels = dataset[targets] if training else None\n        self.batch_size = batch_size\n        self.img_size = (img_size[0], img_size[1], 3)\n        self.img_dir = img_dir\n        self.preprocess = preprocess\n        self.bins = bins\n        self.multi_output = multi_output\n        self.verbose = verbose\n        self.on_epoch_end()\n\n    def __len__(self):\n        return int(np.ceil(len(self.ids) / self.batch_size))\n\n    def __getitem__(self, index):\n        indices = self.indices[index * self.batch_size : (index + 1) * self.batch_size]\n        return self.__data_generation(indices)\n\n    def on_epoch_end(self):\n        self.indices = np.arange(len(self.ids))\n        np.random.shuffle(self.indices)\n\n    def __data_generation(self, indices):\n        X = np.empty((self.batch_size, *self.img_size))\n        if self.training:\n            # if self.multi_output:\n            #    Y = [None,] * self.batch_size\n            # else:\n            Y = np.empty((self.batch_size, len(CLASSES)), dtype=np.float32)\n\n        for i, index in enumerate(indices):\n            filename = self.dataset.iloc[index][\"filename\"][:-4]\n\n            if self.preprocess:\n                final_image, _ = preprocess_dicom(\n                    self.img_dir + filename + \".dcm\",\n                    self.img_size[0],\n                    self.img_size[1],\n                    bins=self.bins,\n                    use_min_max=True,\n                    # windows_type=\"brain\",\n                    windows_type=[\"brain\", \"subdural\", \"brain_bone\"],\n                    verbose=self.verbose,\n                )\n            else:\n                windows_type = [\"brain\", \"subdural\", \"brain_bone\"]\n                final_image = np.empty(self.img_size)\n                for j, window_type in enumerate(windows_type):\n                    image, _ = get_dcm_img(\n                        self.img_dir + filename + \".dcm\", verbose=self.verbose\n                    )\n                    image = normalize_img(image, use_min_max=True)\n                    # Rescale to the defined image size\n                    if image.shape != (self.img_size[0], self.img_size[1]):\n                        image = cv2.resize(\n                            image,\n                            (self.img_size[0], self.img_size[1]),\n                            interpolation=cv2.INTER_NEAREST,\n                        )\n                    final_image[:, :, j] = image.copy()\n            X[i,] = final_image\n            if self.training:\n                # if self.multi_output:\n                #    Y[i] = dict(zip(self.targets_names, self.labels.iloc[index].values))\n                #\n                # else:\n                Y[i,] = self.labels.iloc[index].values\n\n        if self.training:\n            if self.multi_output:\n                return X, [Y[:, i] for i in range(len(self.targets))]\n            else:\n                return X, Y\n        else:\n            return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"bins_mean, mean, std = sample_bins_mean_std(\n    TRAIN_DIR, samples_per_group=5, max_trys=5000, n_bins=100\n)\"\"\"\nbins_mean = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN_FULL_MODEL:\n    weights = None\nelse:\n    weights = \"imagenet\"\n\n# Pre-Trained CNN Model using imagenet dataset for pre-trained weights\nif MODEL_NAME == \"efficientnet\":\n    base_model = EfficientNetB3(\n        input_shape=(img_size[0], img_size[1], 3), weights=weights, include_top=False\n    )\nelif MODEL_NAME == \"inception\":\n    base_model = Xception(\n        input_shape=(img_size[0], img_size[1], 3), weights=weights, include_top=False\n    )\nelse:\n    base_model = vgg16.VGG16(\n        include_top=False,\n        weights=weights,\n        input_tensor=None,\n        input_shape=(img_size[0], img_size[1], 3),\n        pooling=None,\n    )\n\n# Top Model Block\nx = base_model.output\nx = GlobalAveragePooling2D()(x)\n# x = Dense(256, activation='relu')(x)\n# x = BatchNormalization()(x)\nx = Dropout(DROPOUT_RATE, seed=SEED)(x)\npred_2 = Dense(1, activation=\"sigmoid\", name=\"pred_\" + CLASSES[1])(x)\npred_3 = Dense(1, activation=\"sigmoid\", name=\"pred_\" + CLASSES[2])(x)\npred_4 = Dense(1, activation=\"sigmoid\", name=\"pred_\" + CLASSES[3])(x)\npred_5 = Dense(1, activation=\"sigmoid\", name=\"pred_\" + CLASSES[4])(x)\npred_6 = Dense(1, activation=\"sigmoid\", name=\"pred_\" + CLASSES[5])(x)\nx_add_any = Add()([pred_2, pred_3, pred_4, pred_5, pred_6])\nx_add_any = BatchNormalization()(x_add_any)\nx_concat_any = Concatenate()([x, pred_2, pred_3, pred_4, pred_5, pred_6, x_add_any])\npred_1 = Dense(1, activation=\"sigmoid\", name=\"pred_\" + CLASSES[0])(x_concat_any)\n# predictions = Dense(len(CLASSES), activation=\"sigmoid\", name=\"predictions\")(x)\n\n# add your top layer block to your base model\nmodel = Model(base_model.input, [pred_1, pred_2, pred_3, pred_4, pred_5, pred_6])\n# model = Model(base_model.input, predictions)\n\nif TRAIN_FULL_MODEL:\n    config_model_trainable(model, config=\"full\")\nelse:\n    config_model_trainable(model, config=\"top\", base_model=base_model)\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bce_dice_loss_clip(y_true, y_pred):\n    clip_loss = 10\n    return bce_dice_loss(y_true, y_pred, clip_loss=clip_loss)\n\n\ndef jaccard_distance_loss(y_true, y_pred, smooth=100):\n    \"\"\"\n    Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|)\n            = sum(|A*B|)/(sum(|A|)+sum(|B|)-sum(|A*B|))\n    \n    The jaccard distance loss is usefull for unbalanced datasets. This has been\n    shifted so it converges on 0 and is smoothed to avoid exploding or disapearing\n    gradient.\n    \n    Ref: https://en.wikipedia.org/wiki/Jaccard_index\n    \n    @url: https://gist.github.com/wassname/f1452b748efcbeb4cb9b1d059dce6f96\n    @author: wassname\n    \"\"\"\n    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)\n    sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1)\n    jac = (intersection + smooth) / (sum_ - intersection + smooth)\n    return (1 - jac) * smooth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_class_weight(df, targets, prefix=\"pred_\"):\n    class_weight = {}\n    eps = 0.1\n    for i in range(len(targets)):\n        class_name = targets[i]\n        weight = sum(df[class_name]) / len(df[class_name])\n        class_weight[prefix + class_name] = {0: weight + eps, 1: 1 - weight + eps}\n    return class_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def KFolds_stratified(dataframe, k=10, target=\"class\", shuffle=True, seed=None):\n    train_folds = []\n    valid_folds = []\n    Kfolds = []\n\n    stratify_df = dataframe[target] if shuffle else None\n    remain_df, kfold_df = train_test_split(\n        dataframe,\n        test_size=1 / (k),\n        stratify=stratify_df,\n        shuffle=shuffle,\n        random_state=seed,\n    )\n\n    train_folds.append(kfold_df)\n\n    i = 1\n    while i < k - 1:\n        try:\n            stratify_df = remain_df[target] if shuffle else None\n            remain_df, kfold_df = train_test_split(\n                remain_df,\n                test_size=1 / (k - i),\n                stratify=stratify_df,\n                shuffle=shuffle,\n                random_state=seed,\n            )\n        except ValueError as e:\n            print(f\"Stratify is not posible at kfold {i} due to: {e}\")\n            remain_df, kfold_df = train_test_split(\n                remain_df, test_sie=1 / (k - i), shuffle=shuffle, random_state=seed\n            )\n\n        train_folds.append(kfold_df)\n        valid_folds.append(kfold_df)\n\n        i = i + 1\n\n    valid_folds.append(remain_df)\n    i = 0\n    while i < k - 1:\n        Kfolds.append((train_folds[i], valid_folds[i]))\n        i = i + 1\n\n    return Kfolds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"KFOLDS = 6\nKFOLDS_TOP_LAYER_TRAIN = 3\nKFOLDS_START_PREDICTIONS = np.inf  # Currently disable because CPU last to much\nITERATIONS = 2\nSKIP_K = np.inf  # Currently disable\n\npredictions = []\nfor i in range(ITERATIONS):\n    print(\"Starting iteration \", i)\n    # mskf = MultilabelStratifiedKFold(n_splits=KFOLDS, random_state=0)\n    print(\"Creating KFolds\")\n    Kfolds = KFolds_stratified(\n        train_df, k=KFOLDS, target=CLASSES, shuffle=True, seed=SEED\n    )\n    k = 0\n\n    # for train_index, test_index in mskf.split(\n    #    train_df.drop(CLASSES, axis=1), train_df[CLASSES]\n    # ):\n    #    fold_train = train_df.iloc[train_index]\n    #    fold_valid = train_df.iloc[test_index]\n    for fold_train, fold_valid in Kfolds:\n        print(\"Fold \", k)\n        print(\"Initialize generators\")\n        train_gen = DataGenerator(\n            fold_train,\n            training=True,\n            targets=CLASSES,\n            batch_size=BATCH_SIZE,\n            img_size=img_size,\n            img_dir=TRAIN_DIR,\n            bins=bins_mean,\n            multi_output=True,\n            preprocess=False,\n            verbose=False,\n        )\n\n        valid_gen = DataGenerator(\n            fold_valid,\n            training=True,\n            targets=CLASSES,\n            batch_size=BATCH_SIZE,\n            img_size=img_size,\n            img_dir=TRAIN_DIR,\n            bins=bins_mean,\n            multi_output=True,\n            preprocess=False,\n            verbose=False,\n        )\n\n        if k < KFOLDS_TOP_LAYER_TRAIN and not TRAIN_FULL_MODEL and i == 0:\n            print(\"Train top layer\")\n            config_model_trainable(model, config=\"top\", base_model=base_model)\n\n            class_weight = calc_class_weight(fold_train, CLASSES, prefix=\"pred_\")\n\n            optimizer = RAdam(warmup_proportion=WARMUP_PROP, lr=LEARNING_RATE_TOP)\n            loss = \"binary_crossentropy\"  # bce_dice_loss_clip\n            metrics = [\"accuracy\", tf.keras.metrics.AUC()]\n            model.compile(\n                optimizer=optimizer, loss=loss, metrics=metrics,\n            )\n\n            history = model.fit_generator(\n                train_gen,\n                steps_per_epoch=len(fold_train) // BATCH_SIZE,\n                epochs=1,\n                validation_data=valid_gen,\n                validation_steps=len(fold_valid) // BATCH_SIZE,\n                shuffle=False,\n                # class_weight=class_weight,\n                # use_multiprocessing = True,\n                # workers = 2 * multiprocessing.cpu_count(),\n                # callbacks=callbacks_list)\n            )\n        elif k > SKIP_K:\n            break\n        else:\n            if TRAIN_FULL_MODEL:\n                print(\"Train full model\")\n                config_model_trainable(model, config=\"full\")\n            else:\n                print(\"Train partial model\")\n                config_model_trainable(\n                    model, config=\"partial\", last_block=MODELS_LAST_BLOCK[MODEL_NAME]\n                )\n\n                class_weight = calc_class_weight(fold_train, CLASSES, prefix=\"pred_\")\n\n                optimizer = RAdam(warmup_proportion=WARMUP_PROP, lr=LEARNING_RATE)\n                loss = \"binary_crossentropy\"  # bce_dice_loss_clip\n                metrics = [\"accuracy\", tf.keras.metrics.AUC()]\n                model.compile(\n                    optimizer=optimizer, loss=loss, metrics=metrics,\n                )\n\n                history = model.fit_generator(\n                    train_gen,\n                    steps_per_epoch=len(fold_train) // BATCH_SIZE,\n                    epochs=1,\n                    validation_data=valid_gen,\n                    validation_steps=len(fold_valid) // BATCH_SIZE,\n                    shuffle=False,\n                    # class_weight=class_weight,\n                    # use_multiprocessing = True,\n                    # workers = 2 * multiprocessing.cpu_count(),\n                    # callbacks=callbacks_list)\n                )\n\n            if k >= KFOLDS_START_PREDICTIONS or (\n                i > 0 and KFOLDS_START_PREDICTIONS != np.inf\n            ):\n                print(\"Make prediction\")\n                offset = 1 if len(sub_df) % BATCH_SIZE != 0 else 0\n                steps = len(sub_df) // BATCH_SIZE + offset\n                pred = make_pred(model, sub_generator, steps)\n\n                numpy.savetxt(\n                    \"prediction_\" + str(i) + \"_\" + str(k) + \".csv\", pred, delimiter=\",\"\n                )\n                predictions.append(pred)\n                pred_sub = np.mean(predictions, axis=0)\n\n                offset = 1 if len(test_df) % BATCH_SIZE != 0 else 0\n                steps = len(test_df) // BATCH_SIZE + offset\n                print(\n                    \"Evaluation on test set results on: \",\n                    model.evaluate_generator(test_generator, steps=steps, verbose=1,),\n                )\n        k = k + 1\n\n        save_train(model, history, models_dir=MODELS_DIR + f\"iter_{i}_k_{k}/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pred_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#KFolds_gens[0][1][0][1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\npred = pd.DataFrame(columns = ['X','y'])\npred['y'] = KFolds_gens[0][1].classes\npred['X'] = np.array(model.predict_generator(KFolds_gens[0][1],\n                                        steps = KFolds_gens[0][1].n//KFolds_gens[0][1].batch_size,\n                                        verbose = 1))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\npredictions = pd.DataFrame(columns = 'X','y')\ndef get_crossvalidation_pred(model, generator, predictions):\n    pred = pd.DataFrame(columns = 'X','y')\n    pred['y'] = generator.classes\n    pred['X'] = np.array(model.predict_generator(generator,\n                                            steps = generator.n//generator.batch_size,\n                                            verbose = 1))\n    \n    predictions = pd.concat([predictions, pred], ignore_index=True)\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Deprecate stuff"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\n# Save best model\nweights_path = MODELS_DIR + \"model_weights.h5\"\nif not os.path.exists(MODELS_DIR):\n    os.makedirs(MODELS_DIR)\n    \nhistory_top_callback = BatchHistoryEarlyStopping(valid_generator = test_generator\n                                                 targets = CLASSES\n                                                 batch_freq = BATCH_FREQ,\n                                                 reset_on_train = False,\n                                                 early_stopping = True,\n                                                 monitor='val_loss',\n                                                 patience=3,\n                                                 verbose=1,\n                                                 restore_best_weights=True)\n    callbacks_list = [\n        ModelCheckpoint(top_weights_path, monitor='val_loss', verbose=1, save_best_only=True),\n        #EarlyStopping(monitor='val_loss', patience=2, verbose=0),\n        history_top_callback\n    ]\n    # add the best weights from the train top model\n    # at this point we have the pre-train weights of the base model and the trained weight of the new/added top model\n    # we re-load model weights to ensure the best epoch is selected and not the last one.\n    model.load_weights(top_weights_path)\n    '''","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}