{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":5607176,"sourceType":"datasetVersion","datasetId":3135948},{"sourceId":5611745,"sourceType":"datasetVersion","datasetId":3135970}],"dockerImageVersionId":30371,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <span style=\"color:teal\"> RSNA Screening Mammography Breast Cancer Detection<a class=\"anchor\"  id=\"projectTopic\"></a></span>\n### <span style=\"color:teal\"> Detect breast cancers in screening mammograms <a class=\"anchor\"  id=\"detect\"></a></span>\n\n### <span style=\"color:teal\">Developed by : <a class=\"anchor\"  id=\"detect\"></a></span>\n\n* [Gebreyowhans Hailekiros](https://www.kaggle.com/gebreyowhansbahre/)\n* [Mahbub Hasan](https://www.kaggle.com/mahbubhasanuunical)\n* [Muhammad Danish Sadiq](https://www.kaggle.com/muhammaddanishsadiq/)\n\n* This notebook is aimed at developing a neural network model whichi is capable of detecting breast cacers from mamography images.\n\n### <span style=\"color:teal\"> Notebooks <a class=\"anchor\"  id=\"notebooks\"></a></span>\n* Image preprocessing Notebook: [RSNA_BCD_DICOM_PNG_ROI](https://www.kaggle.com/code/gebreyowhansh/rsna-bcd-dicom-png-roi)\n\n* Training Notebook: [RSNA_BCD_Train[TPU_VM]_EfficientNet ](https://www.kaggle.com/code/gebreyowhansh/rsna-bcd-train-tpu-vm/)\n\n* Test Notebook: [RSNA-BCD-GPU-TEST_EfficientNet](https://www.kaggle.com/code/gebreyowhansh/rsna-bcd-gpu-test)\n\n","metadata":{}},{"cell_type":"markdown","source":"## <span style=\"color:teal\">1.Install libraries <a class=\"anchor\"  id=\"libraries\"></a></span>","metadata":{}},{"cell_type":"code","source":"from IPython.display import clear_output\n!pip install -qU --upgrade pip\n!pip install -qU scikit-learn\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:02.666801Z","iopub.execute_input":"2023-06-13T19:00:02.667265Z","iopub.status.idle":"2023-06-13T19:00:11.316079Z","shell.execute_reply.started":"2023-06-13T19:00:02.66723Z","shell.execute_reply":"2023-06-13T19:00:11.314734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q /lib/wheels/tensorflow-2.9.1-cp38-cp38-linux_x86_64.whl\n!pip install -q tensorflow-addons==0.18.0\n!pip install -q tensorflow-probability==0.17.0\n\n!pip install -q opencv-python-headless\n!pip install -q seaborn\n!pip install -q plotly\n\n!pip install -qU pydot\n!pip install -qU graphviz","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:11.318497Z","iopub.execute_input":"2023-06-13T19:00:11.318794Z","iopub.status.idle":"2023-06-13T19:00:44.655494Z","shell.execute_reply.started":"2023-06-13T19:00:11.318764Z","shell.execute_reply":"2023-06-13T19:00:44.654287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Basic python libraries\n%config Completer.use_jedi = False\nimport os, re, math, random, shutil, warnings, gc\nimport glob\nimport numpy as np, pandas as pd\nfrom IPython import display as ipd\nfrom tqdm import tqdm\n\n#Classical Ml tools\nimport sklearn\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_curve, auc\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import train_test_split\n\n#OpenCV Library\nimport cv2\n\n#Plotting tools \nfrom matplotlib.ticker import StrMethodFormatter\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport plotly.offline as py\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\n#Tensorflow and keras tools\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow.keras.backend as K\nfrom keras.models import Sequential,load_model,Model\nfrom keras.layers import Conv2D,MaxPooling2D,Dense,Dropout,BatchNormalization,Flatten,Input\nfrom keras import layers\nfrom tensorflow.keras.utils import plot_model\nfrom tensorflow.python.client import device_lib\nfrom tensorflow.keras.layers.experimental.preprocessing import Rescaling\nfrom tensorflow.keras.applications import EfficientNetB3\nfrom keras.callbacks import EarlyStopping\n\nfrom kaggle_datasets import KaggleDatasets\nfrom kaggle_secrets import UserSecretsClient\n\n#Logging tools\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # to avoid too many logging messages\npd.options.mode.chained_assignment = None\ntf.get_logger().setLevel('ERROR')\npy.init_notebook_mode(connected=True)","metadata":{"cellView":"form","id":"tuOe1ymfHZPu","execution":{"iopub.status.busy":"2023-06-13T19:00:44.65699Z","iopub.execute_input":"2023-06-13T19:00:44.657279Z","iopub.status.idle":"2023-06-13T19:00:44.683168Z","shell.execute_reply.started":"2023-06-13T19:00:44.65725Z","shell.execute_reply":"2023-06-13T19:00:44.682122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('np:', np.__version__)\nprint('pd:', pd.__version__)\nprint('sklearn:', sklearn.__version__)\nprint('tf:',tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:44.684207Z","iopub.execute_input":"2023-06-13T19:00:44.684478Z","iopub.status.idle":"2023-06-13T19:00:44.696895Z","shell.execute_reply.started":"2023-06-13T19:00:44.684452Z","shell.execute_reply":"2023-06-13T19:00:44.696038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">2. Setting basic configurations <a class=\"anchor\"  id=\"configurations\"></a></span>","metadata":{}},{"cell_type":"code","source":"devices = ['TPU', 'GPU']\nclass Config:\n    \n    def __init__(self):\n        \n        self.debug = False\n        self.device=devices[0]\n        self.seed=150\n        \n        self.path='/kaggle/input/bhbahre/'\n        \n        self.train_path= self.path+'train_images'\n        self.train_path = self.path + 'train_images/'\n        self.train_csv = self.path + 'train.csv'\n        self.train_preprocessed_csv = self.path + 'train_df_processed.csv'\n        \n        self.output_path='/kaggle/working/'\n        #number of folds for data-split\n        self.folds = 5\n        # which folds to train\n        self.train_folds = [0, 1, 2]\n        \n        self.oversampling = True\n        self.oversampling_factor = 10\n        self.threshold = 0.6\n        \n        #Model Parameters\n        self.beta=1.5\n        self.batch_size=32\n        self.epochs=15\n        self.dropout=0.5\n        self.optimizer='adam'\n        self.loss='binary_crossentropy'\n        self.patience=5\n        self.img_size =(512,256)\n        self.img_ext = 'png'\n        \n        # ======Augumentation parameters====\n        # pixel-augment\n        self.sat  = [0.7, 1.3]\n        self.cont = [1,2]\n        self.bri  = 0.15\n        self.hue  = 0.05\n\nconfig=Config()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:44.699645Z","iopub.execute_input":"2023-06-13T19:00:44.699955Z","iopub.status.idle":"2023-06-13T19:00:44.710372Z","shell.execute_reply.started":"2023-06-13T19:00:44.699906Z","shell.execute_reply":"2023-06-13T19:00:44.70942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">3. Reproducibility <a class=\"anchor\"  id=\"configurations\"></a></span>","metadata":{}},{"cell_type":"code","source":"def seeding(seed):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    random.seed(seed)\n    \n    print(\"Seeding done\")\n    \nseeding(config.seed)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:44.711412Z","iopub.execute_input":"2023-06-13T19:00:44.711698Z","iopub.status.idle":"2023-06-13T19:00:45.104543Z","shell.execute_reply.started":"2023-06-13T19:00:44.711672Z","shell.execute_reply":"2023-06-13T19:00:45.103548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">4. Device Configuration <a class=\"anchor\"  id=\"deviceconfigurations\"></a></span>","metadata":{}},{"cell_type":"code","source":"if config.device == 'TPU':\n    \n    tpu = 'local' if config.device=='TPU' else None\n    print('Connecting to TPU...')\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu)\n    except ValueError:\n        raise BaseException('ERROR: Not connected to a TPU runtime; please try executing it again or turnon your TPU VM v3-8 in your accelrator section of this note book !')\n    \n    policy =tf.keras.mixed_precision.Policy('float32') \n    tf.keras.mixed_precision.set_global_policy(policy)\n    \n    tf.config.set_soft_device_placement(True)\n    #strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\n    \n    # get number of TPU cores connect at a runtime\n    config.num_devices = strategy.num_replicas_in_sync\n    \n    # batch size of for the TPU\n    config.batch_size = config.batch_size * config.num_devices\n    print(\"Batch size for TPU :\",config.batch_size)\n    print(f'Running on {config.num_devices} TPU devices')\n\n\nif config.device == 'GPU':\n    \n    #Number of GPU core connected\n    num_devices = len(tf.config.list_physical_devices('GPU'))\n    if num_devices > 1:\n        config.num_devices = num_devices\n        strategy = tf.distribute.MirroredStrategy()\n        config.batch_size = config.batch_size * config.num_devices\n        print(f'Running on {num_devices} GPU devices')\n        print(\"Batch size for multi core GPU :\",config.batch_size )\n        \n    elif num_devices == 1:\n        strategy = tf.distribute.get_strategy()\n        print(f'Running on {num_devices} GPU device')\n        \n    else:\n        strategy = tf.distribute.get_strategy()\n        config.device = 'CPU'\n        print(f'Running on CPU')\n    \n    tf.config.optimizer.set_jit(True)\n    \n    if config.device == 'TPU':\n        tf.keras.mixed_precision.set_global_policy(\"mixed_float16\")\n    else:\n        tf.keras.mixed_precision.set_global_policy(policy=\"float32\")","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:45.110176Z","iopub.execute_input":"2023-06-13T19:00:45.110471Z","iopub.status.idle":"2023-06-13T19:00:52.480688Z","shell.execute_reply.started":"2023-06-13T19:00:45.110445Z","shell.execute_reply":"2023-06-13T19:00:52.479318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">5. Load training data set<a class=\"anchor\"  id=\"traindataset\"></a></span>","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(config.train_preprocessed_csv)\n\nif config.debug:\n    train_df = train_df.sample(1000).reset_index(drop=True)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:52.48217Z","iopub.execute_input":"2023-06-13T19:00:52.48284Z","iopub.status.idle":"2023-06-13T19:00:52.709474Z","shell.execute_reply.started":"2023-06-13T19:00:52.482799Z","shell.execute_reply":"2023-06-13T19:00:52.708397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:52.710806Z","iopub.execute_input":"2023-06-13T19:00:52.711195Z","iopub.status.idle":"2023-06-13T19:00:52.731514Z","shell.execute_reply.started":"2023-06-13T19:00:52.711157Z","shell.execute_reply":"2023-06-13T19:00:52.730364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop(['dicom_path'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:52.732889Z","iopub.execute_input":"2023-06-13T19:00:52.733296Z","iopub.status.idle":"2023-06-13T19:00:52.741952Z","shell.execute_reply.started":"2023-06-13T19:00:52.733257Z","shell.execute_reply":"2023-06-13T19:00:52.741116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_path'] = config.train_path + train_df['patient_id'].astype(str) + '/' + train_df['image_id'].astype(str) + '.png'\ntrain_df['image_path']=train_df['image_path'].astype('string')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:52.743188Z","iopub.execute_input":"2023-06-13T19:00:52.743592Z","iopub.status.idle":"2023-06-13T19:00:52.755758Z","shell.execute_reply.started":"2023-06-13T19:00:52.743551Z","shell.execute_reply":"2023-06-13T19:00:52.75499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.io.gfile.exists(train_df.image_path.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:52.756976Z","iopub.execute_input":"2023-06-13T19:00:52.757897Z","iopub.status.idle":"2023-06-13T19:00:52.769136Z","shell.execute_reply.started":"2023-06-13T19:00:52.757852Z","shell.execute_reply":"2023-06-13T19:00:52.767967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:52.7706Z","iopub.execute_input":"2023-06-13T19:00:52.770968Z","iopub.status.idle":"2023-06-13T19:00:52.783665Z","shell.execute_reply.started":"2023-06-13T19:00:52.770939Z","shell.execute_reply":"2023-06-13T19:00:52.782763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">6. Stratified K-Fold Cross Validation <a class=\"anchor\"  id=\"stratified\"></a></span>\n\n* Stratified k-fold cross-validation is the same as just k-fold cross-validation, but Stratified k-fold  cross-validation, it does stratified sampling instead of random sampling. \n\n* This is particularly useful when dealing with imbalanced datasets, where one class or outcome variable is much more prevalent than the others.\n* Without stratification, a standard K-Fold Cross Validation could lead to some folds containing little or no instances of the minority class, which would bias the evaluation of the model's performance.\n* So since our dataset is highly imbalanced we decided to use this kind of cros validation in this project because it ensures that the sample is representative of the population with respect to the actuall population.","metadata":{}},{"cell_type":"code","source":"stratified_columns = train_df.columns[~train_df.columns.isin(['patient_id', 'image_id', \n                                                              'image_path', 'site_id', \n                                                              'age', 'invasive', 'implant', \n                                                              'machine_id'])]\n\ntrain_df['stratify'] = ''\nfor col in stratified_columns:\n    train_df['stratify'] += train_df[col].astype(str)\n     \n\ntrain_df['stratify'] = train_df['stratify'].astype('string')\n\nk_fold = StratifiedGroupKFold(n_splits=config.folds, shuffle=True, random_state=config.seed)\nfor fold, (train_idx, val_idx) in enumerate(k_fold.split(train_df, \n                                                         train_df['stratify'],\n                                                         train_df['patient_id'])):\n    train_df.loc[val_idx, 'fold'] = fold\n    \n    \n\ntrain_df['fold'] = train_df['fold'].astype('int32')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:52.787661Z","iopub.execute_input":"2023-06-13T19:00:52.788128Z","iopub.status.idle":"2023-06-13T19:00:53.389708Z","shell.execute_reply.started":"2023-06-13T19:00:52.788096Z","shell.execute_reply":"2023-06-13T19:00:53.38872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.390947Z","iopub.execute_input":"2023-06-13T19:00:53.391218Z","iopub.status.idle":"2023-06-13T19:00:53.403801Z","shell.execute_reply.started":"2023-06-13T19:00:53.391192Z","shell.execute_reply":"2023-06-13T19:00:53.402953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.groupby(['fold', \"cancer\"]).size())","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.404822Z","iopub.execute_input":"2023-06-13T19:00:53.405121Z","iopub.status.idle":"2023-06-13T19:00:53.416667Z","shell.execute_reply.started":"2023-06-13T19:00:53.405094Z","shell.execute_reply":"2023-06-13T19:00:53.415827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.417656Z","iopub.execute_input":"2023-06-13T19:00:53.417958Z","iopub.status.idle":"2023-06-13T19:00:53.436974Z","shell.execute_reply.started":"2023-06-13T19:00:53.417929Z","shell.execute_reply":"2023-06-13T19:00:53.436154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">6.1. Counts of Samples by Fold and Cancer <a class=\"anchor\"  id=\"sample_plot\"></a></span>","metadata":{}},{"cell_type":"code","source":"counts_df = train_df.groupby(['fold', 'cancer'])['cancer'].count().reset_index(name='count')\n\nfig = px.bar(counts_df, x='fold', y='count', color='cancer', barmode='stack',\n             hover_data={'fold': True, 'cancer': True, 'count': True},\n             labels={'fold': 'Fold', 'count': 'Count', 'cancer': 'Cancer',\n             })\nfig.update_layout(title='Counts of Samples by Fold and Cancer', xaxis_tickangle=-45)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.438056Z","iopub.execute_input":"2023-06-13T19:00:53.438328Z","iopub.status.idle":"2023-06-13T19:00:53.523174Z","shell.execute_reply.started":"2023-06-13T19:00:53.438302Z","shell.execute_reply":"2023-06-13T19:00:53.522113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">7. Data Pipeline <a class=\"anchor\"  id=\"pipline\"></a></span>","metadata":{}},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">7.1 Decode images <a class=\"anchor\"  id=\"decode\"></a></span>\n * **tf.image.decode_png()** and **tf.image.decode_jpeg** are TensorFlow functions that decodes a PNG-encoded   image into a tensor of type uint8.\n \n * These functions takes the following arguments: \n     * **image**: A string tensor containing a PNG or jpeg -encoded image.\n     * **channels**: An optional integer specifying the number of color channels in the decoded image. By  default, this is set to 3,\n     \n\n* The function returns a **uint8** tensor representing the decoded image with the shape of (height, width, channels) \n \n","metadata":{}},{"cell_type":"code","source":"def decode_image(label=True, img_size=config.img_size, ext=config.img_ext):\n    \n    def _decode_image(Input_Image, label=None):\n        image = tf.io.read_file(Input_Image['input_image'])\n        \n        if ext == 'png':\n            ## PNG-encoded image into a tensor of type uint8.\n            image = tf.image.decode_png(image, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            ## jpeg-encoded image into a tensor of type uint8.\n            image = tf.image.decode_jpeg(image, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        \n        ## explicit size needed for TPU\n        image = tf.image.resize(image, img_size)\n        ## convert image to floats in [0, 1] range\n        image = tf.cast(image, tf.float32) / 255.0\n        \n        Input_Image['input_image'] = image\n        \n        if label is None:\n            return Input_Image\n        else:\n            return Input_Image, label\n    \n    if label:\n        return _decode_image\n    else:\n        return lambda x: _decode_image(x, None)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.52426Z","iopub.execute_input":"2023-06-13T19:00:53.524523Z","iopub.status.idle":"2023-06-13T19:00:53.533594Z","shell.execute_reply.started":"2023-06-13T19:00:53.524499Z","shell.execute_reply":"2023-06-13T19:00:53.532717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">7.2 Data augumentation<a class=\"anchor\"  id=\"augumentation\"></a></span>\n* Using Augmentations to reduce overfitting and make model more robust by :\n * 1. random_flip_left_right for applying position transforamtion\n * 2. perofrming some random_hue,random_saturation,random_contrast,random_brightness for pixel transforamtion","metadata":{}},{"cell_type":"code","source":"def data_augment(label=True):\n    def _augment(Input_Image, label=None):\n        image = Input_Image['input_image']\n        #position transforamtion\n        image = tf.image.random_flip_left_right(image)\n        # pixel-augment\n        image = tf.image.random_hue(image, config.hue)\n        image = tf.image.random_saturation(image,config.sat[0], config.sat[1])\n        image = tf.image.random_contrast(image,config.cont[0], config.cont[1])\n        image = tf.image.random_brightness(image,config.bri)\n        Input_Image['input_image'] = image\n        if label is not None:\n            return Input_Image, label\n        else:\n            return Input_Image\n\n    if label:\n        return _augment\n    else:\n        return lambda x: _augment(x, None)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.534622Z","iopub.execute_input":"2023-06-13T19:00:53.535246Z","iopub.status.idle":"2023-06-13T19:00:53.545895Z","shell.execute_reply.started":"2023-06-13T19:00:53.535218Z","shell.execute_reply":"2023-06-13T19:00:53.545116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">7.3 Build tf.data.dataset<a class=\"anchor\"  id=\"dataset\"></a></span>","metadata":{}},{"cell_type":"code","source":"def build_dataset(df, input_features, image_size=config.img_size, batch_size=config.batch_size, \n                  label=True, shuffle=True, augment=False, repeat=False, cache=False, ext=config.img_ext):\n    \n    decode = decode_image(label, img_size=image_size, ext=ext)\n    input_data = {'input_image': df['image_path'].values, \n                  'input_features': df[input_features].values}\n    \n    if label:\n        label_data = df['cancer'].apply(lambda x: int(x)).values\n        dataset = tf.data.Dataset.from_tensor_slices((input_data, label_data))\n    else:\n        dataset = tf.data.Dataset.from_tensor_slices(input_data)\n        \n    dataset = dataset.map(decode, num_parallel_calls=tf.data.AUTOTUNE)\n    \n    if augment:\n        dataset = dataset.map(data_augment(label), num_parallel_calls=tf.data.AUTOTUNE)\n    if shuffle:\n        dataset = dataset.shuffle(batch_size, reshuffle_each_iteration=True)\n    if repeat:\n        dataset = dataset.repeat()\n    if cache:\n        dataset = dataset.cache()\n        \n    dataset = dataset.batch(batch_size, drop_remainder=shuffle)\n    dataset = dataset.prefetch(tf.data.AUTOTUNE)\n    return dataset\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.546858Z","iopub.execute_input":"2023-06-13T19:00:53.547141Z","iopub.status.idle":"2023-06-13T19:00:53.557512Z","shell.execute_reply.started":"2023-06-13T19:00:53.547116Z","shell.execute_reply":"2023-06-13T19:00:53.556583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">8. Defining input features <a class=\"anchor\"  id=\"input-features\"></a></span>\n\n* These inpute features are medical characteristics of the patient that the model has to consider during training in addition the image.","metadata":{}},{"cell_type":"code","source":"input_features = train_df.columns.difference(['patient_id', 'image_id', 'site_id',\n                                              'machine_id','cancer','age', \n                                              'stratify', 'image_path', 'fold'])\ninput_features","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.558491Z","iopub.execute_input":"2023-06-13T19:00:53.558814Z","iopub.status.idle":"2023-06-13T19:00:53.572438Z","shell.execute_reply.started":"2023-06-13T19:00:53.558787Z","shell.execute_reply":"2023-06-13T19:00:53.571485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#### <span style=\"color:teal\">8.1 Normalizating input features <a class=\"anchor\"  id=\"normalization\"></a></span>","metadata":{}},{"cell_type":"code","source":"train_df[input_features] = (train_df[input_features] - train_df[input_features].mean()) / train_df[input_features].std()\ntrain_df[input_features] = train_df[input_features].fillna(0).astype('float32')\ntrain_df[input_features].head(3)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.573537Z","iopub.execute_input":"2023-06-13T19:00:53.573817Z","iopub.status.idle":"2023-06-13T19:00:53.615289Z","shell.execute_reply.started":"2023-06-13T19:00:53.573791Z","shell.execute_reply":"2023-06-13T19:00:53.614268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_data_set = build_dataset(train_df, input_features, batch_size=config.batch_size,\n                          label=True,    \n                          shuffle=False, \n                          augment=True, repeat=False, \n                          cache=False, ext=config.img_ext)\n\nfor _dt in _data_set.take(1):\n    print(_dt)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#### <span style=\"color:teal\">8.2. Save the prepared training dataset for next use <a class=\"anchor\"  id=\"savetraincsv\"></a></span>","metadata":{}},{"cell_type":"code","source":"train_df.to_csv(\"train_df_processed.csv\", sep=',',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.616327Z","iopub.execute_input":"2023-06-13T19:00:53.616638Z","iopub.status.idle":"2023-06-13T19:00:53.644669Z","shell.execute_reply.started":"2023-06-13T19:00:53.616612Z","shell.execute_reply":"2023-06-13T19:00:53.643893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">9. Define evaluation metrics <a class=\"anchor\"  id=\"evaluationmetrix\"></a></span>","metadata":{}},{"cell_type":"code","source":"def p_f1(y_true, y_pred):\n    tp = tf.reduce_sum(tf.cast(y_true * y_pred, tf.float32))\n    tn = tf.reduce_sum(tf.cast((1 - y_true) * (1 - y_pred), tf.float32))\n    fp = tf.reduce_sum(tf.cast((1 - y_true) * y_pred, tf.float32))\n    fn = tf.reduce_sum(tf.cast(y_true * (1 - y_pred), tf.float32))\n\n    p = tp / (tp + fp + tf.keras.backend.epsilon())\n    r = tp / (tp + fn + tf.keras.backend.epsilon())\n    \n    numerator=2*p*r\n    denominator=p+r\n    \n    f1 = numerator / denominator\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n\n    return tf.reduce_mean(f1)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.645636Z","iopub.execute_input":"2023-06-13T19:00:53.645897Z","iopub.status.idle":"2023-06-13T19:00:53.654712Z","shell.execute_reply.started":"2023-06-13T19:00:53.645874Z","shell.execute_reply":"2023-06-13T19:00:53.653677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">10. Model Building <a class=\"anchor\"  id=\"model-building\"></a></span>","metadata":{}},{"cell_type":"code","source":"def build_model(input_features, \n                loss=config.loss, \n                dropout=config.dropout, \n                optimizer=config.optimizer, \n                img_size=config.img_size):\n    with strategy.scope():\n        input_image = tf.keras.layers.Input(shape=(*img_size,3), \n                                            name='input_image')\n        input_features = tf.keras.layers.Input(shape=[len(input_features)],\n                                               name='input_features')\n        \n        efficientNetB3Base_model = tf.keras.applications.EfficientNetB3(input_shape=(*img_size,3),\n                                                 include_top=False, \n                                                 drop_connect_rate=0.3,\n                                                 weights='imagenet')(input_image)\n        #reduce the spatial dimensions of a feature map produced by a pre-trained EfficientNetB3\n        x = tf.keras.layers.GlobalAveragePooling2D()(efficientNetB3Base_model)\n        \n        x = tf.keras.layers.Dense(128,activation=\"relu\")(x)\n        _output = tf.keras.layers.Dropout(dropout)(x)\n        #normalizes its input by subtracting the batch mean and dividing by the batch standard deviation\n        _output = tf.keras.layers.BatchNormalization()(_output)\n        _output = tf.keras.layers.Dense(64,activation=\"relu\")(_output)\n        _output = tf.keras.layers.Dropout(dropout)(_output)\n        _output = tf.keras.layers.BatchNormalization()(_output)\n        _output = tf.keras.layers.Dense(32,activation=\"relu\")(_output)\n        _output = tf.keras.layers.BatchNormalization()(_output)\n        _output = tf.keras.layers.Dense(16,activation=\"relu\")(_output)\n        _output = tf.keras.layers.BatchNormalization()(_output)  \n        _output = tf.keras.layers.Concatenate()([_output, input_features])\n        _output = tf.keras.layers.Dense(1, activation='sigmoid')(_output)\n        \n        model = tf.keras.Model(inputs=[input_image, input_features], outputs=_output)\n        \n        model.compile(optimizer=optimizer,\n                      loss=loss,\n                      metrics=['accuracy', \n                               p_f1])\n\n        return model","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.655864Z","iopub.execute_input":"2023-06-13T19:00:53.656175Z","iopub.status.idle":"2023-06-13T19:00:53.66982Z","shell.execute_reply.started":"2023-06-13T19:00:53.656147Z","shell.execute_reply":"2023-06-13T19:00:53.668816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">10.2 Model summary <a class=\"anchor\"  id=\"model-summary\"></a></span>","metadata":{}},{"cell_type":"code","source":"model = build_model(input_features,config.loss,config.dropout,config.optimizer,config.img_size)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:00:53.671111Z","iopub.execute_input":"2023-06-13T19:00:53.671391Z","iopub.status.idle":"2023-06-13T19:01:14.438882Z","shell.execute_reply.started":"2023-06-13T19:00:53.671365Z","shell.execute_reply":"2023-06-13T19:01:14.437729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">11.3 Ploting the model<a class=\"anchor\"  id=\"plot-model\"></a></span>","metadata":{}},{"cell_type":"code","source":"tf.keras.utils.plot_model(model,to_file='model.png',show_shapes=True, dpi=64)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:01:14.439942Z","iopub.execute_input":"2023-06-13T19:01:14.44022Z","iopub.status.idle":"2023-06-13T19:01:14.839994Z","shell.execute_reply.started":"2023-06-13T19:01:14.440194Z","shell.execute_reply":"2023-06-13T19:01:14.838715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">11. Defining custom callbacks<a class=\"anchor\"  id=\"compilation\"></a></span>\n\n#### <span style=\"color:teal\">11.1 Learning rate scheduler<a class=\"anchor\"  id=\"lr-schedul\"></a></span>\n* In this project we used to implement a custom learning rate scheduler based Super-convergence technique as presented in the [Super-Convergence: Very Fast Training of Neural Networks Using Large Learning Rates](https://arxiv.org/abs/1708.07120) which is a technique that involves training the learning rate varies between a minimum value and a maximum value over a cycle of a certain length (step_size). This technique can speed up training and help the model converge faster compared to other proposed learning rate schedulers like cyclical learning rates as in [Cyclical Learning Rates for Training Neural Networks](https://arxiv.org/abs/1506.01186) or defualt learning rates that comes with the optimzier functions.\n\n\n","metadata":{}},{"cell_type":"code","source":"class LRScheduler(tf.keras.callbacks.Callback):\n    def __init__(self,base_lr=1e-5, max_lr=1e-2, mode='triangular', step_size=10):\n        super(LRScheduler, self).__init__()\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.mode = mode\n        self.step_size = step_size\n        self.history = {}\n\n    def on_train_begin(self, logs={}):\n        self.iterations = 0\n        self.history['lr'] = []\n\n    def on_batch_end(self, batch, logs={}):\n        self.iterations += 1\n        \n        #compute current cycle number during training\n        cycle = np.floor(1 + self.iterations / (2 * self.step_size))\n        \n        # self.iterations -> number of batches that have been processed\n        # self.step_size -> length of half a cycle\n        \n        #computes the relative position within the current cycle\n        x = np.abs(self.iterations / self.step_size - 2 * cycle + 1)\n        \n        if self.mode == 'triangular':\n            self.lr = self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x))\n            \n        elif self.mode == 'super_convergence':\n            self.lr = self.base_lr + (self.max_lr - self.base_lr) * np.minimum(x, 1 - x)\n            \n        self.history['lr'].append(self.lr)\n    \n    def plot_lr(self):\n        plt.plot(self.history['lr'])\n        plt.title('Learning rate schedule')\n        plt.xlabel('Iterations')\n        plt.ylabel('Learning rate')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:01:14.841586Z","iopub.execute_input":"2023-06-13T19:01:14.841946Z","iopub.status.idle":"2023-06-13T19:01:14.85711Z","shell.execute_reply.started":"2023-06-13T19:01:14.841894Z","shell.execute_reply":"2023-06-13T19:01:14.856122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">11.2 Defining modelCheckpoint and earlyStopping callbacks <a class=\"anchor\"  id=\"early-stopping\"></a></span>","metadata":{}},{"cell_type":"code","source":"# Create directory model weightes saving\nsave_model_dir = os.path.join(config.output_path,'models')\nos.makedirs(os.path.dirname(save_model_dir), exist_ok=True)\n\ndef get_callbacks(batch_size, \n                  fold, \n                  patience=config.patience):\n    \n    early_stop = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", \n                                                  mode=\"min\",\n                                                  patience=patience,\n                                                  restore_best_weights=True)\n    \n    save_model = tf.keras.callbacks.ModelCheckpoint(f'{save_model_dir}/model_{fold}.h5', \n                                                         monitor='val_p_f1',\n                                                         mode='max', \n                                                         save_freq='epoch', \n                                                         save_best_only=True, \n                                                         save_weights_only=False,\n                                                         verbose=1)\n\n    callbacks_list = [early_stop, \n                      save_model,\n                      lr_scheduler]\n    \n    return callbacks_list","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:01:14.858228Z","iopub.execute_input":"2023-06-13T19:01:14.858492Z","iopub.status.idle":"2023-06-13T19:01:14.873827Z","shell.execute_reply.started":"2023-06-13T19:01:14.858467Z","shell.execute_reply":"2023-06-13T19:01:14.87306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <span style=\"color:teal\">12. Model training and evaluation <a class=\"anchor\"  id=\"train_evaluate\"></a></span>\n#### <span style=\"color:teal\">12.1 Model training <a class=\"anchor\"  id=\"training\"></a></span>","metadata":{}},{"cell_type":"code","source":"    \nresults = {}\nfor val_fold in range(config.folds):\n    \n    if val_fold in config.train_folds:\n        continue\n    \n    print(f'\\n- Fold [{val_fold}] as validation set, and folds {[i for i in range(config.folds) if i != val_fold]} as training set\\n')\n    \n    train = train_df.query(\"fold != @val_fold\")\n    validation = train_df.query(\"fold == @val_fold\")\n    \n    print(\"number of examples in training : \",len(train))\n    print(\"number of examples in validation : \",len(validation))\n\n    if config.oversampling:\n        number_of_cancers=train.query('cancer == 1')\n        number_of_noncancer=train.query('cancer == 0')\n        \n        print(\"number of cancers :\",len(number_of_cancers))\n        print(\"number of non cancer :\",len(number_of_noncancer))\n        positive = train.query('cancer == 1').sample(frac=config.oversampling_factor, replace=True, \n                                                     random_state=config.seed)\n        \n        negative = train.query('cancer == 0')\n        train = pd.concat([positive, negative], axis=0).reset_index(drop=True)\n        \n        print(\"number of examples in after overs sampling : \",len(train))\n    \n    train_dataset = build_dataset(train, input_features, batch_size=config.batch_size, \n                                  shuffle=True, augment=True, repeat=False, cache=True)\n          \n    val_dataset = build_dataset(validation, input_features, batch_size=config.batch_size, \n                                shuffle=False, augment=False, repeat=False, cache=True)\n    \n    # to ensure the weight for minority class is higher than the weight for majority class\n    class_weight = compute_class_weight(class_weight='balanced',\n                                        classes=train[\"cancer\"].unique(),\n                                        y=train[\"cancer\"].values)\n    \n    model = build_model(input_features)\n    \n    steps_per_epoch = len(train) // config.batch_size\n    \n    # create the lr_scheduler object \n    lr_scheduler = LRScheduler(base_lr=1e-5, \n                               max_lr=1e-1,\n                               mode='super_convergence', \n                               step_size=steps_per_epoch)\n    \n    history = model.fit(train_dataset, \n                        validation_data = val_dataset, \n                        epochs = config.epochs,\n                        callbacks = get_callbacks(config.batch_size, val_fold),\n                        class_weight = dict(zip(train[\"cancer\"].unique(), class_weight)),\n                        steps_per_epoch =steps_per_epoch)\n    \n    \n    print('========================performing out-of-fold predictions ============================================')\n    print('======== Loading saved model ========================')\n    model.load_weights(f'{save_model_dir}/model_{val_fold}.h5')\n\n    print('================Predicting OOF validation set ========================')\n    test_val_dataset= build_dataset(validation, input_features,\n                                  batch_size=config.batch_size, \n                                  shuffle=False, augment=True, \n                                  repeat=False, cache=True)\n    \n    \n    #Remove singleton dimensionality with squeeze function\n    validation['pred'] = np.squeeze(model.predict(test_val_dataset, verbose=1).astype('float32'))\n    validation['pred'] = (validation['pred'] > config.threshold).astype(int)\n    \n    \n    print(' ================Compute metrics========================')\n    pf = p_f1(validation['cancer'], validation['pred'])\n    accuracy = accuracy_score(validation['cancer'], validation['pred'])\n\n    \n    results[val_fold] = {'p_f1': pf,'accuracy': accuracy}\n    print(f'- P-F1: {pf}, Accuracy: {accuracy}\\n')\n   \n    del model\n    gc.collect()\n    tf.keras.backend.clear_session()\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:01:14.874963Z","iopub.execute_input":"2023-06-13T19:01:14.875226Z","iopub.status.idle":"2023-06-13T19:11:30.253927Z","shell.execute_reply.started":"2023-06-13T19:01:14.875201Z","shell.execute_reply":"2023-06-13T19:11:30.252519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">12.2 Plot of the LR_Scheduler <a class=\"anchor\"  id=\"lrScheduler\"></a></span>","metadata":{}},{"cell_type":"code","source":"lr_scheduler.plot_lr()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:11:30.264682Z","iopub.execute_input":"2023-06-13T19:11:30.265023Z","iopub.status.idle":"2023-06-13T19:11:30.579999Z","shell.execute_reply.started":"2023-06-13T19:11:30.264993Z","shell.execute_reply":"2023-06-13T19:11:30.578931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">12.3 Final evaluation results <a class=\"anchor\"  id=\"results\"></a></span>","metadata":{}},{"cell_type":"code","source":"print('\\n Final results:')\nprint(f'- P-F1: {np.mean([results[i][\"p_f1\"] for i in results])}')\nprint(f'- Accuracy: {np.mean([results[i][\"accuracy\"] for i in results])}')","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:11:30.581203Z","iopub.execute_input":"2023-06-13T19:11:30.581499Z","iopub.status.idle":"2023-06-13T19:11:30.587437Z","shell.execute_reply.started":"2023-06-13T19:11:30.581473Z","shell.execute_reply":"2023-06-13T19:11:30.586446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### <span style=\"color:teal\">12.4 Plot of accuracy and loss functions <a class=\"anchor\"  id=\"plotting\"></a></span>\n\n","metadata":{}},{"cell_type":"code","source":"def plot_evaluation_metrics(history):\n    #Accuracy\n    accuracy = history.history[\"accuracy\"]\n    val_accuracy = history.history[\"val_accuracy\"]\n    loss = history.history[\"loss\"]\n    val_loss = history.history[\"val_loss\"]\n    epochs = range(1, len(accuracy) + 1)\n    plt.plot(epochs, accuracy, \"r\", label=\"Training accuracy\")\n    plt.plot(epochs, val_accuracy, \"g\", label=\"Validation accuracy\")\n    \n    plt.title(\"Training and validation accuracy\")\n    plt.legend()\n    plt.figure()\n    plt.plot(epochs, loss, \"r\", label=\"Training loss\")\n    plt.plot(epochs, val_loss, \"g\", label=\"Validation loss\")\n    plt.title(\"Training and validation loss\")\n    plt.legend()\n    plt.show()\n    \n    # PF1-Score\n    pf1 = history.history[\"p_f1\"]\n    val_pf1 = history.history[\"val_p_f1\"]\n    loss = history.history[\"loss\"]\n    val_loss = history.history[\"val_loss\"]\n    epochs = range(1, len(accuracy) + 1)\n    plt.plot(epochs, pf1, \"r\", label=\"Training Pf1\")\n    plt.plot(epochs, val_pf1, \"g\", label=\"Validation Pf1\")\n    plt.title(\"Training and validation pf1\")\n    plt.legend()\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:11:30.588576Z","iopub.execute_input":"2023-06-13T19:11:30.58947Z","iopub.status.idle":"2023-06-13T19:11:30.601811Z","shell.execute_reply.started":"2023-06-13T19:11:30.58944Z","shell.execute_reply":"2023-06-13T19:11:30.600995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_evaluation_metrics(history)","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:11:30.602822Z","iopub.execute_input":"2023-06-13T19:11:30.603089Z","iopub.status.idle":"2023-06-13T19:11:31.164306Z","shell.execute_reply.started":"2023-06-13T19:11:30.603065Z","shell.execute_reply":"2023-06-13T19:11:31.163083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n# experiment.end()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T19:11:31.165527Z","iopub.execute_input":"2023-06-13T19:11:31.165865Z","iopub.status.idle":"2023-06-13T19:11:31.172634Z","shell.execute_reply.started":"2023-06-13T19:11:31.165834Z","shell.execute_reply":"2023-06-13T19:11:31.171694Z"},"trusted":true},"execution_count":null,"outputs":[]}]}