{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Hello fellow Kagglers,\n\nThis notebook demonstrates the processing of the huge 300GB+ dataset of this competition into TFRecords for fast dataloading during training.\n\nTFRecords have the benefit of loading large chunks of data containing many samples instead of loading every image and label seperately.\n\nAll images are resized to 1024x1024 and saved in 100 TFRecords, making each TFRecord contain roughly 550 images.\n\n[RSNA EfficientNetV2 Training Tensorflow TPU](https://www.kaggle.com/code/markwijkhuizen/rsna-efficientnetv2-training-tensorflow-tpu)\n\n**V2**\n\n* 640x512 -> 1024x1024 resolution\n* Cropping images\n* Single image approach, not both CC and MLO image\n\nInference notebook in the making\n\n**V3**\n\n* 1024x1024 -> 768x1344 based on cropped image ratio\n* using PNG encoded images instead of raw tensors to reduce disk space needed\n\n**V5**\n\nThis will be the final update of my notebooks for this competitions, which should achieve a LB score in the low 0.50s. I will continue participating in this competition, however, I will not share my progress anymore to keep this competition competitive.\n\nUpdates in this version:\n\n* Correct linear/sigmoid normalization of images, many thanks to [\nBob de Graaf\n](https://www.kaggle.com/bobdegraaf) which shared this amazing notebook: [DicomSDL & VOI-LUT](https://www.kaggle.com/code/bobdegraaf/dicomsdl-voi-lut)\n* Switching from PNG to JPEG with 95 compression level to stay within 20GB diskspace\n* Adapted cropping algorithm to searches from maxima to threshold instead of edge to threshold (see Example Processing)\n* Padding cropping offsets to image dimensions to keep image information instead of zero padding (see Example Processing)\n* Actually using discomsdl instead of just installing it...\n\nGood luck to all of you in the last month of this excisting competition!","metadata":{}},{"cell_type":"code","source":"%%capture\n# Source: https://www.kaggle.com/code/remekkinas/fast-dicom-processing-1-6-2x-faster?scriptVersionId=113360473\n!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2023-01-25T10:59:26.925789Z","iopub.execute_input":"2023-01-25T10:59:26.926223Z","iopub.status.idle":"2023-01-25T10:59:47.847997Z","shell.execute_reply.started":"2023-01-25T10:59:26.926119Z","shell.execute_reply":"2023-01-25T10:59:47.846522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pylibjpeg\nimport pydicom\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\n\nimport cv2\nimport glob\nimport importlib\nimport os\nimport joblib\nimport sys\nimport dicomsdl\n\nprint(f'Tensorflow Version: {tf.__version__}')\nprint(f'Python Version: {sys.version}')\n\n# Tensorflow and CV2 set number of threads to 1 for speedup in parallell function mapping\ntf.config.threading.set_inter_op_parallelism_threads(num_threads=1)\ncv2.setNumThreads(1)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T10:59:47.849939Z","iopub.execute_input":"2023-01-25T10:59:47.850325Z","iopub.status.idle":"2023-01-25T10:59:54.62554Z","shell.execute_reply.started":"2023-01-25T10:59:47.850292Z","shell.execute_reply":"2023-01-25T10:59:54.624383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# Interactive flag for debugging purposes\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n\n# Dimensions of processed images\nTARGET_HEIGHT = 1344\nTARGET_WIDTH = 768\nN_CHANNELS = 1\nTARGET_HEIGHT_WIDTH_RATIO = TARGET_HEIGHT / TARGET_WIDTH\n\n# Image Normalization Tools, did not improve LB score\n# For a tutorial see: https://docs.opencv.org/4.x/d5/daf/tutorial_py_histogram_equalization.html\nCLAHE = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(32, 32))\nAPPLY_CLAHE = False\nAPPLY_EQ_HIST = False\n\n# Image Format and Config\nIMAGE_FORMAT = 'JPG'\nIMAGE_QUALITY = 95\n\n# Random Generators Seed\nSEED = 42","metadata":{"execution":{"iopub.status.busy":"2023-01-25T10:59:54.626864Z","iopub.execute_input":"2023-01-25T10:59:54.627555Z","iopub.status.idle":"2023-01-25T10:59:54.639937Z","shell.execute_reply.started":"2023-01-25T10:59:54.627522Z","shell.execute_reply":"2023-01-25T10:59:54.637759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Matplotlib Config","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2023-01-25T10:59:54.643941Z","iopub.execute_input":"2023-01-25T10:59:54.644462Z","iopub.status.idle":"2023-01-25T10:59:54.653962Z","shell.execute_reply.started":"2023-01-25T10:59:54.644427Z","shell.execute_reply":"2023-01-25T10:59:54.652529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"if IS_INTERACTIVE:\n    train = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv').head(1024)\nelse:\n    train = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n    \ndef get_file_path(args):\n    patient_id, image_id = args\n    return f'/kaggle/input/rsna-breast-cancer-detection/train_images/{patient_id}/{image_id}.dcm'\n    \ntrain['file_path'] = train[['patient_id', 'image_id']].apply(get_file_path, axis=1)\n    \ndisplay(train.info())\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2023-01-25T10:59:54.655291Z","iopub.execute_input":"2023-01-25T10:59:54.655632Z","iopub.status.idle":"2023-01-25T10:59:54.99674Z","shell.execute_reply.started":"2023-01-25T10:59:54.655601Z","shell.execute_reply":"2023-01-25T10:59:54.995536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# VOI_LUT","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/bobdegraaf/dicomsdl-voi-lut\ndef voi_lut(image, dicom):\n    # Load only the variables we need\n    center = dicom['WindowCenter']\n    width = dicom['WindowWidth']\n    bits_stored = dicom['BitsStored']\n    voi_lut_function = dicom['VOILUTFunction']\n\n    # For sigmoid it's a list, otherwise a single value\n    if isinstance(center, list):\n        center = center[0]\n    if isinstance(width, list):\n        width = width[0]\n\n    # Set y_min, max & range\n    y_min = 0\n    y_max = float(2**bits_stored - 1)\n    y_range = y_max\n\n    # Function with default LINEAR (so for Nan, it will use linear)\n    if voi_lut_function == 'SIGMOID':\n        image = y_range / (1 + np.exp(-4 * (image - center) / width)) + y_min\n    else:\n        # Checks width for < 1 (in our case not necessary, always >= 750)\n        center -= 0.5\n        width -= 1\n\n        below = image <= (center - width / 2)\n        above = image > (center + width / 2)\n        between = np.logical_and(~below, ~above)\n\n        image[below] = y_min\n        image[above] = y_max\n        if between.any():\n            image[between] = (\n                ((image[between] - center) / width + 0.5) * y_range + y_min\n            )\n\n    # Normalize to have 0 as background, some images are reversed where 0 is max intensity\n    if dicom['PhotometricInterpretation'] == 'MONOCHROME1':\n        image = np.max(image) - image\n\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-01-25T10:59:54.998197Z","iopub.execute_input":"2023-01-25T10:59:54.999008Z","iopub.status.idle":"2023-01-25T10:59:55.008762Z","shell.execute_reply.started":"2023-01-25T10:59:54.998973Z","shell.execute_reply":"2023-01-25T10:59:55.007534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Crop Image","metadata":{}},{"cell_type":"code","source":"# Smooth vector used to smoothen sums/stds of axes\ndef smooth(l):\n    # kernel size is 1% of vector\n    kernel_size = int(len(l) * 0.01)\n    kernel = np.ones(kernel_size) / kernel_size\n    return np.convolve(l, kernel, mode='same')\n\n# X Crop offset based on first column with sum below 5% of maximum column sums*std\ndef get_x_offset(image, max_col_sum_ratio_threshold=0.05, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Percentual margin added to offset\n    margin = int(image.shape[1] * 0.00)\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=0).squeeze()) * smooth(image.std(axis=0).squeeze())\n    # Find maximum sum in first 75% of columns\n    vv_argmax = vv[:int(image.shape[1] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_col_sum_ratio_threshold\n    \n    # Find first column after maximum column below threshold value\n    for offset, v in enumerate(vv):\n        # Start searching from vv_argmax\n        if offset < vv_argmax:\n            continue\n        \n        # Column below threshold value found\n        if v < vv_threshold:\n            offset = min(W, offset + margin)\n            break\n            \n    if isinstance(debug, np.ndarray):\n        debug[1].imshow(image)\n        debug[1].set_title('X Offset')\n        vv_scale = H / vv.max() * 0.90\n        # Values\n        debug[1].plot(H - vv * vv_scale , c='red', label='vv')\n        # Threshold\n        debug[1].hlines(H - vv_threshold * vv_scale, 0, W -1, colors='orange', label='threshold')\n        # Max Value\n        debug[1].scatter(vv_argmax, H - vv[vv_argmax] * vv_scale, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[1].scatter(offset, H - vv[offset] * vv_scale, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[1].set_ylim(H, 0)\n        debug[1].legend()\n        debug[1].axis('off')\n        \n    return offset\n\n# Y Crop offset based on first bottom and top rows with sum below 10% of maximum row sum*std\ndef get_y_offsets(image, max_row_sum_ratio_threshold=0.10, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Margin to add to offsets\n    margin = 0\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=1).squeeze()) * smooth(image.std(axis=1).squeeze())\n    # Find maximum sum * std row in inter quartile rows\n    vv_argmax = int(image.shape[0] * 0.25) + vv[int(image.shape[0] * 0.25):int(image.shape[0] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_row_sum_ratio_threshold\n    # Default crop offsets\n    offset_bottom = 0\n    offset_top = H\n\n    # Bottom offset, search from argmax to bottom\n    for offset in reversed(range(0, vv_argmax)):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_bottom = offset\n            break\n    \n    if isinstance(debug, np.ndarray):\n        debug[2].imshow(image)\n        debug[2].set_title('Y Bottom Offset')\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[2].plot(vv * vv_scale, np.arange(H), c='red', label='vv')\n        # Threshold\n        debug[2].vlines(vv_threshold * vv_scale, 0, H -1, colors='orange', label='threshold')\n        # Max Value\n        debug[2].scatter(vv[vv_argmax] * vv_scale, vv_argmax, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[2].scatter(vv[offset_bottom] * vv_scale, offset_bottom, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[2].set_ylim(H, 0)\n        debug[2].legend()\n        debug[2].axis('off')\n            \n    # Top offset, search from argmax to top\n    for offset in range(vv_argmax, H):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_top = offset\n            break\n            \n    if isinstance(debug, np.ndarray):\n        debug[3].imshow(image)\n        debug[3].set_title('Y Top Offset')\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[3].plot(vv * vv_scale, np.arange(H) , c='red', label='vv')\n        # Threshold\n        debug[3].vlines(vv_threshold * vv_scale, 0, H -1, colors='orange', label='threshold')\n        # Max Value\n        debug[3].scatter(vv[vv_argmax] * vv_scale, vv_argmax, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[3].scatter(vv[offset_top] * vv_scale, offset_top, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[2].set_ylim(H, 0)\n        debug[3].legend()\n        debug[3].axis('off')\n            \n    return max(0, offset_bottom - margin), min(image.shape[0], offset_top + margin)\n\n# Crop image and pad offsets to target image height/width ratio to preserve information\ndef crop(image, size=None, debug=False):\n    # Image dimensions\n    H, W = image.shape\n    # Compute x/bottom/top offsets\n    x_offset = get_x_offset(image, debug=debug)\n    offset_bottom, offset_top = get_y_offsets(image[:,:x_offset], debug=debug)\n    # Crop Height and Width\n    h_crop = offset_top - offset_bottom\n    w_crop = x_offset\n    \n    # Pad crop offsets to target aspect ratio\n    if size is not None:\n        # Height too large, pad x offset\n        if (h_crop / w_crop) > TARGET_HEIGHT_WIDTH_RATIO:\n            x_offset += int(h_crop / TARGET_HEIGHT_WIDTH_RATIO - w_crop)\n        else:\n            # Height too small, pad bottom/top offsets\n            offset_bottom -= int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_bottom_correction = max(0, -offset_bottom)\n            offset_bottom += offset_bottom_correction\n\n            offset_top += int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_top += offset_bottom_correction\n        \n    # Crop Image\n    image = image[offset_bottom:offset_top:,:x_offset]\n        \n    return image","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:10:46.337297Z","iopub.execute_input":"2023-01-25T11:10:46.337693Z","iopub.status.idle":"2023-01-25T11:10:46.366616Z","shell.execute_reply.started":"2023-01-25T11:10:46.337659Z","shell.execute_reply":"2023-01-25T11:10:46.365286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility","metadata":{}},{"cell_type":"code","source":"# based on: https://www.kaggle.com/code/remekkinas/fast-dicom-processing-1-6-2x-faster?scriptVersionId=113360473\ndef process(file_path, size=None, dicom_process=True, ret_target=False, crop_image=False, apply_clahe=APPLY_CLAHE, apply_eq_hist=APPLY_EQ_HIST, debug=False):\n    # Read Dicom File\n    dicom = dicomsdl.open(file_path)\n    image = dicom.pixelData()\n    \n    # Save original image for debug purposes\n    if debug:\n        fig, axes = plt.subplots(1, 5, figsize=(20,10))\n        image0 = np.copy(image)\n        axes[0].imshow(image0)\n        axes[0].set_title('Original Image')\n        axes[0].axis('off')\n    else:\n        axes = False\n        \n    # voi_lut\n    image = voi_lut(image, dicom)\n\n    # Normalize [0,1] range\n    image = (image - image.min()) / (image.max() - image.min())\n\n    # Convert to uint8 image in range [0, 255]\n    image = (image * 255).astype(np.uint8)\n    \n    # Normalize to left/right orientation by flipping right/left oriented images\n    h0, w0 = image.shape\n    if image[:,int(-w0 * 0.10):].sum() > image[:,:int(w0 * 0.10)].sum():\n        image = np.flip(image, axis=1)\n    \n    if crop_image:\n        image = crop(image, size=size, debug=axes)\n    \n    # Resize\n    if size is not None:\n        # Pad black pixels to get correct image ratios\n        h, w = image.shape\n        if (h / w) > TARGET_HEIGHT_WIDTH_RATIO:\n            pad = int(h / TARGET_HEIGHT_WIDTH_RATIO - w)\n            image = np.pad(image, [[0,0], [0, pad]])\n            h, w = image.shape\n        else:\n            pad = int(0.50 * (w * TARGET_HEIGHT_WIDTH_RATIO - h))\n            image = np.pad(image, [[pad, pad], [0,0]])\n            h, w = image.shape\n        # Resize\n        image = cv2.resize(image, size, interpolation=cv2.INTER_AREA)\n        \n    # Apply CLAHE contrast enhancement\n    # https://docs.opencv.org/4.x/d5/daf/tutorial_py_histogram_equalization.html\n    if apply_clahe:\n        image = CLAHE.apply(image)\n        \n    # Apply Histogram Equalization\n    # https://docs.opencv.org/4.x/d5/daf/tutorial_py_histogram_equalization.html\n    if apply_eq_hist:\n        image = cv2.equalizeHist(image)\n        \n    if debug:\n        axes[4].imshow(image)\n        axes[4].set_title('Processed Image')\n        axes[4].axis('off')\n        plt.show()\n\n    # Return Cancer Target\n    if ret_target:\n        patient_id = int(file_path.split('/')[-2])\n        image_id = int(file_path.split('/')[-1].split('.')[0])\n\n        target = PATIENT_ID_IMAGE_ID2CANCER[(patient_id, image_id)]\n        \n        return image, target\n    # Return image Only\n    else:\n        if debug:\n            return image0, image\n        else:\n            return image","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:10:46.965083Z","iopub.execute_input":"2023-01-25T11:10:46.965473Z","iopub.status.idle":"2023-01-25T11:10:46.98285Z","shell.execute_reply.started":"2023-01-25T11:10:46.965442Z","shell.execute_reply":"2023-01-25T11:10:46.981254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Processing","metadata":{}},{"cell_type":"code","source":"N = 4 if IS_INTERACTIVE else 10\nfor fp in tqdm(train['file_path'].head(N)):\n    process(\n            fp,\n            crop_image=True,\n            size=(TARGET_WIDTH, TARGET_HEIGHT),\n            debug=True,\n        )","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:10:48.441978Z","iopub.execute_input":"2023-01-25T11:10:48.442409Z","iopub.status.idle":"2023-01-25T11:11:09.165967Z","shell.execute_reply.started":"2023-01-25T11:10:48.442373Z","shell.execute_reply":"2023-01-25T11:11:09.163212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Processed Images","metadata":{}},{"cell_type":"code","source":"def plot_original_processed_examples(rows=48, cols=5):\n    fig, axes = plt.subplots(rows, cols, figsize=(cols * 4, 6 * rows))\n    for r in tqdm(range(rows)):\n        for c in range(cols):\n            idx = (r * cols) + c\n            image = process(\n                    train.loc[idx, 'file_path'],\n                    crop_image=True,\n                    size=(TARGET_WIDTH, TARGET_HEIGHT),\n                    apply_clahe=APPLY_CLAHE,\n                    apply_eq_hist=APPLY_EQ_HIST,\n                    debug=False,\n                )\n            axes[r, c].imshow(image)\n            axes[r, c].set_title(f'{idx} | processed')\n\n    plt.show()\n    \nplot_original_processed_examples(rows=8 if IS_INTERACTIVE else 32)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:00:16.332372Z","iopub.execute_input":"2023-01-25T11:00:16.333153Z","iopub.status.idle":"2023-01-25T11:00:59.425002Z","shell.execute_reply.started":"2023-01-25T11:00:16.333114Z","shell.execute_reply":"2023-01-25T11:00:59.423393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Check if all patients have both CC and MLO view\nif not IS_INTERACTIVE:\n    for g_idx, g in tqdm(train.groupby('patient_id')):\n        if 'CC' not in g['view'].values or 'MLO' not in g['view'].values:\n            display(g)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:00:59.426694Z","iopub.execute_input":"2023-01-25T11:00:59.427256Z","iopub.status.idle":"2023-01-25T11:00:59.435771Z","shell.execute_reply.started":"2023-01-25T11:00:59.427194Z","shell.execute_reply":"2023-01-25T11:00:59.434621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Maps a patient_id and image_id to cancer target\nPATIENT_ID_IMAGE_ID2CANCER = train.set_index(['patient_id', 'image_id'])['cancer'].to_dict()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:00:59.437297Z","iopub.execute_input":"2023-01-25T11:00:59.437641Z","iopub.status.idle":"2023-01-25T11:00:59.459773Z","shell.execute_reply.started":"2023-01-25T11:00:59.437609Z","shell.execute_reply":"2023-01-25T11:00:59.457658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test","metadata":{}},{"cell_type":"code","source":"# Example Test data\ntest = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\n\ndisplay(test.info())\ndisplay(test.head())","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:00:59.461449Z","iopub.execute_input":"2023-01-25T11:00:59.461869Z","iopub.status.idle":"2023-01-25T11:00:59.496877Z","shell.execute_reply.started":"2023-01-25T11:00:59.461831Z","shell.execute_reply":"2023-01-25T11:00:59.49536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample Submission","metadata":{}},{"cell_type":"code","source":"# Example submission\nsample_submission = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/sample_submission.csv')\n\ndisplay(sample_submission.info())\ndisplay(sample_submission.head())","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:00:59.498164Z","iopub.execute_input":"2023-01-25T11:00:59.498523Z","iopub.status.idle":"2023-01-25T11:00:59.525714Z","shell.execute_reply.started":"2023-01-25T11:00:59.498492Z","shell.execute_reply":"2023-01-25T11:00:59.524352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Meta Data","metadata":{}},{"cell_type":"code","source":"# Train data is imbalanced, we have ~50x more negative samples than positive samples\nplt.figure(figsize=(8, 8))\ntrain['cancer'].value_counts().plot(kind='pie', autopct='%1.1f%%', title='Cancer Distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:00:59.527304Z","iopub.execute_input":"2023-01-25T11:00:59.527646Z","iopub.status.idle":"2023-01-25T11:00:59.653921Z","shell.execute_reply.started":"2023-01-25T11:00:59.527607Z","shell.execute_reply":"2023-01-25T11:00:59.652583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Statistics","metadata":{}},{"cell_type":"code","source":"# Patient folder paths\nFOLDER_PATHS = glob.glob('/kaggle/input/rsna-breast-cancer-detection/train_images/*')\nprint(f'Found {len(FOLDER_PATHS)} Train Folders')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:00:59.655489Z","iopub.execute_input":"2023-01-25T11:00:59.657371Z","iopub.status.idle":"2023-01-25T11:00:59.903763Z","shell.execute_reply.started":"2023-01-25T11:00:59.657332Z","shell.execute_reply":"2023-01-25T11:00:59.902744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# File paths\nFILE_PATHS = glob.glob('/kaggle/input/rsna-breast-cancer-detection/train_images/*/*.dcm')\nprint(f'Found {len(FILE_PATHS)} Train Files')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:00:59.904926Z","iopub.execute_input":"2023-01-25T11:00:59.90638Z","iopub.status.idle":"2023-01-25T11:01:55.208963Z","shell.execute_reply.started":"2023-01-25T11:00:59.906333Z","shell.execute_reply":"2023-01-25T11:01:55.206864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.title('Number of Scans Per Patient per Laterality (Left/Right side)')\ntrain.groupby(['patient_id', 'laterality']).apply(len).value_counts().sort_index().plot(kind='bar')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:01:55.210787Z","iopub.execute_input":"2023-01-25T11:01:55.211155Z","iopub.status.idle":"2023-01-25T11:01:55.416733Z","shell.execute_reply.started":"2023-01-25T11:01:55.211125Z","shell.execute_reply":"2023-01-25T11:01:55.415927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View Counts: AT/LM/ML/LMO are rare, too few samples to train on\ndisplay(train['view'].value_counts().to_frame('count'))","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:01:55.417733Z","iopub.execute_input":"2023-01-25T11:01:55.418197Z","iopub.status.idle":"2023-01-25T11:01:55.431105Z","shell.execute_reply.started":"2023-01-25T11:01:55.418151Z","shell.execute_reply":"2023-01-25T11:01:55.429639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Scan View Counts: most patient only have a CC and MLO scan\ndisplay(train.sort_values('view').groupby(['patient_id', 'laterality'])['view'].apply(tuple).value_counts().to_frame('Count'))","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:01:55.432439Z","iopub.execute_input":"2023-01-25T11:01:55.432774Z","iopub.status.idle":"2023-01-25T11:01:55.465299Z","shell.execute_reply.started":"2023-01-25T11:01:55.432743Z","shell.execute_reply":"2023-01-25T11:01:55.463858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Dimensions","metadata":{}},{"cell_type":"code","source":"np.random.seed(42)\n\n# Get height/width statistics\nN = int(16 if IS_INTERACTIVE else 1024)\nWIDTHS = []\nHEIGHTS = []\nfor fp in tqdm(np.random.choice(FILE_PATHS, N)):\n    h, w = process(fp).shape\n    HEIGHTS.append(h)\n    WIDTHS.append(w)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:01:55.467447Z","iopub.execute_input":"2023-01-25T11:01:55.467905Z","iopub.status.idle":"2023-01-25T11:02:06.907947Z","shell.execute_reply.started":"2023-01-25T11:01:55.467861Z","shell.execute_reply":"2023-01-25T11:02:06.906604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Patches are insanely huge!\nplt.figure(figsize=(15,8))\nplt.title('Image Dimensions', size=24)\npd.Series(HEIGHTS).plot(kind='hist', alpha=0.50, label='heights')\npd.Series(WIDTHS).plot(kind='hist', alpha=0.50, label='widths')\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:06.90947Z","iopub.execute_input":"2023-01-25T11:02:06.909866Z","iopub.status.idle":"2023-01-25T11:02:07.185688Z","shell.execute_reply.started":"2023-01-25T11:02:06.909829Z","shell.execute_reply":"2023-01-25T11:02:07.184166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Height to Width Ratio's, height is roughly 1.25x width, that's why resize to (512*1.25)x512 = 640x512\ndisplay(pd.Series(np.array(HEIGHTS) / np.array(WIDTHS)).describe().to_frame('Height/Width Ratio\\'s'))\n\nplt.figure(figsize=(15,8))\npd.Series(np.array(HEIGHTS) / np.array(WIDTHS)).plot(kind='hist')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:07.187536Z","iopub.execute_input":"2023-01-25T11:02:07.187903Z","iopub.status.idle":"2023-01-25T11:02:07.40402Z","shell.execute_reply.started":"2023-01-25T11:02:07.187875Z","shell.execute_reply":"2023-01-25T11:02:07.402836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cropped Image Dimensions","metadata":{}},{"cell_type":"code","source":"np.random.seed(SEED)\n\nN = int(16 if IS_INTERACTIVE else 1024)\nWIDTHS_CROPPED = []\nHEIGHTS_CROPPED = []\n\nfor fp in tqdm(np.random.choice(FILE_PATHS, N)):\n    h, w = process(fp, crop_image=True).shape\n    WIDTHS_CROPPED.append(h)\n    HEIGHTS_CROPPED.append(w)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:07.405568Z","iopub.execute_input":"2023-01-25T11:02:07.406229Z","iopub.status.idle":"2023-01-25T11:02:19.361987Z","shell.execute_reply.started":"2023-01-25T11:02:07.406185Z","shell.execute_reply":"2023-01-25T11:02:19.360804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.title('Cropped Image Dimensions', size=24)\npd.Series(WIDTHS_CROPPED).plot(kind='hist', alpha=0.50, label='cropped heights')\npd.Series(HEIGHTS_CROPPED).plot(kind='hist', alpha=0.50, label='cropped widths')\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:19.363438Z","iopub.execute_input":"2023-01-25T11:02:19.364043Z","iopub.status.idle":"2023-01-25T11:02:19.619276Z","shell.execute_reply.started":"2023-01-25T11:02:19.364009Z","shell.execute_reply":"2023-01-25T11:02:19.618548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Height to Width Ratio's\ndisplay(pd.Series(np.array(WIDTHS_CROPPED) / np.array(HEIGHTS_CROPPED)).describe().to_frame('Cropped Height/Width Ratio\\'s'))\nplt.figure(figsize=(15,8))\npd.Series(np.array(WIDTHS_CROPPED) / np.array(HEIGHTS_CROPPED)).plot(kind='hist')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:19.620525Z","iopub.execute_input":"2023-01-25T11:02:19.620999Z","iopub.status.idle":"2023-01-25T11:02:19.856721Z","shell.execute_reply.started":"2023-01-25T11:02:19.62097Z","shell.execute_reply":"2023-01-25T11:02:19.85589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Chunk Generation","metadata":{}},{"cell_type":"code","source":"# Make Pairs of Views as input to the model\nFILE_PATHS_PAIRS = []\nfor row_idx, row in tqdm(train.iterrows(), total=len(train)):\n        FILE_PATHS_PAIRS.append(row[['patient_id', 'image_id']].values)\n        \nFILE_PATHS_PAIRS = np.array(FILE_PATHS_PAIRS, dtype=object)\nprint(f'FILE_PATHS_PAIRS shape: {FILE_PATHS_PAIRS.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:19.857778Z","iopub.execute_input":"2023-01-25T11:02:19.858216Z","iopub.status.idle":"2023-01-25T11:02:20.288243Z","shell.execute_reply.started":"2023-01-25T11:02:19.858158Z","shell.execute_reply":"2023-01-25T11:02:20.287233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Put every image in a seperate TFRecord file\nN_CHUNKS = 100\nCHUNKS = np.array_split(FILE_PATHS_PAIRS, N_CHUNKS)\n\nprint(f'N_CHUNKS: {N_CHUNKS}, CHUNK len: {len(CHUNKS[0])}, shape: {CHUNKS[0].shape}')","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:20.292558Z","iopub.execute_input":"2023-01-25T11:02:20.292884Z","iopub.status.idle":"2023-01-25T11:02:20.299859Z","shell.execute_reply.started":"2023-01-25T11:02:20.292856Z","shell.execute_reply":"2023-01-25T11:02:20.298341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Single sample processing\ndef process_chunk(args):\n    patient_id, image_id = args\n    # Define file path\n    fp = f'/kaggle/input/rsna-breast-cancer-detection/train_images/{patient_id}/{image_id}.dcm'\n    # Get processed image and target\n    image, target = process(fp, size=(TARGET_WIDTH, TARGET_HEIGHT), ret_target=True, crop_image=True)\n\n    # Make grayscale channel\n    image = np.expand_dims(image, 2)\n    \n    # Encode PNG\n    if IMAGE_FORMAT == 'PNG':\n        image_serialized = tf.io.encode_png(image, compression=9).numpy()\n    # Encode JPEG\n    else:\n        image_serialized = tf.io.encode_jpeg(image, quality=IMAGE_QUALITY, optimize_size=True).numpy()\n    \n    return image_serialized, target, patient_id, image_id","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:20.302574Z","iopub.execute_input":"2023-01-25T11:02:20.302975Z","iopub.status.idle":"2023-01-25T11:02:20.313005Z","shell.execute_reply.started":"2023-01-25T11:02:20.302939Z","shell.execute_reply":"2023-01-25T11:02:20.312092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_tf_records(chunks):\n    for chunk_idx, chunk in enumerate(tqdm(chunks)):\n        print(f'===== GENERATING TFRECORDS {chunk_idx} =====')\n        tfrecord_name = f'batch_{chunk_idx}.tfrecords'\n        \n        # Create the actual TFRecords\n        options = tf.io.TFRecordOptions(compression_type='GZIP', compression_level=9)\n        with tf.io.TFRecordWriter(tfrecord_name, options=options) as file_writer:\n            # Process Samples in Chunk in Parallell\n            jobs = [joblib.delayed(process_chunk)(args) for args in chunk]\n            chunk_processed = joblib.Parallel(\n                n_jobs=cpu_count(),\n                verbose=0,\n                backend='multiprocessing',\n                prefer='threads',\n            )(jobs)\n            \n            # Add Processed Samples to TFRecord\n            for image, target, patient_id, image_id in chunk_processed:\n                record_bytes = tf.train.Example(features=tf.train.Features(feature={\n                    # Image\n                    'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image])),\n\n                    # target\n                    'target': tf.train.Feature(int64_list=tf.train.Int64List(value=[target])),\n                    \n                    # patient_id\n                    'patient_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[patient_id])),\n                    \n                    # image_id\n                    'image_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[image_id])),\n                })).SerializeToString()\n                file_writer.write(record_bytes)\n            \n# Create TFRecords\nif IS_INTERACTIVE:\n    to_tf_records(CHUNKS[:10])\nelse:\n    to_tf_records(CHUNKS)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:02:20.314161Z","iopub.execute_input":"2023-01-25T11:02:20.314503Z","iopub.status.idle":"2023-01-25T11:03:00.649071Z","shell.execute_reply.started":"2023-01-25T11:02:20.314475Z","shell.execute_reply":"2023-01-25T11:03:00.647816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check TFRecords","metadata":{}},{"cell_type":"code","source":"N = 16 if IS_INTERACTIVE else 32","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:03:00.650714Z","iopub.execute_input":"2023-01-25T11:03:00.650957Z","iopub.status.idle":"2023-01-25T11:03:00.656144Z","shell.execute_reply.started":"2023-01-25T11:03:00.650934Z","shell.execute_reply":"2023-01-25T11:03:00.655076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to decode the TFRecords\ndef decode_tfrecord(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64),\n        'patient_id': tf.io.FixedLenFeature([], tf.int64),\n        'image_id': tf.io.FixedLenFeature([], tf.int64),\n    })\n        \n    if IMAGE_FORMAT == 'PNG':\n        image = tf.io.decode_png(features['image'], channels=N_CHANNELS)\n    else:\n        image = tf.io.decode_jpeg(features['image'], channels=N_CHANNELS)\n        \n    image = tf.reshape(image, [TARGET_HEIGHT, TARGET_WIDTH, N_CHANNELS])\n\n    target = features['target']\n    patient_id = features['patient_id']\n    image_id = features['image_id']\n    \n    return image, target, patient_id, image_id","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:13:01.713418Z","iopub.execute_input":"2023-01-25T11:13:01.713845Z","iopub.status.idle":"2023-01-25T11:13:01.723494Z","shell.execute_reply.started":"2023-01-25T11:13:01.713811Z","shell.execute_reply":"2023-01-25T11:13:01.721964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"More on Tensorflow TFRecord Datasets: [TFRecordDataset](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset)","metadata":{}},{"cell_type":"code","source":"# Sample TFRecord Dataset\ndef get_train_dataset():\n    # Read all TFRecord file paths\n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob('./*.tfrecords')\n    # initialize TFRecord dataset\n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=1, compression_type='GZIP')\n    # Decode samples by mapping with decode function\n    train_dataset = train_dataset.map(decode_tfrecord)\n    # Batch samples\n    train_dataset = train_dataset.batch(N)\n    \n    return train_dataset","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:13:02.171694Z","iopub.execute_input":"2023-01-25T11:13:02.172522Z","iopub.status.idle":"2023-01-25T11:13:02.177962Z","shell.execute_reply.started":"2023-01-25T11:13:02.172488Z","shell.execute_reply":"2023-01-25T11:13:02.177067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shows a batch of images\ndef show_batch(dataset, rows=N, cols=1):\n    images, targets, patient_ids, image_ids = next(iter(dataset))\n    images = np.moveaxis(images, 3, 1)\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(cols*6, rows*10))\n    for r in range(rows):\n        for c in range(cols):\n            img = images[r,c]\n            axes[r].imshow(img)\n            if c == 0:\n                target = targets[r]\n                patient_id = patient_ids[r]\n                image_id = image_ids[r]\n                axes[r].set_title(f'target: {target}, patient_id: {patient_id}, image_id: {image_id}', fontsize=12, pad=16)\n        \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:13:02.64269Z","iopub.execute_input":"2023-01-25T11:13:02.643057Z","iopub.status.idle":"2023-01-25T11:13:02.651154Z","shell.execute_reply.started":"2023-01-25T11:13:02.643021Z","shell.execute_reply":"2023-01-25T11:13:02.649851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show Example Batch\ntrain_dataset = get_train_dataset()\nshow_batch(train_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-01-25T11:13:03.136878Z","iopub.execute_input":"2023-01-25T11:13:03.137384Z","iopub.status.idle":"2023-01-25T11:13:04.943075Z","shell.execute_reply.started":"2023-01-25T11:13:03.137344Z","shell.execute_reply":"2023-01-25T11:13:04.942252Z"},"trusted":true},"execution_count":null,"outputs":[]}]}