{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"ls ../input/rsna-breast-cancer-detection/train_images | head -1 -n 4","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:31:39.003285Z","iopub.execute_input":"2023-03-16T22:31:39.004323Z","iopub.status.idle":"2023-03-16T22:31:41.170369Z","shell.execute_reply.started":"2023-03-16T22:31:39.00428Z","shell.execute_reply":"2023-03-16T22:31:41.169249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls ../input/rsna-breast-cancer-detection/train_images/57175","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:31:41.342506Z","iopub.execute_input":"2023-03-16T22:31:41.343356Z","iopub.status.idle":"2023-03-16T22:31:42.379378Z","shell.execute_reply.started":"2023-03-16T22:31:41.343311Z","shell.execute_reply":"2023-03-16T22:31:42.378206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom matplotlib import pyplot as plt\n\nsample_sub = pd.read_csv('../input/rsna-breast-cancer-detection/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:36:38.283715Z","iopub.execute_input":"2023-03-16T22:36:38.284265Z","iopub.status.idle":"2023-03-16T22:36:38.301422Z","shell.execute_reply.started":"2023-03-16T22:36:38.284225Z","shell.execute_reply":"2023-03-16T22:36:38.300443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:36:42.645251Z","iopub.execute_input":"2023-03-16T22:36:42.64572Z","iopub.status.idle":"2023-03-16T22:36:42.664464Z","shell.execute_reply.started":"2023-03-16T22:36:42.645688Z","shell.execute_reply":"2023-03-16T22:36:42.663157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l ../input/rsna-breast-cancer-detection/train_images | wc -l # wc -l outputs one a count of one row too many when used like this! hence we need to subtract one to get the true count","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:32:22.295254Z","iopub.execute_input":"2023-03-16T22:32:22.296159Z","iopub.status.idle":"2023-03-16T22:32:24.120308Z","shell.execute_reply.started":"2023-03-16T22:32:22.296121Z","shell.execute_reply":"2023-03-16T22:32:24.119165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/bobdegraaf/dicomsdl-voi-lut\ndef voi_lut(image, dicom):\n    # Additional Checks\n    if 'WindowWidth' not in dicom.getPixelDataInfo() or 'WindowWidth' not in dicom.getPixelDataInfo():\n        return image\n    \n    # Load only the variables we need\n    center = dicom['WindowCenter']\n    width = dicom['WindowWidth']\n    bits_stored = dicom['BitsStored']\n    voi_lut_function = dicom['VOILUTFunction']\n\n    # For sigmoid it's a list, otherwise a single value\n    if isinstance(center, list):\n        center = center[0]\n    if isinstance(width, list):\n        width = width[0]\n\n    # Set y_min, max & range\n    y_min = 0\n    y_max = float(2**bits_stored - 1)\n    y_range = y_max\n\n    # Function with default LINEAR (so for Nan, it will use linear)\n    if voi_lut_function == 'SIGMOID':\n        image = y_range / (1 + np.exp(-4 * (image - center) / width)) + y_min\n    else:\n        # Checks width for < 1 (in our case not necessary, always >= 750)\n        center -= 0.5\n        width -= 1\n\n        below = image <= (center - width / 2)\n        above = image > (center + width / 2)\n        between = np.logical_and(~below, ~above)\n\n        image[below] = y_min\n        image[above] = y_max\n        if between.any():\n            image[between] = (\n                ((image[between] - center) / width + 0.5) * y_range + y_min\n            )\n\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:34:29.123952Z","iopub.execute_input":"2023-03-16T22:34:29.124501Z","iopub.status.idle":"2023-03-16T22:34:29.160833Z","shell.execute_reply.started":"2023-03-16T22:34:29.124407Z","shell.execute_reply":"2023-03-16T22:34:29.159957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls ../input/rsna-breast-cancer-detection/test_images/10008","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:34:34.209147Z","iopub.execute_input":"2023-03-16T22:34:34.209509Z","iopub.status.idle":"2023-03-16T22:34:35.20452Z","shell.execute_reply.started":"2023-03-16T22:34:34.209478Z","shell.execute_reply":"2023-03-16T22:34:35.203306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport numpy as np\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nfrom pathlib import Path\nimport glob","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:35:00.036874Z","iopub.execute_input":"2023-03-16T22:35:00.037331Z","iopub.status.idle":"2023-03-16T22:35:00.681615Z","shell.execute_reply.started":"2023-03-16T22:35:00.037288Z","shell.execute_reply":"2023-03-16T22:35:00.680665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example = '../input/rsna-breast-cancer-detection/train_images/10006/1459541791.dcm'\npydicom.dcmread(example)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:35:03.903516Z","iopub.execute_input":"2023-03-16T22:35:03.904088Z","iopub.status.idle":"2023-03-16T22:35:03.971212Z","shell.execute_reply.started":"2023-03-16T22:35:03.904049Z","shell.execute_reply":"2023-03-16T22:35:03.970024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source: https://www.kaggle.com/code/allunia/rsna-csf-cervical-spine-fracture-eda/notebook\ndef rescale_img_to_hu(dcm_ds):\n    \"\"\"Rescales the image to Hounsfield unit.\"\"\"\n    data = dcm_ds.pixel_array\n    if dcm_ds.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    return data * dcm_ds.RescaleSlope + dcm_ds.RescaleIntercept\n","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:35:31.945166Z","iopub.execute_input":"2023-03-16T22:35:31.945532Z","iopub.status.idle":"2023-03-16T22:35:31.953262Z","shell.execute_reply.started":"2023-03-16T22:35:31.9455Z","shell.execute_reply":"2023-03-16T22:35:31.95233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_images_for_patient(patient_id):\n    patient_dir = os.path.join('../input/rsna-breast-cancer-detection/train_images', str(patient_id))\n    num_images = len(glob.glob(f\"{patient_dir}/*\"))\n    print(f\"Number of images for patient: {num_images}\")\n    fig, axs = plt.subplots(2, 2, figsize=(24,15))\n    axs = axs.flatten()\n    for i, img_path in enumerate(list(Path(patient_dir).iterdir())):\n        ds = pydicom.dcmread(img_path)\n        axs[i].imshow(rescale_img_to_hu(ds), cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:35:47.95473Z","iopub.execute_input":"2023-03-16T22:35:47.955085Z","iopub.status.idle":"2023-03-16T22:35:47.962164Z","shell.execute_reply.started":"2023-03-16T22:35:47.955048Z","shell.execute_reply":"2023-03-16T22:35:47.961103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images_for_patient(10006)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:35:57.832949Z","iopub.execute_input":"2023-03-16T22:35:57.833295Z","iopub.status.idle":"2023-03-16T22:36:16.329246Z","shell.execute_reply.started":"2023-03-16T22:35:57.833264Z","shell.execute_reply":"2023-03-16T22:36:16.328304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/rsna-breast-cancer-detection/train.csv')\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:36:56.0739Z","iopub.execute_input":"2023-03-16T22:36:56.074266Z","iopub.status.idle":"2023-03-16T22:36:56.163344Z","shell.execute_reply.started":"2023-03-16T22:36:56.074234Z","shell.execute_reply":"2023-03-16T22:36:56.162408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('../input/rsna-breast-cancer-detection/test.csv')\ntest_csv","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:36:59.82477Z","iopub.execute_input":"2023-03-16T22:36:59.825115Z","iopub.status.idle":"2023-03-16T22:36:59.848053Z","shell.execute_reply.started":"2023-03-16T22:36:59.825087Z","shell.execute_reply":"2023-03-16T22:36:59.847212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.shape[0], train_csv.patient_id.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:37:15.378552Z","iopub.execute_input":"2023-03-16T22:37:15.378951Z","iopub.status.idle":"2023-03-16T22:37:15.393862Z","shell.execute_reply.started":"2023-03-16T22:37:15.378918Z","shell.execute_reply":"2023-03-16T22:37:15.392639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nplt.subplot(1,2,1)\nax1 = sns.countplot(data=train_csv, x='cancer')\nfor container in ax1.containers:\n    ax1.bar_label(container)\nplt.title('Distribution of targets');","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:37:28.383345Z","iopub.execute_input":"2023-03-16T22:37:28.383783Z","iopub.status.idle":"2023-03-16T22:37:28.598886Z","shell.execute_reply.started":"2023-03-16T22:37:28.383748Z","shell.execute_reply":"2023-03-16T22:37:28.597806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.data.all import *\nfrom fastai.vision.all import *\n\npath = '../input/rsna-mammography-images-as-pngs/images_as_pngs/train_images_processed'\n\ntrain_csv = pd.read_csv('../input/rsna-breast-cancer-detection/train.csv')\nfn2label = {fn: cancer_or_not for fn, cancer_or_not in zip(train_csv['image_id'].astype('str'), train_csv['cancer'])}\n\ndef label_func(path):\n    return fn2label[path.stem]\n\ndblock = DataBlock(\n    blocks    = (ImageBlock, CategoryBlock),\n    get_items = get_image_files,\n    get_y = label_func,\n    splitter  = RandomSplitter()\n)\ndsets = dblock.datasets(path)\ndls = dblock.dataloaders(path)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:39:49.158748Z","iopub.execute_input":"2023-03-16T22:39:49.159221Z","iopub.status.idle":"2023-03-16T22:40:55.035395Z","shell.execute_reply.started":"2023-03-16T22:39:49.159178Z","shell.execute_reply":"2023-03-16T22:40:55.034192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:40:55.04106Z","iopub.execute_input":"2023-03-16T22:40:55.043442Z","iopub.status.idle":"2023-03-16T22:40:56.499403Z","shell.execute_reply.started":"2023-03-16T22:40:55.043401Z","shell.execute_reply":"2023-03-16T22:40:56.498429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = vision_learner(dls, resnet18, metrics=error_rate, pretrained=False)\nlearn.fit_one_cycle(1, 1e-2)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:41:05.240824Z","iopub.execute_input":"2023-03-16T22:41:05.241276Z","iopub.status.idle":"2023-03-16T22:45:41.53818Z","shell.execute_reply.started":"2023-03-16T22:41:05.241236Z","shell.execute_reply":"2023-03-16T22:45:41.536901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls ../input/rsna-breast-cancer-detection/test_images","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:45:41.543961Z","iopub.execute_input":"2023-03-16T22:45:41.546809Z","iopub.status.idle":"2023-03-16T22:45:42.773658Z","shell.execute_reply.started":"2023-03-16T22:45:41.546763Z","shell.execute_reply":"2023-03-16T22:45:42.772225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(data={'prediction_id': test_csv['prediction_id'], 'cancer': np.random.rand(test_csv.shape[0])}).drop_duplicates(subset='prediction_id')\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:45:42.777725Z","iopub.execute_input":"2023-03-16T22:45:42.778158Z","iopub.status.idle":"2023-03-16T22:45:42.810697Z","shell.execute_reply.started":"2023-03-16T22:45:42.778112Z","shell.execute_reply":"2023-03-16T22:45:42.809585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T22:45:50.816062Z","iopub.execute_input":"2023-03-16T22:45:50.816555Z","iopub.status.idle":"2023-03-16T22:45:50.823922Z","shell.execute_reply.started":"2023-03-16T22:45:50.816516Z","shell.execute_reply":"2023-03-16T22:45:50.822856Z"},"trusted":true},"execution_count":null,"outputs":[]}]}