{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n #   for filename in filenames:\n  #      print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-27T11:01:08.245893Z","iopub.execute_input":"2023-02-27T11:01:08.247092Z","iopub.status.idle":"2023-02-27T11:01:08.276733Z","shell.execute_reply.started":"2023-02-27T11:01:08.246976Z","shell.execute_reply":"2023-02-27T11:01:08.275713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install -U pylibjpeg pylibjpeg-openjpeg pylibjpeg-libjpeg pydicom python-gdcm","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:01:08.278878Z","iopub.execute_input":"2023-02-27T11:01:08.279652Z","iopub.status.idle":"2023-02-27T11:01:08.284898Z","shell.execute_reply.started":"2023-02-27T11:01:08.279613Z","shell.execute_reply":"2023-02-27T11:01:08.283854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -q ../input/timm-with-dependencies/timm_all -d timm-with-dependencies\n!pip install --no-index --find-links timm-with-dependencies timm\n\n\n# This is a dependency that is needed for reading DICOM images\n\ntry:\n    import pylibjpeg\nexcept:\n    !rm -rf /root/.cache/torch/hub/checkpoints/\n    !mkdir -p /root/.cache/torch/hub/checkpoints/\n    !pip install /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n    !pip install /kaggle/input/rsna-2022-whl/{torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl,torchvision-0.13.1-cp37-cp37m-manylinux1_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:01:08.286706Z","iopub.execute_input":"2023-02-27T11:01:08.287539Z","iopub.status.idle":"2023-02-27T11:03:46.417844Z","shell.execute_reply.started":"2023-02-27T11:01:08.287483Z","shell.execute_reply":"2023-02-27T11:03:46.416554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# copying the pretrained weights\nif not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n!cp '/kaggle/input/timm-pretrained-model-weights/resnet26d-69e92c46.pth' '/root/.cache/torch/hub/checkpoints/resnet26d-69e92c46.pth'","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:03:46.420537Z","iopub.execute_input":"2023-02-27T11:03:46.421219Z","iopub.status.idle":"2023-02-27T11:03:48.230313Z","shell.execute_reply.started":"2023-02-27T11:03:46.421168Z","shell.execute_reply":"2023-02-27T11:03:48.228819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.learner import *\nfrom fastai.data.all import *\nfrom fastai.vision.all import *\nimport timm\n\nfrom collections import defaultdict\nimport pandas as pd\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:03:50.361147Z","iopub.execute_input":"2023-02-27T11:03:50.361575Z","iopub.status.idle":"2023-02-27T11:03:53.73694Z","shell.execute_reply.started":"2023-02-27T11:03:50.361532Z","shell.execute_reply":"2023-02-27T11:03:53.735716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_path = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs/train_images_processed'\n\ntrain_csv = pd.read_csv('../input/rsna-breast-cancer-detection/train.csv')\nfn2label = {fn: cancer_or_not for fn, cancer_or_not in zip(train_csv['image_id'].astype('str'), train_csv['cancer'])}\n\ndef label_func(path):\n    return fn2label[path.stem]\n\ndblock = DataBlock(\n    blocks    = (ImageBlock, CategoryBlock),\n    get_items = get_image_files,\n    get_y = label_func,\n    splitter  = RandomSplitter(),\n    item_tfms=[Resize(256, method='squish')],\n    batch_tfms=[IntToFloatTensor(div=2**16-1), *aug_transforms()]\n)\ndsets = dblock.datasets(train_image_path)\ndls = dblock.dataloaders(train_image_path)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:04:05.019536Z","iopub.execute_input":"2023-02-27T11:04:05.019919Z","iopub.status.idle":"2023-02-27T11:04:56.61951Z","shell.execute_reply.started":"2023-02-27T11:04:05.019886Z","shell.execute_reply":"2023-02-27T11:04:56.618363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tst_files = '/kaggle/input/rsna-breast-cancer-detection/test_images'","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:06:57.157647Z","iopub.execute_input":"2023-02-27T11:06:57.158352Z","iopub.status.idle":"2023-02-27T11:06:57.167322Z","shell.execute_reply.started":"2023-02-27T11:06:57.158298Z","shell.execute_reply":"2023-02-27T11:06:57.165792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(arch, size, item=Resize(480, method='squish'), accum=1, finetune=True, epochs=12):\n    dls = ImageDataLoaders.from_folder(trn_path, valid_pct=0.2, item_tfms=item,\n        batch_tfms=aug_transforms(size=size, min_scale=0.75), bs=64//accum)\n    cbs = GradientAccumulation(64) if accum else []\n    learn = vision_learner(dls, arch, metrics=error_rate, cbs=cbs).to_fp16()\n    if finetune:\n        learn.fine_tune(epochs, 0.01)\n        return learn.tta(dl=dls.test_dl(tst_files))\n    else:\n        learn.unfreeze()\n        learn.fit_one_cycle(epochs, 0.01)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:06:58.964704Z","iopub.execute_input":"2023-02-27T11:06:58.965315Z","iopub.status.idle":"2023-02-27T11:06:58.97819Z","shell.execute_reply.started":"2023-02-27T11:06:58.965268Z","shell.execute_reply":"2023-02-27T11:06:58.976949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = ImageDataLoaders.from_folder(train_image_path, valid_pct=0.2, seed=42,\n    item_tfms=Resize(480, method='squish'),\n    batch_tfms=aug_transforms(size=128, min_scale=0.75))\n\ndls.show_batch(max_n=6)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:07:42.796932Z","iopub.execute_input":"2023-02-27T11:07:42.797402Z","iopub.status.idle":"2023-02-27T11:08:01.01463Z","shell.execute_reply.started":"2023-02-27T11:07:42.797362Z","shell.execute_reply":"2023-02-27T11:08:01.013545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = vision_learner(dls, 'resnet26d', metrics=error_rate, path='.').to_fp16()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:08:22.183706Z","iopub.execute_input":"2023-02-27T11:08:22.184214Z","iopub.status.idle":"2023-02-27T11:08:22.895383Z","shell.execute_reply.started":"2023-02-27T11:08:22.184171Z","shell.execute_reply":"2023-02-27T11:08:22.894273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn = vision_learner(dls, 'resnet26d', metrics=error_rate, pretrained=True).to_fp16()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:10:00.097969Z","iopub.execute_input":"2023-02-27T11:10:00.098525Z","iopub.status.idle":"2023-02-27T11:10:00.104817Z","shell.execute_reply.started":"2023-02-27T11:10:00.09845Z","shell.execute_reply":"2023-02-27T11:10:00.10345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.lr_find(suggest_funcs=(valley, slide))","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:10:00.546388Z","iopub.execute_input":"2023-02-27T11:10:00.546921Z","iopub.status.idle":"2023-02-27T11:10:00.552421Z","shell.execute_reply.started":"2023-02-27T11:10:00.546877Z","shell.execute_reply":"2023-02-27T11:10:00.550954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.fine_tune(1, 0.1)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:10:01.236125Z","iopub.execute_input":"2023-02-27T11:10:01.236612Z","iopub.status.idle":"2023-02-27T11:10:01.245536Z","shell.execute_reply.started":"2023-02-27T11:10:01.236568Z","shell.execute_reply":"2023-02-27T11:10:01.244488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = ImageDataLoaders.from_folder(train_image_path, valid_pct=0.2, seed=42,\n    item_tfms=Resize(192, method=ResizeMethod.Pad, pad_mode=PadMode.Zeros))\ndls.show_batch(max_n=3)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:11:15.397077Z","iopub.execute_input":"2023-02-27T11:11:15.397512Z","iopub.status.idle":"2023-02-27T11:11:33.866786Z","shell.execute_reply.started":"2023-02-27T11:11:15.397461Z","shell.execute_reply":"2023-02-27T11:11:33.865776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(arch, item, batch, epochs=1):\n    dls = ImageDataLoaders.from_folder(train_image_path, valid_pct=0.1, item_tfms=item, batch_tfms=batch)\n    learn = vision_learner(dls, arch, metrics=error_rate).to_fp16()\n    learn.fine_tune(epochs, 0.01)\n    return learn","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:13:15.733Z","iopub.execute_input":"2023-02-27T11:13:15.733452Z","iopub.status.idle":"2023-02-27T11:13:15.740626Z","shell.execute_reply.started":"2023-02-27T11:13:15.733405Z","shell.execute_reply":"2023-02-27T11:13:15.739547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = train('resnet26d', item=Resize(192),\n              batch=aug_transforms(size=128, min_scale=0.75))","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:13:17.347581Z","iopub.execute_input":"2023-02-27T11:13:17.348037Z","iopub.status.idle":"2023-02-27T11:28:52.2059Z","shell.execute_reply.started":"2023-02-27T11:13:17.347996Z","shell.execute_reply":"2023-02-27T11:28:52.20473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn = train_images_path(arch, item=Resize((256,192), method=ResizeMethod.Pad, pad_mode=PadMode.Zeros),\n      #batch=aug_transforms(size=256))","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:28:54.751685Z","iopub.execute_input":"2023-02-27T11:28:54.752177Z","iopub.status.idle":"2023-02-27T11:28:54.760035Z","shell.execute_reply.started":"2023-02-27T11:28:54.752131Z","shell.execute_reply":"2023-02-27T11:28:54.758517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\n    \nfrom pathlib import Path\nfrom PIL import Image\nimport multiprocessing as mp\n\nRESIZE_TO = (256, 256)\n!rm -rf test_resized_{RESIZE_TO[0]}\n\n# https://www.kaggle.com/code/tanlikesmath/brain-tumor-radiogenomic-classification-eda/notebook\ndef dicom_file_to_ary(path):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n\ndirectories = list(Path('/kaggle/input/rsna-breast-cancer-detection/test_images').iterdir())\n\ndef process_directory(directory_path):\n    parent_directory = str(directory_path).split('/')[-1]\n    !mkdir -p test_resized_{RESIZE_TO[0]}/{parent_directory}\n    for image_path in directory_path.iterdir():\n        processed_ary = dicom_file_to_ary(image_path)\n        im = Image.fromarray(processed_ary).resize(RESIZE_TO)\n        im.save(f'test_resized_{RESIZE_TO[0]}/{parent_directory}/{image_path.stem}.png')\n\nwith mp.Pool(mp.cpu_count()) as p:\n    p.map(process_directory, directories)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:28:56.260937Z","iopub.execute_input":"2023-02-27T11:28:56.261545Z","iopub.status.idle":"2023-02-27T11:29:03.166586Z","shell.execute_reply.started":"2023-02-27T11:28:56.261473Z","shell.execute_reply":"2023-02-27T11:29:03.16466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = learn.dls.test_dl(get_image_files(f'test_resized_{RESIZE_TO[0]}'))\npreds, _ = learn.get_preds(dl=test_dl)\n\nimage_ids = [path.stem for path in test_dl.items]\npreds = np.array(preds)[:, 1]","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:29:07.242316Z","iopub.execute_input":"2023-02-27T11:29:07.242886Z","iopub.status.idle":"2023-02-27T11:29:07.810706Z","shell.execute_reply.started":"2023-02-27T11:29:07.242833Z","shell.execute_reply":"2023-02-27T11:29:07.809534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id2pred = defaultdict(lambda: 0)\nfor image_id, pred in zip(image_ids, preds):\n    image_id2pred[int(image_id)] = pred","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:29:09.060267Z","iopub.execute_input":"2023-02-27T11:29:09.061141Z","iopub.status.idle":"2023-02-27T11:29:09.070844Z","shell.execute_reply.started":"2023-02-27T11:29:09.061086Z","shell.execute_reply":"2023-02-27T11:29:09.06991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\n\nprediction_ids = []\npreds = []\n\nfor _, row in test_csv.iterrows():\n    prediction_ids.append(row.prediction_id)\n    preds.append(image_id2pred[row.image_id])\n\nsubmission = pd.DataFrame(data={'prediction_id': prediction_ids, 'cancer': preds}).groupby('prediction_id').mean().reset_index()\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:29:11.160691Z","iopub.execute_input":"2023-02-27T11:29:11.161153Z","iopub.status.idle":"2023-02-27T11:29:11.220736Z","shell.execute_reply.started":"2023-02-27T11:29:11.161115Z","shell.execute_reply":"2023-02-27T11:29:11.219672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T11:29:12.560263Z","iopub.execute_input":"2023-02-27T11:29:12.561141Z","iopub.status.idle":"2023-02-27T11:29:12.574207Z","shell.execute_reply.started":"2023-02-27T11:29:12.561096Z","shell.execute_reply":"2023-02-27T11:29:12.572921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}