{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"references:\n\n- [fast ai](https://www.kaggle.com/code/igorlashkov/rsna-miccai-btumor-classification-finished/notebook)\n- [FAST dicom processing](https://www.kaggle.com/code/remekkinas/fast-dicom-processing-1-6-2x-faster/)","metadata":{"papermill":{"duration":0.027355,"end_time":"2021-10-24T18:56:44.934968","exception":false,"start_time":"2021-10-24T18:56:44.907613","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import sys\nimport os\nimport platform\nprint(sys.version)\nprint(os.name)\nprint(platform.system())\nprint(platform.release())","metadata":{"papermill":{"duration":0.041954,"end_time":"2021-10-24T18:56:45.005241","exception":false,"start_time":"2021-10-24T18:56:44.963287","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:23.653501Z","iopub.execute_input":"2022-12-25T11:30:23.653962Z","iopub.status.idle":"2022-12-25T11:30:23.681506Z","shell.execute_reply.started":"2022-12-25T11:30:23.653872Z","shell.execute_reply":"2022-12-25T11:30:23.680466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nis_cuda_enabled = torch.cuda.is_available()\nprint('Cuda enabled', is_cuda_enabled)\nif torch.cuda.is_available():\n    print(torch.cuda.current_device())\n    print(torch.cuda.device(0))\n    print(torch.cuda.device_count())\n    print(torch.cuda.get_device_name(0))","metadata":{"papermill":{"duration":1.43331,"end_time":"2021-10-24T18:56:46.465785","exception":false,"start_time":"2021-10-24T18:56:45.032475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:30.426566Z","iopub.execute_input":"2022-12-25T11:30:30.427524Z","iopub.status.idle":"2022-12-25T11:30:32.373181Z","shell.execute_reply.started":"2022-12-25T11:30:30.427477Z","shell.execute_reply":"2022-12-25T11:30:32.371389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvcc --version","metadata":{"papermill":{"duration":0.704667,"end_time":"2021-10-24T18:56:47.197829","exception":false,"start_time":"2021-10-24T18:56:46.493162","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:32.37519Z","iopub.execute_input":"2022-12-25T11:30:32.375747Z","iopub.status.idle":"2022-12-25T11:30:33.356563Z","shell.execute_reply.started":"2022-12-25T11:30:32.375709Z","shell.execute_reply":"2022-12-25T11:30:33.355371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"papermill":{"duration":0.853962,"end_time":"2021-10-24T18:56:48.07905","exception":false,"start_time":"2021-10-24T18:56:47.225088","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:33.358841Z","iopub.execute_input":"2022-12-25T11:30:33.359163Z","iopub.status.idle":"2022-12-25T11:30:34.38271Z","shell.execute_reply.started":"2022-12-25T11:30:33.359131Z","shell.execute_reply":"2022-12-25T11:30:34.381629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pydicom\nimport pandas as pd\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom tqdm import tqdm\nimport binascii\nfrom PIL import Image\n\nfrom fastai.vision.all import *\nimport numpy as np\nimport pandas as pd\nimport random\nnp.set_printoptions(threshold=sys.maxsize)","metadata":{"papermill":{"duration":1.565577,"end_time":"2021-10-24T18:56:49.677142","exception":false,"start_time":"2021-10-24T18:56:48.111565","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:42.272201Z","iopub.execute_input":"2022-12-25T11:30:42.272696Z","iopub.status.idle":"2022-12-25T11:30:43.684818Z","shell.execute_reply.started":"2022-12-25T11:30:42.27263Z","shell.execute_reply":"2022-12-25T11:30:43.683829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pytorch_lightning as pl\nrandom_seed=1\npl.seed_everything(random_seed)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T11:30:43.686501Z","iopub.execute_input":"2022-12-25T11:30:43.687692Z","iopub.status.idle":"2022-12-25T11:30:46.20483Z","shell.execute_reply.started":"2022-12-25T11:30:43.687634Z","shell.execute_reply":"2022-12-25T11:30:46.203904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"papermill":{"duration":0.067379,"end_time":"2021-10-24T18:56:49.771774","exception":false,"start_time":"2021-10-24T18:56:49.704395","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:46.206892Z","iopub.execute_input":"2022-12-25T11:30:46.207829Z","iopub.status.idle":"2022-12-25T11:30:46.262061Z","shell.execute_reply.started":"2022-12-25T11:30:46.207792Z","shell.execute_reply":"2022-12-25T11:30:46.260721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load labels","metadata":{"papermill":{"duration":0.026922,"end_time":"2021-10-24T18:56:49.826788","exception":false,"start_time":"2021-10-24T18:56:49.799866","status":"completed"},"tags":[]}},{"cell_type":"code","source":"EPOCHS = 10\nINPUT_PATH = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_512/train_images_processed_512'\nLABELS_PATH = '/kaggle/input/rsna-breast-cancer-detection/train.csv'\nMODEL_EXPORT = 'trained_model'\n\ndf = pd.read_csv(LABELS_PATH)","metadata":{"papermill":{"duration":0.090731,"end_time":"2021-10-24T18:56:49.944474","exception":false,"start_time":"2021-10-24T18:56:49.853743","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:46.264521Z","iopub.execute_input":"2022-12-25T11:30:46.265441Z","iopub.status.idle":"2022-12-25T11:30:46.413741Z","shell.execute_reply.started":"2022-12-25T11:30:46.265307Z","shell.execute_reply":"2022-12-25T11:30:46.412752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"papermill":{"duration":0.074969,"end_time":"2021-10-24T18:56:50.046629","exception":false,"start_time":"2021-10-24T18:56:49.97166","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:46.722553Z","iopub.execute_input":"2022-12-25T11:30:46.722935Z","iopub.status.idle":"2022-12-25T11:30:46.797934Z","shell.execute_reply.started":"2022-12-25T11:30:46.722902Z","shell.execute_reply":"2022-12-25T11:30:46.796986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# values distribution\nplt.figure(figsize=(5, 4))\nsns.countplot(data=df, x=\"cancer\")","metadata":{"papermill":{"duration":0.252681,"end_time":"2021-10-24T18:56:50.327818","exception":false,"start_time":"2021-10-24T18:56:50.075137","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:47.492885Z","iopub.execute_input":"2022-12-25T11:30:47.49324Z","iopub.status.idle":"2022-12-25T11:30:47.756617Z","shell.execute_reply.started":"2022-12-25T11:30:47.49321Z","shell.execute_reply":"2022-12-25T11:30:47.755877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create output dataset folders\nos.makedirs('./train', exist_ok = True)\nprint('Train folder created')\n\nos.makedirs('./test', exist_ok = True)\nprint('Test folder created')","metadata":{"papermill":{"duration":0.071934,"end_time":"2021-10-24T18:56:50.432681","exception":false,"start_time":"2021-10-24T18:56:50.360747","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:48.708133Z","iopub.execute_input":"2022-12-25T11:30:48.708491Z","iopub.status.idle":"2022-12-25T11:30:48.766063Z","shell.execute_reply.started":"2022-12-25T11:30:48.708458Z","shell.execute_reply":"2022-12-25T11:30:48.765141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['file'] = df.apply(lambda x: f'{x[\"patient_id\"]}/{x[\"image_id\"]}.png', axis=1)\ndf.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T11:30:49.77782Z","iopub.execute_input":"2022-12-25T11:30:49.778603Z","iopub.status.idle":"2022-12-25T11:30:50.553634Z","shell.execute_reply.started":"2022-12-25T11:30:49.778564Z","shell.execute_reply":"2022-12-25T11:30:50.552703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"help(ImageDataLoaders.from_df)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T11:30:50.756339Z","iopub.execute_input":"2022-12-25T11:30:50.757404Z","iopub.status.idle":"2022-12-25T11:30:50.810891Z","shell.execute_reply.started":"2022-12-25T11:30:50.757362Z","shell.execute_reply":"2022-12-25T11:30:50.809867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a DataLoaders object is a combination of training and validation data\nimage_data = ImageDataLoaders.from_df(df[['file','cancer']], item_tfms=Resize(224), bs=64, label_col=1, fn_col=0, path=INPUT_PATH)","metadata":{"papermill":{"duration":4.811744,"end_time":"2021-10-24T18:59:08.699444","exception":false,"start_time":"2021-10-24T18:59:03.8877","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:30:51.566256Z","iopub.execute_input":"2022-12-25T11:30:51.567204Z","iopub.status.idle":"2022-12-25T11:31:00.177699Z","shell.execute_reply.started":"2022-12-25T11:30:51.567168Z","shell.execute_reply":"2022-12-25T11:31:00.176728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at the data\nimage_data.show_batch()","metadata":{"papermill":{"duration":1.041027,"end_time":"2021-10-24T18:59:09.772078","exception":false,"start_time":"2021-10-24T18:59:08.731051","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:31:00.179714Z","iopub.execute_input":"2022-12-25T11:31:00.180175Z","iopub.status.idle":"2022-12-25T11:31:01.959711Z","shell.execute_reply.started":"2022-12-25T11:31:00.180137Z","shell.execute_reply":"2022-12-25T11:31:01.958748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training stage","metadata":{"papermill":{"duration":0.033494,"end_time":"2021-10-24T18:59:09.838759","exception":false,"start_time":"2021-10-24T18:59:09.805265","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import torch \nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass Net(nn.Module):\n    def __init__(self, pretrained=False):\n        super().__init__()\n        # 3 input image channel, 6 output channels, 5x5 square convolution\n        # kernel\n        self.conv1 = nn.Conv2d(3, 6, 5)\n        self.pool = nn.MaxPool2d(2, 2)\n        self.conv2 = nn.Conv2d(6, 16, 5)\n        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 10)\n\n    def forward(self, x):\n        x = self.pool(F.relu(self.conv1(x)))\n        x = self.pool(F.relu(self.conv2(x)))\n        x = torch.flatten(x, 1) # flatten all dimensions except batch\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = F.relu(self.fc3(x))\n        return x","metadata":{"papermill":{"duration":0.096605,"end_time":"2021-10-24T18:59:09.968261","exception":false,"start_time":"2021-10-24T18:59:09.871656","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:31:01.961157Z","iopub.execute_input":"2022-12-25T11:31:01.961892Z","iopub.status.idle":"2022-12-25T11:31:02.058763Z","shell.execute_reply.started":"2022-12-25T11:31:01.961852Z","shell.execute_reply":"2022-12-25T11:31:02.057769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Net()\nprint(model)","metadata":{"papermill":{"duration":0.10169,"end_time":"2021-10-24T18:59:10.103208","exception":false,"start_time":"2021-10-24T18:59:10.001518","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:31:02.066713Z","iopub.execute_input":"2022-12-25T11:31:02.069605Z","iopub.status.idle":"2022-12-25T11:31:02.147554Z","shell.execute_reply.started":"2022-12-25T11:31:02.069568Z","shell.execute_reply":"2022-12-25T11:31:02.14672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = list(model.parameters())\nprint(len(params))\nprint(params[0].size())  # conv1's .weight","metadata":{"papermill":{"duration":0.093462,"end_time":"2021-10-24T18:59:10.230611","exception":false,"start_time":"2021-10-24T18:59:10.137149","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:31:02.151409Z","iopub.execute_input":"2022-12-25T11:31:02.154086Z","iopub.status.idle":"2022-12-25T11:31:02.227699Z","shell.execute_reply.started":"2022-12-25T11:31:02.154051Z","shell.execute_reply":"2022-12-25T11:31:02.226791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# chooses an appropriate loss function\n#https://docs.fast.ai/metrics.html\nlearn = vision_learner(image_data, Net, metrics=[error_rate, accuracy], model_dir=\"/tmp/model/\").to_fp16()\n# auc_score = RocAuc()\n# f1 = F1Score()\n# learn = vision_learner(image_data, Net, metrics=[auc_score, f1], model_dir=\"/tmp/model/\").to_fp16()","metadata":{"papermill":{"duration":0.191923,"end_time":"2021-10-24T18:59:10.457334","exception":false,"start_time":"2021-10-24T18:59:10.265411","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:31:02.232196Z","iopub.execute_input":"2022-12-25T11:31:02.235133Z","iopub.status.idle":"2022-12-25T11:31:02.368314Z","shell.execute_reply.started":"2022-12-25T11:31:02.235097Z","shell.execute_reply":"2022-12-25T11:31:02.367399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.lr_find()","metadata":{"papermill":{"duration":37.266086,"end_time":"2021-10-24T18:59:47.799589","exception":false,"start_time":"2021-10-24T18:59:10.533503","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:31:02.372877Z","iopub.execute_input":"2022-12-25T11:31:02.373778Z","iopub.status.idle":"2022-12-25T11:32:02.972831Z","shell.execute_reply.started":"2022-12-25T11:31:02.373741Z","shell.execute_reply":"2022-12-25T11:32:02.971611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print model's state_dict\nprint(\"Model's state_dict:\")\nfor param_tensor in model.state_dict():\n    print(param_tensor, \"\\t\", model.state_dict()[param_tensor].size())","metadata":{"papermill":{"duration":0.109149,"end_time":"2021-10-24T18:59:47.945083","exception":false,"start_time":"2021-10-24T18:59:47.835934","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T11:32:02.974729Z","iopub.execute_input":"2022-12-25T11:32:02.975096Z","iopub.status.idle":"2022-12-25T11:32:03.037992Z","shell.execute_reply.started":"2022-12-25T11:32:02.975059Z","shell.execute_reply":"2022-12-25T11:32:03.036749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlearn.fit_one_cycle(EPOCHS, lr_max=1e-2)","metadata":{"papermill":{"duration":29.299863,"end_time":"2021-10-24T19:00:17.28136","exception":false,"start_time":"2021-10-24T18:59:47.981497","status":"completed"},"tags":[],"scrolled":true,"execution":{"iopub.status.busy":"2022-12-25T11:32:03.039609Z","iopub.execute_input":"2022-12-25T11:32:03.039981Z","iopub.status.idle":"2022-12-25T12:30:01.805955Z","shell.execute_reply.started":"2022-12-25T11:32:03.039944Z","shell.execute_reply":"2022-12-25T12:30:01.80474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show results of prediction\nlearn.show_results()","metadata":{"papermill":{"duration":1.677093,"end_time":"2021-10-24T19:00:19.01842","exception":false,"start_time":"2021-10-24T19:00:17.341327","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T12:30:01.809826Z","iopub.execute_input":"2022-12-25T12:30:01.810872Z","iopub.status.idle":"2022-12-25T12:30:03.080608Z","shell.execute_reply.started":"2022-12-25T12:30:01.810832Z","shell.execute_reply":"2022-12-25T12:30:03.079585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save model to disk\nlearn.save(MODEL_EXPORT)","metadata":{"papermill":{"duration":0.104405,"end_time":"2021-10-24T19:00:19.168585","exception":false,"start_time":"2021-10-24T19:00:19.06418","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T12:30:03.082059Z","iopub.execute_input":"2022-12-25T12:30:03.08263Z","iopub.status.idle":"2022-12-25T12:30:03.142435Z","shell.execute_reply.started":"2022-12-25T12:30:03.082592Z","shell.execute_reply":"2022-12-25T12:30:03.141424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_top_losses(9, figsize=(15,11))","metadata":{"papermill":{"duration":1.564351,"end_time":"2021-10-24T19:00:20.773945","exception":false,"start_time":"2021-10-24T19:00:19.209594","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T12:30:03.145886Z","iopub.execute_input":"2022-12-25T12:30:03.146166Z","iopub.status.idle":"2022-12-25T12:31:12.083381Z","shell.execute_reply.started":"2022-12-25T12:30:03.146141Z","shell.execute_reply":"2022-12-25T12:31:12.082384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare test images","metadata":{}},{"cell_type":"code","source":"%%capture\n\n!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2022-12-25T13:22:43.485702Z","iopub.execute_input":"2022-12-25T13:22:43.486081Z","iopub.status.idle":"2022-12-25T13:23:48.908332Z","shell.execute_reply.started":"2022-12-25T13:22:43.485997Z","shell.execute_reply":"2022-12-25T13:23:48.907019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl as dicoml\nimport cv2\nimport pydicom\n\nfrom joblib import Parallel, delayed\nimport glob","metadata":{"execution":{"iopub.status.busy":"2022-12-25T13:27:38.355046Z","iopub.execute_input":"2022-12-25T13:27:38.355548Z","iopub.status.idle":"2022-12-25T13:27:38.851458Z","shell.execute_reply.started":"2022-12-25T13:27:38.355503Z","shell.execute_reply":"2022-12-25T13:27:38.850359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom pathlib import Path\nimport numpy as np\nfrom PIL import Image\n\nRESIZE_TO = (512, 512)\n!rm -rf test\n!mkdir test\n\n# https://www.kaggle.com/code/tanlikesmath/brain-tumor-radiogenomic-classification-eda/notebook\ndef dicom_file_to_ary_v1(path):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n\ndef dicom_file_to_ary(path):\n    dicom = dicoml.open(path)\n    data = dicom.pixelData()\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n\ndirectories = list(Path('/kaggle/input/rsna-breast-cancer-detection/test_images').iterdir())\n\ndef process_directory(directory_path):\n    parent_directory = str(directory_path).split('/')[-1]\n    !mkdir -p test/{parent_directory}\n    for image_path in directory_path.iterdir():\n        processed_ary = dicom_file_to_ary(str(image_path))\n        im = Image.fromarray(processed_ary).resize(RESIZE_TO)\n        im.save(f'test/{parent_directory}/{image_path.stem}.png')\n        \nimport multiprocessing as mp\n\nwith mp.Pool(64) as p:\n    p.map(process_directory, directories)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T13:38:08.795223Z","iopub.execute_input":"2022-12-25T13:38:08.795713Z","iopub.status.idle":"2022-12-25T13:38:17.497908Z","shell.execute_reply.started":"2022-12-25T13:38:08.79567Z","shell.execute_reply":"2022-12-25T13:38:17.496241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")\ndf_test['file'] = df_test.apply(lambda x: f'{x[\"patient_id\"]}/{x[\"image_id\"]}.png', axis=1)\ndf_test.head(2)","metadata":{"papermill":{"duration":0.120071,"end_time":"2021-10-24T19:00:20.941485","exception":false,"start_time":"2021-10-24T19:00:20.821414","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-25T13:38:28.777479Z","iopub.execute_input":"2022-12-25T13:38:28.778009Z","iopub.status.idle":"2022-12-25T13:38:28.812585Z","shell.execute_reply.started":"2022-12-25T13:38:28.777962Z","shell.execute_reply":"2022-12-25T13:38:28.810045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict stage","metadata":{"papermill":{"duration":0.045747,"end_time":"2021-10-24T19:00:21.032654","exception":false,"start_time":"2021-10-24T19:00:20.986907","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# load weights\n#learn = cnn_learner(image_data, Net, metrics=[error_rate, accuracy], model_dir=\"/tmp/model/\").to_fp16()\n#learn_new = learn.load(MODEL_EXPORT)","metadata":{"papermill":{"duration":0.102848,"end_time":"2021-10-24T19:00:21.18075","exception":false,"start_time":"2021-10-24T19:00:21.077902","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npreds = []\nfor _, row in df_test.iterrows():\n    full_path = f'./test/{row[\"file\"]}'\n    prediction = learn.predict(full_path)\n    probability = prediction[2][1].item()\n    print(probability)\n    preds.append(probability)","metadata":{"papermill":{"duration":2.291883,"end_time":"2021-10-24T19:00:23.518459","exception":false,"start_time":"2021-10-24T19:00:21.226576","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['cancer']=preds","metadata":{"papermill":{"duration":0.165168,"end_time":"2021-10-24T19:00:23.78773","exception":false,"start_time":"2021-10-24T19:00:23.622562","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_output = df_test[['prediction_id', 'cancer']].groupby('prediction_id').mean().reset_index()\ndf_output.to_csv('submission.csv', index=False)\ndf_output.head()","metadata":{"papermill":{"duration":0.169518,"end_time":"2021-10-24T19:00:24.334656","exception":false,"start_time":"2021-10-24T19:00:24.165138","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Print requirements","metadata":{"papermill":{"duration":0.103388,"end_time":"2021-10-24T19:00:24.541679","exception":false,"start_time":"2021-10-24T19:00:24.438291","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# taken from here https://stackoverflow.com/a/49199019\nimport pkg_resources\nimport types\ndef get_imports():\n    for name, val in globals().items():\n        if isinstance(val, types.ModuleType):\n            # Split ensures you get root package, \n            # not just imported function\n            name = val.__name__.split(\".\")[0]\n\n        elif isinstance(val, type):\n            name = val.__module__.split(\".\")[0]\n\n        # Some packages are weird and have different\n        # imported names vs. system/pip names. Unfortunately,\n        # there is no systematic way to get pip names from\n        # a package's imported name. You'll have to add\n        # exceptions to this list manually!\n        poorly_named_packages = {\n            \"PIL\": \"Pillow\",\n            \"sklearn\": \"scikit-learn\"\n        }\n        if name in poorly_named_packages.keys():\n            name = poorly_named_packages[name]\n\n        yield name\nimports = list(set(get_imports()))\n\n# The only way I found to get the version of the root package\n# from only the name of the package is to cross-check the names \n# of installed packages vs. imported packages\nrequirements = []\nfor m in pkg_resources.working_set:\n    if m.project_name in imports and m.project_name!=\"pip\":\n        requirements.append((m.project_name, m.version))\n\nfor r in requirements:\n    print(\"{}=={}\".format(*r))","metadata":{"papermill":{"duration":0.178156,"end_time":"2021-10-24T19:00:24.823652","exception":false,"start_time":"2021-10-24T19:00:24.645496","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}