{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip download pylibjpeg pylibjpeg-libjpeg pydicom python-gdcm","metadata":{"_uuid":"c9425651-2442-44c7-8f2f-3ecea7038358","_cell_guid":"54404b16-4371-46f7-af76-ec3dd4b4e8c0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:17:03.90597Z","iopub.execute_input":"2023-02-15T11:17:03.906376Z","iopub.status.idle":"2023-02-15T11:17:03.927635Z","shell.execute_reply.started":"2023-02-15T11:17:03.906298Z","shell.execute_reply":"2023-02-15T11:17:03.926727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /opt/conda/lib/python3.7/site-packages/nvidia/dali/plugin/","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:17:03.929563Z","iopub.execute_input":"2023-02-15T11:17:03.930484Z","iopub.status.idle":"2023-02-15T11:17:04.981548Z","shell.execute_reply.started":"2023-02-15T11:17:03.930448Z","shell.execute_reply":"2023-02-15T11:17:04.980243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -q ../input/timm-with-dependencies/timm_all -d timm-with-dependencies\n!pip install --no-index --find-links timm-with-dependencies timm","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:29:11.945621Z","iopub.execute_input":"2023-02-15T12:29:11.94613Z","iopub.status.idle":"2023-02-15T12:30:04.141941Z","shell.execute_reply.started":"2023-02-15T12:29:11.946087Z","shell.execute_reply":"2023-02-15T12:30:04.140574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n\n!pip install /kaggle/input/nvidia-dali-wheel/nvidia_dali_nightly_cuda110-1.22.0.dev20221213-6757685-py3-none-manylinux2014_x86_64.whl\n!pip install /kaggle/input/nvidia-dali-wheel/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!cp /kaggle/input/modified-pytorchpy/pytorch.py /opt/conda/lib/python3.7/site-packages/nvidia/dali/plugin/pytorch.py","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:17:04.983666Z","iopub.execute_input":"2023-02-15T11:17:04.984063Z","iopub.status.idle":"2023-02-15T11:18:23.475458Z","shell.execute_reply.started":"2023-02-15T11:17:04.984023Z","shell.execute_reply":"2023-02-15T11:18:23.474126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/rsna-python-libraries/pydicom-2.3.1-py3-none-any.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install ../input/rsna-python-libraries/numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install ../input/rsna-python-libraries/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install ../input/rsna-python-libraries/python_gdcm-3.0.21-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg_libjpeg-1.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg_openjpeg-1.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:18:23.477106Z","iopub.execute_input":"2023-02-15T11:18:23.477546Z","iopub.status.idle":"2023-02-15T11:22:25.872332Z","shell.execute_reply.started":"2023-02-15T11:18:23.477512Z","shell.execute_reply":"2023-02-15T11:22:25.870965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !ln -s ../input/rsna-breast-cancer-256-pngs/ ./processed_images","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:22:25.884081Z","iopub.execute_input":"2023-02-15T11:22:25.884824Z","iopub.status.idle":"2023-02-15T11:22:25.899309Z","shell.execute_reply.started":"2023-02-15T11:22:25.884783Z","shell.execute_reply":"2023-02-15T11:22:25.898202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !mkdir processed_images\n# !find ../input/rsna-breast-cancer-256-pngs/ -name \"*\" -exec cp -ruf \"{}\" ./processed_images/ \\;\n!ln -s ../input/rsna-breast-cancer-256-pngs/ ./processed_images","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:22:25.904626Z","iopub.execute_input":"2023-02-15T11:22:25.905338Z","iopub.status.idle":"2023-02-15T11:22:27.344063Z","shell.execute_reply.started":"2023-02-15T11:22:25.905303Z","shell.execute_reply":"2023-02-15T11:22:27.342339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gdcm\n\nimport importlib\nimportlib.reload(__import__(\"gdcm\"))\n\nfrom gdcm import DataElement\nimport pandas as pd\nimport os\nfrom pathlib import Path","metadata":{"_uuid":"335b7335-b926-45d0-8de9-d81f870d6a9f","_cell_guid":"1ec149b9-f104-4a70-a25a-ce825992541a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:27.345929Z","iopub.execute_input":"2023-02-15T11:22:27.346297Z","iopub.status.idle":"2023-02-15T11:22:27.422647Z","shell.execute_reply.started":"2023-02-15T11:22:27.346256Z","shell.execute_reply":"2023-02-15T11:22:27.421649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_name = \"rsna-breast-cancer-detection\"\njson = False\nlines = False\nsubset_rows = None\nfile_path = f\"/kaggle/input/{competition_name}\"\n\niskaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\nif iskaggle:\n    path = Path(file_path)\nelse:\n    path = Path('titanic')\n    if not path.exists():\n        import zipfile\n        import kaggle\n        kaggle.api.competition_download_cli(str(path))\n        zipfile.ZipFile(f'{path}.zip').extractall(path)\n\n\n# load test and train data\n# [train/test]_images/[patient_id]/[image_id].dcm \nif json:\n    train = pd.read_json(f\"{path}/train.json\", lines=lines, nrows=subset_rows)\n    test = pd.read_json(f\"{path}/test.json\", lines=lines, nrows=subset_rows)\nelse:\n    train_csv = pd.read_csv(f\"{path}/train.csv\", nrows=subset_rows)\n    test_csv = pd.read_csv(f\"{path}/test.csv\", nrows=subset_rows)","metadata":{"_uuid":"97aad611-eecb-4117-b5cf-853cb59c099d","_cell_guid":"dedef351-d1a9-41b4-a977-bf3ccaabdca4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:27.427162Z","iopub.execute_input":"2023-02-15T11:22:27.429493Z","iopub.status.idle":"2023-02-15T11:22:27.562951Z","shell.execute_reply.started":"2023-02-15T11:22:27.429454Z","shell.execute_reply":"2023-02-15T11:22:27.561913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv['test'] = False\ntest_csv['test'] = True","metadata":{"_uuid":"2fbbcf2e-d222-4cb4-8ea8-131912885d94","_cell_guid":"722b751b-ec50-409a-ba0e-0e364c29a89d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:27.56775Z","iopub.execute_input":"2023-02-15T11:22:27.570117Z","iopub.status.idle":"2023-02-15T11:22:27.583539Z","shell.execute_reply.started":"2023-02-15T11:22:27.570075Z","shell.execute_reply":"2023-02-15T11:22:27.582401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.basics import *\nfrom fastai.callback.all import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\nfrom sklearn.metrics import accuracy_score, f1_score\n\nimport pydicom\n\nimport pandas as pd\n\nfrom pydicom import dcmread\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport gdcm","metadata":{"_uuid":"d682ef44-9fdd-4610-b60c-07e07f8683ed","_cell_guid":"7d28427a-3916-47b2-a9ed-1f87ed7ab77b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:27.588249Z","iopub.execute_input":"2023-02-15T11:22:27.590532Z","iopub.status.idle":"2023-02-15T11:22:31.567392Z","shell.execute_reply.started":"2023-02-15T11:22:27.590495Z","shell.execute_reply":"2023-02-15T11:22:31.5663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 1\ndcm_path = f'{path}/test_images/{test_csv.loc[row,\"patient_id\"]}/{test_csv.loc[row, \"image_id\"]}.dcm'\ndcm = dcmread(dcm_path)#, force=True)\n# transfer syntaxes https://pydicom.github.io/pydicom/stable/old/image_data_handlers.html\n\nimage = Image.fromarray(dcm.pixel_array.astype(float))\nplt.imshow(dcm.pixel_array, cmap=plt.cm.bone)","metadata":{"_uuid":"46dd73a1-9137-4ee5-8880-661a349b347c","_cell_guid":"375c6757-d769-429c-b478-ea58e6303886","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:31.572455Z","iopub.execute_input":"2023-02-15T11:22:31.575081Z","iopub.status.idle":"2023-02-15T11:22:33.457199Z","shell.execute_reply.started":"2023-02-15T11:22:31.575038Z","shell.execute_reply":"2023-02-15T11:22:33.456221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id_col = test_csv.columns.get_loc('patient_id')\nimage_id_col = test_csv.columns.get_loc('image_id')\nprint(image_id_col)","metadata":{"_uuid":"9ab8fcb1-8237-48b4-969d-f83d911dd868","_cell_guid":"0bcddf67-42bb-43b2-a785-8fb6ee9e4798","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:33.461578Z","iopub.execute_input":"2023-02-15T11:22:33.463865Z","iopub.status.idle":"2023-02-15T11:22:33.473863Z","shell.execute_reply.started":"2023-02-15T11:22:33.463825Z","shell.execute_reply":"2023-02-15T11:22:33.472849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_csv.loc[train_csv.loc[:, 'cancer'] == 1]))\nprint(len(train_csv.loc[train_csv.loc[:, 'cancer'] == 0]))\n","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:22:33.477475Z","iopub.execute_input":"2023-02-15T11:22:33.478835Z","iopub.status.idle":"2023-02-15T11:22:33.501996Z","shell.execute_reply.started":"2023-02-15T11:22:33.478798Z","shell.execute_reply":"2023-02-15T11:22:33.501079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp ../input/rsna-image-preprocessing/RSNA_image_preprocessing.py ./preprocessing.py","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:22:33.509769Z","iopub.execute_input":"2023-02-15T11:22:33.51226Z","iopub.status.idle":"2023-02-15T11:22:34.657187Z","shell.execute_reply.started":"2023-02-15T11:22:33.512216Z","shell.execute_reply":"2023-02-15T11:22:34.65565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !mkdir test-folder\n# !chmod 777 test-folder","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:22:34.662934Z","iopub.execute_input":"2023-02-15T11:22:34.665391Z","iopub.status.idle":"2023-02-15T11:22:34.672371Z","shell.execute_reply.started":"2023-02-15T11:22:34.665343Z","shell.execute_reply":"2023-02-15T11:22:34.671284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl\nfrom nvidia.dali.plugin.pytorch import feed_ndarray, to_torch_type\nfrom pydicom.filebase import DicomBytesIO\nfrom nvidia.dali.types import DALIDataType\nfrom nvidia.dali import pipeline_def\nimport nvidia.dali.types as types\nimport nvidia.dali.fn as fn\nimport torch.nn.functional as F\nimport torch\nimport os\nimport sys\nimport cv2\nimport glob\nimport gdcm\nimport json\nimport shutil\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed\n\nDEBUG = False\n\nIMG_PATH = \"/kaggle/input/rsna-breast-cancer-detection/test_images/\"\ntest_images = glob.glob(f\"{IMG_PATH}*/*.dcm\")\n\nif DEBUG:\n    IMG_PATH = \"/kaggle/input/rsna-breast-cancer-detection/train_images/\"\n#     test_images = glob.glob(f\"{IMG_PATH}*/*.dcm\")[:1000]\n    test_images = glob.glob(f\"{IMG_PATH}10042/*.dcm\")\n\nprint(\"Number of images :\", len(test_images))\n\nSAVE_FOLDER = \"./test_processed_images/\"\nSIZE = 1024\n\nos.makedirs(SAVE_FOLDER, exist_ok=True)\n\nif len(test_images) > 100:\n    N_CHUNKS = 4\nelse:\n    N_CHUNKS = 1\n\nCHUNKS = [(len(test_images) / N_CHUNKS * k, len(test_images) /\n           N_CHUNKS * (k + 1)) for k in range(N_CHUNKS)]\nCHUNKS = np.array(CHUNKS).astype(int)\n\nJ2K_FOLDER = \"/tmp/j2k/\"\n\n\ndef convert_dicom_to_j2k(file, save_folder=\"\"):\n    patient = file.split('/')[-2]\n    image = file.split('/')[-1][:-4]\n    dcmfile = pydicom.dcmread(file)\n\n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        # <---- the jpeg2000 header info we're looking for\n        offset = ds.PixelData.find(b\"\\x00\\x00\\x00\\x0C\")\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jp2\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n\n\n@pipeline_def\ndef j2k_decode_pipeline(j2kfiles):\n    jpegs, _ = fn.readers.file(files=j2kfiles)\n    images = fn.experimental.decoders.image(\n        jpegs, device='mixed', output_type=types.ANY_DATA, dtype=DALIDataType.UINT16)\n    return images\n\n\nfor chunk in tqdm(CHUNKS):\n    os.makedirs(J2K_FOLDER, exist_ok=True)\n\n    _ = Parallel(n_jobs=2)(\n        delayed(convert_dicom_to_j2k)(img, save_folder=J2K_FOLDER)\n        for img in test_images[chunk[0]: chunk[1]]\n    )\n\n    j2kfiles = glob.glob(J2K_FOLDER + \"*.jp2\")\n\n    if not len(j2kfiles):\n        continue\n\n    pipe = j2k_decode_pipeline(\n        j2kfiles, batch_size=1, num_threads=2, device_id=0, debug=True)\n    pipe.build()\n\n    for i, f in enumerate(j2kfiles):\n        patient, image = f.split('/')[-1][:-4].split('_')\n        dicom = pydicom.dcmread(IMG_PATH + f\"{patient}/{image}.dcm\")\n\n        out = pipe.run()\n\n        # Dali -> Torch\n        img = out[0][0]\n        img_torch = torch.empty(img.shape(), dtype=torch.int16, device=\"cuda\")\n        feed_ndarray(img, img_torch,\n                     cuda_stream=torch.cuda.current_stream(device=0))\n        img = img_torch.float()\n\n        # Scale, resize, invert on GPU !\n        min_, max_ = img.min(), img.max()\n        img = (img - min_) / (max_ - min_)\n\n        if SIZE:\n            img = F.interpolate(img.view(1, 1, img.size(0), img.size(\n                1)), (SIZE, SIZE), mode=\"bilinear\")[0, 0]\n\n        if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n            img = 1 - img\n\n        # Back to CPU + SAVE\n        img = (img * 255).cpu().numpy().astype(np.uint8)\n\n        cv2.imwrite(SAVE_FOLDER + f\"{patient}_{image}.png\", img)\n\n    shutil.rmtree(J2K_FOLDER)\n\n\ndef dicomsdl_to_numpy_image(dicom, index=0):\n    info = dicom.getPixelDataInfo()\n    dtype = info['dtype']\n    if info['SamplesPerPixel'] != 1:\n        raise RuntimeError('SamplesPerPixel != 1')\n    else:\n        shape = [info['Rows'], info['Cols']]\n    outarr = np.empty(shape, dtype=dtype)\n    dicom.copyFrameData(index, outarr)\n    return outarr\n\n\ndef load_img_dicomsdl(f):\n    return dicomsdl_to_numpy_image(dicomsdl.open(f))\n\n\ndef process(f, size=256, save_folder=\"./test_processed_images/\"):\n    patient = f.split('/')[-2]\n    image = f.split('/')[-1][:-4]\n    \n\n    dicom = pydicom.dcmread(f)\n\n    try:\n        img = load_img_dicomsdl(f)\n    except:\n        img = dicom.pixel_array\n\n    img = (img - img.min()) / (img.max() - img.min())\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img\n\n    img = cv2.resize(img, (size, size))\n\n    file_name = f\"{save_folder}{patient}_{image}.png\"\n    print(file_name)\n    print(type((img * 255).astype(np.uint8)))\n#     cv2.imwrite(f\"{file_name}\",(img * 255).astype(np.uint8))\n    res = cv2.imwrite(f\"{file_name}\", img)\n    print(res)\n\n_ = Parallel(n_jobs=2)(\n    delayed(process)(img, size=SIZE, save_folder=SAVE_FOLDER)\n    for img in tqdm(test_images)\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:22:34.675844Z","iopub.execute_input":"2023-02-15T11:22:34.677054Z","iopub.status.idle":"2023-02-15T11:22:44.088743Z","shell.execute_reply.started":"2023-02-15T11:22:34.677015Z","shell.execute_reply":"2023-02-15T11:22:44.087431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id_column = train_csv.columns.get_loc('patient_id')\nimage_id_column = train_csv.columns.get_loc('image_id')\ncancer_column = train_csv.columns.get_loc('cancer')","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:22:44.094202Z","iopub.execute_input":"2023-02-15T11:22:44.096575Z","iopub.status.idle":"2023-02-15T11:22:44.104086Z","shell.execute_reply.started":"2023-02-15T11:22:44.096529Z","shell.execute_reply":"2023-02-15T11:22:44.103151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\n\npre_processed = True\n\ndef get_x(x):\n    if pre_processed:\n        sub_path = 'test_processed_images'  if x[-1] else 'processed_images'\n        return f\"./{sub_path}/{x[patient_id_column]}_{x[image_id_column]}.png\"\n    \n    sub_path = 'test_images'  if x[-1] else 'train_images'\n    return f\"{path}/{sub_path}/{x[patient_id_col]}/{x[image_id_col]}.dcm\"\n\ndef get_y(y):\n    return y[cancer_column]\n\n\ncancer = DataBlock(\n        blocks=(\n            ImageBlock(cls=PILImage),\n            CategoryBlock\n        ),\n        get_x=get_x,\n        get_y=get_y,\n        splitter=RandomSplitter(),\n        item_tfms=[Resize(224, resamples= (Image.Resampling.NEAREST,0))],\n        batch_tfms=[\n#             IntToFloatTensor(div=2**16-1),\n            *aug_transforms(size=224),\n            Normalize.from_stats(*imagenet_stats)\n        ]\n    )","metadata":{"_uuid":"50377e80-7a03-4441-82f0-dacae3ff3c46","_cell_guid":"5820b86c-70da-4182-a4c0-254d0a45cf42","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T12:32:30.487035Z","iopub.execute_input":"2023-02-15T12:32:30.487485Z","iopub.status.idle":"2023-02-15T12:32:30.506252Z","shell.execute_reply.started":"2023-02-15T12:32:30.487446Z","shell.execute_reply":"2023-02-15T12:32:30.505131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\nprint('_________________________')\nprint (datetime.datetime.now())\nprint('dataloaders')\nprint('_________________________')","metadata":{"_uuid":"1e9b5073-5880-4156-a5e3-ca9839098402","_cell_guid":"6214dc00-ccc5-47f5-9b97-6aaff1b1f7f1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:44.13605Z","iopub.execute_input":"2023-02-15T11:22:44.138363Z","iopub.status.idle":"2023-02-15T11:22:44.146817Z","shell.execute_reply.started":"2023-02-15T11:22:44.138325Z","shell.execute_reply":"2023-02-15T11:22:44.145795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = cancer.dataloaders(train_csv.values, num_workers=0) ","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:24:39.719647Z","iopub.execute_input":"2023-02-15T11:24:39.720193Z","iopub.status.idle":"2023-02-15T11:24:48.405414Z","shell.execute_reply.started":"2023-02-15T11:24:39.720146Z","shell.execute_reply":"2023-02-15T11:24:48.404275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.device = \"cuda\"","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:27:52.780339Z","iopub.execute_input":"2023-02-15T11:27:52.78082Z","iopub.status.idle":"2023-02-15T11:27:52.786873Z","shell.execute_reply.started":"2023-02-15T11:27:52.78078Z","shell.execute_reply":"2023-02-15T11:27:52.785863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch(max_n=32, nrows=2, unique=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:27:53.822329Z","iopub.execute_input":"2023-02-15T11:27:53.822772Z","iopub.status.idle":"2023-02-15T11:27:56.606819Z","shell.execute_reply.started":"2023-02-15T11:27:53.822732Z","shell.execute_reply":"2023-02-15T11:27:56.605942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# download https://download.pytorch.org/models/resnet34-b627a593.pth & upload as data source / https://www.kaggle.com/datasets/pytorch/resnet34\nimport os\nif not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n!cp '../input/resnet34/resnet34.pth' '/root/.cache/torch/hub/checkpoints/resnet34-b627a593.pth'","metadata":{"_uuid":"0beb0796-a356-41ce-9ba2-5a24a603876d","_cell_guid":"1c45a99b-42e4-4694-9321-f008d1910356","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:25:52.611091Z","iopub.execute_input":"2023-02-15T11:25:52.611813Z","iopub.status.idle":"2023-02-15T11:25:55.612564Z","shell.execute_reply.started":"2023-02-15T11:25:52.611768Z","shell.execute_reply":"2023-02-15T11:25:55.611019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\nprint('_________________________')\nprint (datetime.datetime.now())\nprint('lr_find')\nprint('_________________________')","metadata":{"_uuid":"28805024-f498-47fc-8d1f-6b16469853a1","_cell_guid":"0cc361e2-61a7-410f-9ad3-b094fa870c9e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:25:55.618007Z","iopub.execute_input":"2023-02-15T11:25:55.620488Z","iopub.status.idle":"2023-02-15T11:25:55.630753Z","shell.execute_reply.started":"2023-02-15T11:25:55.62042Z","shell.execute_reply":"2023-02-15T11:25:55.629676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from timm.models.layers.adaptive_avgmax_pool import SelectAdaptivePool2d\nfrom torch.nn import Flatten\n\nlearn = vision_learner(\n    dls,\n    resnet34,\n#     custom_head=nn.Sequential(SelectAdaptivePool2d(pool_type='avg', flatten=Flatten()), nn.Linear(1280, 2)), \n    metrics=[\n        error_rate,\n#         AccumMetric(pfbeta_torch, activation=ActivationType.Softmax, flatten=False),\n#         AccumMetric(pfbeta_torch_thresh, activation=ActivationType.Softmax, flatten=False)\n    ],\n    loss_func=CrossEntropyLossFlat(weight=torch.tensor([1,50]).float()),\n    pretrained=True,\n    normalize=False\n).to_fp16()\n\n\nlearn.lr_find()\nlearn.fit_one_cycle(3)\nlearn.export('learner.pkl')","metadata":{"_uuid":"21bfb103-8d1e-4de5-a8d2-822b0c50d47b","_cell_guid":"42d453de-859e-4cc7-827a-4fdeb9cb2b37","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T12:32:45.551676Z","iopub.execute_input":"2023-02-15T12:32:45.552791Z","iopub.status.idle":"2023-02-15T12:52:15.844659Z","shell.execute_reply.started":"2023-02-15T12:32:45.552744Z","shell.execute_reply":"2023-02-15T12:52:15.84194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_confusion_matrix()","metadata":{"execution":{"iopub.status.busy":"2023-02-15T12:24:17.522646Z","iopub.execute_input":"2023-02-15T12:24:17.523116Z","iopub.status.idle":"2023-02-15T12:24:17.603133Z","shell.execute_reply.started":"2023-02-15T12:24:17.523073Z","shell.execute_reply":"2023-02-15T12:24:17.600548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\nprint('_________________________')\nprint (datetime.datetime.now())\nprint('interp')\nprint('_________________________')","metadata":{"_uuid":"b2063a76-5fc7-4c22-a08c-fb7f9539c7c5","_cell_guid":"1c44c0a9-6bab-49e4-92bd-056f63cd3976","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:44.715702Z","iopub.status.idle":"2023-02-15T11:22:44.716177Z","shell.execute_reply.started":"2023-02-15T11:22:44.715939Z","shell.execute_reply":"2023-02-15T11:22:44.715962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv","metadata":{"_uuid":"ad7ddbcb-8cfd-4dff-a221-8c21b3a0831b","_cell_guid":"4452e1ae-8655-4999-90f3-044015da453b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:44.724711Z","iopub.status.idle":"2023-02-15T11:22:44.725188Z","shell.execute_reply.started":"2023-02-15T11:22:44.72495Z","shell.execute_reply":"2023-02-15T11:22:44.724974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 10008_736471439\n# !ls ./processed_images/10008*\n!ls ../input/rsna-breast-cancer-detection/","metadata":{"execution":{"iopub.status.busy":"2023-02-15T11:22:44.726922Z","iopub.status.idle":"2023-02-15T11:22:44.727407Z","shell.execute_reply.started":"2023-02-15T11:22:44.727164Z","shell.execute_reply":"2023-02-15T11:22:44.727187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = learn.dls.test_dl(test_csv.values)\npredictions, _, decoded = learn.get_preds(dl=test_dl, with_decoded=True)\nprint('predictions:')\nprint(predictions)\nprint('decoded')\nprint(decoded)","metadata":{"_uuid":"515dcbc0-6717-483e-93e2-fb7876e522db","_cell_guid":"ee24f49b-852c-45c7-8d12-048e383cd8fc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:44.744458Z","iopub.status.idle":"2023-02-15T11:22:44.745033Z","shell.execute_reply.started":"2023-02-15T11:22:44.744721Z","shell.execute_reply":"2023-02-15T11:22:44.744748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# submit_df = predictions[['prediction_id', 'cancer']]\n# submission = pd.DataFrame([])\n# submission['cancer'] = pd.DataFrame(predictions.numpy())[0]\n# submission['prediction_id']  = test_csv['patient_id'].astype(str) + \"-\" + test_csv['laterality']\n# # submission.columns = ['predictions_id', 'cancer']\n# submission.sort_index()\n# # subsmission.groupby('prediction_id')","metadata":{"_uuid":"6e47da43-15ad-4f98-b75d-85c4bdf5becb","_cell_guid":"2e886b22-4c02-4b6c-810d-5eb29dc0ca05","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:44.746119Z","iopub.status.idle":"2023-02-15T11:22:44.746627Z","shell.execute_reply.started":"2023-02-15T11:22:44.746341Z","shell.execute_reply":"2023-02-15T11:22:44.746364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame([])\nsubmission['prediction_id'] = test_csv['patient_id'].astype(str) + \"_\" + test_csv['laterality']\nsubmission['cancer'] = pd.DataFrame(predictions.numpy())[0]\nsubmission = submission.groupby('prediction_id').mean().reset_index()\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()\n! head submission.csv","metadata":{"_uuid":"e48cf73b-1b48-42d2-934d-d3d1838cffe4","_cell_guid":"39aa0070-a76b-4517-b6a8-0acd01cbc0aa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-15T11:22:44.751053Z","iopub.status.idle":"2023-02-15T11:22:44.751536Z","shell.execute_reply.started":"2023-02-15T11:22:44.751275Z","shell.execute_reply":"2023-02-15T11:22:44.751297Z"},"trusted":true},"execution_count":null,"outputs":[]}]}