{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!unzip -q ../input/timm-with-dependencies/timm_all -d timm-with-dependencies\n!pip install --no-index --find-links timm-with-dependencies timm\n!pip install /kaggle/input/dicomsdl-offline-installer/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install wandb\n\n#import wandb\n#from kaggle_secrets import UserSecretsClient\n\n\nfrom fastai.vision.learner import *\nfrom fastai.data.all import *\nfrom fastai.vision.all import *\nfrom fastai.metrics import ActivationType\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom collections import defaultdict\nimport pandas as pd\nimport numpy as np\nfrom pdb import set_trace","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:29:16.698928Z","iopub.execute_input":"2023-02-19T16:29:16.699475Z","iopub.status.idle":"2023-02-19T16:30:43.275594Z","shell.execute_reply.started":"2023-02-19T16:29:16.699365Z","shell.execute_reply":"2023-02-19T16:30:43.274405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#user_secrets = UserSecretsClient()\n\n#wandb_api = user_secrets.get_secret('wandb') \n\n#wandb.login(key=wandb_api)\n\n#####\n\n#add the callback to your learner:\n\n#wandb.init(project='my_project')\n# To log only during one training phase\n#learn.fit(..., cbs=WandbCallback())\n# To log continuously for all training phases\n#learn = learner(..., cbs=WandbCallback())","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.062114Z","iopub.status.idle":"2023-02-19T16:28:42.067692Z","shell.execute_reply.started":"2023-02-19T16:28:42.065232Z","shell.execute_reply":"2023-02-19T16:28:42.065259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_EPOCHS = 4\nNUM_SPLITS = 4\n\nRESIZE_TO = (512, 512)\n\nDATA_PATH = '/kaggle/input/rsna-breast-cancer-detection'\nTRAIN_IMAGE_DIR = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_cv2_512'\n# TRAIN_IMAGE_DIR = '/kaggle/input/rsna-train-images-png'\nTEST_DICOM_DIR = '/kaggle/input/rsna-breast-cancer-detection/test_images'\nMODEL_PATH = '/kaggle/input/rsna-trained-model-weights/tf_effv2_s_208_402/tf_effv2_s_208_402'\n# MODEL_PATH = '/kaggle/input/rsna-trained-model-weights/res18_095_74/res18_095_74'\n\nlabel_smoothing_weights = torch.tensor([1,10]).float()\nif torch.cuda.is_available():\n    label_smoothing_weights = label_smoothing_weights.cuda()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.072172Z","iopub.status.idle":"2023-02-19T16:28:42.073087Z","shell.execute_reply.started":"2023-02-19T16:28:42.072835Z","shell.execute_reply":"2023-02-19T16:28:42.07286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.is_available()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.077373Z","iopub.status.idle":"2023-02-19T16:28:42.080037Z","shell.execute_reply.started":"2023-02-19T16:28:42.079612Z","shell.execute_reply":"2023-02-19T16:28:42.079642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv(f'{DATA_PATH}/train.csv')\npatient_id_any_cancer = train_csv.groupby('patient_id').cancer.max().reset_index()\nskf = StratifiedKFold(NUM_SPLITS, shuffle=True, random_state=42)\nsplits = list(skf.split(patient_id_any_cancer.patient_id, patient_id_any_cancer.cancer))","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.083084Z","iopub.status.idle":"2023-02-19T16:28:42.096799Z","shell.execute_reply.started":"2023-02-19T16:28:42.096547Z","shell.execute_reply":"2023-02-19T16:28:42.096571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splits","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.098147Z","iopub.status.idle":"2023-02-19T16:28:42.098902Z","shell.execute_reply.started":"2023-02-19T16:28:42.098659Z","shell.execute_reply":"2023-02-19T16:28:42.098682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/369267  \ndef pfbeta_torch(preds, labels, beta=1):\n    if preds.dim() != 2 or (preds.dim() == 2 and preds.shape[1] !=2): raise ValueError('Houston, we got a problem')\n    preds = preds[:, 1]\n    preds = preds.clip(0, 1)\n    y_true_count = labels.sum()\n    ctp = preds[labels==1].sum()\n    cfp = preds[labels==0].sum()\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0.0\n\n# https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/369886    \ndef pfbeta_torch_thresh(preds, labels):\n    optimized_preds = optimize_preds(preds, labels)\n    return pfbeta_torch(optimized_preds, labels)\n\ndef optimize_preds(preds, labels=None, thresh=None, return_thresh=False, print_results=False):\n    preds = preds.clone()\n    if labels is not None: without_thresh = pfbeta_torch(preds, labels)\n    \n    if not thresh and labels is not None:\n        threshs = np.linspace(0, 1, 101)\n        f1s = [pfbeta_torch((preds > thr).float(), labels) for thr in threshs]\n        idx = np.argmax(f1s)\n        thresh, best_pfbeta = threshs[idx], f1s[idx]\n\n    preds = (preds > thresh).float()\n\n    if print_results:\n        print(f'without optimization: {without_thresh}')\n        pfbeta = pfbeta_torch(preds, labels)\n        print(f'with optimization: {pfbeta}')\n        print(f'best_thresh = {thresh}')\n    if return_thresh:\n        return thresh\n    return preds\n\nfn2label = {fn: cancer_or_not for fn, cancer_or_not in zip(train_csv['image_id'].astype('str'), train_csv['cancer'])}\n\ndef splitting_func(paths):\n    train = []\n    valid = []\n    for idx, path in enumerate(paths):\n        if int(path.parent.name) in patient_id_any_cancer.iloc[splits[SPLIT][0]].patient_id.values:\n            train.append(idx)\n        else:\n            valid.append(idx)\n    return train, valid\n\ndef label_func(path):\n    return fn2label[path.stem]\n\ndef get_items(image_dir_path):\n    items = []\n    for p in get_image_files(image_dir_path):\n        items.append(p)\n        if p.stem in fn2label and int(p.parent.name) in patient_id_any_cancer.iloc[splits[SPLIT][0]].patient_id.values:\n            if label_func(p) == 1:\n                for _ in range(5):\n                    items.append(p)\n    return items","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-02-19T16:28:42.107419Z","iopub.status.idle":"2023-02-19T16:28:42.108346Z","shell.execute_reply.started":"2023-02-19T16:28:42.108107Z","shell.execute_reply":"2023-02-19T16:28:42.108132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from timm.models.layers.adaptive_avgmax_pool import SelectAdaptivePool2d\nfrom torch.nn import Flatten\n\ndef get_dataloaders():\n    train_image_path = TRAIN_IMAGE_DIR\n\n    dblock = DataBlock(\n        blocks    = (ImageBlock, CategoryBlock),\n        get_items = get_items,\n        get_y = label_func,\n        splitter  = splitting_func,\n        batch_tfms=[Flip()],\n    )\n    dsets = dblock.datasets(train_image_path)\n    return dblock.dataloaders(train_image_path, batch_size=32)\n\nimport wandb\nfrom fastai.callback.wandb import *\n\ndef get_learner(arch=resnet18):\n    learner = vision_learner(\n        get_dataloaders(),\n        arch,\n        custom_head=nn.Sequential(SelectAdaptivePool2d(pool_type='avg', flatten=Flatten()), nn.Linear(1280, 2)),\n        metrics=[\n            error_rate,\n            AccumMetric(pfbeta_torch, activation=ActivationType.Softmax, flatten=False),\n            AccumMetric(pfbeta_torch_thresh, activation=ActivationType.Softmax, flatten=False)\n        ],\n        loss_func=CrossEntropyLossFlat(weight=torch.tensor([1,50]).float()),\n        pretrained=True,\n        normalize=False\n    ).to_fp16()\n    return learner","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.10997Z","iopub.status.idle":"2023-02-19T16:28:42.1135Z","shell.execute_reply.started":"2023-02-19T16:28:42.113087Z","shell.execute_reply":"2023-02-19T16:28:42.113116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This is a dependency that is needed for reading DICOM images\n\ntry:\n    import pylibjpeg\nexcept:\n    !rm -rf /root/.cache/torch/hub/checkpoints/\n    !mkdir -p /root/.cache/torch/hub/checkpoints/\n    !pip install /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n    !pip install /kaggle/input/rsna-2022-whl/{torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl,torchvision-0.13.1-cp37-cp37m-manylinux1_x86_64.whl}\n\n# copying the pretrained weights\n\nif not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n!cp '/kaggle/input/pretrained-model-weights-for-fastai/resnet18-f37072fd.pth' '/root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth'\n!cp '/kaggle/input/pretrained-model-weights-for-fastai/tf_efficientnetv2_s-eb54923e.pth' '/root/.cache/torch/hub/checkpoints/tf_efficientnetv2_s-eb54923e.pth'","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.117796Z","iopub.status.idle":"2023-02-19T16:28:42.122711Z","shell.execute_reply.started":"2023-02-19T16:28:42.12108Z","shell.execute_reply":"2023-02-19T16:28:42.121113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npreds, labels = [], []\n\nSPLIT = 0 # our learner needs this to construct its dataloaders...\nlearn = get_learner('tf_efficientnetv2_s')\n\n# instead of training, to conserve pipeline time, I am uploading models trained locally\n# uncomment the lines below for training\n  \n# for SPLIT in range(NUM_SPLITS):\n#     learn = get_learner()\n#     learn.unfreeze()\n#     learn.fit_one_cycle(NUM_EPOCHS, 1e-4, pct_start=0.1)\n#     learn.save(f'{MODEL_PATH}/{SPLIT}')\n        \n#     output = learn.get_preds()\n#     preds.append(output[0])\n#     labels.append(output[1])","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.125012Z","iopub.status.idle":"2023-02-19T16:28:42.128351Z","shell.execute_reply.started":"2023-02-19T16:28:42.126675Z","shell.execute_reply":"2023-02-19T16:28:42.126908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# threshold = optimize_preds(torch.cat(preds), torch.cat(labels), return_thresh=True, print_results=True)\nthreshold = 0.402","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.133774Z","iopub.status.idle":"2023-02-19T16:28:42.134656Z","shell.execute_reply.started":"2023-02-19T16:28:42.134352Z","shell.execute_reply":"2023-02-19T16:28:42.134376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pydicom\n# from pydicom.pixel_data_handlers.util import apply_voi_lut\nimport dicomsdl\n    \nfrom pathlib import Path\nimport multiprocessing as mp\nimport cv2\n\n!rm -rf test_resized_{RESIZE_TO[0]}\n\ndef dicom_file_to_ary(path):\n    dcm_file = dicomsdl.open(str(path))\n    data = dcm_file.pixelData()\n\n    data = (data - data.min()) / (data.max() - data.min())\n\n    if dcm_file.getPixelDataInfo()['PhotometricInterpretation'] == \"MONOCHROME1\":\n        data = 1 - data\n\n    data = cv2.resize(data, RESIZE_TO)\n    data = (data * 255).astype(np.uint8)\n    return data\n\ndirectories = list(Path(TEST_DICOM_DIR).iterdir())\n\ndef process_directory(directory_path):\n    parent_directory = str(directory_path).split('/')[-1]\n    !mkdir -p test_resized_{RESIZE_TO[0]}/{parent_directory}\n    for image_path in directory_path.iterdir():\n        processed_ary = dicom_file_to_ary(image_path)\n        cv2.imwrite(\n            f'test_resized_{RESIZE_TO[0]}/{parent_directory}/{image_path.stem}.png',\n            processed_ary\n        )\n\nwith mp.Pool(mp.cpu_count()) as p:\n    p.map(process_directory, directories)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-02-19T16:28:42.140541Z","iopub.status.idle":"2023-02-19T16:28:42.146735Z","shell.execute_reply.started":"2023-02-19T16:28:42.146469Z","shell.execute_reply":"2023-02-19T16:28:42.146496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npreds_all = []\n\ntest_dl = learn.dls.test_dl(get_image_files(f'test_resized_{RESIZE_TO[0]}'))\nfor SPLIT in range(NUM_SPLITS):\n    learn.load(f'{MODEL_PATH}/{SPLIT}')\n    preds, _ = learn.get_preds(dl=test_dl)\n    preds_all.append(preds)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.148847Z","iopub.status.idle":"2023-02-19T16:28:42.151557Z","shell.execute_reply.started":"2023-02-19T16:28:42.150159Z","shell.execute_reply":"2023-02-19T16:28:42.150187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = torch.zeros_like(preds_all[0])\nfor pred in preds_all:\n    preds += pred\n\npreds /= NUM_SPLITS\n\n\npreds = optimize_preds(preds, thresh=threshold)\nimage_ids = [path.stem for path in test_dl.items]\n\nimage_id2pred = defaultdict(lambda: 0)\nfor image_id, pred in zip(image_ids, preds[:, 1]):\n    image_id2pred[int(image_id)] = pred.item()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.15489Z","iopub.status.idle":"2023-02-19T16:28:42.155858Z","shell.execute_reply.started":"2023-02-19T16:28:42.155612Z","shell.execute_reply":"2023-02-19T16:28:42.155636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\n\nprediction_ids = []\npreds = []\n\nfor _, row in test_csv.iterrows():\n    prediction_ids.append(row.prediction_id)\n    preds.append(image_id2pred[row.image_id])\n\nsubmission = pd.DataFrame(data={'prediction_id': prediction_ids, 'cancer': preds}).groupby('prediction_id').max().reset_index()\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.160998Z","iopub.status.idle":"2023-02-19T16:28:42.164459Z","shell.execute_reply.started":"2023-02-19T16:28:42.162976Z","shell.execute_reply":"2023-02-19T16:28:42.163236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:28:42.169122Z","iopub.status.idle":"2023-02-19T16:28:42.17207Z","shell.execute_reply.started":"2023-02-19T16:28:42.171776Z","shell.execute_reply":"2023-02-19T16:28:42.171803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-02-19T16:41:53.902852Z","iopub.execute_input":"2023-02-19T16:41:53.904016Z","iopub.status.idle":"2023-02-19T16:41:54.192034Z","shell.execute_reply.started":"2023-02-19T16:41:53.903957Z","shell.execute_reply":"2023-02-19T16:41:54.189214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}