{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"try:\n    import pylibjpeg\nexcept:\n    !pip install -q /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n!pip install -q segmentation_models_pytorch\nimport cv2\nimport os\nimport glob\nimport torch\nimport gdcm\nimport pydicom\nimport pylibjpeg\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nfrom sklearn.metrics import precision_score, recall_score\nfrom torchvision.io import read_image\nfrom joblib import Parallel, delayed\nfrom sklearn.model_selection import StratifiedShuffleSplit, StratifiedGroupKFold, StratifiedKFold\nfrom tqdm.notebook import tqdm\nfrom torchvision.models import EfficientNet\nimport sys\nsys.path.append('/kaggle/input/timm-0-6-9/pytorch-image-models-master')\nimport timm","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:21.856313Z","iopub.execute_input":"2022-12-06T12:33:21.856804Z","iopub.status.idle":"2022-12-06T12:33:58.031203Z","shell.execute_reply.started":"2022-12-06T12:33:21.856708Z","shell.execute_reply":"2022-12-06T12:33:58.029745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    save_folder = \"/kaggle/working/image_output/\"\n    torch_im_models_path = '/root/.cache/torch/hub/checkpoints/'\n    train_im_path = \"/kaggle/input/rsna-breast-cancer-256-pngs\"\n    train_df_path = \"/kaggle/input/rsna-breast-cancer-detection/train.csv\"\n    test_df_path = '/kaggle/input/rsna-breast-cancer-detection/test.csv'\n    pretrained_state_dict_path = \"/kaggle/input/rsna-baseline-resnet18-5-epochs/res18_ep5.pth\"\n    device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n    extension = \"png\"\n    train_size = 1000\n    size = 224\n    seed = 42\n    n_epochs = 10\n    n_splits = 2\n    batch_size = 16\n    learning_rate = 1e-6\n    threshold = 0.16\n    num_workers = os.cpu_count()\n    amp = True\n    using_torch = True","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:58.034195Z","iopub.execute_input":"2022-12-06T12:33:58.035272Z","iopub.status.idle":"2022-12-06T12:33:58.042407Z","shell.execute_reply.started":"2022-12-06T12:33:58.035232Z","shell.execute_reply":"2022-12-06T12:33:58.041341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed: int):\n    import random, os\n    import numpy as np\n    import torch\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nseed_everything(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:58.043945Z","iopub.execute_input":"2022-12-06T12:33:58.044329Z","iopub.status.idle":"2022-12-06T12:33:58.062145Z","shell.execute_reply.started":"2022-12-06T12:33:58.044298Z","shell.execute_reply":"2022-12-06T12:33:58.061301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CancerDataset(Dataset):\n    def __init__(self, df, transform=None):\n        super(CancerDataset, self).__init__()\n        self.df = df.copy()\n        self.transform = transform\n        self.path_to_dcms = CFG.train_im_path\n        \n    def __getitem__(self, idx):\n        pid = self.df.loc[idx, 'patient_id']\n        im_id = self.df.loc[idx, 'image_id']\n        im_path = CFG.train_im_path + f\"/{pid}_{im_id}.png\"\n        img = cv2.imread(im_path)\n        if self.transform:\n            img = self.transform(img)\n        label = self.df.loc[idx, \"cancer\"]\n        label = torch.tensor(label, dtype=torch.float32).unsqueeze(0)\n        return img, label\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:58.065094Z","iopub.execute_input":"2022-12-06T12:33:58.066231Z","iopub.status.idle":"2022-12-06T12:33:58.075126Z","shell.execute_reply.started":"2022-12-06T12:33:58.066194Z","shell.execute_reply":"2022-12-06T12:33:58.073696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model**","metadata":{}},{"cell_type":"code","source":"import segmentation_models_pytorch as smp\n\ndef build_model():\n    model = smp.Unet(\n        encoder_name=\"resnet18\",      \n        encoder_weights=\"imagenet\",     \n        in_channels=3,                 \n        classes=1,\n        activation=None,\n    )\n    model.to(CFG.device)\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:58.077383Z","iopub.execute_input":"2022-12-06T12:33:58.077931Z","iopub.status.idle":"2022-12-06T12:33:59.469441Z","shell.execute_reply.started":"2022-12-06T12:33:58.077886Z","shell.execute_reply":"2022-12-06T12:33:59.468408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RSNAModel(nn.Module):\n    def __init__(self, out_features = 1, pretrained = True):\n        super(RSNAModel, self).__init__()\n#         backbone = timm.create_model(\"vit_base_patch16_224\", pretrained=False)\n#         if pretrained:\n#             backbone.load_state_dict(torch.load(\"../input/vit-base-models-pretrained-pytorch/jx_vit_base_p16_224-80ecf9dd.pth\"))\n        backbone = build_model()\n#         backbone.head = nn.Linear(768, out_features)\n        self.backbone = backbone\n        \n    def forward(self, dcm):\n        out = self.backbone(dcm)\n        out = torch.sigmoid(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:59.470856Z","iopub.execute_input":"2022-12-06T12:33:59.471203Z","iopub.status.idle":"2022-12-06T12:33:59.478495Z","shell.execute_reply.started":"2022-12-06T12:33:59.471172Z","shell.execute_reply":"2022-12-06T12:33:59.477379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_batch(batch, model, criterion, optimizer, threshold=CFG.threshold, scaler = None):\n    optimizer.zero_grad()\n    \n    dcms, labels = batch\n    dcms, labels = dcms.to(CFG.device), labels.to(CFG.device)\n    \n    if scaler:\n        with torch.cuda.amp.autocast():\n            preds = model(dcms)\n            cpu_labels = labels.detach().cpu().numpy()\n            cpu_preds = preds.detach().cpu().numpy()\n#             print(\"Labels are: \", cpu_labels)\n#             print(cpu_preds)\n            precision = precision_score(cpu_labels, cpu_preds > threshold, zero_division=0)\n            recall = recall_score(cpu_labels, cpu_preds > threshold, zero_division=0)\n            loss = criterion(preds, labels)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n    else:\n        preds = model(dcms)\n        cpu_labels = labels.detach().cpu().numpy()\n        cpu_preds = preds.detach().cpu().numpy()\n        precision = precision_score(cpu_labels, cpu_preds > threshold, zero_division=0)\n        recall = recall_score(cpu_labels, cpu_preds > threshold, zero_division=0)\n\n        loss = criterion(preds, labels)\n        loss.backward()\n        optimizer.step()\n        \n    return loss, precision, recall","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-12-06T12:33:59.480466Z","iopub.execute_input":"2022-12-06T12:33:59.48085Z","iopub.status.idle":"2022-12-06T12:33:59.496379Z","shell.execute_reply.started":"2022-12-06T12:33:59.480797Z","shell.execute_reply":"2022-12-06T12:33:59.495452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_batch(batch, model, criterion, threshold=CFG.threshold):\n    dcms, labels = batch\n    dcms, labels = dcms.to(CFG.device), labels.to(CFG.device)\n    \n    preds = model(dcms)\n    \n    cpu_labels = labels.detach().cpu().numpy()\n    cpu_preds = preds.detach().cpu().numpy()\n    precision = precision_score(cpu_labels, cpu_preds > threshold, zero_division=0)\n    recall = recall_score(cpu_labels, cpu_preds > threshold, zero_division=0)\n    \n    loss = criterion(preds, labels)\n    return loss, precision, recall","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-12-06T12:33:59.497622Z","iopub.execute_input":"2022-12-06T12:33:59.498424Z","iopub.status.idle":"2022-12-06T12:33:59.513898Z","shell.execute_reply.started":"2022-12-06T12:33:59.498382Z","shell.execute_reply":"2022-12-06T12:33:59.512731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_loop(train_dataset, val_dataset, n_epoch):\n    batch_size = CFG.batch_size\n    lr = CFG.learning_rate\n    num_workers = CFG.num_workers\n    model = RSNAModel().to(CFG.device)\n    criterion = nn.BCEWithLogitsLoss(reduction=\"mean\")\n    optimizer = torch.optim.Adam(model.parameters(), lr = lr)\n    \n    train_dataloader = DataLoader(train_dataset, \n                                  batch_size=batch_size, \n                                  num_workers=num_workers, \n                                  shuffle=True)\n    \n    val_dataloader = DataLoader(val_dataset, \n                                batch_size=batch_size, \n                                num_workers=num_workers, \n                                shuffle=False)\n    \n    for epoch in range(1, n_epoch + 1):\n        print(f\"Epoch {epoch}\\n\", \"-\" * 50)\n        \n        # Train loop\n        train_loss = 0\n        train_precision = 0\n        train_recall = 0\n    \n        if CFG.amp:\n            scaler = torch.cuda.amp.GradScaler()\n        model.train()\n        for batch in train_dataloader:\n            loss, precision, recall = train_batch(batch, model, criterion, optimizer, scaler = scaler)\n            \n            with torch.no_grad():\n                train_loss += loss\n                train_precision += precision\n                train_recall += recall\n            \n        \n        print(f\"Train Loss: {train_loss / len(train_dataloader)} Precision: {train_precision / len(train_dataloader)} Recall: {train_recall / len(train_dataloader)}\")\n        \n        # Validation loop\n        val_loss = 0\n        val_precision = 0\n        val_recall = 0\n        model.eval()\n        for batch in val_dataloader:\n            with torch.no_grad():\n                loss, precision, recall = eval_batch(batch, model, criterion)\n                val_loss += loss\n                val_precision += precision\n                val_recall += recall\n                \n        print(f\"Val Loss: {val_loss / len(val_dataloader)} Precision: {val_precision / len(val_dataloader)} Recall: {val_recall / len(val_dataloader)}\")\n        torch.save(model.state_dict(), f\"res50_ep{epoch}.pth\")\n        print(\"\"*50)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:59.515841Z","iopub.execute_input":"2022-12-06T12:33:59.51637Z","iopub.status.idle":"2022-12-06T12:33:59.530831Z","shell.execute_reply.started":"2022-12-06T12:33:59.516299Z","shell.execute_reply":"2022-12-06T12:33:59.529564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(cv, train_df):\n    cols_to_keep = [\"patient_id\", \"image_id\", \"cancer\"]\n    train_df = train_df[cols_to_keep]\n    X = train_df[\"image_id\"]\n    y = train_df['cancer']\n    groups = train_df[\"patient_id\"]\n    train_transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.RandomAffine(degrees=45, translate=(0.1, 0.1), scale=(0.5, 1.5)),\n        transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.5),\n        transforms.RandomGrayscale(p=0.5),\n        transforms.RandomPerspective(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n        transforms.Resize((CFG.size, CFG.size))\n    ])\n    val_transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.RandomAffine(degrees=45, translate=(0.1, 0.1), scale=(0.5, 1.5)),\n        transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.5),\n        transforms.RandomGrayscale(p=0.5),\n        transforms.RandomPerspective(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n        transforms.Resize((CFG.size, CFG.size))\n    ])\n    if CFG.amp:\n            print(\"[LOG] AMP Mode is ON\")\n    for fold, (train_idx, valid_idx) in enumerate(cv.split(X = X,  y = y, groups = groups)):\n        print(\"-\"*20, f\"Training started for fold number {fold}\", \"-\"*20,)\n        train_data = train_df.iloc[train_idx].reset_index(drop = True)\n        validation_data = train_df.iloc[valid_idx].reset_index(drop = True)\n        train_dataset = CancerDataset(df = train_data, \n                                      transform = train_transform)\n        val_dataset = CancerDataset(df = validation_data, \n                                    transform = val_transform)\n        train_loop(\n            train_dataset = train_dataset,\n            val_dataset = val_dataset,\n            n_epoch = CFG.n_epochs,\n        )\n        print(\" \"*50)\n        print(f\"Training for fold {fold} over\")\n        print(f\"Strating training for fold {fold + 1}...\")","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:59.533973Z","iopub.execute_input":"2022-12-06T12:33:59.534326Z","iopub.status.idle":"2022-12-06T12:33:59.549469Z","shell.execute_reply.started":"2022-12-06T12:33:59.534295Z","shell.execute_reply":"2022-12-06T12:33:59.54822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if CFG.using_torch:\n#     if not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n#         os.makedirs('/root/.cache/torch/hub/checkpoints/')\n#     !cp '/kaggle/input/vit-base-models-pretrained-pytorch/jx_vit_base_p16_384-83fb41ba.pth' '/root/.cache/torch/hub/checkpoints/vit_b_16-c867db91.pth'\n#     print(\"TorchVision Model Appended\")\n# else:\n#     import sys\n#     sys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\n#     from efficientnet_pytorch import EfficientNet\n#     print(\"Pretrained Model Loaded\")","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:59.551095Z","iopub.execute_input":"2022-12-06T12:33:59.551807Z","iopub.status.idle":"2022-12-06T12:33:59.565127Z","shell.execute_reply.started":"2022-12-06T12:33:59.55177Z","shell.execute_reply":"2022-12-06T12:33:59.56364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(CFG.train_df_path)\nif CFG.train_size != \"full\":\n    df_wout_cancer = train_df[train_df[\"cancer\"] == 0]\n    df_w_cancer = train_df[train_df[\"cancer\"] == 1]\n    train_this = df_wout_cancer.sample(n = CFG.train_size).append(df_w_cancer)\n    del df_wout_cancer\n    del df_w_cancer\ncv = StratifiedKFold(n_splits=CFG.n_splits, shuffle=True, random_state=CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:59.566775Z","iopub.execute_input":"2022-12-06T12:33:59.567491Z","iopub.status.idle":"2022-12-06T12:33:59.723906Z","shell.execute_reply.started":"2022-12-06T12:33:59.567454Z","shell.execute_reply":"2022-12-06T12:33:59.722896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.train_size != \"full\":\n    print(f\"[LOG] Training on a dataset of size {train_this.shape[0]}\")\n    train(cv = cv, train_df = train_this)\nelse:\n    train(cv = cv, train_df = train_df)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:33:59.725204Z","iopub.execute_input":"2022-12-06T12:33:59.725646Z","iopub.status.idle":"2022-12-06T12:34:10.603827Z","shell.execute_reply.started":"2022-12-06T12:33:59.725606Z","shell.execute_reply":"2022-12-06T12:34:10.602169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.iloc[32818]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:34:10.605622Z","iopub.status.idle":"2022-12-06T12:34:10.606216Z","shell.execute_reply.started":"2022-12-06T12:34:10.605923Z","shell.execute_reply":"2022-12-06T12:34:10.605951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T12:34:10.608088Z","iopub.status.idle":"2022-12-06T12:34:10.608823Z","shell.execute_reply.started":"2022-12-06T12:34:10.608601Z","shell.execute_reply":"2022-12-06T12:34:10.608623Z"},"trusted":true},"execution_count":null,"outputs":[]}]}