{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q timm\n!pip install -q pytorch-lightning-bolts\n!pip install -q command","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:28:16.473703Z","iopub.execute_input":"2022-10-31T07:28:16.474405Z","iopub.status.idle":"2022-10-31T07:28:51.382131Z","shell.execute_reply.started":"2022-10-31T07:28:16.474317Z","shell.execute_reply":"2022-10-31T07:28:51.380962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport os\nfrom glob import glob\nimport copy\nimport time\nimport math\nimport command\nimport random\n\nimport cv2\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nmpl.rcParams['figure.figsize'] = 12, 8\n\nfrom skimage import img_as_ubyte\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import Dataset, DataLoader\nimport pytorch_lightning as pl\nimport pl_bolts as pb\nimport timm\n\ndef seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:28:51.384558Z","iopub.execute_input":"2022-10-31T07:28:51.384939Z","iopub.status.idle":"2022-10-31T07:28:57.143172Z","shell.execute_reply.started":"2022-10-31T07:28:51.384899Z","shell.execute_reply":"2022-10-31T07:28:57.142073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIR = '/kaggle/input/'\n\ndata = pd.read_csv(f'{DIR}/rsna-2022-cervical-spine-fracture-detection/train.csv')\ntrue = data.copy()\ntruth = data.copy()\nbox = pd.read_csv(f'{DIR}/rsna-2022-cervical-spine-fracture-detection/train_bounding_boxes.csv')\n\npatient_to_classes = np.load(f\"{DIR}/classes-volumes-b1-v10/classes_volumes_b1v10.npy\", allow_pickle=1).item()\n\nidxs = [data[data.StudyInstanceUID==p].index.item() for p in pd.unique(box.StudyInstanceUID)]\nleftout_data = data.drop(idxs)\nleftout_data = leftout_data[leftout_data.patient_overall==0]\ndata = data.iloc[idxs].reset_index().drop('index', axis=1)\ndata = data.append(leftout_data[:280])\n\nStudyInstanceUIDs = []\nslice_numbers = []\nfractured = []\nfor patient in data.StudyInstanceUID:\n    paths = sorted(glob(f\"{DIR}/rsna-2022-cervical-spine-fracture-detection/train_images/{patient}/*\"), key=lambda x: int(x.split('/')[-1].split('.')[0]))\n    sls = box[box.StudyInstanceUID==patient].slice_number.values\n    for i in range(1, len(paths)+1):#[int(p.split('/')[-1].split('.')[0]) for p in paths]:\n        StudyInstanceUIDs.append(patient)\n        slice_numbers.append(i)\n        if i in sls:\n            fractured.append(1)\n        else: fractured.append(0)\n\n            \ndata = pd.DataFrame({'StudyInstanceUID': StudyInstanceUIDs, 'slice_number': slice_numbers, 'fractured': fractured})\n    \nrows = []\nfor i, row in tqdm(data.iterrows()):\n    x = patient_to_classes[row.StudyInstanceUID]\n    if x[row.slice_number-1] not in [100, 8] or row.fractured:\n        rows.append(row)\n        \ndata = pd.DataFrame(rows)\n    \ndata","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:28:57.144746Z","iopub.execute_input":"2022-10-31T07:28:57.145696Z","iopub.status.idle":"2022-10-31T07:30:19.84954Z","shell.execute_reply.started":"2022-10-31T07:28:57.145663Z","shell.execute_reply":"2022-10-31T07:30:19.848626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = 1 # CHANGE THIS TO 0 IF WANT TO ACTUALLY TRAIN","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:30:19.852237Z","iopub.execute_input":"2022-10-31T07:30:19.852647Z","iopub.status.idle":"2022-10-31T07:30:19.857439Z","shell.execute_reply.started":"2022-10-31T07:30:19.85261Z","shell.execute_reply":"2022-10-31T07:30:19.85642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DDP = 0\n    DDP_INIT_DONE = 0\n    \n    SEED = 3407\n    SPLITS = 5\n    FOLD = 0\n    \n    SZ_H = 456\n    SZ_W = 456\n    \n    TRN_BS = 4 #WAS ORIGINALLY TRAINED ON 128 GLOBAL BATCH SIZE\n    VAL_BS = 4\n    ACCUMS = 1\n    \n    EPOCHS = 2 if DEBUG else 12\n    LR = 1e-3\n    WARMUP_EPOCHS = 4\n    WARMUP_LR = 1e-6\n    \n    FOLDER = \"/kaggle/working/\"\n    NAME = \"b5\"\n    V = \"5\"\n    \n#pl.seed_everything(CFG.SEED)\nseed_everything(CFG.SEED)\nOUTPUT_FOLDER = f\"{CFG.FOLDER}/{CFG.NAME}_v{CFG.V}/\"\n\nCFG.cache_dir = OUTPUT_FOLDER + '/cache/'\n\nos.makedirs(OUTPUT_FOLDER, exist_ok=1)\nos.makedirs(CFG.cache_dir, exist_ok=1)","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:35:07.523005Z","iopub.execute_input":"2022-10-31T07:35:07.523366Z","iopub.status.idle":"2022-10-31T07:35:07.530964Z","shell.execute_reply.started":"2022-10-31T07:35:07.523336Z","shell.execute_reply":"2022-10-31T07:35:07.529897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SpineDataset(Dataset):\n    def __init__(self, data, transforms=None):\n        self.data = data\n        self.transforms = transforms\n        self.is_train = 1 if len(data)>25000 else 0\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, i):\n        row = self.data.iloc[i]\n            \n        patient = row.StudyInstanceUID\n        sl = row.slice_number\n        fractured = row.fractured\n        \n        #print(patient, sl)\n        \n        #bone = patient_to_classes[patient][sl-1]\n        \n        #if bone==100: bone = 1\n        #if bone==8: bone = 7\n        \n        #label = np.zeros((7,), dtype=np.float32)\n        #label[bone-1] = fractured\n        \n        try:\n            image = np.load(f\"{DIR}/rsna-2022-train-images-kaggle/train_images_kaggle/{patient}_{sl}.npy\")\n            \n            try: image_last = np.load(f\"{DIR}/rsna-2022-train-images-kaggle/train_images_kaggle/{patient}_{sl-1}.npy\")\n            except: image_last = image\n            \n            try: image_next = np.load(f\"{DIR}/rsna-2022-train-images-kaggle/train_images_kaggle/{patient}_{sl+1}.npy\")\n            except: image_next = image\n            \n            mask = np.load(f\"{DIR}/rsna-2022-prediction-masks-kaggle/prediction_masks_kaggle/{patient}_{sl}.npy\").clip(0, 1)\n            \n            #mask = cv2.resize(mask, (image.shape[1], image.shape[0]))\n            \n        except:\n            image = np.zeros((512, 512), dtype=np.float32)\n            image_last = np.zeros((512, 512), dtype=np.float32)\n            image_next = np.zeros((512, 512), dtype=np.float32)\n            mask = np.zeros((512, 512), dtype=np.float32)\n        \n        #image_last *= mask\n        #image *= mask\n        #image_next *= mask\n        \n        #print(image.shape, image_last.shape, image_next.shape)\n        \n        image = np.stack([image_last, image, image_next], -1)\n        \n        #print(mask.shape, image.shape)\n        \n        try:\n            ymin, ymax = np.min(np.where(mask)[0])/mask.shape[1], np.max(np.where(mask)[0])/mask.shape[1]\n            xmin, xmax = np.min(np.where(mask)[1])/mask.shape[0], np.max(np.where(mask)[1])/mask.shape[0]\n            \n            if self.is_train:\n                xmin = xmin * np.random.choice(np.arange(0.9, 1., 0.01))\n                ymin = ymin * np.random.choice(np.arange(0.9, 1., 0.01))\n                xmax = xmax * np.random.choice(np.arange(1., 1.1, 0.01))\n                ymax = ymax * np.random.choice(np.arange(1., 1.1, 0.01))\n                #pass\n            else:\n                xmin = xmin * 0.95\n                ymin = ymin * 0.95\n                xmax = xmax * 1.05\n                ymax = ymax * 1.05\n            \n            image = image[int(ymin*image.shape[0]):int(ymax*image.shape[0]), int(xmin*image.shape[1]):int(xmax*image.shape[1])]\n            #mask = mask[int(ymin*mask.shape[0]):int(ymax*mask.shape[0]), int(xmin*mask.shape[1]):int(xmax*mask.shape[1])]\n            \n        except:\n            pass\n            #image = np.zeros((CFG.SZ_H, CFG.SZ_W), dtype=np.uint8)\n        \n        #masks_.append(image)\n        #continue\n\n        if image.shape[0]==0 or image.shape[1]==0:\n            image = np.zeros((CFG.SZ_H, CFG.SZ_W, 3), dtype=np.uint8)\n        \n        #image = np.stack([image], -1)\n        \n        if self.transforms:\n            transformed = self.transforms(image=image)\n            image = transformed['image']\n            if image.dtype==torch.uint8: image = image.float()/255\n            \n        return image, torch.as_tensor([fractured]).float(), patient+'_'+str(sl)","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:30:54.920546Z","iopub.execute_input":"2022-10-31T07:30:54.920948Z","iopub.status.idle":"2022-10-31T07:30:54.939778Z","shell.execute_reply.started":"2022-10-31T07:30:54.920917Z","shell.execute_reply":"2022-10-31T07:30:54.938426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = [*StratifiedGroupKFold(n_splits=CFG.SPLITS).split(data, data.fractured, groups=data.StudyInstanceUID)]\n\ndef get_loaders():\n    \n    train_paths = data.iloc[folds[CFG.FOLD][0]]\n    valid_paths = data.iloc[folds[CFG.FOLD][1]]\n    \n    #train_paths = train_paths.append([train_paths[train_paths.fractured==1]] * 1)\n    \n    train_augs = A.Compose([\n        #A.Resize(CFG.SZ_H, CFG.SZ_W),\n        A.LongestMaxSize(CFG.SZ_H),\n        A.PadIfNeeded(CFG.SZ_H, CFG.SZ_W, border_mode=0, p=1),\n        #A.RandomResizedCrop(CFG.SZ_H, CFG.SZ_W, ratio=[0.95, 1.05], scale=[0.95, 1.05], p=0.5),\n        #A.OneOf([\n        #    A.Resize(CFG.SZ_H, CFG.SZ_W),\n        #    A.RandomResizedCrop(CFG.SZ_H, CFG.SZ_W, ratio=[0.95, 1.05], scale=[0.95, 1.05]),\n        #], p=1.),\n        A.Perspective(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        #A.VerticalFlip(p=0.25),\n        #A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=0, p=.25),\n        A.Rotate(p=0.5, limit=(45, -45)),\n        #A.RandomContrast(limit=(0.5, 0.5), p=1.),\n        A.RandomBrightnessContrast(p=0.5),\n        #A.Cutout(p=0.25, max_h_size=CFG.SZ_H//4, max_w_size=CFG.SZ_W//4, num_holes=4),\n        #A.Normalize(),\n        ToTensorV2(),\n    ])\n    \n    valid_augs = A.Compose([\n        #A.Resize(CFG.SZ_H, CFG.SZ_W),\n        A.LongestMaxSize(CFG.SZ_H),\n        A.PadIfNeeded(CFG.SZ_H, CFG.SZ_W, border_mode=0, p=1),\n        #A.RandomContrast(limit=(0.2, 0.2), p=1.),\n        #A.Normalize(),\n        ToTensorV2()\n    ])\n    \n    train_dataset = SpineDataset(train_paths, train_augs)\n    valid_dataset = SpineDataset(valid_paths, valid_augs)\n    \n    if CFG.DDP and CFG.DDP_INIT_DONE:\n        train_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_dataset, shuffle=True)\n        train_loader = DataLoader(train_dataset, batch_size=CFG.TRN_BS, sampler=train_sampler, num_workers=8, pin_memory=False)\n        \n        valid_sampler = torch.utils.data.distributed.DistributedSampler(dataset=valid_dataset, shuffle=False)\n        valid_loader = DataLoader(valid_dataset, batch_size=CFG.TRN_BS, sampler=valid_sampler, shuffle=False, num_workers=8, pin_memory=False)\n    else:\n        train_loader = DataLoader(train_dataset, batch_size=CFG.TRN_BS, shuffle=True, num_workers=8, pin_memory=False)\n        valid_loader = DataLoader(valid_dataset, batch_size=CFG.VAL_BS, shuffle=False, num_workers=0, pin_memory=False)\n    \n    return train_loader, valid_loader#, train_data, valid_data\n\ntrain_loader, valid_loader = get_loaders()\nfor d in valid_loader: break\nplt.imshow(d[0][0].numpy().transpose(1, 2, 0)[:, :, :3], cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:30:55.106076Z","iopub.execute_input":"2022-10-31T07:30:55.106945Z","iopub.status.idle":"2022-10-31T07:30:55.814259Z","shell.execute_reply.started":"2022-10-31T07:30:55.106908Z","shell.execute_reply":"2022-10-31T07:30:55.813302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _score(preds, targets):\n    scores = []\n    difs = []\n    b = targets.copy()\n    for th in np.arange(0.01, 0.99, 0.001):\n        a = preds.copy()\n        #th = 0.638\n        a[a>th] = 1\n        a[a<th] = 0\n        #print(np.sum(np.round(a[targets==1])==1) / np.sum(targets==1), np.sum(np.round(a[targets==0])==0) / np.sum(targets==0))\n        #print(np.sum(np.round(targets[a==1])==1) / np.sum(a==1), np.sum(np.round(targets[a==0])==0) / np.sum(a==0))\n\n        s1, s2 = np.sum(np.round(a[b==1])==1) / np.sum(b==1), np.sum(np.round(a[b==0])==0) / np.sum(b==0) #made for data?\n        #s1, s2 = np.sum(np.round(b[a==1])==1) / np.sum(a==1), np.sum(np.round(b[a==0])==0) / np.sum(a==0) #made for model?\n\n        s = (s1 * 0.18) + (s2 * 0.82)\n\n        #difs.append(max([s1, s2]) - min([s1, s2]))\n        #scores.append([s1, s2])\n\n        scores.append(s)\n\n    #scores = np.array(scores)\n    #return scores[np.argmin(difs)]\n\n    return np.max(scores)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return nn.functional.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'\n\nclass PcamPool(nn.Module):\n    def __init__(self, f):\n        super(PcamPool, self).__init__()\n        self.cam = nn.Conv2d(f, 1, kernel_size=1, stride=1, padding=0, bias=True)\n        \n    def forward(self, feat_map):\n        logit_map = self.cam(feat_map)\n        \n        assert logit_map is not None\n\n        prob_map = torch.sigmoid(logit_map)\n        weight_map = prob_map / prob_map.sum(dim=2, keepdim=True)            .sum(dim=3, keepdim=True)\n        feat = (feat_map * weight_map).sum(dim=2, keepdim=True)            .sum(dim=3, keepdim=True)\n\n        return feat\n    \nclass Model(pl.LightningModule):\n    def __init__(self):\n        super(Model, self).__init__()\n        \n        #timm -> current version: 0.6.7, last version: 0.5.4\n        self.feature_extractor = timm.models.tf_efficientnet_b5_ns(in_chans=3, pretrained=True, num_classes=0, global_pool='')\n        \n        f = self.feature_extractor.num_features\n        \n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        #self.maxpool = nn.AdaptiveMaxPool2d(1)\n        #self.gempool = GeM()\n        #self.pcampool = PcamPool(f)\n        \n        self.dropout = nn.Dropout(0.2)\n        \n        self.classifier = nn.Linear(f*1, 1)\n        \n        self.flatten = nn.Flatten()\n        self.sigmoid = nn.Sigmoid()\n        #self.softmax = nn.Softmax(-1)\n        \n    def forward(self, inp):\n        features = self.feature_extractor(inp)\n        \n        features = self.avgpool(features)\n        #features2 = self.pcampool(features)\n        #features = torch.cat([features1, features2], 1)\n        \n        features = self.flatten(features)\n        \n        features = self.dropout(features)\n        \n        logits = self.classifier(features)\n        \n        return logits\n    \n    \ndef define_criterion_optimizer_scheduler_scaler(model):\n    criterion = nn.BCEWithLogitsLoss(pos_weight=torch.as_tensor([1.5,]).cuda()).cuda()\n    optimizer = optim.AdamW(model.parameters(), lr=CFG.LR, weight_decay=1e-5)\n    \n    #optimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-5)\n\n    scheduler = pb.optimizers.lr_scheduler.LinearWarmupCosineAnnealingLR(optimizer, \n                                                                         warmup_epochs=CFG.WARMUP_EPOCHS, \n                                                                         max_epochs=CFG.EPOCHS,\n                                                                         warmup_start_lr=CFG.WARMUP_LR)\n\n    scaler = torch.cuda.amp.GradScaler(enabled=True)\n    \n    return criterion, optimizer, scheduler, scaler","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:30:56.242455Z","iopub.execute_input":"2022-10-31T07:30:56.243761Z","iopub.status.idle":"2022-10-31T07:30:56.265371Z","shell.execute_reply.started":"2022-10-31T07:30:56.243717Z","shell.execute_reply":"2022-10-31T07:30:56.262191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, loader):\n    model.train()\n    \n    running_loss = 0.0\n\n    tq = tqdm(loader)\n    \n    for i, batch in enumerate(tq):\n        images, targets, ids = batch\n        \n        #targets[targets==1] = 0.99\n        #targets[targets==0] = 0.01\n        \n        optimizer.zero_grad()\n\n        with torch.cuda.amp.autocast():\n            images, targets = images.cuda(), targets.cuda()\n\n            outputs = model(images)\n            loss = criterion(outputs, targets)\n        \n        scaler.scale(loss).backward()\n        \n        scaler.step(optimizer)\n        scaler.update()\n        \n        running_loss += loss.item()\n        \n        tq.set_postfix(loss=running_loss/(i+1))\n        \n        if DEBUG and i==10: break\n    \n    if 1:\n        torch.save(model.state_dict(), f\"{OUTPUT_FOLDER}/curr_f{CFG.FOLD}.pth\")\n        \ndef test_one_epoch(path, loader):\n    model = Model()\n    st = torch.load(path, map_location=f\"cpu\")\n    model.eval()\n    model.cuda()\n    model.load_state_dict(st)\n    \n    tq = tqdm(loader)\n    \n    running_loss = 0.\n    \n    OUTPUTS = []\n    TARGETS = []\n    IDS = []\n    \n    for i, batch in enumerate(tq):\n        with torch.no_grad():\n            images, targets, ids = batch\n            images, targets = images.cuda(), targets.cuda()\n            \n            outputs = model(images)\n            \n            #loss = criterion(outputs, targets)\n            \n            outputs = torch.max(outputs, -1)[0].unsqueeze(-1)\n            targets = torch.max(targets, -1)[0].unsqueeze(-1)\n            \n            OUTPUTS.extend(model.sigmoid(outputs).detach().cpu().numpy()[:, :])\n            TARGETS.extend(targets.detach().cpu().numpy()[:, :])\n            IDS.extend(ids)\n            \n        #running_loss += loss.item()\n        \n        #tq.set_postfix(valid_loss=running_loss/(i+1))\n        \n        if DEBUG and i==30: break\n    \n    #return np.array(OUTPUTS), np.array(TARGETS)\n    \n    OUTPUTS = np.array(OUTPUTS)\n    TARGETS = np.array(TARGETS)\n    IDS = np.array(IDS)\n    \n    score = _score(OUTPUTS, TARGETS)\n    \n    print(score)\n    \n    score = roc_auc_score(TARGETS, OUTPUTS)\n    \n    print(score)\n    \n    return score\n    \n        \ndef run(model, train_loader, valid_loader):\n    epochs = []\n    scores = []\n    \n    best_score = float('-inf')\n    for epoch in range(CFG.EPOCHS):\n        train_one_epoch(model, train_loader)\n        \n        score = test_one_epoch(f\"{OUTPUT_FOLDER}/curr_f{CFG.FOLD}.pth\", valid_loader)\n        print(f\"EPOCH {epoch+1} SCORE:\", score)\n            \n        epochs.append(epoch)\n        scores.append(score)\n            \n        pd.DataFrame({'epoch': epochs, 'score': scores}).to_csv(f\"{OUTPUT_FOLDER}/log_f{CFG.FOLD}.csv\", index=False)\n\n        if score>best_score:\n            torch.save(model.state_dict(), f\"{OUTPUT_FOLDER}/best_f{CFG.FOLD}.pth\")\n            best_score = score\n\n        try:\n            command.run(['rm', '-r', CFG.cache_dir])\n            pass\n        except:\n            pass\n\n        os.makedirs(CFG.cache_dir, exist_ok=1)\n            \n        scheduler.step()\n        \n        #break","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:34:54.830909Z","iopub.execute_input":"2022-10-31T07:34:54.831324Z","iopub.status.idle":"2022-10-31T07:34:54.861258Z","shell.execute_reply.started":"2022-10-31T07:34:54.831289Z","shell.execute_reply":"2022-10-31T07:34:54.86021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for F in range(5):\n    \n    print(f\"FOLD {F}\")\n    \n    CFG.FOLD = F\n    train_loader, valid_loader = get_loaders()\n    \n    model = Model().cuda()\n    \n    criterion, optimizer, scheduler, scaler = define_criterion_optimizer_scheduler_scaler(model)\n\n    run(model, train_loader, valid_loader)\n    \n    print()\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:36:03.920733Z","iopub.execute_input":"2022-10-31T07:36:03.92113Z","iopub.status.idle":"2022-10-31T07:37:52.455799Z","shell.execute_reply.started":"2022-10-31T07:36:03.921092Z","shell.execute_reply":"2022-10-31T07:37:52.454046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **PSEUDO TRAIN**","metadata":{}},{"cell_type":"code","source":"DIR = '/kaggle/input/'\n\ndata = pd.read_csv(f'{DIR}/rsna-2022-cervical-spine-fracture-detection/train.csv')\ntrue = data.copy()\ntruth = data.copy()\nbox = pd.read_csv(f'{DIR}/rsna-2022-cervical-spine-fracture-detection/train_bounding_boxes.csv')\n\n\npseudo = pd.read_csv(f'{DIR}/try3-pseudo-round1/try3_pseudo_round1.csv')\n\npseudo.fractured = 0\npseudo = pseudo.fillna(-1)\nrows = pseudo[pseudo.prediction!=-1]\nthresh = np.quantile(rows.prediction, .4)\nidxs = rows[rows.prediction>thresh].index.values\nfor i in idxs: pseudo.loc[i, 'fractured'] = 1\n\npatient_to_classes = np.load(f\"{DIR}/classes-volumes-b1-v10/classes_volumes_b1v10.npy\", allow_pickle=1).item()\n\nidxs = [data[data.StudyInstanceUID==p].index.item() for p in pd.unique(box.StudyInstanceUID)]\nleftout_data = data.drop(idxs)\nleftout_data = leftout_data[leftout_data.patient_overall==0]\ndata = data.iloc[idxs].reset_index().drop('index', axis=1)\ndata = data.append(leftout_data[:280])\n\nStudyInstanceUIDs = []\nslice_numbers = []\nfractured = []\nfor patient in data.StudyInstanceUID:\n    paths = sorted(glob(f\"{DIR}/rsna-2022-cervical-spine-fracture-detection/train_images/{patient}/*\"), key=lambda x: int(x.split('/')[-1].split('.')[0]))\n    sls = box[box.StudyInstanceUID==patient].slice_number.values\n    for i in range(1, len(paths)+1):#[int(p.split('/')[-1].split('.')[0]) for p in paths]:\n        StudyInstanceUIDs.append(patient)\n        slice_numbers.append(i)\n        if i in sls:\n            fractured.append(1)\n        else: fractured.append(0)\n\n            \ndata = pd.DataFrame({'StudyInstanceUID': StudyInstanceUIDs, 'slice_number': slice_numbers, 'fractured': fractured})\n    \nrows = []\nfor i, row in tqdm(data.iterrows()):\n    x = patient_to_classes[row.StudyInstanceUID]\n    if x[row.slice_number-1] not in [100, 8] or row.fractured:\n        rows.append(row)\n        \ndata = pd.DataFrame(rows)\n    \ndata","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:42:43.724518Z","iopub.execute_input":"2022-10-31T07:42:43.725621Z","iopub.status.idle":"2022-10-31T07:43:06.916138Z","shell.execute_reply.started":"2022-10-31T07:42:43.725568Z","shell.execute_reply":"2022-10-31T07:43:06.914999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DDP = 0\n    DDP_INIT_DONE = 0\n    \n    SEED = 3407\n    SPLITS = 5\n    FOLD = 0\n    \n    SZ_H = 456\n    SZ_W = 456\n    \n    TRN_BS = 4 #WAS ORIGINALLY TRAINED ON 128 GLOBAL BATCH SIZE\n    VAL_BS = 4\n    ACCUMS = 1\n    \n    EPOCHS = 3\n    LR = 1e-5\n    WARMUP_EPOCHS = 3\n    WARMUP_LR = 1e-5\n    \n    FOLDER = \"/kaggle/working/\"\n    NAME = \"b5\"\n    V = \"5_pseudo_round1\"\n    \n#pl.seed_everything(CFG.SEED)\nseed_everything(CFG.SEED)\nOUTPUT_FOLDER = f\"{CFG.FOLDER}/{CFG.NAME}_v{CFG.V}/\"\n\nCFG.cache_dir = OUTPUT_FOLDER + '/cache/'\n\nos.makedirs(OUTPUT_FOLDER, exist_ok=1)\nos.makedirs(CFG.cache_dir, exist_ok=1)","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:44:21.14731Z","iopub.execute_input":"2022-10-31T07:44:21.147702Z","iopub.status.idle":"2022-10-31T07:44:21.155702Z","shell.execute_reply.started":"2022-10-31T07:44:21.147667Z","shell.execute_reply":"2022-10-31T07:44:21.154654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = [*StratifiedGroupKFold(n_splits=CFG.SPLITS).split(data, data.fractured, groups=data.StudyInstanceUID)]\nfolds_pseudo = [*StratifiedGroupKFold(n_splits=CFG.SPLITS).split(pseudo, pseudo.fractured, groups=pseudo.StudyInstanceUID)]\n\ndef get_loaders():\n    \n    train_paths = data.iloc[folds[CFG.FOLD][0]]\n    valid_paths = data.iloc[folds[CFG.FOLD][1]]\n    \n    pseudo_paths = pseudo.iloc[folds_pseudo[CFG.FOLD][0]]\n    train_paths = train_paths.append(pseudo_paths)\n    \n    #train_paths = train_paths.append([train_paths[train_paths.fractured==1]] * 1)\n    \n    train_augs = A.Compose([\n        #A.Resize(CFG.SZ_H, CFG.SZ_W),\n        A.LongestMaxSize(CFG.SZ_H),\n        A.PadIfNeeded(CFG.SZ_H, CFG.SZ_W, border_mode=0, p=1),\n        #A.RandomResizedCrop(CFG.SZ_H, CFG.SZ_W, ratio=[0.9, 1.1], scale=[0.9, 1.1]),\n        #A.OneOf([\n        #    A.Resize(CFG.SZ_H, CFG.SZ_W),\n        #    A.RandomResizedCrop(CFG.SZ_H, CFG.SZ_W, ratio=[0.6, 1.4], scale=[0.5, 1.5]),\n        #], p=1.),\n        A.Perspective(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        #A.VerticalFlip(p=0.25),\n        #A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=0, p=.2),\n        A.Rotate(p=0.5, limit=(45, -45)),\n        #A.RandomContrast(limit=(0.5, 0.5), p=1.),\n        A.RandomBrightnessContrast(p=0.5),\n        #A.Cutout(p=0.25, max_h_size=CFG.SZ_H//4, max_w_size=CFG.SZ_W//4, num_holes=4),\n        #A.Normalize(),\n        ToTensorV2(),\n    ])\n    \n    valid_augs = A.Compose([\n        #A.Resize(CFG.SZ_H, CFG.SZ_W),\n        A.LongestMaxSize(CFG.SZ_H),\n        A.PadIfNeeded(CFG.SZ_H, CFG.SZ_W, border_mode=0, p=1),\n        #A.RandomContrast(limit=(0.2, 0.2), p=1.),\n        #A.Normalize(),\n        ToTensorV2()\n    ])\n    \n    train_dataset = SpineDataset(train_paths, train_augs)\n    valid_dataset = SpineDataset(valid_paths, valid_augs)\n    \n    if CFG.DDP and CFG.DDP_INIT_DONE:\n        train_sampler = torch.utils.data.distributed.DistributedSampler(dataset=train_dataset, shuffle=True)\n        train_loader = DataLoader(train_dataset, batch_size=CFG.TRN_BS, sampler=train_sampler, num_workers=8, pin_memory=False)\n        \n        valid_sampler = torch.utils.data.distributed.DistributedSampler(dataset=valid_dataset, shuffle=False)\n        valid_loader = DataLoader(valid_dataset, batch_size=CFG.TRN_BS, sampler=valid_sampler, shuffle=False, num_workers=8, pin_memory=False)\n    else:\n        train_loader = DataLoader(train_dataset, batch_size=CFG.TRN_BS, shuffle=True, num_workers=8, pin_memory=False)\n        valid_loader = DataLoader(valid_dataset, batch_size=CFG.VAL_BS, shuffle=False, num_workers=8, pin_memory=False)\n    \n    return train_loader, valid_loader#, train_data, valid_data\n\ntrain_loader, valid_loader = get_loaders()\nfor d in valid_loader: break\nplt.imshow(d[0][0].numpy().transpose(1, 2, 0)[:, :, :3], cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:45:38.396761Z","iopub.execute_input":"2022-10-31T07:45:38.397218Z","iopub.status.idle":"2022-10-31T07:45:40.701588Z","shell.execute_reply.started":"2022-10-31T07:45:38.397172Z","shell.execute_reply":"2022-10-31T07:45:40.700499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for F in range(5):\n    \n    print(f\"FOLD {F}\")\n    \n    CFG.FOLD = F\n    train_loader, valid_loader = get_loaders()\n    \n    model = Model().cuda()\n    \n    model.load_state_dict(torch.load(f\"{DIR}/try17-b5-v5/best_f{CFG.FOLD}.pth\", map_location='cpu'), strict=False)\n    \n    criterion, optimizer, scheduler, scaler = define_criterion_optimizer_scheduler_scaler(model)\n\n    run(model, train_loader, valid_loader)\n    \n    print()\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T07:47:54.548438Z","iopub.execute_input":"2022-10-31T07:47:54.5496Z","iopub.status.idle":"2022-10-31T07:50:45.056168Z","shell.execute_reply.started":"2022-10-31T07:47:54.549556Z","shell.execute_reply":"2022-10-31T07:50:45.054955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"THE AUC'S IN SOME OF THE FOLDS ABOVE ARE REALLY HIGH, THAT IS JUST BECAUSE WE ARE BREAKING AT FIRST 30 STEPS OF VALIDATION, TURNING DEBUG OFF AND TRAINING WILL FIX THAT","metadata":{}},{"cell_type":"markdown","source":"THIS NOTEBOOK IS ADVICED NOT TO BE RUN ON KAGGLE AS THE TIME LIMIT OF 9 HOURS WILL SUPRESS IT FROM FINISHING, LOCAL HARDWARE OR NO TIME LIMIT CLOUD SERVICES MUST BE USED TO AVOID THIS LIMIT","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}