{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"package_path = '../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)\n!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nfrom efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\n\nCFG = {\n    'train': True,\n    \n    'train_img_path': '../input/rsna-str-pulmonary-embolism-detection/train',\n    'test_img_path': '../input/rsna-str-pulmonary-embolism-detection/test',\n    'cv_fold_path': '../input/stratified-validation-strategy/rsna_train_splits_fold_20.csv',\n    'train_path': '../input/rsna-str-pulmonary-embolism-detection/train.csv',\n    'test_path': '../input/rsna-str-pulmonary-embolism-detection/test.csv',\n    \n    'image_target_cols': [\n        'pe_present_on_image', # only image level\n    ],\n    \n    'exam_target_cols': [\n        'negative_exam_for_pe', # exam level\n        #'qa_motion',\n        #'qa_contrast',\n        #'flow_artifact',\n        'rv_lv_ratio_gte_1', # exam level\n        'rv_lv_ratio_lt_1', # exam level\n        'leftsided_pe', # exam level\n        'chronic_pe', # exam level\n        #'true_filling_defect_not_pe',\n        'rightsided_pe', # exam level\n        'acute_and_chronic_pe', # exam level\n        'central_pe', # exam level\n        'indeterminate' # exam level\n    ], \n   \n    'img_size': 256,\n    'lr': 0.0005,\n    'epochs': 1,\n    'device': 'cuda', # cuda, cpu\n    'train_bs': 64,\n    'valid_bs': 256,\n    'accum_iter': 1,\n    'verbose_step': 1,\n    'num_workers': 4,\n    'efbnet': 'efficientnet-b0',\n    \n    'train_folds': [np.arange(0,1),#np.arange(0,16),\n                    #np.concatenate([np.arange(0,12), np.arange(16,20)]),\n                    #np.concatenate([np.arange(0,8), np.arange(12,20)]),\n                    #np.concatenate([np.arange(0,4), np.arange(8,20)]),\n                    #np.arange(4,20),\n                   ],#[np.arange(0,16)],\n    \n    'valid_folds': [np.arange(16, 17),\n                    #np.arange(16,20),\n                    #np.arange(12,16),\n                    #np.arange(8,12),\n                    #np.arange(4,8),\n                    #np.arange(0,4)\n                   ],#[np.arange(16,20)],\n    \n    'model_path': '../input/kh-rsna-model',\n    'tag': 'efb0_stage1_example'\n}\n\nSEED = 42321\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\ndef window_min_max(img, min_, max_, WL=50, WW=350):\n    upper, lower = WL+WW//2, WL-WW//2\n    X = np.clip(img.copy(), lower, upper)\n    #min_ = max(min_, lower)\n    #max_ = min(max_, upper)\n    #X = (X - min_) / (max_-min_)\n    X = X - np.min(X)\n    X = X / np.max(X)\n    \n    #X = (X*255.0).astype('uint8')\n    return X\n\ndef get_img_min_max(path, min_, max_):\n    # min_: patient level pixel min\n    # max_: patient level pixel max\n    \n    d = pydicom.read_file(path)\n    '''\n    res = cv2.resize((d.pixel_array - d.RescaleIntercept) / (d.RescaleSlope * 1000), (CFG['img_size'], CFG['img_size'])), d.ImagePositionPatient[2]\n    '''\n    \n    '''\n    RED channel / LUNG window / level=-600, width=1500\n    GREEN channel / PE window / level=100, width=700\n    BLUE channel / MEDIASTINAL window / level=40, width=400\n    '''\n    \n    img = (d.pixel_array * d.RescaleSlope) + d.RescaleIntercept\n    \n    r = window_min_max(img, min_, max_, -600, 1500)\n    g = window_min_max(img, min_, max_, 100, 700)\n    b = window_min_max(img, min_, max_, 40, 400)\n    \n    res = np.concatenate([r[:, :, np.newaxis],\n                          g[:, :, np.newaxis],\n                          b[:, :, np.newaxis]], axis=-1)\n    \n    #res = (res*255.0).astype('uint8')\n    res = zoom(res, [CFG['img_size']/res.shape[0], CFG['img_size']/res.shape[1], 1.], prefilter=False, order=1)\n    #res = res.astype(np.float32)/255.\n    \n    return res\n\n    '''\n    \n    img -= img.min()\n    img /= img.max()\n    return img[:, :, np.newaxis]\n    '''\n\ndef get_meta(path):\n    x = pydicom.read_file(path)\n    loc = x.ImagePositionPatient[2]\n    img_min = x.pixel_array.min()\n    img_max = x.pixel_array.max()\n    return (loc, img_min, img_max)\n\ndef update_image_metas(df, data_root):\n    from multiprocessing import Pool\n    from tqdm import tqdm\n    \n    t = time.time()\n    paths = data_root + \"/\" + df.StudyInstanceUID.apply(str) + \"/\" + df.SeriesInstanceUID.apply(str) + \"/\" + df.SOPInstanceUID.apply(str) + \".dcm\"\n    print(type(paths))\n    print('paths num = {:d}'.format(len(paths)))\n    \n    with Pool(CFG['num_workers']) as pool:\n        locs = []\n        img_mins = []\n        img_maxs = []\n        \n        for p in tqdm(df.StudyInstanceUID.unique()):\n            #print(paths[df.StudyInstanceUID==p])\n            meta = list(pool.map(get_meta, list(paths[df.StudyInstanceUID==p])))\n            locs_, img_mins_, img_maxs_ = map(list, zip(*meta))\n            locs += locs_\n            img_mins += img_mins_\n            img_maxs += img_maxs_\n    \n    assert len(locs) == df.shape[0]\n    df['zpos'] = locs\n    df['img_min'] = img_mins\n    df['img_max'] = img_maxs\n    df.img_min = df.StudyInstanceUID.map(df.groupby('StudyInstanceUID')['img_min'].min()) # group into patient level\n    df.img_max = df.StudyInstanceUID.map(df.groupby('StudyInstanceUID')['img_max'].max())\n    \n    print(\"Update meta complete: {:.4f} secs\".format(time.time()-t))\n    \n    '''\n    for p in df.StudyInstanceUID.unique():\n        df_ = df.loc[(df.StudyInstanceUID==p) & (df.pe_present_on_image == 1),]\n        if df_.shape[0] > 1:\n            print(df_.zpos.min(), df_.zpos.max(), (df_.zpos.max()-df_.zpos.min())/df_.shape[0])\n    '''        \n    return df\n\nclass RSNADataset(Dataset):\n    def __init__(\n        self, df, label_smoothing, data_root, \n        image_subsampling=True, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        #self.df = update_image_metas(self.df, data_root)\n        \n        self.label_smoothing = label_smoothing\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index][CFG['image_target_cols'][0]]\n          \n        path = \"{}/{}/{}/{}.dcm\".format(self.data_root, \n                                        self.df.iloc[index]['StudyInstanceUID'], \n                                        self.df.iloc[index]['SeriesInstanceUID'], \n                                        self.df.iloc[index]['SOPInstanceUID'])\n        \n        img  = get_img_min_max(path, 0, 0)\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        # do label smoothing\n        if self.output_label == True:\n            target = np.clip(target, self.label_smoothing, 1 - self.label_smoothing)\n            \n            return img, target\n        else:\n            return img\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate \n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            RandomRotate90(p=0.5),\n            #ShiftScaleRotate(p=0.5),\n            #RandomRotate90(p=0.5),\n            #HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            #RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            #RandomResizedCrop(CFG['img_size'], CFG['img_size'], scale=(0.9, 1.0), ratio=(0.9, 1.1), p=0.5),\n            #Cutout(p=1),\n            #CoarseDropout(p=0.5),\n            #Normalize(mean=(0.456, 0.456, 0.456), std=(0.224, 0.224, 0.224), max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n    '''\n    return transforms.Compose([\n            transforms.Lambda(lambda imgs: torch.stack([transforms.ToTensor()(img) for img in imgs])),\n            transforms.Lambda(lambda imgs: torch.stack([transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                                             std=[0.229, 0.224, 0.225])(img) for img in imgs])),\n           \n        ])\n    '''   \n        \ndef get_valid_transforms():\n    return Compose([\n            #Normalize(mean=(0.456, 0.456, 0.456), std=(0.224, 0.224, 0.224), max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n    '''\n    return transforms.Compose([\n            transforms.Lambda(lambda imgs: torch.stack([transforms.ToTensor()(img) for img in imgs])),\n            transforms.Lambda(lambda imgs: torch.stack([transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                                             std=[0.229, 0.224, 0.225])(img) for img in imgs])),\n           \n        ])\n    '''  \n\n    \n    \nclass RNSAImageFeatureExtractor(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn_model = EfficientNet.from_pretrained(CFG['efbnet'], in_channels=3)\n        #print(self.cnn_model, CFG['efbnet'])\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        \n    def get_dim(self):\n        return self.cnn_model._fc.in_features\n        \n    def forward(self, x):\n        feats = self.cnn_model.extract_features(x)\n        return self.pooling(feats).view(x.shape[0], -1)                         \n\nclass RSNAImgClassifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn_model = RNSAImageFeatureExtractor()\n        self.image_predictors = nn.Linear(self.cnn_model.get_dim(), 1)\n        \n    def forward(self, imgs):\n        #print(images.shape)\n        imgs_embdes = self.cnn_model(imgs) # bs * efb_feat_size\n        #print(imgs_embdes.shape)\n        image_preds = self.image_predictors(imgs_embdes)\n        \n        return image_preds\n'''\nclass RSNAImgClassifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        import torchvision\n        self.cnn_model = torchvision.models.resnext101_32x8d(pretrained=True, progress=True)\n        self.cnn_model.fc = nn.Linear(self.cnn_model.fc.in_features, 1)\n        \n    def get_dim(self):\n        return self.cnn_model.fc.in_features\n        \n    def forward(self, x):\n        return self.cnn_model(x)\n'''\n\n#RSNAClassifier(64)\ndef rsna_wloss_inference(y_true_img, y_pred_img):\n    bce_func = torch.nn.BCELoss(reduction='sum')\n    image_loss = bce_func(y_pred_img, y_true_img)\n    correct_count = ((y_pred_img>0) == y_true_img).sum()\n    counts = y_pred_img.shape[0]\n    return image_loss, correct_count, counts\n\ndef rsna_wloss_train(y_true_img, y_pred_img, device):\n    bce_func = torch.nn.BCEWithLogitsLoss(reduction='sum').to(device)\n    y_pred_img = y_pred_img.flatten()\n    image_loss = bce_func(y_pred_img, y_true_img)\n    correct_count = ((y_pred_img>0) == (y_true_img>0.5)).sum(axis=0)\n    counts = y_true_img.size()[0]\n    \n    return image_loss, correct_count, counts\n\ndef rsna_wloss_valid(y_true_img, y_pred_img, device):\n    return rsna_wloss_train(y_true_img, y_pred_img, device)\n\ndef prepare_train_dataloader(train, cv_df, train_fold, valid_fold):\n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_patients = cv_df.loc[cv_df.fold.isin(train_fold), 'StudyInstanceUID'].unique()\n    valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()\n\n    train_ = train.loc[train.StudyInstanceUID.isin(train_patients),:].reset_index(drop=True)\n    valid_ = train.loc[train.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)\n\n    # train mode to do image-level subsampling\n    train_ds = RSNADataset(train_, 0.0, CFG['train_img_path'],  image_subsampling=False, transforms=get_train_transforms(), output_label=True) \n    valid_ds = RSNADataset(valid_, 0.0, CFG['train_img_path'],  image_subsampling=False, transforms=get_valid_transforms(), output_label=True)\n\n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_[CFG['image_target_cols'][0]].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    #print(len(train_loader), len(val_loader))\n\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, device, scaler, optimizer, train_loader):\n    model.train()\n\n    t = time.time()\n    loss_sum = 0\n    acc_sum = 0\n    loss_w_sum = 0\n\n    for step, (imgs, image_labels) in enumerate(train_loader):\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).float()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds = model(imgs)   #output = model(input)\n            #print(image_preds.shape, exam_pred.shape)\n\n            image_loss, correct_count, counts = rsna_wloss_train(image_labels, image_preds, device)\n            \n            loss = image_loss/counts\n            scaler.scale(loss).backward()\n\n            loss_sum += image_loss.detach().item()\n            acc_sum += correct_count.detach().item()\n            loss_w_sum += counts\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()                \n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                print(\n                    f'epoch {epoch} train step {step+1}/{len(train_loader)}, ' + \\\n                    f'loss: {loss_sum/loss_w_sum:.4f}, ' + \\\n                    f'acc: {acc_sum/loss_w_sum:.4f}, ' + \\\n                    f'time: {(time.time() - t):.4f}', end= '\\r' if (step + 1) != len(train_loader) else '\\n'\n                )\n\ndef valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    acc_sum = 0\n    loss_w_sum = 0\n\n    for step, (imgs, image_labels) in enumerate(val_loader):\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n\n        image_loss, correct_count, counts = rsna_wloss_valid(image_labels, image_preds, device)\n\n        loss = image_loss/counts\n        \n        loss_sum += image_loss.detach().item()\n        acc_sum += correct_count.detach().item()\n        loss_w_sum += counts     \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            print(\n                f'epoch {epoch} valid Step {step+1}/{len(val_loader)}, ' + \\\n                f'loss: {loss_sum/loss_w_sum:.4f}, ' + \\\n                f'acc: {acc_sum/loss_w_sum:.4f}, ' + \\\n                f'time: {(time.time() - t):.4f}', end='\\r' if (step + 1) != len(val_loader) else '\\n'\n            )\n    \n    if schd_loss_update:\n        scheduler.step(loss_sum/loss_w_sum)\n    else:\n        scheduler.step()\n        \ndef inference(model, device, df, root_path):\n    model.eval()\n\n    t = time.time()\n\n    ds = RSNADataset(df, 0.0, root_path,  image_subsampling=False, transforms=get_valid_transforms(), output_label=False)\n    \n    dataloader = torch.utils.data.DataLoader(\n        ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=True,\n    )\n    \n    for step, (imgs, locs, img_num, index, seq_ix) in enumerate(dataloader):\n        imgs = imgs.to(device).float()\n        locs = locs.to(device).float()\n        \n        index = index.detach().numpy()[0]\n        seq_ix = seq_ix.detach().numpy()[0,:]\n        \n        patient_filt = (df.StudyInstanceUID == patients[index])\n        \n        patient_df = pd.DataFrame()\n        patient_df['SOPInstanceUID'] = df.loc[patient_filt, 'SOPInstanceUID'].values[seq_ix]\n        patient_df['SeriesInstanceUID'] = df.loc[patient_filt, 'SeriesInstanceUID'].values # no need to sort\n        patient_df['StudyInstanceUID'] = patients[index] # single value\n        \n        for c in CFG['image_target_cols']+CFG['exam_target_cols']:\n            patient_df[c] = 0.0\n\n        #with autocast():\n        image_preds, exam_pred = model(imgs, locs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        \n        exam_pred, image_preds = post_process(exam_pred, image_preds)\n        \n        exam_pred = torch.sigmoid(exam_pred).cpu().detach().numpy()\n        image_preds = torch.sigmoid(image_preds).cpu().detach().numpy()\n\n        patient_df[CFG['exam_target_cols']] = exam_pred[0]\n        patient_df[CFG['image_target_cols']] = image_preds[0,:]\n        res_dfs += [patient_df]\n\n        '''\n        res_df = res_df.merge(patient_df, on=['SOPInstanceUID', 'StudyInstanceUID'], how='left')\n        '''\n        # naive slow version\n        '''\n        res_df.loc[patient_filt, CFG['exam_target_cols']] = exam_pred[0]\n        for si, sop_id in enumerate(sop_ids):\n            sop_filt = (patient_filt) & (res_df.SOPInstanceUID == sop_id)\n            res_df.loc[sop_filt, CFG['image_target_cols']] = image_preds[0, si]\n        '''\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(dataloader)):\n            print(\n                f'Inference Step {step+1}/{len(dataloader)}, ' + \\\n                f'time: {(time.time() - t):.4f}', end='\\r' if (step + 1) != len(dataloader) else '\\n'\n            )\n                \n    res_dfs = pd.concat(res_dfs, axis=0).reset_index(drop=True)\n    res_dfs = df[['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID']].merge(res_dfs, on=['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID'], how='left')\n    print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].head(5))\n    print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].tail(5))\n    assert res_dfs.shape[0] == df.shape[0]\n    check_label_consistency(res_dfs)\n    \n    return res_dfs\n    \n    \nif __name__ == '__main__':\n    if CFG['train']:\n        from  torch.cuda.amp import autocast, GradScaler # for training only, need nightly build pytorch\n\n    seed_everything(SEED)\n    \n    if CFG['train']:\n        # read train file\n        train_df = pd.read_csv(CFG['train_path'])\n\n        # read cv file\n        cv_df = pd.read_csv(CFG['cv_fold_path'])\n\n        # img must be sorted before feeding into NN for correct orders\n    else:\n        #assert False, \"This kernel is for training only!\"\n        # read test file\n        test_df = pd.read_csv(CFG['test_path'])\n    \n    if CFG['train']:\n        \n        for fold, (train_fold, valid_fold) in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])):\n            if fold < 0:\n                continue\n            print(fold)   \n            \n            train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, train_fold, valid_fold)\n\n            device = torch.device(CFG['device'])\n            model = RSNAImgClassifier().to(device)\n            scaler = GradScaler()   \n            optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'])\n            #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=1); schd_loss_update=True\n            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False\n\n            for epoch in range(CFG['epochs']):\n                train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)\n\n                with torch.no_grad():\n                    valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=schd_loss_update)\n\n            torch.save(model.state_dict(),'{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n            #torch.save(model.cnn_model.state_dict(),'{}/cnn_model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n            del model, optimizer, train_loader, val_loader, scaler, scheduler\n            torch.cuda.empty_cache()\n         \n        # train a final stage 1 model for testing\n        '''\n        train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, np.arange(0, 20), np.array([]))\n        #print(len(train_loader), len(val_loader))\n        device = torch.device(CFG['device'])\n        model = RSNAImgClassifier().to(device)\n        scaler = GradScaler()   \n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'])\n        #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=1); schd_loss_update=True\n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False\n\n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)\n\n        torch.save(model.state_dict(),'{}/model_{}'.format(CFG['model_path'], CFG['tag']))\n        '''\n    else:\n        assert False","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}