{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPUs Attempt at running  Effnet on this massive dataset\n\nuses the Ian Pan Jpeg datset:\n\nhttps://www.kaggle.com/vaillant/rsna-str-pe-detection-jpeg-256\n\nand some tricks from this Pytorch notebook:\n\nhttps://www.kaggle.com/orkatz2/pulmonary-embolism-pytorch-train\n\nas well as some standard Pytorch from the tutorials page:\n\nhttps://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html#load-data\n\nWith TPU support from this Notebook:\n\nhttps://www.kaggle.com/tanulsingh077/pytorch-xla-understanding-tpu-s-and-xla"},{"metadata":{},"cell_type":"markdown","source":"## Results so far\n\n* got it running on one core but still takes 55mins per epoch!\n* now parallelising to run on the 8 cores available."},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\nfrom sklearn.model_selection import train_test_split\nimport transformers\nfrom transformers import AdamW\n\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla\nimport torch_xla.debug.metrics as met\n\nimport torch_xla.utils.utils as xu\n\nimport torch_xla.test.test_utils as test_utils\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U git+git://github.com/lilohuang/PyTurboJPEG.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from turbojpeg import TurboJPEG, TJPF_GRAY, TJSAMP_GRAY\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torch_optimizer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport cv2\nimport os\nfrom matplotlib import pyplot as plt\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom skimage.color import gray2rgb\nimport functools\nimport torch\nfrom torch import Tensor\nfrom torchvision import transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.nn.parameter import Parameter\nfrom torch.optim import lr_scheduler\nimport torch_optimizer as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nfrom tqdm.auto import tqdm\nfrom matplotlib import animation, rc\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\nimport pydicom as dcm\nimport gdcm\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#torch.backends.cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/rsna-str-pulmonary-embolism-detection/train.csv'\ntest_path = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/test.csv\")\njpeg_path = '../input/rsna-str-pe-detection-jpeg-256/train-jpegs'\nfiles = glob.glob('../input/rsna-str-pulmonary-embolism-detection/train/*/*/*.dcm')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=pd.read_csv(train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_of_jpgs= [i for i in range(1,10)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns_only_for_info = [\"qa_motion\",\"qa_contrast\",\"flow_artifact\",\"true_filling_defect_not_pe\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Can take the onlyforinfo cols out as they are not needed."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.iloc[list_of_jpgs,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_columns = ['StudyInstanceUID','SeriesInstanceUID','SOPInstanceUID','pe_present_on_image', 'negative_exam_for_pe', 'rv_lv_ratio_gte_1', \n                  'rv_lv_ratio_lt_1','leftsided_pe', 'chronic_pe','rightsided_pe', \n                  'acute_and_chronic_pe', 'central_pe', 'indeterminate']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[target_columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class jpg_dataset(Dataset):\n    def __init__(self, root_dir, df,  transforms = None):\n        \n        super().__init__()\n        self.df = df[target_columns]\n        self.root_dir = root_dir\n        self.transforms = transforms\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, ndx):\n        row = self.df.iloc[ndx,:]\n        in_file = open(glob.glob(f\"{root_dir}/{row[0]}/{row[1]}/*{row[2]}.jpg\")[0], 'rb')\n        img = jpeg.decode(in_file.read())\n        in_file.close()\n        label = row[3:].astype(int)\n        label[2:] = label[2:] if label[0]==1 else 0\n        #img /= 255.0\n        \n        #Retriving class label\n\n        \n        #Applying transforms on image\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n\n        return (img,label.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root_dir=jpeg_path\njpeg = TurboJPEG()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Trial of dataset class"},{"metadata":{},"cell_type":"markdown","source":"## Real Dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"StudyInstanceUID = list(set(train_df['StudyInstanceUID']))\nprint(len(StudyInstanceUID))\nt_df = train_df[train_df['StudyInstanceUID'].isin(StudyInstanceUID[0:500])]\nv_df = train_df[train_df['StudyInstanceUID'].isin(StudyInstanceUID[500:600])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.environ['XLA_USE_BF16']=\"1\"\nos.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data augmentation and normalization for training\n# Just normalization for validation\ndata_transforms = {\n    'train': A.Compose(\n    [\n        #A.SmallestMaxSize(max_size=160),\n        #A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=15, p=0.5),\n        A.RandomCrop(height=128, width=128),\n        #A.RGBShift(r_shift_limit=15, g_shift_limit=15, b_shift_limit=15, p=0.5),\n        A.RandomBrightnessContrast(p=0.5),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2(),\n    ]\n),\n    'val': A.Compose(\n    [\n        A.SmallestMaxSize(max_size=160),\n        A.CenterCrop(height=128, width=128),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2(),\n    ]\n),\n}\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build model"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetEncoderHead(nn.Module):\n    def __init__(self, depth, num_classes):\n        super(EfficientNetEncoderHead, self).__init__()\n        self.depth = depth\n        self.base = EfficientNet.from_pretrained(f'efficientnet-b{self.depth}')\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.output_filter = self.base._fc.in_features\n        self.classifier = nn.Linear(self.output_filter, num_classes)\n    def forward(self, x):\n        x = self.base.extract_features(x)\n        x = self.avg_pool(x).squeeze(-1).squeeze(-1)\n        x = self.classifier(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = EfficientNetEncoderHead(depth=0, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, no_epochs=25):\n    since = time.time()\n    criterion = torch.nn.BCEWithLogitsLoss()\n    batch_size = 16\n    no_epochs = 4\n    num_workers = 0\n    lr = 1e-3\n    device = xm.xla_device()\n    model = model.to(device)\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    # Not sure if creating datasets and dataloaders in the train fn will cause OOM\n    #print(\"Initializing Datasets and Dataloaders...\")\n\n    # Create training and validation datasets\n    image_datasets = {}\n    image_datasets['train'] = jpg_dataset(root_dir=jpeg_path,df=t_df, transforms = data_transforms['train'])\n    image_datasets['val'] = jpg_dataset(root_dir=jpeg_path,df=v_df, transforms = data_transforms['val'])\n    dataloaders_dict={}\n    # Create training and validation dataloaders\n    train_sampler = None\n    if xm.xrt_world_size() > 1:\n        #print ('more than one core found')\n        train_sampler = torch.utils.data.distributed.DistributedSampler(\n              image_datasets['train'],\n              num_replicas=xm.xrt_world_size(),\n              rank=xm.get_ordinal(),\n              shuffle=True)\n    dataloaders_dict['train'] = torch.utils.data.DataLoader(\n                image_datasets['train'],\n                batch_size=batch_size,\n                sampler=train_sampler,\n                shuffle=False if train_sampler else True,\n                num_workers=num_workers)\n    dataloaders_dict['val'] = torch.utils.data.DataLoader(\n            image_datasets['val'],\n            batch_size=batch_size,\n            shuffle=False,\n            num_workers=num_workers)\n    dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n    device_loader={}\n    para_loader={}\n    #device_loader['train'] = pl.MpDeviceLoader(dataloaders_dict['train'], device)\n    #device_loader['val'] = pl.MpDeviceLoader(dataloaders_dict['val'], device)\n    lr = lr * xm.xrt_world_size()\n    optimizer = radam(model.parameters(), lr=lr, betas=(0.9,0.999), eps=1e-3, weight_decay=1e-4)\n    scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(dataloaders_dict['train'])*no_epochs, eta_min=1e-6) # dataloader should be train one - change later\n\n    # Decay LR by a factor of 0.1 every 7 epochs\n    exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n        \n    for epoch in range(no_epochs):\n        para_loader['train'] = pl.ParallelLoader(dataloaders_dict['train'], [device]).per_device_loader(device)\n        para_loader['val'] = pl.ParallelLoader(dataloaders_dict['val'], [device]).per_device_loader(device)\n\n        xm.master_print('Epoch {}/{}'.format(epoch, no_epochs - 1))\n        xm.master_print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            current_loss_mean = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            tqdm_loader = tqdm(para_loader[phase])\n\n            for batch_idx, (inputs,labels) in enumerate(tqdm_loader):\n                inputs = inputs.to(device).float()\n                labels = labels.to(device).float()\n                \n                #inputs, labels = inputs.cuda().float(), labels.cuda().float() \n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history only if in train\n                with torch.set_grad_enabled(phase == 'train'):\n\n                    outputs = model(inputs)\n                    loss = criterion(outputs.float(), labels.float())\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        #if batch_idx % 10 == 0:\n                        #    xm.master_print(f'batch_idx={batch_idx} of length {len(para_loader[phase])}, loss={loss}')\n                        #optimizer.step()\n                        xm.optimizer_step(optimizer)\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                current_loss_mean = (current_loss_mean * batch_idx + loss) / (batch_idx + 1)\n                if batch_idx % 10 == 0:\n                    xm.master_print(f'batch_idx={batch_idx} of length {len(para_loader[phase])}, loss={current_loss_mean}')\n                running_corrects += torch.sum(outputs == labels.data)\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            score = 1-current_loss_mean\n            xm.master_print('metric {}'.format(score))\n            \n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            xm.master_print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time() - since\n    xm.master_print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    xm.master_print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    xm.save(model.state_dict(),f'model{k}.bin')\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def radam(parameters, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n    if isinstance(betas, str):\n        betas = eval(betas)\n    return optim.RAdam(parameters,\n                      lr=lr,\n                      betas=betas,\n                      eps=eps,\n                      weight_decay=weight_decay)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"flags={}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_model(model, criterion, optimizer, scheduler, no_epochs=no_epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n    global acc_list\n    torch.set_default_tensor_type('torch.FloatTensor')\n    a = train_model(model=model, no_epochs=3)\n\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}