{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch > /dev/null\n!pip install -q albumentations > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install pytorch-lightning==0.9.1rc4","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Imports\nimport torch\nimport torch.nn as nn\n\nimport torchvision\nimport torchvision.transforms as transforms\n\nimport torch_xla\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.debug.metrics as met\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport cv2\nimport matplotlib.pyplot as plt\nfrom collections import defaultdict \nfrom efficientnet_pytorch import EfficientNet\n\n \nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom torch import optim\nfrom torchvision import datasets, transforms, models\n\n\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom torchvision.transforms import ToTensor, RandomHorizontalFlip, Resize\nfrom efficientnet_pytorch import EfficientNet\nfrom transformers import AdamW, get_cosine_schedule_with_warmup\nfrom albumentations import *\nfrom albumentations.pytorch import ToTensor\nfrom tqdm import tqdm\nimport json\nimport time\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"saved_df = pd.read_csv('/kaggle/input/rsna-train-df-with-jpg/train_df_with_jpg_file_names.csv')\ntrain_df = saved_df[['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID', 'pe_present_on_image', 'new_file_names']]\ntrain_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['path'] = '/kaggle/input/rsna-str-pe-detection-jpeg-256/train-jpegs/' \\\n                        + train_df['StudyInstanceUID'].astype(str) + '/'\\\n                        + train_df['SeriesInstanceUID'].astype(str) +'/'\\\n                        + train_df['new_file_names']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image = cv2.imread(train_df.loc[1]['path'])\nimage.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# RNSA Model building part"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import wandb\n# wandb.login(key ='38308edac7fc6d1cdb1f4753fa958995f23cd110')\n# causes the training to not end... :(","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run = wandb.init(project='SNAR bare min logger',\n#                  name = '1k rows Adam',\n#                  notes = 'R EB0 train_df[:600000]',\n#                  config={  # and include hyperparameters and metadata\n#                      \"learning_rate\": 1e-3, # from lr rate finder\n#                      \"epochs\": 2\n#                  })\n# config = wandb.config  # We'll use this to configure our experiment","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"https://www.kaggle.com/c/rsna-intracranial-hemorrhage-detection/discussion/112290  "},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nimport pytorch_lightning as pl\nfrom torch.utils.data import DataLoader, random_split\nfrom torch.nn import functional as F\nfrom pytorch_lightning.metrics.functional import accuracy\nfrom torchvision import datasets, transforms\n# import os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# or use it below like\n# train_df_subset = train_df[:4000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SimpleDataset(Dataset):\n            def __init__(self, image_ids_df, labels_df, transform=None):\n                self.image_ids = image_ids_df\n                self.labels = labels_df\n                self.transform = transform\n\n            def __getitem__(self, idx):\n                image = cv2.imread(self.image_ids.values[idx])\n                label = self.labels.values[idx]\n\n                sample = {\n                    'image': image,\n                    'label': label\n                }\n\n                if self.transform:\n                    sample = self.transform(**sample)\n\n                image, label = sample['image'], sample['label']\n\n                return image, label\n\n            def __len__(self):\n                return len(self.image_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nimport torch\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader\nfrom torchvision.datasets import MNIST\nfrom torchvision import transforms\nimport pytorch_lightning as pl\n\nclass RNSAModel(pl.LightningModule):\n\n    def __init__(self,  train_df = train_df):\n        super(RNSAModel, self).__init__()\n        \n        n_channels_dict = {'efficientnet-b0': 1280, 'efficientnet-b1': 1280, 'efficientnet-b2': 1408,\n                           'efficientnet-b3': 1536, 'efficientnet-b4': 1792, 'efficientnet-b5': 2048,\n                           'efficientnet-b6': 2304, 'efficientnet-b7': 2560}\n        self.encoder='efficientnet-b0'\n        self.net = EfficientNet.from_pretrained(self.encoder)\n        self.df = train_df\n        \n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.classifier = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(in_features=1280, out_features=2, bias=True) # hard code it  for now\n        )\n\n    def forward(self, x):\n        x = self.net.extract_features(x)\n        x = self.avg_pool(x)\n        #out =  torch.log_softmax(self.classifier(x))\n        out = self.classifier(x)\n\n        return out\n\n    def training_step(self, batch, batch_nb):\n        # REQUIRED\n        x, y = batch\n        y_hat = self(x)\n        loss = F.cross_entropy(y_hat, y)\n        tensorboard_logs = {'train_loss': loss}\n        return {'loss': loss, 'log': tensorboard_logs}\n\n    def validation_step(self, batch, batch_nb):\n        # OPTIONAL\n        x, y = batch\n        y_hat = self(x)\n        val_accuracy = accuracy(y_hat,y,num_classes = 2)\n        \n        return {'val_loss': F.cross_entropy(y_hat, y),\n                'val_accuracy' : val_accuracy\n               }\n\n    def validation_epoch_end(self, outputs):\n        # OPTIONAL\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        avg_accuracy = torch.stack([x['val_accuracy'] for x in outputs]).mean()\n        \n        tensorboard_logs = {'val_loss': avg_loss}\n        \n        # wandb.log({'val_loss': avg_loss})\n        # wandb.log({'val_acc': avg_accuracy})\n        print(f\"'avg_val_loss': {avg_loss} and 'val_acc': {avg_accuracy}\")\n        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n\n    def test_step(self, batch, batch_nb):\n        # OPTIONAL\n        x, y = batch\n        y_hat = self(x)\n        return {'test_loss': F.cross_entropy(y_hat, y)}\n\n    def test_epoch_end(self, outputs):\n        # OPTIONAL\n        avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()\n        logs = {'test_loss': avg_loss}\n        return {'avg_test_loss': avg_loss, 'log': logs, 'progress_bar': logs}\n\n    def configure_optimizers(self):\n        # REQUIRED\n        # can return multiple optimizers and learning_rate schedulers\n        # (LBFGS it is automatically supported, no need for closure function)\n        return torch.optim.Adam(self.parameters(), lr=0.001)\n    \n    def prepare_data(self):\n        # self.mnist_train = MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor())\n        # self.mnist_test = MNIST(os.getcwd(), train=False, download=True, transform=transforms.ToTensor())\n        image_ids = self.df['path']\n        labels = self.df['pe_present_on_image']\n\n        X_train, X_test, y_train, y_test = train_test_split(image_ids, labels, \n                                                            test_size=0.25, \n                                                            random_state=42, \n                                                            stratify =labels)\n        \n        train_transform = Compose([\n            Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225), always_apply=True),\n            ToTensor()\n            ])\n        \n        test_transform = Compose([\n            Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225), always_apply=True),\n            ToTensor()\n            ])\n        \n        self.train_ds = SimpleDataset(X_train, y_train, transform = train_transform)\n        self.val_ds = SimpleDataset(X_test, y_test, transform = test_transform)\n\n    def train_dataloader(self):\n        loader = DataLoader(self.train_ds, batch_size=16, num_workers=4)\n        return loader\n\n    def val_dataloader(self):\n        loader = DataLoader(self.val_ds, batch_size=16, num_workers=4)\n        return loader\n\n    def test_dataloader(self):\n        loader = DataLoader(self.test_ds, batch_size=16, num_workers=4)\n        # todo processing of dicom images\n        return loader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_batch_num = 1\ndata_batch_size  = 300000\n\nstart = data_batch_num*data_batch_size\nend = (data_batch_num+1)*data_batch_size\n\nif end > train_df.shape[0]:\n    end = train_df.shape[0]\n\n\ntrain_df_subset = train_df[start: end]\n\n(train_df_subset.shape,train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = RNSAModel(train_df = train_df_subset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pytorch_lightning.callbacks import ModelCheckpoint\ncheckpoint_callback = ModelCheckpoint(filepath='/kaggle/working/checkpoints',\n                                        save_top_k=1,\n                                        verbose=True,\n                                        monitor='val_loss',\n                                        mode='min',\n                                        prefix=''\n                                     )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\nPath('/kaggle/working/checkpoints').mkdir(parents = True, exist_ok =True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/input/rsna-pytorch-lightning-chkpts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run batch size scaling, result overrides hparams.batch_size\n# trainer = pl.Trainer(tpu_cores=8, \n#                      progress_bar_refresh_rate=20,\n#                      max_epochs=2,\n#                      auto_scale_batch_size='binsearch',# not available\n#                      checkpoint_callback=checkpoint_callback,\n#                      default_root_dir='/kaggle/working/checkpoints'\n#                     )\n\ntrainer = pl.Trainer(\n                     tpu_cores=8, \n#                      progress_bar_refresh_rate=20,\n                     max_epochs=2,\n#                      auto_scale_batch_size='binsearch',# not available\n                     checkpoint_callback=checkpoint_callback,\n                     resume_from_checkpoint='/kaggle/input/rsna-pytorch-lightning-chkpts/zero_300k_pass1_checkpoints-v0.ckpt',\n                     default_root_dir='/kaggle/working/checkpoints'\n                    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# call tune to find the batch size\ntrainer.fit(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# MyModel = RNSAModel(train_df = train_df[:1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # new_model = MyModel.load_from_checkpoint(checkpoint_path=\"/kaggle/working/checkpoints.ckpt\")\n# new_model = RNSAModel()\n# new_model.load_from_checkpoint(\"/kaggle/working/checkpoints.ckpt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run batch size scaling, result overrides hparams.batch_size\n# trainer = pl.Trainer(tpu_cores=8, \n#                      progress_bar_refresh_rate=20,\n#                      max_epochs=2,\n#                      auto_scale_batch_size='binsearch',# not available\n#                      checkpoint_callback=checkpoint_callback,\n#                      default_root_dir='/kaggle/working/checkpoints'\n#                     )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# trainer = pl.Trainer(resume_from_checkpoint='/kaggle/working/checkpoints.ckpt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# trainer.fit(model = new_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def weights_update(model, checkpoint):\n#     model_dict = model.state_dict()\n#     pretrained_dict = {k: v for k, v in checkpoint['state_dict'].items() if k in model_dict}\n#     model_dict.update(pretrained_dict)\n#     model.load_state_dict(model_dict)\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # https://github.com/PyTorchLightning/pytorch-lightning/issues/924\n# model = weights_update(model=EfficientNet.from_pretrained('efficientnet-b0'),\n#                        checkpoint=torch.load(\"/kaggle/working/checkpoints.ckpt\"))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}